Removed and refactored the database to import data directly from Arsenal.

.gitignore

@@ -1,2 +1 @@
-commands.raw
-commands.state
+data/commands.*

README.md

@@ -48,7 +48,7 @@ echo 'source ~/fms/integration.zsh' >> ~/.zshrc
 
 ## How to configure
 
-Each command is stored in the file commands.yaml. To add new commands or modify existing ones, edit commands.yaml with the following fields:
+Each command is stored in the file data/commands.yaml. To add new commands or modify existing ones, edit data/commands.yaml with the following fields:
 ```yaml
   - cmd: <the command to insert in your prompt>
     lang: <the language of your command, sh, ps1>
@@ -74,9 +74,14 @@ Then, to update the database and apply the changes, run the following commands:
 
 Warning, this may take a long time. 
 
-The Fast Memo Shell comes with a list of commands, but you are highly encouraged to update it according to your needs.
+# Populate database from Arsenal data
 
-The database provided as an example comes from a quick extract of the [Arsenal](https://github.com/Orange-Cyberdefense/arsenal) tool's database.
+The Fast Memo Shell can import data from the [Arsenal](https://github.com/Orange-Cyberdefense/arsenal) project, but you are highly encouraged to update it according to your needs.
+
+To generate a FMS-compatible commands.yaml file from Arsenal use, the following script:
+```
+./data/generate_arsenal_commands.sh
+```
 
 ## How to use
 
@@ -96,4 +101,3 @@ Fast Memo Shell can also display all available manuals. To switch between both m
 
 * Allow more language like java, cmd .. (with adapted comment chars for description)
 * Multithreaded update
-* Write a clean extractor for Arsenal

data/arsenal_parser.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import sys
+import yaml
+
+if len(sys.argv) < 2:
+    script = sys.argv[0]
+    print(f"Usage: {script} <arsenal_data_file>")
+    exit(1)
+
+commands = []
+cmd = ""
+tags = "pentest"
+lang = ""
+desc = ""
+with open(sys.argv[1], "r") as f:
+    in_code_block = False
+    for line in f.readlines():
+        if line.startswith("% "):
+            tags = line[2:].strip()
+        elif line.startswith("## "):
+            desc = line[3:].strip()
+        elif line.startswith("#plateform/"):
+            plateform = "".join(line[11:].strip().split(" ")[0])
+            if plateform == "linux":
+                lang = "sh"
+            elif plateform == "windows":
+                lang = "powershell"
+        elif line.startswith("```"):
+            in_code_block = not in_code_block
+        elif in_code_block:
+            if "arsenal" not in tags and "internal" not in tags:
+                cmd = line.strip().replace("\n","")
+                if len(cmd) > 0 and len(desc) > 0 and len(lang) > 0:
+                    entry = {}
+                    entry["cmd"] = cmd
+                    entry["tags"] = tags
+                    entry["lang"] = lang
+                    entry["desc"] = desc
+                    commands.append(entry)
+if len(commands) > 0:
+    print(yaml.safe_dump(commands,width=1000))

data/generate_arsenal_commands.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+SCRIPT=$(readlink -f "$0")
+SCRIPTPATH=$(dirname "$SCRIPT")
+for c in python git; do
+        if ! command -v ${c} &>/dev/null; then
+                echo "$command ${c} could not be found"
+                exit 1
+        fi
+done
+
+cd /tmp
+git clone --depth 1 https://github.com/Orange-Cyberdefense/arsenal
+echo -e "title: Commands for Fast Memo Shell from Arsenal\ncommands:" >"${SCRIPTPATH}/commands.yaml"
+find arsenal/arsenal/data/cheats/ -iname '*.md' ! -name 'README.md' -exec ${SCRIPTPATH}/arsenal_parser.py {} \; >>"${SCRIPTPATH}/commands.yaml"
+rm -rf /tmp/arsenal

fms.sh

@@ -14,9 +14,9 @@ done
 SWITCH_MENU=$(mktemp)
 
 function update() {
-	touch commands.state
-	if [[ "$(stat -c%Y commands.yaml)" != "$(cat commands.state)" ]] || [[ $1 == "force" ]]; then
-		cat /dev/null >commands.raw
+	touch data/commands.state
+	if [[ "$(stat -c%Y data/commands.yaml)" != "$(cat data/commands.state)" ]] || [[ $1 == "force" ]]; then
+		cat /dev/null >data/commands.raw
 		i=1
 		#set -eux
 		while read -r line; do
@@ -28,15 +28,15 @@ function update() {
 				echo "Error on line: ${line}"
 				exit
 			fi
-			echo "${cmd} # [${tags} ${lang} ${i}] ${desc}" | tee -a commands.tmp.${lang} >/dev/null
+			echo "${cmd} # [${tags} ${lang} ${i}] ${desc}" | tee -a data/commands.tmp.${lang} >/dev/null
 			i=$(($i + 1))
-		done < <(cat commands.yaml | yq e -o=j -I=0 .commands[])
-		for filename in $(ls -t commands.tmp.*); do
+		done < <(cat data/commands.yaml | yq e -o=j -I=0 .commands[])
+		for filename in $(ls -t data/commands.tmp.*); do
 			lang=$(echo ${filename} | rev | cut -d'.' -f1 | rev)
-			cat ${filename} | bat --color=always -l ${lang} -pp >>commands.raw
+			cat ${filename} | bat --color=always -l ${lang} -pp >>data/commands.raw
 			rm -f ${filename}
 		done
-		stat -c%Y commands.yaml >commands.state
+		stat -c%Y data/commands.yaml >data/commands.state
 	fi
 }
 
@@ -44,7 +44,7 @@ function notes() {
 	update
 	echo -n 0 >${SWITCH_MENU}
 	unset RES
-	RES=$(cat commands.raw | sk --reverse --ansi --inline-info \
+	RES=$(cat data/commands.raw | sk --reverse --ansi --inline-info \
 		--bind "shift-left:execute-silent(echo -n 1 > ${SWITCH_MENU})+abort" \
 		--bind "shift-right:execute-silent(echo -n 1 > ${SWITCH_MENU})+abort")
 	SWITCH=$(cat ${SWITCH_MENU})