diff --git a/.bin/trickest-patcher.py b/.bin/trickest-patcher.py new file mode 100755 index 00000000..609c5001 --- /dev/null +++ b/.bin/trickest-patcher.py @@ -0,0 +1,69 @@ +#!/usr/bin/python3 + +import os,shutil + +print("[+] trickest wordlist patcher") + +ROOT=".working_space" +INPUT_TECHNOLOGIES=os.path.join(ROOT,"wordlists/technologies/") +INPUT_ROBOTS=os.path.join(ROOT,"wordlists/robots/") +OUTPUT_TECHNOLOGIES="Discovery/Web-Content/CMS/trickest-cms-wordlist/" +OUTPUT_ROBOTS="Discovery/Web-Content/trickest-robots-disallowed-wordlists/" + +if not os.path.isdir(".working_space"): + print("[!] Working dir not found!") + exit(2) + +if not os.path.isdir(os.path.join(ROOT,"wordlists")): + print("[!] wordlists dir not found!") + exit(2) + +if not os.path.isdir(OUTPUT_TECHNOLOGIES): + os.makedirs(OUTPUT_TECHNOLOGIES) + +if not os.path.isdir(OUTPUT_ROBOTS): + os.makedirs(OUTPUT_ROBOTS) + +for i in os.listdir(INPUT_TECHNOLOGIES): + path=os.path.join(INPUT_TECHNOLOGIES,i) + + if os.path.isfile(path): + shutil.copy(path,OUTPUT_TECHNOLOGIES) + else: + shutil.copytree(path,OUTPUT_TECHNOLOGIES,dirs_exist_ok=True) + +for i in os.listdir(INPUT_ROBOTS): + path=os.path.join(INPUT_ROBOTS,i) + + if os.path.isfile(path): + shutil.copy(path,OUTPUT_ROBOTS) + else: + shutil.copytree(path,OUTPUT_ROBOTS,dirs_exist_ok=True) + +print("[+] Copied all the files") +for i in [OUTPUT_ROBOTS,OUTPUT_TECHNOLOGIES]: + for root,_,file_list in os.walk(i): + for file in file_list: + + path=os.path.join(root,file) + contents=open(path,"rb").read() + + if contents.endswith(b"\n"): + print("[!] %s ends with new line"%(path)) + contents=contents[:-1] + open(path,"wb").write(contents) + + patch_content=[] + counter=0 + for content in contents.split(b"\n"): + counter+=1 + if not content: + print("[+] %s has an empty line at %i"%(path,counter)) + continue + patch_content.append(content) + + if len(contents)!=len(patch_content): + open(path,"wb").write(b"\n".join(patch_content)) + + + diff --git a/.bin/trickest-updater.sh b/.bin/trickest-updater.sh new file mode 100755 index 00000000..203d3df9 --- /dev/null +++ b/.bin/trickest-updater.sh @@ -0,0 +1,12 @@ +#!/usr/bin/bash + +set -e +set -o pipefail + +mkdir -p .working_space +cd .working_space +git clone --depth=1 https://github.com/trickest/wordlists.git +cd ../ + +./.bin/trickest-patcher.py +rm -rf .working_space \ No newline at end of file diff --git a/.github/workflows/wordlist-updater_trickest-wordlists.yml b/.github/workflows/wordlist-updater_trickest-wordlists.yml new file mode 100644 index 00000000..725bbdd9 --- /dev/null +++ b/.github/workflows/wordlist-updater_trickest-wordlists.yml @@ -0,0 +1,32 @@ +name: Wordlist Updater - Trickest wordlists updater + +on: + schedule: + - cron: 0 0 * * * + + workflow_dispatch: + +jobs: + update-files: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Update lists + run: ./.bin/trickest-updater.sh + + - name: Commit files if changed + run: | + git add -N . + + if [ -z "$(git ls-files --modified)" ]; then + echo "[+] No files were changed" + else + echo "[+] Files were changed! Pushing changed..." + git add -A + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY + git config --local user.email "example@github.com" + git config --local user.name "GitHub Action" + git commit -m "[Github Action] Automated trickest wordlists update." + git push + fi diff --git a/.gitignore b/.gitignore index 4183a8ca..7b78b08a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .*.icloud .gitkeep .idea +.working_space/