Collect stars and last push during the weekly audit

The audit already asks GitHub about every linked repository, so the same
query can carry back a star count and a last-push date for free. Results go
to repo_meta.json, which lets a search result say which of two hundred
linked PoCs is worth opening.

2.1 MB for 45,008 repositories, refreshed weekly.
This commit is contained in:
0xMarcio committed 2026-08-30 21:53:20 +02:00
1 parent 5452bc8353
commit 576b0e4735
3 files changed
+72 -19

No files matched your search

+1
View File
@@ -39,6 +39,7 @@ jobs:
git add --all -- \
.github/link_audit_state.json \
github.txt \
repo_meta.json \
':(top,glob)[12][0-9][0-9][0-9]/CVE-*.md'
if git diff --cached --quiet; then
+1
View File
@@ -0,0 +1 @@
{"0-9194/node-poc-http-smuggling":[0,"2022-12-12"],"0-d3y/cve-2023-29489":[13,"2024-08-25"],"0-d3y/exploit-ye":[9,"2024-06-28"],"0-d3y/langflow-rce-exploit":[7,"2025-06-23"],"0-d3y/xss_1915":[13,"2024-08-25"],"0-gram/cve-2022-41040":[0,"2024-11-23"],"0-x-2-2/cve-2021-44228":[66,"2021-12-12"],"0000000o0oo/wordpress-cve-2020-25213":[0,"2021-03-13"],"00000rest/py_trustwallet_wasm":[0,"2024-05-08"],"00052/spectre-attack-example":[12,"2018-01-05"],"000pp/pwnfaces":[19,"2023-04-18"],"000pp/wsob":[26,"2023-05-23"],"000sushant/hacking_windows7_using_metasploit":[2,"2023-04-04"],"000tonio/cve-2021-1675":[0,"2022-11-07"],"007alice/crashes":[0,"2020-01-15"],"007bsd/ml-kem-key-recovery":[0,"2026-08-15"],"007criptografia/c-cve-2024-0044":[4,"2024-07-11"],"00mjk/exploit-cve-2017-7494":[1,"2022-05-08"],"00theway/ghostcat-cnvd-2020-10487":[423,"2020-03-09"],"00xcanelo/cve-2024-47533-poc":[1,"2025-08-13"],"00xcanelo/cve-2025-27591":[2,"2025-07-23"],"00xcanelo/cve-2025-32778":[3,"2025-08-18"],"00xcanelo/cve-2025-49113":[8,"2025-07-19"],"00xnetrunner/shodan_cheet-sheet":[60,"2025-11-14"],"00xph4ntom/epsseeker":[2,"2023-03-28"],"02manas-jha/metasploitable-nessus-vulnerability-assessment":[2,"2024-10-02"],"03hice-collab/cve":[0,"2025-08-24"],"040840308-liu/icse2025-olasan-artifact":[0,"2024-11-13"],"04shivam/cve-2023-30253-exploit":[0,"2024-05-27"],"0510green-hand/cve":[0,"2025-08-21"],"06-ux/cve-2026-9256-poc":[1,"2026-06-03"],"0730nophone/cve-2022-22947-":[60,"2022-05-16"],"0730nophone/e-cology-workflowservicexml-":[5,"2021-05-21"],"0733wcr/5":[0,"2023-08-31"],"0759104103/cd-cve-2019-11932":[0,"2023-03-29"],"090913/cve":[0,"2024-11-06"],"09ria09/ftc-2020-2021":[0,"2023-01-16"],"0ang3el/aem-hacker":[813,"2024-07-28"],"0ang3el/drupalgeddon2":[8,"2018-04-17"],"0ca/swftools_crashes":[0,"2020-01-16"],"0cool-design/pwnentaho":[0,"2023-03-27"],"0d000721999/evc1":[0,"2026-05-13"],"0day1day/yarasigs":[16,"2013-06-05"],"0dayan0n/rce_cve-2024-7954-":[2,"2024-12-28"],"0dayctf/cve-2020-9484":[24,"2024-09-16"],"0dayninja/cve-2021-24155.rb":[10,"2021-07-30"],"0dayninja/cve-2021-3560":[9,"2021-07-30"],"0dteam/cve-2024-22024":[31,"2024-02-09"],"0e0w/gohacktools":[12,"2021-10-06"],"0e0w/learnpython":[45,"2024-04-05"],"0hmsec/nvms-1000-directory-traversal-bash":[0,"2025-02-06"],"0hmsec/searchor-2.4.0-arbitrary-cmd-injection-python":[1,"2025-02-01"],"0housefly0/printnightmare":[0,"2024-07-22"],"0init/cve-2026-45185":[0,"2026-08-07"],"0katz/cve-2019-12476":[43,"2020-04-20"],"0n3m4ns4rmy/whatthebug":[0,"2024-11-13"],"0ndras3k/cve-2020-11883":[0,"2021-03-01"],"0neb1n/cve-2020-1349":[11,"2020-07-28"],"0neb1n/cve-2020-1493":[25,"2020-08-25"],"0neb1n/cve-2020-16947":[122,"2020-10-20"],"0neday/multi_path":[4,"2018-06-06"],"0nightsedge0/cve-2024-9950-poc":[0,"2025-01-22"],"0nin0hanz0/cve-2024-24919-poc":[12,"2024-07-27"],"0nion1/cve-2021-3129":[6,"2022-10-11"],"0nopnop/qualysparser":[0,"2015-09-23"],"0nsec/cve-2023-35078":[1,"2025-08-21"],"0nyx-hkr/cve-2023-4350":[0,"2023-09-11"],"0nyx-my7h/cve-2025-43919":[0,"2025-04-20"],"0nyx-my7h/cve-2025-43920":[1,"2025-04-20"],"0nyx-my7h/cve-2025-43921":[0,"2025-04-20"],"0o-de-lally/inferno":[1,"2026-03-21"],"0p5cur/cve-2026-24061-poc":[7,"2026-08-11"],"0pc0defr/wordpress-sploit-framework":[33,"2016-09-19"],"0pensrcerer/jfrogadvsec-demo":[0,"2024-05-02"],"0pensrcerer/struts2-rce-public":[0,"2021-07-21"],"0pensrcerer/struts2rce":[0,"2022-02-22"],"0ps/cobaltstrike_cat_4.5":[2,"2023-01-13"],"0pts/0pts-bitpixie":[3,"2025-06-29"],"0ptyx/cve-2024-0582":[0,"2024-05-28"],"0range1337/cve-2022-1015":[0,"2024-03-18"],"0range1337/cve-cve-2023-4622":[1,"2024-06-26"],"0rx1/cve-2021-43858":[3,"2022-01-03"],"0spwn/cve-2023-27372-poc":[6,"2023-07-05"],"0th3rs-security-team/cve-2019-11043":[14,"2019-11-06"],"0thm4n3/cve-2020-14882":[2,"2020-10-31"],"0timeday/exploit-js2py":[1,"2025-12-13"],"0truust/iperius-backup-security-advisories":[0,"2026-03-25"],"0utl4nder/another-metabase-rce-cve-2023-38646":[1,"2023-12-12"],"0v3rride/0v3rride.github.io":[1,"2024-12-18"],"0v3rride/week-7":[0,"2018-04-11"],"0velychk0/my_bashrc":[0,"2025-03-25"],"0vercl0k/cve-2019-11708":[624,"2020-06-13"],"0vercl0k/cve-2019-9810":[227,"2019-12-28"],"0vercl0k/cve-2021-24086":[236,"2021-04-15"],"0vercl0k/cve-2021-28476":[226,"2021-06-01"],"0vercl0k/cve-2021-31166":[827,"2021-06-12"],"0vercl0k/cve-2021-32537":[55,"2021-06-26"],"0vercl0k/cve-2022-21971":[302,"2022-02-26"],"0vercl0k/cve-2022-21974":[57,"2022-02-26"],"0vercl0k/cve-2022-28281":[73,"2022-04-08"],"0vercl0k/paracosme":[90,"2023-10-08"],"0vercl0k/pwn2own2023-miami":[62,"2023-08-05"],"0vercl0k/zenith":[131,"2022-04-25"],"0x-nope/cve-2019-12180":[4,"2020-02-07"],"0x00-0x00/-cve-2017-9805":[15,"2020-11-26"],"0x00-0x00/cve-2014-6271":[3,"2017-11-23"],"0x00-0x00/cve-2015-3224":[2,"2018-02-08"],"0x00-0x00/cve-2016-10033":[6,"2018-02-09"],"0x00-0x00/cve-2016-2098":[16,"2018-02-09"],"0x00-0x00/cve-2017-5638":[6,"2018-02-15"],"0x00-0x00/cve-2018-1000001":[31,"2018-02-09"],"0x00-0x00/cve-2018-10517":[5,"2018-11-0Line truncated
+70 -19
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Remove PoC links whose GitHub repository no longer exists."""
"""Prune PoC links to deleted repositories and refresh the metadata the site shows."""
from __future__ import annotations
@@ -25,37 +25,51 @@ from update_cves import (
ROOT = Path(__file__).resolve().parents[1]
STATE_FILE = ROOT / ".github" / "link_audit_state.json"
GITHUB_LIST = ROOT / "github.txt"
REPO_META = ROOT / "repo_meta.json"
GITHUB_SECTION = "#### Github"
REFERENCE_SECTION = "#### Reference"
GITHUB_EMPTY = "No PoCs found on GitHub currently."
BATCH = 20
def collect_repositories() -> dict[str, list[tuple[Path, str]]]:
"""Map every referenced repository to the entries and links citing it."""
references: dict[str, list[tuple[Path, str]]] = {}
def collect_repositories() -> tuple[dict[str, list[tuple[Path, str]]], set[str]]:
"""Map prunable links to their entries, and name every repository the site shows.
Only the GitHub section is prunable; a reference comes from the CVE record
itself, so it stays even when it rots. Both sections need star counts.
"""
prunable: dict[str, list[tuple[Path, str]]] = {}
referenced: set[str] = set()
for path in sorted(ROOT.glob("[12][0-9][0-9][0-9]/CVE-*.md")):
text = path.read_text(encoding="utf-8", errors="replace")
for url in section_links(text, GITHUB_SECTION):
full_name = github_repo_from_url(url)
if full_name and "/" in full_name:
references.setdefault(full_name, []).append((path, url))
return references
prunable.setdefault(full_name, []).append((path, url))
referenced.add(full_name)
for url in section_links(text, REFERENCE_SECTION):
full_name = github_repo_from_url(url)
if full_name and "/" in full_name:
referenced.add(full_name)
return prunable, referenced
def missing_repositories(client: GitHubClient, names: list[str]) -> set[str]:
"""Return the repositories GitHub reports as NOT_FOUND.
def survey(client: GitHubClient, names: list[str]) -> tuple[set[str], dict[str, list]]:
"""Return the repositories GitHub reports as NOT_FOUND, and stars/last push for the rest.
Anything else — a network error, a throttle, an unexpected response — leaves
the repository out of the result, so an audit failure never drops a link.
the repository out of both results, so an audit failure never drops a link.
"""
batches = [names[index : index + BATCH] for index in range(0, len(names), BATCH)]
missing: set[str] = set()
metadata: dict[str, list] = {}
checked = 0
def check(batch: list[str]) -> set[str]:
def check(batch: list[str]) -> tuple[set[str], dict[str, list]]:
aliases = " ".join(
f"r{index}: repository(owner: {json.dumps(name.split('/', 1)[0])}, "
f"name: {json.dumps(name.split('/', 1)[1])}) {{ nameWithOwner }}"
f"name: {json.dumps(name.split('/', 1)[1])}) "
"{ nameWithOwner stargazerCount pushedAt }"
for index, name in enumerate(batch)
)
payload = http_json(
@@ -70,20 +84,49 @@ def missing_repositories(client: GitHubClient, names: list[str]) -> set[str]:
index = str(path[0])[1:]
if index.isdigit() and int(index) < len(batch):
gone.add(batch[int(index)])
return gone
found: dict[str, list] = {}
data = payload.get("data") or {}
for index, name in enumerate(batch):
repo = data.get(f"r{index}")
if not repo:
continue
pushed = str(repo.get("pushedAt") or "")[:10]
found[name.lower()] = [
int(repo.get("stargazerCount") or 0),
pushed,
]
return gone, found
with ThreadPoolExecutor(max_workers=4) as executor:
futures = {executor.submit(check, batch): batch for batch in batches}
for future in as_completed(futures):
batch = futures[future]
try:
missing |= future.result()
gone, found = future.result()
missing |= gone
metadata.update(found)
except Exception as exc:
print(f"Kept {len(batch)} repositories after a failed check: {exc}", file=sys.stderr)
checked += len(batch)
if checked % 5000 < len(batch):
print(f"Checked {checked} of {len(names)} repositories")
return missing
return missing, metadata
def save_metadata(metadata: dict[str, list], alive: set[str], *, dry_run: bool) -> int:
"""Merge this run's readings over the stored ones and forget deleted repositories."""
try:
stored = json.loads(REPO_META.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
stored = {}
stored.update(metadata)
stored = {name: value for name, value in stored.items() if name.lower() in alive}
if not dry_run:
REPO_META.write_text(
json.dumps(stored, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n",
encoding="utf-8",
)
return len(stored)
def prune_entries(
@@ -136,14 +179,19 @@ def save_cursor(cursor: str, *, dry_run: bool) -> None:
def main() -> int:
parser = argparse.ArgumentParser(description="Drop PoC links to deleted repositories")
parser = argparse.ArgumentParser(
description="Drop PoC links to deleted repositories and refresh repo_meta.json"
)
parser.add_argument("--limit", type=int, default=0, help="Repositories to check this run")
parser.add_argument("--dry-run", action="store_true", help="Report without writing files")
args = parser.parse_args()
references = collect_repositories()
names = sorted(references)
print(f"{sum(len(v) for v in references.values())} links across {len(names)} repositories")
references, referenced = collect_repositories()
names = sorted(referenced)
print(
f"{sum(len(v) for v in references.values())} prunable links | "
f"{len(names)} repositories to survey"
)
cursor = load_cursor()
start = next((i for i, name in enumerate(names) if name > cursor), 0) if cursor else 0
@@ -153,14 +201,17 @@ def main() -> int:
print(f"Checking {len(scope)} repositories starting after {cursor or 'the beginning'}")
token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") or ""
missing = missing_repositories(GitHubClient(token), scope)
missing, metadata = survey(GitHubClient(token), scope)
print(f"{len(missing)} repositories no longer exist")
entries, links, dead_urls = prune_entries(references, missing, dry_run=args.dry_run)
inventory = prune_inventory(dead_urls, dry_run=args.dry_run)
alive = {name.lower() for name in names if name not in missing}
tracked = save_metadata(metadata, alive, dry_run=args.dry_run)
save_cursor(scope[-1] if scope else "", dry_run=args.dry_run)
print(f"Removed {links} links from {entries} entries | {inventory} inventory lines")
print(f"Refreshed stars and last-push for {len(metadata)} repositories | {tracked} tracked")
return 0