mirror of
https://github.com/0xMarcio/cve.git
synced 2026-10-27 16:51:03 +00:00
Collect stars and last push during the weekly audit
The audit already asks GitHub about every linked repository, so the same query can carry back a star count and a last-push date for free. Results go to repo_meta.json, which lets a search result say which of two hundred linked PoCs is worth opening. 2.1 MB for 45,008 repositories, refreshed weekly.
This commit is contained in:
1 parent
5452bc8353
commit
576b0e4735
3 files changed
+72
-19
No files matched your search
@@ -39,6 +39,7 @@ jobs:
|
||||
git add --all -- \
|
||||
.github/link_audit_state.json \
|
||||
github.txt \
|
||||
repo_meta.json \
|
||||
':(top,glob)[12][0-9][0-9][0-9]/CVE-*.md'
|
||||
|
||||
if git diff --cached --quiet; then
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"0-9194/node-poc-http-smuggling":[0,"2022-12-12"],"0-d3y/cve-2023-29489":[13,"2024-08-25"],"0-d3y/exploit-ye":[9,"2024-06-28"],"0-d3y/langflow-rce-exploit":[7,"2025-06-23"],"0-d3y/xss_1915":[13,"2024-08-25"],"0-gram/cve-2022-41040":[0,"2024-11-23"],"0-x-2-2/cve-2021-44228":[66,"2021-12-12"],"0000000o0oo/wordpress-cve-2020-25213":[0,"2021-03-13"],"00000rest/py_trustwallet_wasm":[0,"2024-05-08"],"00052/spectre-attack-example":[12,"2018-01-05"],"000pp/pwnfaces":[19,"2023-04-18"],"000pp/wsob":[26,"2023-05-23"],"000sushant/hacking_windows7_using_metasploit":[2,"2023-04-04"],"000tonio/cve-2021-1675":[0,"2022-11-07"],"007alice/crashes":[0,"2020-01-15"],"007bsd/ml-kem-key-recovery":[0,"2026-08-15"],"007criptografia/c-cve-2024-0044":[4,"2024-07-11"],"00mjk/exploit-cve-2017-7494":[1,"2022-05-08"],"00theway/ghostcat-cnvd-2020-10487":[423,"2020-03-09"],"00xcanelo/cve-2024-47533-poc":[1,"2025-08-13"],"00xcanelo/cve-2025-27591":[2,"2025-07-23"],"00xcanelo/cve-2025-32778":[3,"2025-08-18"],"00xcanelo/cve-2025-49113":[8,"2025-07-19"],"00xnetrunner/shodan_cheet-sheet":[60,"2025-11-14"],"00xph4ntom/epsseeker":[2,"2023-03-28"],"02manas-jha/metasploitable-nessus-vulnerability-assessment":[2,"2024-10-02"],"03hice-collab/cve":[0,"2025-08-24"],"040840308-liu/icse2025-olasan-artifact":[0,"2024-11-13"],"04shivam/cve-2023-30253-exploit":[0,"2024-05-27"],"0510green-hand/cve":[0,"2025-08-21"],"06-ux/cve-2026-9256-poc":[1,"2026-06-03"],"0730nophone/cve-2022-22947-":[60,"2022-05-16"],"0730nophone/e-cology-workflowservicexml-":[5,"2021-05-21"],"0733wcr/5":[0,"2023-08-31"],"0759104103/cd-cve-2019-11932":[0,"2023-03-29"],"090913/cve":[0,"2024-11-06"],"09ria09/ftc-2020-2021":[0,"2023-01-16"],"0ang3el/aem-hacker":[813,"2024-07-28"],"0ang3el/drupalgeddon2":[8,"2018-04-17"],"0ca/swftools_crashes":[0,"2020-01-16"],"0cool-design/pwnentaho":[0,"2023-03-27"],"0d000721999/evc1":[0,"2026-05-13"],"0day1day/yarasigs":[16,"2013-06-05"],"0dayan0n/rce_cve-2024-7954-":[2,"2024-12-28"],"0dayctf/cve-2020-9484":[24,"2024-09-16"],"0dayninja/cve-2021-24155.rb":[10,"2021-07-30"],"0dayninja/cve-2021-3560":[9,"2021-07-30"],"0dteam/cve-2024-22024":[31,"2024-02-09"],"0e0w/gohacktools":[12,"2021-10-06"],"0e0w/learnpython":[45,"2024-04-05"],"0hmsec/nvms-1000-directory-traversal-bash":[0,"2025-02-06"],"0hmsec/searchor-2.4.0-arbitrary-cmd-injection-python":[1,"2025-02-01"],"0housefly0/printnightmare":[0,"2024-07-22"],"0init/cve-2026-45185":[0,"2026-08-07"],"0katz/cve-2019-12476":[43,"2020-04-20"],"0n3m4ns4rmy/whatthebug":[0,"2024-11-13"],"0ndras3k/cve-2020-11883":[0,"2021-03-01"],"0neb1n/cve-2020-1349":[11,"2020-07-28"],"0neb1n/cve-2020-1493":[25,"2020-08-25"],"0neb1n/cve-2020-16947":[122,"2020-10-20"],"0neday/multi_path":[4,"2018-06-06"],"0nightsedge0/cve-2024-9950-poc":[0,"2025-01-22"],"0nin0hanz0/cve-2024-24919-poc":[12,"2024-07-27"],"0nion1/cve-2021-3129":[6,"2022-10-11"],"0nopnop/qualysparser":[0,"2015-09-23"],"0nsec/cve-2023-35078":[1,"2025-08-21"],"0nyx-hkr/cve-2023-4350":[0,"2023-09-11"],"0nyx-my7h/cve-2025-43919":[0,"2025-04-20"],"0nyx-my7h/cve-2025-43920":[1,"2025-04-20"],"0nyx-my7h/cve-2025-43921":[0,"2025-04-20"],"0o-de-lally/inferno":[1,"2026-03-21"],"0p5cur/cve-2026-24061-poc":[7,"2026-08-11"],"0pc0defr/wordpress-sploit-framework":[33,"2016-09-19"],"0pensrcerer/jfrogadvsec-demo":[0,"2024-05-02"],"0pensrcerer/struts2-rce-public":[0,"2021-07-21"],"0pensrcerer/struts2rce":[0,"2022-02-22"],"0ps/cobaltstrike_cat_4.5":[2,"2023-01-13"],"0pts/0pts-bitpixie":[3,"2025-06-29"],"0ptyx/cve-2024-0582":[0,"2024-05-28"],"0range1337/cve-2022-1015":[0,"2024-03-18"],"0range1337/cve-cve-2023-4622":[1,"2024-06-26"],"0rx1/cve-2021-43858":[3,"2022-01-03"],"0spwn/cve-2023-27372-poc":[6,"2023-07-05"],"0th3rs-security-team/cve-2019-11043":[14,"2019-11-06"],"0thm4n3/cve-2020-14882":[2,"2020-10-31"],"0timeday/exploit-js2py":[1,"2025-12-13"],"0truust/iperius-backup-security-advisories":[0,"2026-03-25"],"0utl4nder/another-metabase-rce-cve-2023-38646":[1,"2023-12-12"],"0v3rride/0v3rride.github.io":[1,"2024-12-18"],"0v3rride/week-7":[0,"2018-04-11"],"0velychk0/my_bashrc":[0,"2025-03-25"],"0vercl0k/cve-2019-11708":[624,"2020-06-13"],"0vercl0k/cve-2019-9810":[227,"2019-12-28"],"0vercl0k/cve-2021-24086":[236,"2021-04-15"],"0vercl0k/cve-2021-28476":[226,"2021-06-01"],"0vercl0k/cve-2021-31166":[827,"2021-06-12"],"0vercl0k/cve-2021-32537":[55,"2021-06-26"],"0vercl0k/cve-2022-21971":[302,"2022-02-26"],"0vercl0k/cve-2022-21974":[57,"2022-02-26"],"0vercl0k/cve-2022-28281":[73,"2022-04-08"],"0vercl0k/paracosme":[90,"2023-10-08"],"0vercl0k/pwn2own2023-miami":[62,"2023-08-05"],"0vercl0k/zenith":[131,"2022-04-25"],"0x-nope/cve-2019-12180":[4,"2020-02-07"],"0x00-0x00/-cve-2017-9805":[15,"2020-11-26"],"0x00-0x00/cve-2014-6271":[3,"2017-11-23"],"0x00-0x00/cve-2015-3224":[2,"2018-02-08"],"0x00-0x00/cve-2016-10033":[6,"2018-02-09"],"0x00-0x00/cve-2016-2098":[16,"2018-02-09"],"0x00-0x00/cve-2017-5638":[6,"2018-02-15"],"0x00-0x00/cve-2018-1000001":[31,"2018-02-09"],"0x00-0x00/cve-2018-10517":[5,"2018-11-0Line truncated
|
||||
+70
-19
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Remove PoC links whose GitHub repository no longer exists."""
|
||||
"""Prune PoC links to deleted repositories and refresh the metadata the site shows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -25,37 +25,51 @@ from update_cves import (
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STATE_FILE = ROOT / ".github" / "link_audit_state.json"
|
||||
GITHUB_LIST = ROOT / "github.txt"
|
||||
REPO_META = ROOT / "repo_meta.json"
|
||||
GITHUB_SECTION = "#### Github"
|
||||
REFERENCE_SECTION = "#### Reference"
|
||||
GITHUB_EMPTY = "No PoCs found on GitHub currently."
|
||||
BATCH = 20
|
||||
|
||||
|
||||
def collect_repositories() -> dict[str, list[tuple[Path, str]]]:
|
||||
"""Map every referenced repository to the entries and links citing it."""
|
||||
references: dict[str, list[tuple[Path, str]]] = {}
|
||||
def collect_repositories() -> tuple[dict[str, list[tuple[Path, str]]], set[str]]:
|
||||
"""Map prunable links to their entries, and name every repository the site shows.
|
||||
|
||||
Only the GitHub section is prunable; a reference comes from the CVE record
|
||||
itself, so it stays even when it rots. Both sections need star counts.
|
||||
"""
|
||||
prunable: dict[str, list[tuple[Path, str]]] = {}
|
||||
referenced: set[str] = set()
|
||||
for path in sorted(ROOT.glob("[12][0-9][0-9][0-9]/CVE-*.md")):
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
for url in section_links(text, GITHUB_SECTION):
|
||||
full_name = github_repo_from_url(url)
|
||||
if full_name and "/" in full_name:
|
||||
references.setdefault(full_name, []).append((path, url))
|
||||
return references
|
||||
prunable.setdefault(full_name, []).append((path, url))
|
||||
referenced.add(full_name)
|
||||
for url in section_links(text, REFERENCE_SECTION):
|
||||
full_name = github_repo_from_url(url)
|
||||
if full_name and "/" in full_name:
|
||||
referenced.add(full_name)
|
||||
return prunable, referenced
|
||||
|
||||
|
||||
def missing_repositories(client: GitHubClient, names: list[str]) -> set[str]:
|
||||
"""Return the repositories GitHub reports as NOT_FOUND.
|
||||
def survey(client: GitHubClient, names: list[str]) -> tuple[set[str], dict[str, list]]:
|
||||
"""Return the repositories GitHub reports as NOT_FOUND, and stars/last push for the rest.
|
||||
|
||||
Anything else — a network error, a throttle, an unexpected response — leaves
|
||||
the repository out of the result, so an audit failure never drops a link.
|
||||
the repository out of both results, so an audit failure never drops a link.
|
||||
"""
|
||||
batches = [names[index : index + BATCH] for index in range(0, len(names), BATCH)]
|
||||
missing: set[str] = set()
|
||||
metadata: dict[str, list] = {}
|
||||
checked = 0
|
||||
|
||||
def check(batch: list[str]) -> set[str]:
|
||||
def check(batch: list[str]) -> tuple[set[str], dict[str, list]]:
|
||||
aliases = " ".join(
|
||||
f"r{index}: repository(owner: {json.dumps(name.split('/', 1)[0])}, "
|
||||
f"name: {json.dumps(name.split('/', 1)[1])}) {{ nameWithOwner }}"
|
||||
f"name: {json.dumps(name.split('/', 1)[1])}) "
|
||||
"{ nameWithOwner stargazerCount pushedAt }"
|
||||
for index, name in enumerate(batch)
|
||||
)
|
||||
payload = http_json(
|
||||
@@ -70,20 +84,49 @@ def missing_repositories(client: GitHubClient, names: list[str]) -> set[str]:
|
||||
index = str(path[0])[1:]
|
||||
if index.isdigit() and int(index) < len(batch):
|
||||
gone.add(batch[int(index)])
|
||||
return gone
|
||||
found: dict[str, list] = {}
|
||||
data = payload.get("data") or {}
|
||||
for index, name in enumerate(batch):
|
||||
repo = data.get(f"r{index}")
|
||||
if not repo:
|
||||
continue
|
||||
pushed = str(repo.get("pushedAt") or "")[:10]
|
||||
found[name.lower()] = [
|
||||
int(repo.get("stargazerCount") or 0),
|
||||
pushed,
|
||||
]
|
||||
return gone, found
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
futures = {executor.submit(check, batch): batch for batch in batches}
|
||||
for future in as_completed(futures):
|
||||
batch = futures[future]
|
||||
try:
|
||||
missing |= future.result()
|
||||
gone, found = future.result()
|
||||
missing |= gone
|
||||
metadata.update(found)
|
||||
except Exception as exc:
|
||||
print(f"Kept {len(batch)} repositories after a failed check: {exc}", file=sys.stderr)
|
||||
checked += len(batch)
|
||||
if checked % 5000 < len(batch):
|
||||
print(f"Checked {checked} of {len(names)} repositories")
|
||||
return missing
|
||||
return missing, metadata
|
||||
|
||||
|
||||
def save_metadata(metadata: dict[str, list], alive: set[str], *, dry_run: bool) -> int:
|
||||
"""Merge this run's readings over the stored ones and forget deleted repositories."""
|
||||
try:
|
||||
stored = json.loads(REPO_META.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
stored = {}
|
||||
stored.update(metadata)
|
||||
stored = {name: value for name, value in stored.items() if name.lower() in alive}
|
||||
if not dry_run:
|
||||
REPO_META.write_text(
|
||||
json.dumps(stored, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return len(stored)
|
||||
|
||||
|
||||
def prune_entries(
|
||||
@@ -136,14 +179,19 @@ def save_cursor(cursor: str, *, dry_run: bool) -> None:
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Drop PoC links to deleted repositories")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Drop PoC links to deleted repositories and refresh repo_meta.json"
|
||||
)
|
||||
parser.add_argument("--limit", type=int, default=0, help="Repositories to check this run")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Report without writing files")
|
||||
args = parser.parse_args()
|
||||
|
||||
references = collect_repositories()
|
||||
names = sorted(references)
|
||||
print(f"{sum(len(v) for v in references.values())} links across {len(names)} repositories")
|
||||
references, referenced = collect_repositories()
|
||||
names = sorted(referenced)
|
||||
print(
|
||||
f"{sum(len(v) for v in references.values())} prunable links | "
|
||||
f"{len(names)} repositories to survey"
|
||||
)
|
||||
|
||||
cursor = load_cursor()
|
||||
start = next((i for i, name in enumerate(names) if name > cursor), 0) if cursor else 0
|
||||
@@ -153,14 +201,17 @@ def main() -> int:
|
||||
print(f"Checking {len(scope)} repositories starting after {cursor or 'the beginning'}")
|
||||
|
||||
token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") or ""
|
||||
missing = missing_repositories(GitHubClient(token), scope)
|
||||
missing, metadata = survey(GitHubClient(token), scope)
|
||||
print(f"{len(missing)} repositories no longer exist")
|
||||
|
||||
entries, links, dead_urls = prune_entries(references, missing, dry_run=args.dry_run)
|
||||
inventory = prune_inventory(dead_urls, dry_run=args.dry_run)
|
||||
alive = {name.lower() for name in names if name not in missing}
|
||||
tracked = save_metadata(metadata, alive, dry_run=args.dry_run)
|
||||
save_cursor(scope[-1] if scope else "", dry_run=args.dry_run)
|
||||
|
||||
print(f"Removed {links} links from {entries} entries | {inventory} inventory lines")
|
||||
print(f"Refreshed stars and last-push for {len(metadata)} repositories | {tracked} tracked")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user