From c8f213af17aff19e271e7aca033c5df197b3f7c2 Mon Sep 17 00:00:00 2001 From: 0xMarcio Date: Fri, 4 Sep 2026 03:46:17 +0200 Subject: [PATCH] Find this year's PoCs within the hour and stop indexing bare shells The daily sweep found new repositories within 2 to 9 hours where their name carried the CVE, and up to 30 hours where only the README did. The hourly run now searches repositories pushed in the last day for this year's and last year's CVEs, reads their READMEs and applies the same classifier, so a PoC published this morning is on the site within the hour. The daily run keeps covering every year and the CVE record references; the hourly run skips those, and skipping leaves the delta checkpoint alone. A sample of two hundred indexed repositories held six with nothing in them: an empty README or none, a placeholder, a link to a paywall. A repository with no directory, no code, no file named for the CVE or as a PoC, and a README with no code block, command, request or real write-up is bare, and neither discovery nor the daily audit keeps it. The audit now reads each repository's README and root to judge that, and sweeps 4,000 repositories a day instead of 1,750, since one link in eight pointed at a deleted repository under the fortnightly cycle. The upstream PoC-in-GitHub collection is judged daily rather than weekly: fourteen of the last thirty-nine additions came from it, which means it finds repositories our own search does not. --- .github/workflows/audit_poc_links.yml | 11 ++++---- .github/workflows/hot_cves.yml | 11 ++++++++ .github/workflows/sync_pocingithub.yml | 7 ++--- scripts/audit_poc_links.py | 30 ++++++++++++++++----- scripts/update_cves.py | 37 ++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 15 deletions(-) diff --git a/.github/workflows/audit_poc_links.yml b/.github/workflows/audit_poc_links.yml index d817fad4f61..011aa323188 100644 --- a/.github/workflows/audit_poc_links.yml +++ b/.github/workflows/audit_poc_links.yml @@ -16,7 +16,7 @@ concurrency: jobs: audit: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 50 steps: - name: Checkout repository uses: actions/checkout@v7 @@ -26,14 +26,13 @@ jobs: with: python-version: '3.13' - # A seventh of the old weekly batch every day: the 12,000-and-12,000 - # weekly run measured an hour, and it holds the repository-updates lock - # the whole time, which stalls the hourly refresh and can cancel a daily - # sync queued behind it. Same sweep period, about nine minutes a day. + # 4,000 repositories and 3,000 reference links a day sweeps the whole + # index about weekly, in around half an hour. A sample found one link in + # eight pointing at a deleted repository under the old fortnightly sweep. - name: Drop dead links and refresh repository metadata env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: python3 scripts/audit_poc_links.py --limit 1750 --references 1750 + run: python3 scripts/audit_poc_links.py --limit 4000 --references 3000 - name: Verify the generated index run: python3 scripts/build_site.py diff --git a/.github/workflows/hot_cves.yml b/.github/workflows/hot_cves.yml index e9d9bf4620b..a177b3495f1 100644 --- a/.github/workflows/hot_cves.yml +++ b/.github/workflows/hot_cves.yml @@ -26,6 +26,15 @@ jobs: with: python-version: '3.13' + # Every repository pushed in the last day that names a CVE of this year + # or last, with its README read and the classifier applied, so a PoC + # published this morning is on the site within the hour. The daily run + # keeps covering every year and the CVE record references. + - name: Discover this year's PoCs pushed in the last day + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 scripts/update_cves.py --lookback-days 1 --years 2 --skip-cvelist + - name: Ingest the previous hour's candidates env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -52,6 +61,8 @@ jobs: README.md \ index/trending.json \ index/github.txt \ + index/references.txt \ + index/reference_pocs.txt \ index/cve_dates.json \ index/cve_record_state.json \ docs/hero.svg \ diff --git a/.github/workflows/sync_pocingithub.yml b/.github/workflows/sync_pocingithub.yml index d25dada0fba..a40bff5d54e 100644 --- a/.github/workflows/sync_pocingithub.yml +++ b/.github/workflows/sync_pocingithub.yml @@ -2,7 +2,7 @@ name: Sync PoC-in-GitHub on: schedule: - - cron: '10 5 * * 3' + - cron: '10 5 * * *' workflow_dispatch: permissions: @@ -26,10 +26,11 @@ jobs: with: python-version: '3.13' - # Every candidate is judged again each week rather than remembered as + # Every candidate is judged again each day rather than remembered as # rejected: a repository that was an empty shell in March is often a # working exploit by June, and re-reading 6,800 of them costs five - # minutes. + # minutes. Daily because the collection keeps finding repositories our + # own search missed. - name: Judge upstream PoC candidates env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/audit_poc_links.py b/scripts/audit_poc_links.py index 48d1472a88a..340c036110b 100644 --- a/scripts/audit_poc_links.py +++ b/scripts/audit_poc_links.py @@ -16,6 +16,8 @@ from urllib.parse import urlsplit sys.path.insert(0, str(Path(__file__).resolve().parent)) from update_cves import ( + readme_text, + repo_is_bare, GITHUB_GRAPHQL_URL, kev_badge, USER_AGENT, @@ -72,11 +74,24 @@ def collect_repositories() -> tuple[dict[str, list[tuple[Path, str]]], set[str]] return prunable, referenced -def survey(client: GitHubClient, names: list[str]) -> tuple[set[str], dict[str, list]]: - """Return the repositories GitHub reports as NOT_FOUND, and stars/last push for the rest. +REPO_FIELDS = ( + "{ nameWithOwner stargazerCount pushedAt " + 'readmeMd: object(expression: "HEAD:README.md") { ... on Blob { text } } ' + 'readmeUpper: object(expression: "HEAD:README.MD") { ... on Blob { text } } ' + 'readmeRst: object(expression: "HEAD:README.rst") { ... on Blob { text } } ' + 'readmeBare: object(expression: "HEAD:README") { ... on Blob { text } } ' + 'root: object(expression: "HEAD:") { ... on Tree { entries { name type } } } }' +) - Anything else, a network error or a throttle or an unexpected response, leaves - the repository out of both results, so an audit failure never drops a link. + +def survey(client: GitHubClient, names: list[str]) -> tuple[set[str], dict[str, list]]: + """Return the repositories to drop, and stars/last push for the rest. + + Dropped means GitHub reports NOT_FOUND, or the repository is bare: no code, + no directory, no artifact and a README with nothing runnable in it, which + the discovery gate would reject today. Anything else, a network error or a + throttle or an unexpected response, leaves the repository out of both + results, so an audit failure never drops a link. """ batches = [names[index : index + BATCH] for index in range(0, len(names), BATCH)] missing: set[str] = set() @@ -87,7 +102,7 @@ def survey(client: GitHubClient, names: list[str]) -> tuple[set[str], dict[str, aliases = " ".join( f"r{index}: repository(owner: {json.dumps(name.split('/', 1)[0])}, " f"name: {json.dumps(name.split('/', 1)[1])}) " - "{ nameWithOwner stargazerCount pushedAt }" + f"{REPO_FIELDS}" for index, name in enumerate(batch) ) payload = http_json( @@ -108,6 +123,9 @@ def survey(client: GitHubClient, names: list[str]) -> tuple[set[str], dict[str, repo = data.get(f"r{index}") if not repo: continue + if repo_is_bare(repo, readme_text(repo)): + gone.add(name) + continue pushed = str(repo.get("pushedAt") or "")[:10] found[name.lower()] = [ int(repo.get("stargazerCount") or 0), @@ -418,7 +436,7 @@ def main() -> int: GitHubClient(token), [name for name in scope if name not in blocked], ) - print(f"{len(missing)} repositories no longer exist") + print(f"{len(missing)} repositories no longer exist or hold nothing runnable") if blocked: print(f"Pruning {len(blocked)} explicitly blacklisted repositories") missing |= blocked diff --git a/scripts/update_cves.py b/scripts/update_cves.py index 1d8f3a80e84..3fde86c48b9 100755 --- a/scripts/update_cves.py +++ b/scripts/update_cves.py @@ -291,6 +291,41 @@ def root_has_code(repo: dict[str, Any]) -> bool: return any(CODE_FILE_RE.search(name) for name in root_entry_names(repo)) +TEXT_ONLY_RE = re.compile( + r"^(?:readme|license|licence|security|contributing|changelog|code_of_conduct|" + r"\.gitignore|\.gitattributes|funding\.yml)(?:\.|$)|\.(?:md|txt|rst)$", + re.IGNORECASE, +) +PAYLOAD_RE = re.compile(r"```|~~~|^ {4}\S", re.MULTILINE) +COMMAND_RE = re.compile( + r"\b(?:curl|wget|python3?|php|perl|ruby|nc|ncat|bash|msfconsole|nuclei|sqlmap|" + r"GET /|POST /|PUT /)\b|https?://[^\s)]*\?" +) + + +def repo_is_bare(repo: dict[str, Any], readme: str) -> bool: + """Nothing runnable, and nothing that says how. + + A sample of two hundred indexed repositories held fourteen with a bare + README or none at all: empty shells, placeholders, a link to a paywall. + Anything with a directory, a file that is not documentation, a file named + for the CVE or as a PoC, or a README that carries a code block, a command, + a request or a real write-up is not bare. Where the root is unknown the + question is not asked. + """ + root = repo.get("root") + if not isinstance(root, dict): + return False + for entry in root.get("entries") or []: + name = str(entry.get("name") or "") + if str(entry.get("type") or "") == "tree" or not TEXT_ONLY_RE.search(name): + return False + if POC_RE.search(normalise_identity(name)) or CODE_FILE_RE.search(name): + return False + text = readme or "" + return not (len(text) >= 1000 or PAYLOAD_RE.search(text) or COMMAND_RE.search(text)) + + def truncated_ids(cve_ids: Collection[str]) -> set[str]: """Identifiers that a longer identifier from the same year extends.""" numbers: dict[str, set[str]] = defaultdict(set) @@ -343,6 +378,8 @@ def qualifying_repo_cves( description = str(repo.get("description") or "") topics = " ".join(topic_names(repo)) readme = readme_text(repo) + if repo_is_bare(repo, readme): + return set() all_name_cves = extract_cves(full_name) | extract_cves(normalise_identity(full_name)) all_identity_cves = ( all_name_cves