diff --git a/.github/workflows/audit_poc_links.yml b/.github/workflows/audit_poc_links.yml index d817fad4f61..011aa323188 100644 --- a/.github/workflows/audit_poc_links.yml +++ b/.github/workflows/audit_poc_links.yml @@ -16,7 +16,7 @@ concurrency: jobs: audit: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 50 steps: - name: Checkout repository uses: actions/checkout@v7 @@ -26,14 +26,13 @@ jobs: with: python-version: '3.13' - # A seventh of the old weekly batch every day: the 12,000-and-12,000 - # weekly run measured an hour, and it holds the repository-updates lock - # the whole time, which stalls the hourly refresh and can cancel a daily - # sync queued behind it. Same sweep period, about nine minutes a day. + # 4,000 repositories and 3,000 reference links a day sweeps the whole + # index about weekly, in around half an hour. A sample found one link in + # eight pointing at a deleted repository under the old fortnightly sweep. - name: Drop dead links and refresh repository metadata env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: python3 scripts/audit_poc_links.py --limit 1750 --references 1750 + run: python3 scripts/audit_poc_links.py --limit 4000 --references 3000 - name: Verify the generated index run: python3 scripts/build_site.py diff --git a/.github/workflows/hot_cves.yml b/.github/workflows/hot_cves.yml index e9d9bf4620b..a177b3495f1 100644 --- a/.github/workflows/hot_cves.yml +++ b/.github/workflows/hot_cves.yml @@ -26,6 +26,15 @@ jobs: with: python-version: '3.13' + # Every repository pushed in the last day that names a CVE of this year + # or last, with its README read and the classifier applied, so a PoC + # published this morning is on the site within the hour. The daily run + # keeps covering every year and the CVE record references. + - name: Discover this year's PoCs pushed in the last day + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 scripts/update_cves.py --lookback-days 1 --years 2 --skip-cvelist + - name: Ingest the previous hour's candidates env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -52,6 +61,8 @@ jobs: README.md \ index/trending.json \ index/github.txt \ + index/references.txt \ + index/reference_pocs.txt \ index/cve_dates.json \ index/cve_record_state.json \ docs/hero.svg \ diff --git a/.github/workflows/sync_pocingithub.yml b/.github/workflows/sync_pocingithub.yml index d25dada0fba..a40bff5d54e 100644 --- a/.github/workflows/sync_pocingithub.yml +++ b/.github/workflows/sync_pocingithub.yml @@ -2,7 +2,7 @@ name: Sync PoC-in-GitHub on: schedule: - - cron: '10 5 * * 3' + - cron: '10 5 * * *' workflow_dispatch: permissions: @@ -26,10 +26,11 @@ jobs: with: python-version: '3.13' - # Every candidate is judged again each week rather than remembered as + # Every candidate is judged again each day rather than remembered as # rejected: a repository that was an empty shell in March is often a # working exploit by June, and re-reading 6,800 of them costs five - # minutes. + # minutes. Daily because the collection keeps finding repositories our + # own search missed. - name: Judge upstream PoC candidates env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/audit_poc_links.py b/scripts/audit_poc_links.py index 48d1472a88a..340c036110b 100644 --- a/scripts/audit_poc_links.py +++ b/scripts/audit_poc_links.py @@ -16,6 +16,8 @@ from urllib.parse import urlsplit sys.path.insert(0, str(Path(__file__).resolve().parent)) from update_cves import ( + readme_text, + repo_is_bare, GITHUB_GRAPHQL_URL, kev_badge, USER_AGENT, @@ -72,11 +74,24 @@ def collect_repositories() -> tuple[dict[str, list[tuple[Path, str]]], set[str]] return prunable, referenced -def survey(client: GitHubClient, names: list[str]) -> tuple[set[str], dict[str, list]]: - """Return the repositories GitHub reports as NOT_FOUND, and stars/last push for the rest. +REPO_FIELDS = ( + "{ nameWithOwner stargazerCount pushedAt " + 'readmeMd: object(expression: "HEAD:README.md") { ... on Blob { text } } ' + 'readmeUpper: object(expression: "HEAD:README.MD") { ... on Blob { text } } ' + 'readmeRst: object(expression: "HEAD:README.rst") { ... on Blob { text } } ' + 'readmeBare: object(expression: "HEAD:README") { ... on Blob { text } } ' + 'root: object(expression: "HEAD:") { ... on Tree { entries { name type } } } }' +) - Anything else, a network error or a throttle or an unexpected response, leaves - the repository out of both results, so an audit failure never drops a link. + +def survey(client: GitHubClient, names: list[str]) -> tuple[set[str], dict[str, list]]: + """Return the repositories to drop, and stars/last push for the rest. + + Dropped means GitHub reports NOT_FOUND, or the repository is bare: no code, + no directory, no artifact and a README with nothing runnable in it, which + the discovery gate would reject today. Anything else, a network error or a + throttle or an unexpected response, leaves the repository out of both + results, so an audit failure never drops a link. """ batches = [names[index : index + BATCH] for index in range(0, len(names), BATCH)] missing: set[str] = set() @@ -87,7 +102,7 @@ def survey(client: GitHubClient, names: list[str]) -> tuple[set[str], dict[str, aliases = " ".join( f"r{index}: repository(owner: {json.dumps(name.split('/', 1)[0])}, " f"name: {json.dumps(name.split('/', 1)[1])}) " - "{ nameWithOwner stargazerCount pushedAt }" + f"{REPO_FIELDS}" for index, name in enumerate(batch) ) payload = http_json( @@ -108,6 +123,9 @@ def survey(client: GitHubClient, names: list[str]) -> tuple[set[str], dict[str, repo = data.get(f"r{index}") if not repo: continue + if repo_is_bare(repo, readme_text(repo)): + gone.add(name) + continue pushed = str(repo.get("pushedAt") or "")[:10] found[name.lower()] = [ int(repo.get("stargazerCount") or 0), @@ -418,7 +436,7 @@ def main() -> int: GitHubClient(token), [name for name in scope if name not in blocked], ) - print(f"{len(missing)} repositories no longer exist") + print(f"{len(missing)} repositories no longer exist or hold nothing runnable") if blocked: print(f"Pruning {len(blocked)} explicitly blacklisted repositories") missing |= blocked diff --git a/scripts/update_cves.py b/scripts/update_cves.py index 1d8f3a80e84..3fde86c48b9 100755 --- a/scripts/update_cves.py +++ b/scripts/update_cves.py @@ -291,6 +291,41 @@ def root_has_code(repo: dict[str, Any]) -> bool: return any(CODE_FILE_RE.search(name) for name in root_entry_names(repo)) +TEXT_ONLY_RE = re.compile( + r"^(?:readme|license|licence|security|contributing|changelog|code_of_conduct|" + r"\.gitignore|\.gitattributes|funding\.yml)(?:\.|$)|\.(?:md|txt|rst)$", + re.IGNORECASE, +) +PAYLOAD_RE = re.compile(r"```|~~~|^ {4}\S", re.MULTILINE) +COMMAND_RE = re.compile( + r"\b(?:curl|wget|python3?|php|perl|ruby|nc|ncat|bash|msfconsole|nuclei|sqlmap|" + r"GET /|POST /|PUT /)\b|https?://[^\s)]*\?" +) + + +def repo_is_bare(repo: dict[str, Any], readme: str) -> bool: + """Nothing runnable, and nothing that says how. + + A sample of two hundred indexed repositories held fourteen with a bare + README or none at all: empty shells, placeholders, a link to a paywall. + Anything with a directory, a file that is not documentation, a file named + for the CVE or as a PoC, or a README that carries a code block, a command, + a request or a real write-up is not bare. Where the root is unknown the + question is not asked. + """ + root = repo.get("root") + if not isinstance(root, dict): + return False + for entry in root.get("entries") or []: + name = str(entry.get("name") or "") + if str(entry.get("type") or "") == "tree" or not TEXT_ONLY_RE.search(name): + return False + if POC_RE.search(normalise_identity(name)) or CODE_FILE_RE.search(name): + return False + text = readme or "" + return not (len(text) >= 1000 or PAYLOAD_RE.search(text) or COMMAND_RE.search(text)) + + def truncated_ids(cve_ids: Collection[str]) -> set[str]: """Identifiers that a longer identifier from the same year extends.""" numbers: dict[str, set[str]] = defaultdict(set) @@ -343,6 +378,8 @@ def qualifying_repo_cves( description = str(repo.get("description") or "") topics = " ".join(topic_names(repo)) readme = readme_text(repo) + if repo_is_bare(repo, readme): + return set() all_name_cves = extract_cves(full_name) | extract_cves(normalise_identity(full_name)) all_identity_cves = ( all_name_cves