From 89fbf68e218da40c8439ef58707a704967d0f476 Mon Sep 17 00:00:00 2001 From: 0xMarcio Date: Tue, 6 Oct 2026 21:09:05 +0200 Subject: [PATCH] Fix workflow queues and index consistency --- .github/workflows/audit_poc_links.yml | 3 ++ .github/workflows/hot_cves.yml | 3 ++ .github/workflows/site.yml | 8 ++- .github/workflows/sync_collections.yml | 3 ++ .github/workflows/sync_cve_pocs.yml | 3 ++ .github/workflows/sync_exploits.yml | 3 ++ .github/workflows/sync_metadata.yml | 3 ++ .github/workflows/sync_nuclei.yml | 3 ++ .github/workflows/sync_pocingithub.yml | 3 ++ .gitignore | 3 ++ docs/logic.js | 2 +- scripts/audit_poc_links.py | 24 +++++++-- scripts/build_pages.py | 2 +- tests/test_audit_poc_links.py | 63 ++++++++++++++++++++++ tests/test_build_pages.py | 30 +++++++++++ tests/test_search.js | 72 ++++++++++++++++++++++++++ 16 files changed, 220 insertions(+), 8 deletions(-) create mode 100644 tests/test_audit_poc_links.py create mode 100644 tests/test_build_pages.py create mode 100644 tests/test_search.js diff --git a/.github/workflows/audit_poc_links.yml b/.github/workflows/audit_poc_links.yml index 011aa323188..d83c1460585 100644 --- a/.github/workflows/audit_poc_links.yml +++ b/.github/workflows/audit_poc_links.yml @@ -11,6 +11,7 @@ permissions: concurrency: group: repository-updates + queue: max cancel-in-progress: false jobs: @@ -20,6 +21,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + ref: main - name: Set up Python uses: actions/setup-python@v7 diff --git a/.github/workflows/hot_cves.yml b/.github/workflows/hot_cves.yml index a177b3495f1..ce270eeba9d 100644 --- a/.github/workflows/hot_cves.yml +++ b/.github/workflows/hot_cves.yml @@ -11,6 +11,7 @@ permissions: concurrency: group: repository-updates + queue: max cancel-in-progress: false jobs: @@ -20,6 +21,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + ref: main - name: Set up Python uses: actions/setup-python@v7 diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml index 2205b748d9e..39752bf565f 100644 --- a/.github/workflows/site.yml +++ b/.github/workflows/site.yml @@ -15,7 +15,7 @@ permissions: concurrency: group: 'pages' - cancel-in-progress: false + cancel-in-progress: true jobs: build: @@ -30,6 +30,11 @@ jobs: with: python-version: '3.13' + - name: Verify code + run: | + python3 -m unittest discover -s tests + node --test tests/test_search.js + - name: Build site data run: python3 scripts/build_site.py @@ -68,6 +73,7 @@ jobs: deploy: needs: build runs-on: ubuntu-latest + timeout-minutes: 10 environment: name: github-pages url: ${{ steps.deploy.outputs.page_url }} diff --git a/.github/workflows/sync_collections.yml b/.github/workflows/sync_collections.yml index 188d60f6ef6..6d57a5dceea 100644 --- a/.github/workflows/sync_collections.yml +++ b/.github/workflows/sync_collections.yml @@ -11,6 +11,7 @@ permissions: concurrency: group: repository-updates + queue: max cancel-in-progress: false jobs: @@ -20,6 +21,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + ref: main - name: Set up Python uses: actions/setup-python@v7 diff --git a/.github/workflows/sync_cve_pocs.yml b/.github/workflows/sync_cve_pocs.yml index b1fce3329e7..1658e0e96ab 100644 --- a/.github/workflows/sync_cve_pocs.yml +++ b/.github/workflows/sync_cve_pocs.yml @@ -11,6 +11,7 @@ permissions: concurrency: group: repository-updates + queue: max cancel-in-progress: false jobs: @@ -20,6 +21,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + ref: main - name: Set up Python uses: actions/setup-python@v7 diff --git a/.github/workflows/sync_exploits.yml b/.github/workflows/sync_exploits.yml index 38557f60d16..6b17b0fc982 100644 --- a/.github/workflows/sync_exploits.yml +++ b/.github/workflows/sync_exploits.yml @@ -11,6 +11,7 @@ permissions: concurrency: group: repository-updates + queue: max cancel-in-progress: false jobs: @@ -20,6 +21,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + ref: main - name: Set up Python uses: actions/setup-python@v7 diff --git a/.github/workflows/sync_metadata.yml b/.github/workflows/sync_metadata.yml index 06b04a6275d..77a6a58d95b 100644 --- a/.github/workflows/sync_metadata.yml +++ b/.github/workflows/sync_metadata.yml @@ -18,6 +18,7 @@ permissions: concurrency: group: repository-updates + queue: max cancel-in-progress: false jobs: @@ -27,6 +28,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + ref: main - name: Set up Python uses: actions/setup-python@v7 diff --git a/.github/workflows/sync_nuclei.yml b/.github/workflows/sync_nuclei.yml index d21df114bb5..d8a6e0625d2 100644 --- a/.github/workflows/sync_nuclei.yml +++ b/.github/workflows/sync_nuclei.yml @@ -11,6 +11,7 @@ permissions: concurrency: group: repository-updates + queue: max cancel-in-progress: false jobs: @@ -20,6 +21,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + ref: main - name: Set up Python uses: actions/setup-python@v7 diff --git a/.github/workflows/sync_pocingithub.yml b/.github/workflows/sync_pocingithub.yml index a40bff5d54e..a895a9ea915 100644 --- a/.github/workflows/sync_pocingithub.yml +++ b/.github/workflows/sync_pocingithub.yml @@ -11,6 +11,7 @@ permissions: concurrency: group: repository-updates + queue: max cancel-in-progress: false jobs: @@ -20,6 +21,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + ref: main - name: Set up Python uses: actions/setup-python@v7 diff --git a/.gitignore b/.gitignore index ede59cec80e..f087424365f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,12 +6,15 @@ docs/repo_meta.json docs/trending_poc.json docs/nuclei.json docs/cve_metadata.json +docs/cvss.json +docs/advisories.json docs/epss.json docs/robots.txt docs/opensearch.xml docs/sitemap.xml docs/sitemap-*.xml docs/CVE-*.html +docs/[12][0-9][0-9][0-9].html docs/404.html docs/social-card.png __pycache__/ diff --git a/docs/logic.js b/docs/logic.js index 82279cda24d..a3179443287 100644 --- a/docs/logic.js +++ b/docs/logic.js @@ -1328,7 +1328,7 @@ async function loadJSON(url, options) { loadJSON('/epss.json', { cache: 'no-cache' }).then(data => { epss = data || {}; - if (state.ready && (state.query || hasActiveFilters())) renderResults(null); + if (state.ready && (state.query || hasActiveFilters())) render(); }).catch(err => console.warn(err.message)); loadJSON('/repo_meta.json', { cache: 'no-cache' }).then(data => { diff --git a/scripts/audit_poc_links.py b/scripts/audit_poc_links.py index 340c036110b..2d8ba191158 100644 --- a/scripts/audit_poc_links.py +++ b/scripts/audit_poc_links.py @@ -111,16 +111,30 @@ def survey(client: GitHubClient, names: list[str]) -> tuple[set[str], dict[str, data={"query": "query { " + aliases + " rateLimit { remaining resetAt } }"}, ) gone: set[str] = set() + failed_aliases: set[str] = set() for entry in payload.get("errors") or []: path = entry.get("path") or [] - if str(entry.get("type") or "") == "NOT_FOUND" and path: - index = str(path[0])[1:] - if index.isdigit() and int(index) < len(batch): - gone.add(batch[int(index)]) + if not path: + return set(), {} + alias = str(path[0]) + failed_aliases.add(alias) + index = alias[1:] + if ( + str(entry.get("type") or "") == "NOT_FOUND" + and len(path) == 1 + and alias.startswith("r") + and index.isdigit() + and int(index) < len(batch) + ): + gone.add(batch[int(index)]) found: dict[str, list] = {} data = payload.get("data") or {} for index, name in enumerate(batch): - repo = data.get(f"r{index}") + alias = f"r{index}" + # A failed README or root lookup is not evidence of a bare repo. + if alias in failed_aliases: + continue + repo = data.get(alias) if not repo: continue if repo_is_bare(repo, readme_text(repo)): diff --git a/scripts/build_pages.py b/scripts/build_pages.py index 3da88048d86..1e4c954eac4 100644 --- a/scripts/build_pages.py +++ b/scripts/build_pages.py @@ -230,7 +230,7 @@ def link_rows(urls: list, repo_meta: dict, *, trusted: bool = False) -> str: rows = [] for url in urls: label = url[len(GITHUB):] if url.startswith(GITHUB) else url - meta = repo_meta.get(label.lower()) if url.startswith(GITHUB) else None + meta = repo_meta.get(repo_of(url) or "") note = f"{meta[0]}★ · {esc(meta[1])}" if meta else "" rows.append( f'
  • ' diff --git a/tests/test_audit_poc_links.py b/tests/test_audit_poc_links.py new file mode 100644 index 00000000000..4b4dddfc326 --- /dev/null +++ b/tests/test_audit_poc_links.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import audit_poc_links +from update_cves import GitHubClient + + +class RepositorySurveyTests(unittest.TestCase): + def repo(self, filename: str = "README.md") -> dict: + return { + "nameWithOwner": "owner/poc", + "stargazerCount": 10, + "pushedAt": "2026-10-06T12:00:00Z", + "root": {"entries": [{"name": filename, "type": "blob"}]}, + "readmeMd": None, + } + + def survey(self, payload: dict, names: list[str] | None = None) -> tuple: + with patch.object(audit_poc_links, "http_json", return_value=payload): + return audit_poc_links.survey(GitHubClient("test-token"), names or ["owner/poc"]) + + def test_preserves_repository_when_readme_lookup_fails(self) -> None: + self.assertEqual(self.survey({ + "data": {"r0": self.repo()}, + "errors": [{"type": "INTERNAL", "path": ["r0", "readmeMd"]}], + }), (set(), {})) + + def test_nested_not_found_does_not_mean_repository_is_deleted(self) -> None: + self.assertEqual(self.survey({ + "data": {"r0": self.repo("exploit.py")}, + "errors": [{"type": "NOT_FOUND", "path": ["r0", "readmeMd"]}], + }), (set(), {})) + + def test_pathless_error_preserves_the_batch(self) -> None: + self.assertEqual(self.survey({ + "data": {"r0": self.repo()}, + "errors": [{"type": "INTERNAL", "message": "Temporary failure"}], + }), (set(), {})) + + def test_confirmed_missing_repository_is_removed(self) -> None: + self.assertEqual(self.survey({ + "data": {"r0": None}, + "errors": [{"type": "NOT_FOUND", "path": ["r0"]}], + }), ({"owner/poc"}, {})) + + def test_complete_bare_repository_is_removed(self) -> None: + self.assertEqual(self.survey({"data": {"r0": self.repo()}}), ({"owner/poc"}, {})) + + def test_partial_failure_keeps_other_repository_metadata(self) -> None: + self.assertEqual(self.survey({ + "data": {"r0": self.repo(), "r1": self.repo("exploit.py")}, + "errors": [{"type": "INTERNAL", "path": ["r0", "readmeMd"]}], + }, ["owner/poc", "owner/healthy"]), (set(), {"owner/healthy": [10, "2026-10-06"]})) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_build_pages.py b/tests/test_build_pages.py new file mode 100644 index 00000000000..0bdee95c131 --- /dev/null +++ b/tests/test_build_pages.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import build_pages + + +class RepositoryMetadataTests(unittest.TestCase): + def test_repository_metadata_is_shown_for_repository_paths(self) -> None: + metadata = {"owner/repo": [123, "2026-10-06", "2025-01-01"]} + for suffix in ("", "/blob/main/poc.py", "/tree/main", ".git"): + url = f"https://github.com/Owner/Repo{suffix}" + with self.subTest(url=url): + row = build_pages.link_rows([url], metadata) + self.assertIn(f'href="{url}"', row) + self.assertIn("123★ · 2026-10-06", row) + + def test_links_without_repository_metadata_are_still_rendered(self) -> None: + url = "https://example.com/advisory" + row = build_pages.link_rows([url], {}) + self.assertIn(f'href="{url}"', row) + self.assertNotIn("★", row) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_search.js b/tests/test_search.js new file mode 100644 index 00000000000..18160c804ea --- /dev/null +++ b/tests/test_search.js @@ -0,0 +1,72 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const vm = require('node:vm'); +const { MessageChannel } = require('node:worker_threads'); + +const script = fs.readFileSync(path.join(__dirname, '../docs/logic.js'), 'utf8'); + +for (const sort of ['relevance', 'newest']) { + test(`late EPSS data refreshes results while respecting ${sort} sort`, async t => { + const elements = new Map(); + const document = { + addEventListener() {}, + querySelectorAll: () => [], + querySelector(selector) { + if (!elements.has(selector)) { + elements.set(selector, { + value: '', + innerHTML: '', + addEventListener() {}, + setAttribute() {}, + querySelectorAll: () => [] + }); + } + return elements.get(selector); + } + }; + let finishEpss; + const pendingEpss = new Promise(resolve => { finishEpss = resolve; }); + const records = ['CVE-2025-1234', 'CVE-2024-1234'].map(cve => ({ + cve, + desc: 'widget remote execution', + poc: [`https://github.com/owner/${cve}`] + })); + const context = vm.createContext({ + console, + document, + performance, + MessageChannel, + URLSearchParams, + location: { pathname: '/', search: `?q=widget&sort=${sort}` }, + window: { matchMedia: () => ({ matches: false }) }, + setTimeout() {}, + clearTimeout() {}, + async fetch(url) { + const data = url === '/epss.json' ? await pendingEpss + : url === '/CVE_list.json' ? records + : url === '/trending_poc.json' + ? { items: [], generated: '2026-10-06', total_cves: 2, with_pocs: 2 } + : {}; + return { ok: true, json: async () => data }; + } + }); + vm.runInContext(script, context); + t.after(() => vm.runInContext('yieldPort.port1.close(); yieldPort.port2.close();', context)); + const resultIds = () => [...elements.get('[data-results]').innerHTML + .matchAll(/class="result-id" href="\/([^"]+)"/g)].map(match => match[1]); + + await new Promise(setImmediate); + assert.deepEqual(resultIds(), ['CVE-2025-1234', 'CVE-2024-1234']); + + finishEpss({ 'CVE-2025-1234': [0.01, 0.1], 'CVE-2024-1234': [0.99, 0.999] }); + await new Promise(setImmediate); + assert.deepEqual(resultIds(), sort === 'relevance' + ? ['CVE-2024-1234', 'CVE-2025-1234'] + : ['CVE-2025-1234', 'CVE-2024-1234']); + assert.match(elements.get('[data-results]').innerHTML, /EPSS 99%/); + }); +}