Track PoC releases independently and expand verified sources

scripts/build_pages.py

@@ -22,8 +22,10 @@ import os
 import sys
 from collections import defaultdict
 from datetime import date, datetime, timezone
+from decimal import ROUND_HALF_UP, Decimal
 from string import Template
 from urllib import error, request
+from urllib.parse import unquote, urlsplit
 
 ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir)
 sys.path.insert(0, os.path.join(ROOT, "scripts"))
@@ -36,6 +38,7 @@ GITHUB = "https://github.com/"
 SEVERITIES = {"NONE", "LOW", "MEDIUM", "HIGH", "CRITICAL"}
 LASTMOD = "page_lastmod.json"
 PAGE_STATE = "page_state.json"
+TREND_ROWS = 20
 
 SOURCES = (
     ("nuclei", "Nuclei templates", "projectdiscovery/nuclei-templates"),
@@ -45,6 +48,11 @@ SOURCES = (
     ("collections", "Exploit collections", ""),
 )
 
+# As logic.js labels curated paths: by leaf for these (CURATED there), and
+# below these collection roots for the rest (COLLECTION_SOURCES there).
+NAMED_BY_LEAF = {"zan8in/afrog", "chaitin/xray", "helloexp/0day", "tzwlhack/vulnerability"}
+COLLECTION_ROOTS = {"google/security-research": "pocs/", "github/securitylab": "SecurityExploits/"}
+
 
 def load(name: str) -> dict | list:
     with open(os.path.join(DOCS, name), encoding="utf-8") as handle:
@@ -166,13 +174,25 @@ def repo_of(url: str) -> str | None:
     return f"{parts[0].lower()}/{re.sub(r'[.]git$', '', parts[1], flags=re.I).lower()}"
 
 
+def artifact_label(url: str) -> str:
+    """A curated link's path inside its repository, as logic.js labels it.
+    Empty for a repository root or a link that is not into a repository."""
+    repo = repo_of(url)
+    if not repo:
+        return ""
+    parts = [part for part in urlsplit(url).path.split("/") if part][2:]
+    if parts[:1] in (["tree"], ["blob"]):
+        parts = parts[2:]
+    if repo in NAMED_BY_LEAF:
+        parts = parts[-1:]
+    path = unquote("/".join(parts))
+    root = COLLECTION_ROOTS.get(repo, "")
+    return path[len(root):] if root and path.startswith(root) and len(path) > len(root) else path
+
+
 def page_lastmod(entry: dict, data: dict) -> str:
-    """Bootstrap date before a page has a published content fingerprint."""
-    stamps = [entry.get("modified") or "", entry.get("published") or ""]
-    for url in entry.get("poc") or []:
-        meta = data["repo_meta"].get(repo_of(url) or "")
-        if meta and len(meta) > 1 and isinstance(meta[1], str):
-            stamps.append(meta[1][:10])
+    """Bootstrap date before a page has a published content fingerprint: the record's own."""
+    stamps = (entry.get("modified") or "", entry.get("published") or "")
     return max((s for s in stamps if len(s) == 10), default=data["today"])
 
 
@@ -196,38 +216,46 @@ def previous_page_state() -> dict:
     return payload["pages"]
 
 
-def tracked_state(key: str, payload: object, today: str, fallback: str, previous: dict) -> list[str]:
-    fingerprint = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+def digest(payload: object) -> str:
+    return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+
+
+def tracked_state(key: str, payload: object, today: str, fallback: str, previous: dict,
+                  legacy=None) -> list[str]:
+    fingerprint = digest(payload)
     old = previous.get(key)
     if old is not None:
         if not isinstance(old, list) or len(old) != 2:
             raise ValueError(f"Invalid published content date for {key}")
         date.fromisoformat(old[1])
-        lastmod = old[1] if old[0] == fingerprint else today
+        unchanged = old[0] == fingerprint or (legacy is not None and old[0] == digest(legacy()))
+        lastmod = old[1] if unchanged else today
     else:
         lastmod = today if previous else fallback
     return [fingerprint, lastmod]
 
 
 def content_state(entry: dict, data: dict, previous: dict) -> list[str]:
-    """Track significant content, including curated links absent from repo_meta.
+    """Track significant content: the record, every link, CVSS, advisories,
+    Nuclei and KEV.
 
-    Daily EPSS fluctuations, star counts and site-wide totals are deliberately
-    excluded; they are not a new exploit, assessment, or vulnerability record.
+    Daily EPSS fluctuations, star counts, repository pushes and site-wide totals
+    are deliberately excluded; they are not a new exploit, assessment, or
+    vulnerability record.
     """
     cid = entry["cve"]
-    payload = {
-        "entry": entry,
-        "cvss": (data["meta"].get(cid) or {}).get("cvss"),
-        "advisories": (data["meta"].get(cid) or {}).get("advisories"),
-        "nuclei": data["nuclei"].get(cid),
-        "kev": data["kev"].get(cid),
-        "pushed": {
-            repo: meta[1] for url in entry.get("poc") or []
-            if (repo := repo_of(url)) and (meta := data["repo_meta"].get(repo)) and len(meta) > 1
-        },
-    }
-    return tracked_state(cid, payload, data["today"], page_lastmod(entry, data), previous)
+    meta = data["meta"].get(cid) or {}
+    payload = {"entry": entry, "cvss": meta.get("cvss"), "advisories": meta.get("advisories"),
+               "nuclei": data["nuclei"].get(cid), "kev": data["kev"].get(cid)}
+
+    def legacy() -> dict:
+        # Manifests published while pushes were fingerprinted still match, so they keep their dates.
+        repos = data.get("repo_meta") or {}
+        pushed = {repo: stored[1] for url in entry.get("poc") or []
+                  if (repo := repo_of(url)) and (stored := repos.get(repo)) and len(stored) > 1}
+        return {**payload, "pushed": pushed}
+
+    return tracked_state(cid, payload, data["today"], page_lastmod(entry, data), previous, legacy)
 
 
 def build_related(cves: list) -> dict:
@@ -269,7 +297,7 @@ def build_related(cves: list) -> dict:
     return related
 
 
-def link_rows(urls: list, repo_meta: dict, *, trusted: bool = False) -> str:
+def link_rows(urls: list, repo_meta: dict, *, trusted: bool = False, collection: bool = False) -> str:
     """One row per source link.
 
     The repository list is crowd-sourced: anyone can publish a repository named
@@ -277,13 +305,21 @@ def link_rows(urls: list, repo_meta: dict, *, trusted: bool = False) -> str:
     Metasploit, Vulhub and the CVE and NVD records are curated destinations the
     index deliberately vouches for, and Google's guidance is to leave a link
     unqualified when the association is intended.
+
+    A curated path shows neither its repository's stars nor its push date.
+    Collection paths are labelled as search labels them, with the repository
+    as the note.
     """
     rel = "noopener" if trusted else "nofollow noopener"
     rows = []
     for url in urls:
         label = url[len(GITHUB):] if url.startswith(GITHUB) else url
-        meta = repo_meta.get(repo_of(url) or "")
+        path = artifact_label(url) if trusted else ""
+        meta = None if path else repo_meta.get(repo_of(url) or "")
         note = f"<span>{meta[0]}★ · {esc(meta[1])}</span>" if meta else ""
+        if path and collection:
+            repository = "/".join(url[len(GITHUB):].split("/")[:2])
+            label, note = path, f"<span>{esc(repository)}</span>"
         rows.append(
             f'<li><a href="{esc(url)}" rel="{rel}" target="_blank">'
             f"{esc(short(label, 90))}</a>{note}</li>"
@@ -402,7 +438,7 @@ def page(entry: dict, data: dict) -> str:
         if urls:
             blocks.append(
                 f"<h2>{heading} ({len(urls)})</h2>"
-                f'<ul class="cve-links">{link_rows(urls, data["repo_meta"], trusted=True)}</ul>'
+                f'<ul class="cve-links">{link_rows(urls, data["repo_meta"], trusted=True, collection=key == "collections")}</ul>'
             )
 
     siblings = "".join(
@@ -633,6 +669,52 @@ def hub_pages(cves: list, data: dict) -> dict:
     return {"pages": pages, "lastmod": lastmod, "states": states}
 
 
+def utc_day(value: object) -> str:
+    """UTC date of an ISO date or timestamp, read as logic.js reads it: a time
+    without a zone is UTC. Empty when there is no readable date."""
+    if not isinstance(value, str) or not re.match(r"\d{4}-\d{2}-\d{2}", value):
+        return ""
+    try:
+        moment = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return ""
+    return (moment.astimezone(timezone.utc) if moment.tzinfo else moment).date().isoformat()
+
+
+def star_label(stars: int) -> str:
+    """formatStars in logic.js. toFixed rounds the exact binary value half up,
+    so 1,250 stars is 1.3k there where Python's format would print 1.2k."""
+    if stars < 1000:
+        return str(stars)
+    return f"{Decimal(stars / 1000).quantize(Decimal('0.1'), ROUND_HALF_UP)}k"
+
+
+def trend_rows(items: object, landed: bool) -> str:
+    """A front-page list as logic.js renders it, in feed order. Just landed is
+    dated by first release, Trending by latest artifact commit."""
+    rows = []
+    for item in (items if isinstance(items, list) else [])[:TREND_ROWS]:
+        # A path in a collection does not have the collection's stars.
+        stars = None if item.get("artifact") else item.get("stars")
+        star_cell = "" if stars is None else f'{star_label(stars)} <span class="star">★</span>'
+        popular = " is-popular" if stars is not None and stars >= 500 else ""
+        # Off GitHub the last path segment is a bare id, so it keeps its host.
+        source = str(item.get("source") or "").removeprefix("www.")
+        name = (f"{source}/{item.get('name') or ''}" if item.get("artifact") and source and source != "github.com"
+                else item.get("name"))
+        flagged = '<span class="trend-kev" title="CISA known exploited">KEV</span>' if item.get("kev") else ""
+        detail = (f'<a class="trend-detail" href="{esc(item["page"])}">{esc(item.get("cve"))} details</a>'
+                  if item.get("page") else "")
+        rows.append(
+            f'<div class="trend-row"><span class="trend-stars{popular}">{star_cell}</span>'
+            f'<span class="trend-age">{utc_day(item.get("released" if landed else "pushed"))}</span>'
+            f'<span class="trend-name-cell">{flagged}<a class="trend-name" href="{esc(item.get("url"))}" '
+            f'target="_blank" rel="nofollow noopener">{esc(name)}</a>{detail}</span>'
+            f'<span class="trend-desc">{esc(item.get("desc"))}</span></div>'
+        )
+    return "".join(rows) or '<div class="empty">No recent PoCs.</div>'
+
+
 def homepage(cves: list, kev: dict, trending: dict) -> str:
     """Publish the same content before JavaScript or a crawler render runs."""
     indexed = {entry["cve"] for entry in cves}
@@ -640,35 +722,6 @@ def homepage(cves: list, kev: dict, trending: dict) -> str:
         raise ValueError("Homepage counts disagree with the searchable corpus")
     generated = datetime.fromisoformat(trending["generated"].replace("Z", "+00:00"))
 
-    def rank(item: dict) -> float:
-        try:
-            pushed = datetime.fromisoformat(item["pushed"].replace("Z", "+00:00"))
-            if pushed.tzinfo is None:
-                pushed = pushed.replace(tzinfo=timezone.utc)
-            age = max(0, (generated - pushed).total_seconds() / 3600)
-        except (KeyError, TypeError, ValueError):
-            age = 8760
-        return item["stars"] / (age + 6) ** 0.45
-
-    rows = []
-    for item in sorted(trending["items"], key=rank, reverse=True)[:20]:
-        cid = item.get("cve")
-        stars = item["stars"]
-        star_label = f"{stars / 1000:.1f}k" if stars >= 1000 else str(stars)
-        popular = " is-popular" if stars >= 500 else ""
-        flagged = '<span class="trend-kev" title="CISA known exploited">KEV</span>' if cid in kev else ""
-        detail = (f'<a class="trend-detail" href="/{esc(cid)}">{esc(cid)} details</a>'
-                  if cid in indexed else "")
-        rows.append(
-            f'<div class="trend-row"><span class="trend-stars{popular}">{star_label} <span class="star">★</span></span>'
-            f'<span class="trend-age">{esc((item.get("pushed") or "")[:10])}</span>'
-            f'<span class="trend-name-cell">{flagged}<a class="trend-name" href="{esc(item["url"])}" '
-            f'target="_blank" rel="nofollow noopener">{esc(item["name"])}</a>{detail}</span>'
-            f'<span class="trend-desc">{esc(item.get("desc"))}</span></div>'
-        )
-    if not rows:
-        rows.append('<div class="trend-row"><span class="trend-desc">No recent PoCs.</span></div>')
-
     website = {
         "@context": "https://schema.org",
         "@type": "WebSite",
@@ -687,7 +740,8 @@ def homepage(cves: list, kev: dict, trending: dict) -> str:
         description=esc(DESCRIPTION), site=esc(SITE), fonts=FONTS,
         sources_line=SOURCES_LINE, repo_link=REPO_LINK, website_schema=json_ld(website),
         total_cves=f"{trending['total_cves']:,}", with_pocs=f"{len(indexed):,}", kev=f"{len(kev):,}",
-        trending_rows="\n".join(rows), generated=esc(generated.strftime("%Y-%m-%d %H:%M UTC")),
+        landed_rows=trend_rows(trending.get("landed"), True),
+        generated=esc(generated.strftime("%Y-%m-%d %H:%M UTC")),
         year_links=" ".join(f'<a href="/{esc(year)}">{esc(year)}</a>' for year in years),
     )
 

scripts/build_site.py

@@ -36,6 +36,7 @@ METADATA_OUTPUT = DOCS_DIR / "cve_metadata.json"
 CVSS_OUTPUT = DOCS_DIR / "cvss.json"
 ADVISORIES_OUTPUT = DOCS_DIR / "advisories.json"
 VERIFIED_REFERENCES = INDEX / "reference_pocs.txt"
+REVIEWED_REFERENCES = INDEX / "reference_poc_verified.json"
 EPSS_OUTPUT = DOCS_DIR / "epss.json"
 EPSS_FEED = "https://epss.empiricalsecurity.com/epss_scores-current.csv.gz"
 GITHUB_NON_REPOS = {
@@ -103,11 +104,11 @@ def load_metadata() -> Dict[str, object]:
 
 @lru_cache(maxsize=1)
 def load_verified_references() -> set[tuple[str, str]]:
-    """CVE Program references with direct exploit evidence."""
+    """Independent CVE Program and reviewed-content reference evidence."""
     try:
         lines = VERIFIED_REFERENCES.read_text(encoding="utf-8").splitlines()
     except OSError:
-        return set()
+        lines = []
     verified: set[tuple[str, str]] = set()
     for line in lines:
         if " - " not in line:
@@ -115,6 +116,11 @@ def load_verified_references() -> set[tuple[str, str]]:
         cve_id, url = line.split(" - ", 1)
         if cve_id and url:
             verified.add((cve_id, link_key(url)))
+    if REVIEWED_REFERENCES.exists():
+        reviewed = json.loads(REVIEWED_REFERENCES.read_text(encoding="utf-8"))
+        if not isinstance(reviewed, dict) or any(not isinstance(urls, list) for urls in reviewed.values()):
+            raise ValueError("Invalid reviewed reference ledger")
+        verified.update((cve_id, link_key(url)) for cve_id, urls in reviewed.items() for url in urls)
     return verified
 
 
@@ -189,7 +195,8 @@ def repo_from_url(url: str) -> str:
     except Exception:
         path = url
     parts = path.strip("/").split("/")
-    if len(parts) < 2 or parts[0].lower() in GITHUB_NON_REPOS:
+    if (len(parts) < 2 or parts[0].lower() in GITHUB_NON_REPOS
+            or [part.lower() for part in parts[2:4]] == ["security", "advisories"]):
         return ""
     owner, repo = parts[0].lower(), re.sub(r"\.git$", "", parts[1], flags=re.I).lower()
     return f"{owner}/{repo}" if repo else ""
@@ -299,8 +306,12 @@ def reference_is_verified_poc(
     verified: Collection[tuple[str, str]],
 ) -> bool:
     key = link_key(url)
+    if (cve_id, key) in load_reference_exclusions():
+        return False
     if (cve_id, key) in verified:
         return True
+    if (cve_id, key) in load_managed_references():
+        return False
     for row in (metadata.get(cve_id) or {}).get("advisories", []):
         if not isinstance(row, list) or not row:
             continue
@@ -310,6 +321,18 @@ def reference_is_verified_poc(
     return False
 
 
+@lru_cache(maxsize=1)
+def load_reference_exclusions() -> set[tuple[str, str]]:
+    import sync_reference_pocs
+    return sync_reference_pocs.load_exclusions()
+
+
+@lru_cache(maxsize=1)
+def load_managed_references() -> set[tuple[str, str]]:
+    import sync_reference_pocs
+    return {(row["cve"], link_key(row["url"])) for row in sync_reference_pocs.load_reviews()}
+
+
 def collect_links(block: str, *, blacklist: Optional[Collection[str]] = None) -> List[str]:
     links: List[str] = []
     blacklist = blacklist or set()
@@ -525,6 +548,35 @@ def build_trending(blacklist: Collection[str]) -> List[Dict[str, object]]:
     ]
 
 
+def build_landed(cve_payload: list[dict], repo_meta: dict, kev: dict,
+                 ledger: dict | None = None, *, now: str | None = None, candidates: list[dict] | None = None) -> list[dict]:
+    import releases
+    ledger = releases.load_ledger() if ledger is None else ledger
+    entries = {entry["cve"]: entry for entry in cve_payload}
+    published = {releases.key(cve, url) for cve, url in releases.published_pairs(cve_payload)}
+    candidates = {releases.key(row["cve"], row["url"]): row for row in candidates or []}
+    rows = []
+    for artifact in releases.landed(ledger, now=now):
+        cve, url = artifact["cve"], artifact["url"]
+        if releases.key(cve, url) not in published:
+            continue
+        parsed = urlparse(url)
+        parts = unquote(parsed.path).strip("/").split("/")
+        repository = repo_from_url(url)
+        root_link = parsed.hostname == "github.com" and len(parts) == 2
+        meta = repo_meta.get(repository or "") if root_link else None
+        current = candidates.get(releases.key(cve, url), {})
+        stars = current.get("stars", meta[0] if meta else None) if root_link else None
+        rows.append({
+            "year": int(cve.split("-")[1]), "cve": cve, "url": url,
+            "name": parts[-1] or cve, "stars": stars,
+            "desc": entries[cve].get("desc", ""), "released": artifact["released"],
+            "basis": artifact["basis"], "kev": cve in kev, "page": f"/{cve}",
+            "landed": True, "source": parsed.hostname, "artifact": not root_link,
+        })
+    return rows[:100]
+
+
 def write_json(path: Path, data, *, indent: Optional[int] = None) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     with path.open("w", encoding="utf-8") as handle:
@@ -558,8 +610,12 @@ def main() -> int:
     epss = build_epss(cve_payload)
     write_json(EPSS_OUTPUT, epss)
 
-    trending_items = build_trending(blacklist)
     indexed = {entry["cve"] for entry in cve_payload}
+    frontpage_candidates = build_trending(blacklist)
+    trending_items = [row for row in frontpage_candidates
+                      if row.get("trending", True) and row.get("cve") in indexed]
+    trending_items.sort(key=lambda row: row.get("score", 0), reverse=True)
+    landed_items = build_landed(cve_payload, repo_meta, kev, candidates=frontpage_candidates)
     for item in trending_items:
         # Only link to detail pages that this build will actually publish.
         item["page"] = f"/{item['cve']}" if item.get("cve") in indexed else None
@@ -570,6 +626,8 @@ def main() -> int:
             "total_cves": total_cves,
             "with_pocs": len(cve_payload),
             "items": trending_items,
+            "landed": landed_items,
+            "ranking": "stars_weighted_by_artifact_release_age",
         },
         indent=2,
     )

scripts/releases.py

@@ -0,0 +1,728 @@
+#!/usr/bin/env python3
+"""Durable release evidence for published CVE/artifact pairs.
+
+Observing a link is not evidence of its publication date. Imports remain
+undated until source history supplies evidence; removed links stay recorded.
+"""
+from __future__ import annotations
+
+import argparse
+import base64
+import copy
+import hashlib
+import json
+import os
+import re
+import tempfile
+import threading
+import time
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from urllib.parse import quote, unquote, urlencode, urlsplit
+from urllib.request import Request, urlopen
+
+import source_artifacts
+from update_cves import CODE_FILE_RE, INDEX, POC_RE, poc_link_key
+
+DIRECTORY = INDEX / "releases"
+VERSION = 1
+DATE_ORDER = {"commit": 1, "source": 2, "created": 3, "public": 4, "merged": 5, "seen": 6}
+EXCLUDED_DIRS = {".git", ".github", "docs", "doc", "images", "img", "assets", "screenshots", "media",
+                 "tests", "test", "node_modules", "vendor", "third_party", "__pycache__"}
+BUILD_FILES = {"setup.py", "conftest.py", "__init__.py", "requirements.txt", "package.json", "package-lock.json",
+               "go.mod", "go.sum", "cargo.toml", "cargo.lock", "makefile", "cmakelists.txt", "dockerfile",
+               "docker-compose.yml", "pyproject.toml"}
+DATA_SUFFIXES = {".json", ".yml", ".yaml", ".toml", ".lock", ".txt", ".csv", ".xml", ".ini", ".cfg", ".md",
+                 ".rst", ".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".gz", ".zip", ".xz", ".bz2",
+                 ".tar", ".7z", ".o", ".so", ".a", ".dll", ".exe"}
+PAPERWORK = re.compile(r"^(?:readme|license|licence|security|contributing|changelog|code.of.conduct)(?:\.|$)", re.I)
+SUPPORT_FILE = re.compile(r"^(?:setup|install|example|demo|test|conftest|helper|util|common)(?:[_.-]|$)", re.I)
+HISTORY_VERSION = 3
+HISTORY_FIELDS = {"paths", "rev", "blobs", "commit", "sha", "commit_verified", "history_paths",
+                  "merged", "merged_verified", "history_version", "bulk"}
+
+
+class Ledger(dict):
+    """Keep identity comparisons local to one CVE during large backfills."""
+    def __init__(self, values=()):
+        super().__init__()
+        self.by_cve = defaultdict(set)
+        for identity, row in dict(values).items():
+            self[identity] = row
+
+    def __setitem__(self, identity, row):
+        super().__setitem__(identity, row)
+        self.by_cve[row["cve"]].add(identity)
+
+    def __deepcopy__(self, memo):
+        return Ledger((identity, copy.deepcopy(row, memo)) for identity, row in self.items())
+
+
+def utcnow() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def timestamp(value: object) -> datetime | None:
+    if not isinstance(value, str):
+        return None
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed.astimezone(timezone.utc)
+    except ValueError:
+        return None
+
+
+def key(cve: str, url: str) -> str:
+    if not re.fullmatch(r"CVE-\d{4}-\d{4,}", cve):
+        raise ValueError(f"Invalid CVE: {cve}")
+    parsed = urlsplit(url)
+    if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
+        raise ValueError("Invalid artifact URL")
+    return f"{cve} {poc_link_key(url)}"
+
+
+def artifact_paths(entries: list[dict], cve: str, *, readme_path: str | None = None,
+                   readme_qualified: bool = False, limit: int = 12) -> list[str]:
+    """Select paths in an already qualified repository, before applying a cap.
+
+    A selected directory still needs its qualifying files checked for dating.
+    README fallback is explicit: an advisory/claim alone is not an artifact.
+    """
+    ranked = []
+    for entry in entries:
+        path = str(entry.get("path") or entry.get("name") or "")
+        parts = path.split("/")
+        name = parts[-1].lower()
+        if (not path or any(part.lower() in EXCLUDED_DIRS for part in parts)
+                or name in BUILD_FILES or PAPERWORK.match(name) or SUPPORT_FILE.match(name)):
+            continue
+        kind = str(entry.get("type") or "blob").lower()
+        code = bool(CODE_FILE_RE.search(name)) or name.endswith((".html", ".asm", ".lua"))
+        if not code and Path(name).suffix in DATA_SUFFIXES:
+            continue
+        rank = (0 if cve.lower() in name else 1 if code else 2 if POC_RE.search(name)
+                else 3 if kind == "tree" else 4)
+        ranked.append((rank, path))
+    selected = [path for _, path in sorted(set(ranked))[:limit]]
+    return selected or ([readme_path] if readme_qualified and readme_path else [])
+
+
+def artifact_identity(entries: list[dict], paths: list[str]) -> dict:
+    """Fingerprint selected content, independently of path moves or paperwork."""
+    selected = {entry.get("path") or entry.get("name"): entry for entry in entries}
+    objects, blobs = [], []
+    for path in paths:
+        entry = selected.get(path) or {}
+        oid = entry.get("sha256") or entry.get("oid") or entry.get("sha")
+        if not isinstance(oid, str) or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", oid):
+            continue
+        kind = str(entry.get("type") or "blob").lower()
+        objects.append((kind, oid))
+        size = entry.get("size", entry.get("byteSize", 0))
+        if kind == "blob" and isinstance(size, int) and size >= 256:
+            blobs.append(oid)
+    result = {"paths": sorted(paths)}
+    if objects:
+        result["rev"] = hashlib.sha256(json.dumps(sorted(set(objects)), separators=(",", ":")).encode()).hexdigest()
+    if blobs:
+        result["blobs"] = sorted(set(blobs))
+    return result
+
+
+def date_release(evidence: dict, *, now: str | None = None) -> tuple[str | None, str]:
+    """Latest evidenced lower bound; client commit clocks have strict ceilings."""
+    if evidence.get("pending"):
+        return None, "pending"
+    current = timestamp(now or utcnow())
+    ceiling = current
+    for name in ("seen", "pushed_at"):
+        if parsed := timestamp(evidence.get(name)):
+            ceiling = min(ceiling, parsed)
+    bounds = []
+    for name in ("created", "public", "commit", "merged", "source"):
+        value = evidence.get(name)
+        moment = timestamp(value)
+        if moment is None or moment > ceiling:
+            continue
+        if name in {"commit", "merged"} and evidence.get(f"{name}_verified") is not True:
+            continue
+        bounds.append((moment, DATE_ORDER[name], value, name))
+    chosen = max(bounds) if bounds else None
+    absent, seen = timestamp(evidence.get("absent")), timestamp(evidence.get("seen"))
+    # Parser/gate changes and import baselines never establish absence.
+    if (not evidence.get("imported") and evidence.get("absence_verified") is True
+            and evidence.get("previous_head") and absent and seen and absent < seen <= current
+            and (chosen is None or chosen[0] <= absent)):
+        return evidence["seen"], "seen"
+    return (chosen[2], chosen[3]) if chosen else (None, "unknown")
+
+
+def load_ledger(directory: Path | None = None) -> dict[str, dict]:
+    result = Ledger()
+    for path in sorted((directory or DIRECTORY).glob("[12][0-9][0-9][0-9].json")):
+        payload = json.loads(path.read_text(encoding="utf-8"))
+        if payload.get("version") != VERSION or not isinstance(payload.get("entries"), dict):
+            raise ValueError(f"Invalid release shard: {path}")
+        for identity, row in payload["entries"].items():
+            if (not isinstance(row, dict) or identity != key(row.get("cve", ""), row.get("url", ""))
+                    or row["cve"].split("-")[1] != path.stem or identity in result):
+                raise ValueError(f"Invalid release entry: {path}")
+            result[identity] = row
+    return result
+
+
+def save_ledger(ledger: dict[str, dict], directory: Path | None = None) -> None:
+    directory = directory or DIRECTORY
+    previous = load_ledger(directory)
+    if not previous.keys() <= ledger.keys():
+        raise ValueError("Release history cannot shrink; retain removed entries as tombstones")
+    for identity, old in previous.items():
+        if old.get("seen") != ledger[identity].get("seen") or old.get("imported") != ledger[identity].get("imported"):
+            raise ValueError("First release observation and import provenance are immutable")
+    shards = defaultdict(dict)
+    for identity, row in sorted(ledger.items()):
+        if identity != key(row["cve"], row["url"]):
+            raise ValueError("Release key differs from its artifact")
+        shards[row["cve"].split("-")[1]][identity] = row
+    directory.mkdir(parents=True, exist_ok=True)
+    for year, entries in shards.items():
+        # One sorted entry per line keeps backfills and hourly diffs reviewable.
+        body = '{"version":1,"entries":{\n' + ",\n".join(
+            json.dumps(identity) + ":" + json.dumps(row, sort_keys=True, separators=(",", ":"))
+            for identity, row in entries.items()) + "\n}}\n"
+        path = directory / f"{year}.json"
+        if path.exists() and path.read_text(encoding="utf-8") == body:
+            continue
+        temporary = path.with_suffix(".tmp")
+        temporary.write_text(body, encoding="utf-8")
+        temporary.replace(path)
+
+
+def _repository_artifact(url: str) -> tuple[str, str] | None:
+    parsed = urlsplit(url)
+    parts = unquote(parsed.path).strip("/").split("/")
+    if parsed.hostname not in {"github.com", "www.github.com"} or len(parts) < 2:
+        return None
+    if len(parts) == 2:
+        return "/".join(parts[:2]).lower(), ""
+    if len(parts) >= 5 and parts[2] in {"blob", "tree"}:
+        return "/".join(parts[:2]).lower(), "/".join(parts[4:])
+    return None
+
+
+def _same_artifact(row: dict, candidate: dict) -> bool:
+    left, right = _repository_artifact(row["url"]), _repository_artifact(candidate["url"])
+    if (left and right and row.get("rev") and row.get("rev") == candidate.get("rev")
+            and (left[0] == right[0] or row.get("repo") and row.get("repo") == candidate.get("repo"))):
+        return True
+    # Repository IDs identify transfers, but never collapse different variants.
+    return bool(left and right and row.get("repo") and row.get("repo") == candidate.get("repo")
+                and left[1] == right[1] and left[0] != right[0])
+
+
+def record(ledger: dict[str, dict], cve: str, url: str, evidence: dict | None = None, *,
+           observed_at: str | None = None, import_mode: bool = False) -> dict:
+    observed_at = observed_at or utcnow()
+    identity = key(cve, url)
+    old = ledger.get(identity)
+    row = copy.deepcopy(old) if old else {"cve": cve, "url": url, "basis": "unknown", "released": None}
+    row.pop("gone", None)
+    if not old:
+        if import_mode:
+            row["imported"] = True
+        else:
+            row["seen"] = observed_at
+    evidence = evidence or {}
+    if evidence.get("error") and row.get("released"):
+        # A failed refresh may advance retry bookkeeping, never valid evidence.
+        evidence = {name: value for name, value in evidence.items() if name in {"error", "checked", "head"}}
+    method = evidence.get("history_version", 0)
+    method = method if type(method) is int and method > 0 else 0
+    old_method = row.get("history_version", 0)
+    old_method = old_method if type(old_method) is int and old_method > 0 else 0
+    moment = timestamp(evidence.get("commit"))
+    ceiling = min(timestamp(value) for value in (
+        observed_at, row.get("seen"), evidence.get("pushed_at", row.get("pushed_at"))
+    ) if timestamp(value))
+    verified_history = bool(not evidence.get("error") and evidence.get("commit_verified") is True
+                            and moment and moment <= ceiling)
+    correction = bool(old and verified_history and method > old_method)
+    if old_method and evidence.get("commit_verified") and method < old_method:
+        evidence = {name: value for name, value in evidence.items() if name not in HISTORY_FIELDS}
+    if correction:
+        for name in HISTORY_FIELDS | {"copy", "absence_verified"}:
+            row.pop(name, None)
+        row.update(released=None, basis="unknown")
+    for name in ("created", "public", "repo", "paths", "rev", "blobs", "source", "absent", "previous_head",
+                 "absence_verified", "commit", "sha", "commit_verified", "merged", "merged_verified", "pushed_at", "bulk", "checked", "head", "history_paths"):
+        if name not in evidence:
+            continue
+        value = evidence[name]
+        if name in {"paths", "rev", "blobs", "absent", "previous_head", "absence_verified"} and name in row:
+            continue
+        if name in {"commit", "merged", "source"} and timestamp(row.get(name)):
+            if not timestamp(value) or timestamp(value) >= timestamp(row[name]):
+                continue
+        if name == "sha" and row.get("commit") != evidence.get("commit"):
+            continue
+        row[name] = value
+    if verified_history and method >= old_method and method:
+        row["history_version"] = method
+    if evidence.get("error"):
+        if not row.get("released"):
+            row.update(basis="pending", pending=True)
+    else:
+        if evidence.get("commit_verified") or evidence.get("source") or evidence.get("absence_verified"):
+            row.pop("pending", None)
+        released, basis = date_release(row, now=observed_at)
+        if released:
+            # Re-adds and subsequent observations are not another release.
+            if row.get("basis") != "seen" or not row.get("released"):
+                row.update(released=released, basis=basis)
+    if (not old or correction or not any(old.get(field) for field in ("repo", "rev", "blobs"))) and any(row.get(field) for field in ("repo", "rev", "blobs")):
+        identities = ledger.by_cve.get(cve, ()) if isinstance(ledger, Ledger) else ledger.keys()
+        candidates = [(other_key, ledger[other_key]) for other_key in identities
+                      if other_key != identity and ledger[other_key]["cve"] == cve]
+        matches = [(other_key, other) for other_key, other in candidates if _same_artifact(row, other)]
+        copies = [(other_key, other) for other_key, other in candidates
+                  if row.get("blobs") and set(row["blobs"]) == set(other.get("blobs", []))]
+        if matches or copies:
+            other_key, other = min(matches or copies, key=lambda item: (item[1].get("released") or "9999", item[0]))
+            row["copy"] = other.get("copy", other_key)
+            row["released"], row["basis"] = other.get("released"), other.get("basis", "unknown")
+    elif row.get("copy") in ledger:
+        original = ledger[row["copy"]]
+        row["released"], row["basis"] = original.get("released"), original.get("basis", "unknown")
+    ledger[identity] = row
+    return row
+
+
+def mark_bulk(ledger: dict[str, dict]) -> None:
+    groups = defaultdict(list)
+    for identity, row in ledger.items():
+        source = row.get("repo") or (_repository_artifact(row["url"]) or (row["url"],))[0]
+        if row.get("sha") and row.get("commit_verified"):
+            groups[(str(source), row["sha"], "commit")].append(identity)
+        elif row.get("basis") == "seen":
+            groups[(str(source), row["seen"], "seen")].append(identity)
+    for (*_, basis), identities in groups.items():
+        count = len({ledger[identity]["cve"] for identity in identities}) if basis == "commit" else len(identities)
+        if count > (10 if basis == "commit" else 25):
+            for identity in identities:
+                ledger[identity]["bulk"] = True
+
+
+def resolve_copies(ledger: dict[str, dict], *, now: str | None = None) -> None:
+    """Resolve identities after a batch, independent of enrichment order.
+
+    Entire selected nontrivial blob sets must match. One shared helper cannot
+    collapse distinct variants. Tombstones still supply original provenance.
+    """
+    groups, parents = defaultdict(list), {}
+    for identity, row in ledger.items():
+        location = _repository_artifact(row["url"])
+        if blobs := row.get("blobs"):
+            groups[(row["cve"], "content", tuple(sorted(set(blobs))))].append(identity)
+        if row.get("rev") and location:
+            groups[(row["cve"], "revision", str(row.get("repo") or location[0]), row["rev"])].append(identity)
+        if row.get("repo") and location:
+            groups[(row["cve"], "transfer", str(row["repo"]), location[1])].append(identity)
+
+    def root(identity):
+        parents.setdefault(identity, identity)
+        if parents[identity] != identity:
+            parents[identity] = root(parents[identity])
+        return parents[identity]
+
+    for signature, identities in groups.items():
+        if len(identities) < 2:
+            continue
+        if signature[1] == "transfer" and len({_repository_artifact(ledger[item]["url"])[0] for item in identities}) < 2:
+            continue
+        first = min(identities)
+        for identity in identities:
+            parents[root(identity)] = root(first)
+    components = defaultdict(list)
+    for identity in parents:
+        components[root(identity)].append(identity)
+    for identities in components.values():
+        own_dates = {identity: date_release(ledger[identity], now=now) for identity in identities}
+        original = min(identities, key=lambda identity: (
+            timestamp(own_dates[identity][0]) or datetime.max.replace(tzinfo=timezone.utc), identity))
+        released, basis = own_dates[original]
+        for identity in identities:
+            row = ledger[identity]
+            row["released"], row["basis"] = released, basis
+            if identity == original:
+                row.pop("copy", None)
+            else:
+                row["copy"] = original
+    for identity, row in ledger.items():
+        if row.get("copy") and identity not in parents:
+            row.pop("copy", None)
+            row["released"], row["basis"] = date_release(row, now=now)
+
+
+def reconcile(ledger: dict[str, dict], pairs: list[tuple[str, str]], *, evidence: dict[str, dict] | None = None,
+              observed_at: str | None = None, import_mode: bool = False, complete: bool = True) -> dict[str, dict]:
+    observed_at = observed_at or utcnow()
+    result = copy.deepcopy(ledger) if isinstance(ledger, Ledger) else Ledger(copy.deepcopy(ledger))
+    present = set()
+    for cve, url in sorted(set(pairs)):
+        identity = key(cve, url)
+        present.add(identity)
+        record(result, cve, url, (evidence or {}).get(identity), observed_at=observed_at, import_mode=import_mode)
+    if complete:
+        for identity, row in result.items():
+            if identity not in present:
+                row.setdefault("gone", observed_at)
+    resolve_copies(result, now=observed_at)
+    mark_bulk(result)
+    return result
+
+
+def landed(ledger: dict[str, dict], *, now: str | None = None, days: int = 10) -> list[dict]:
+    moment = timestamp(now or utcnow())
+    found = [(identity, row) for identity, row in ledger.items()
+             if not any(row.get(flag) for flag in ("gone", "copy", "bulk"))
+             and row.get("basis") not in {"unknown", "pending"}
+             and timestamp(row.get("released")) is not None
+             and moment - timedelta(days=days) <= timestamp(row["released"]) <= moment]
+    found.sort(key=lambda item: item[0])
+    found.sort(key=lambda item: timestamp(item[1]["released"]), reverse=True)
+    return [dict(row) for _, row in found]
+
+
+def published_pairs(entries: list[dict] | None = None) -> list[tuple[str, str]]:
+    if entries is None:
+        # Lazy import also lets the site builder consume this ledger.
+        from build_site import build_cve_list, load_blacklist
+        entries, _ = build_cve_list(load_blacklist())
+    return sorted({(row["cve"], url) for row in entries
+                   for field in ("poc", "nuclei", "edb", "msf", "vulhub", "collections")
+                   for url in row.get(field, [])})
+
+
+def qualifying_content(repository: str, cve: str, path: str, source: bytes, reviews: list[dict]) -> bool:
+    """Confirm that a historical file contains reproduction material, not a stub.
+
+    This only dates already accepted links. It is not a discovery/approval gate.
+    Ambiguous historical content stays undated until reviewed.
+    """
+    if cve in source_artifacts.approved_cves_for_artifact(repository, path, source, reviews=reviews):
+        return True
+    if source_artifacts.identity_cves(path) - {cve}:
+        return False
+    if len(source.strip()) < 40:
+        return False
+    text = source.decode("utf-8", "replace")
+    name = Path(path).name.lower()
+    if repository.lower() == "github/securitylab" and name in {"bug.pdf", "fuzzer-poc.djvu", "poc.bin"}:
+        return bool(source)
+    if repository.lower() == "github/securitylab" and name.endswith(".cue"):
+        return bool(re.search(r"(?m)^\s*(?:FILE|TRACK|INDEX)\b", text))
+    if name.endswith((".md", ".rst")) or name == "readme":
+        context = ""
+        for chunk in re.split(r"(```[^\n]*\n.*?```)", text, flags=re.S):
+            if not chunk.startswith("```"):
+                context = chunk[-1000:]
+                continue
+            if not POC_RE.search(context) or re.search(r"(?im)^#+\s*(?:install|setup|requirements|mitigation)", context):
+                continue
+            if re.search(r"(?m)^\s*(?:curl\s+(?!.*(?:install|update)\.).*(?:https?://|--url)|(?:GET|POST|PUT)\s+/|.*(?:requests\.(?:get|post)|\.sendall)\s*\()", chunk):
+                return True
+        return False
+    if repository in {"tenable/poc", "pedrib/PoC"}:
+        from sync_collections import author_artifact_ids
+        if cve not in author_artifact_ids(repository, path, text):
+            return False
+    if name in {"diff.txt", "stroke_patch.txt"} and repository.lower() == "github/securitylab":
+        return bool(re.search(r"(?m)^@@ .*@@", text) and re.search(r"(?m)^\+[^+]", text))
+    if not (CODE_FILE_RE.search(name) or name.endswith((".html", ".asm", ".lua"))):
+        return False
+    # Imports/comments alone and placeholder text do not establish introduction.
+    executable = "\n".join(line for line in text.splitlines()
+                           if line.strip() and not re.match(r"\s*(?:#|//|/\*|\*|import\b|from\s+\S+\s+import\b)", line))
+    # Function/class declarations, pass/return stubs and arbitrary method calls
+    # are not historical exploit evidence. Require an actual action/payload.
+    return bool(re.search(r"(?:\b(?:socket|send|sendto|sendmsg|sendmmsg|write|ioctl|syscall|mmap|mprotect|ptrace|"
+                          r"prctl|setsockopt|msgsnd|msgrcv|setxattr|execve|system|memcpy|strcpy|gets|"
+                          r"remote|process|fetch|eval|unserialize)\s*\(|"
+                          r"\b(?:requests|urllib|http|axios|Net::HTTP)\W.{0,40}\b(?:get|post|request|open|start)\s*\(|"
+                          r"\.(?:send|sendall|send_request_cgi|write|Write|writeObject|Dial|NewRequest|"
+                          r"connect|Connect|exec|execute|communicate)\s*\(|"
+                          r"<(?:form|script)\b|\bcurl\s+(?:.*(?:https?://|--url)))", executable))
+
+
+class GitHubHistory:
+    """Pinned, bounded history reads. Failed requests never become absence."""
+    def __init__(self, headers: dict, *, fetch=None):
+        self.headers, self.fetch, self.cache = headers, fetch, {}
+        self.reviews = source_artifacts.load_reviews()
+        self.budget = threading.local()
+
+    def begin(self, *, budget_seconds: float = 45, max_requests: int = 24) -> None:
+        self.budget.deadline = time.monotonic() + budget_seconds
+        self.budget.remaining = max_requests
+
+    def get(self, path: str):
+        if path not in self.cache:
+            if not hasattr(self.budget, "deadline"):
+                self.begin()
+            remaining = self.budget.deadline - time.monotonic()
+            if remaining <= 0 or self.budget.remaining <= 0:
+                raise TimeoutError("Artifact history inspection budget exhausted")
+            self.budget.remaining -= 1
+            url = "https://api.github.com" + path
+            if self.fetch is not None:
+                payload = self.fetch(url, headers=self.headers, timeout=min(15, remaining))
+            else:
+                request = Request(url, headers={"User-Agent": "pocindex-release-evidence", **self.headers})
+                with urlopen(request, timeout=min(15, remaining)) as response:
+                    payload = json.load(response)
+            self.cache[path] = payload
+        return self.cache[path]
+
+    def tree(self, repository: str, revision: str, scope: str = "") -> list[dict]:
+        expression = revision + (":" + scope if scope else "")
+        payload = self.get(f"/repos/{repository}/git/trees/{quote(expression, safe='')}?recursive=1")
+        if not isinstance(payload, dict) or payload.get("truncated") is not False or not isinstance(payload.get("tree"), list):
+            raise ValueError("Incomplete artifact tree")
+        result = []
+        for item in payload["tree"]:
+            if not isinstance(item, dict) or not source_artifacts.valid_path(str(item.get("path") or "")):
+                raise ValueError("Invalid artifact tree entry")
+            if item.get("type") == "blob" and item.get("mode") in {"100644", "100755"}:
+                result.append({**item, "path": (scope + "/" if scope else "") + item["path"]})
+        return result
+
+    def blob(self, repository: str, entry: dict) -> bytes:
+        if not isinstance(entry.get("size"), int) or not 0 < entry["size"] <= 2_000_000:
+            raise ValueError("Artifact blob is empty or exceeds the history inspection bound")
+        payload = self.get(f"/repos/{repository}/git/blobs/{entry['sha']}")
+        if payload.get("encoding") != "base64" or not isinstance(payload.get("content"), str):
+            raise ValueError("Missing artifact blob")
+        source = base64.b64decode("".join(payload["content"].split()), validate=True)
+        digest = hashlib.sha1(f"blob {len(source)}\0".encode() + source).hexdigest()
+        if len(source) != entry["size"] or digest != entry["sha"]:
+            raise ValueError("Artifact blob differs from its pinned tree")
+        return source
+
+    def qualifying(self, repository: str, revision: str, paths: list[str], cve: str) -> list[dict]:
+        # Read each parent tree once; a requested directory is expanded into its
+        # real files so its initial README/config commit cannot date the PoC.
+        found = {}
+        for path in paths:
+            parent = path.rsplit("/", 1)[0] if "/" in path else ""
+            rows = self.tree(repository, revision, parent)
+            candidates = [r for r in rows if r["path"] == path or r["path"].startswith(path.rstrip("/") + "/")]
+            explicit = [r for r in candidates if r["path"] == path]
+            selected = {r["path"] for r in explicit} or set(artifact_paths(candidates, cve))
+            if repository.lower() == "github/securitylab":
+                selected.update(r["path"] for r in candidates if Path(r["path"]).name.lower() in {
+                    "bug.pdf", "fuzzer-poc.djvu", "poc.bin", "diff.txt", "stroke_patch.txt"} or r["path"].endswith(".cue"))
+            for entry in candidates:
+                size = entry.get("size")
+                if not isinstance(size, int) or not 0 < size <= 2_000_000:
+                    continue
+                if entry["path"] in selected and qualifying_content(repository, cve, entry["path"], self.blob(repository, entry), self.reviews):
+                    found[entry["path"]] = entry
+        return list(found.values())
+
+    def history(self, repository: str, revision: str, path: str) -> list[dict]:
+        result = []
+        for page in range(1, 21):
+            query = urlencode({"sha": revision, "path": path, "per_page": 100, "page": page})
+            rows = self.get(f"/repos/{repository}/commits?{query}")
+            if not isinstance(rows, list) or any(not isinstance(row, dict) or not re.fullmatch(r"[a-f0-9]{40}", str(row.get("sha") or "")) for row in rows):
+                raise ValueError("Incomplete artifact history")
+            result.extend(rows)
+            if len(rows) < 100:
+                return list(reversed(result))
+        raise ValueError("Artifact history exceeds the bounded scan; use the local backfill")
+
+    def introduction(self, repository: str, revision: str, path: str, cve: str, *, depth: int = 0) -> dict:
+        if depth > 8:
+            raise ValueError("Artifact rename history exceeds inspection bound")
+        for commit in self.history(repository, revision, path):
+            # First path existence alone is insufficient: inspect those bytes.
+            entries = self.qualifying(repository, commit["sha"], [path], cve)
+            if not entries:
+                continue
+            detail = self.get(f"/repos/{repository}/commits/{commit['sha']}")
+            files = detail.get("files")
+            if not isinstance(files, list) or not any(item.get("filename") == path for item in files):
+                raise ValueError("Missing introduction diff")
+            renamed = next((item.get("previous_filename") for item in files
+                            if item.get("filename") == path and item.get("status") == "renamed"), None)
+            if renamed and detail.get("parents"):
+                parent = detail["parents"][0]["sha"]
+                if self.qualifying(repository, parent, [renamed], cve):
+                    return self.introduction(repository, parent, renamed, cve, depth=depth + 1)
+            result = {"commit": commit["commit"]["committer"]["date"], "sha": commit["sha"],
+                      "commit_verified": True, "history_paths": [entry["path"] for entry in entries]}
+            introduced = {cve for item in files if item.get("status") == "added"
+                          for cve in source_artifacts.identity_cves(item.get("filename", ""))}
+            if len(introduced) > 10:
+                result["bulk"] = True
+            return result
+        raise ValueError("No verified qualifying artifact introduction in source history")
+
+    def absent(self, repository: str, revision: str, paths: list[str]) -> bool:
+        # A complete known commit tree is stronger than an ambiguous 404 from
+        # a contents endpoint (which may mean the commit/repository is missing).
+        entries = self.tree(repository, revision)
+        present = {entry["path"] for entry in entries}
+        return all(path not in present for path in paths)
+
+
+def current_repository_evidence(repo: dict, paths: list[str], headers: dict, *, cve: str | None = None,
+                                cache_dir: Path | None = None, observed_at: str | None = None,
+                                history: GitHubHistory | None = None, budget_seconds: float = 45,
+                                max_requests: int = 24) -> dict:
+    """Verify first qualifying file history for an already accepted repository.
+
+    The supplied HEAD must be captured by the caller. Complete results are
+    cached by HEAD, paths and review rules; errors return pending evidence.
+    """
+    name = str(repo.get("full_name") or repo.get("nameWithOwner") or "")
+    revision = str(repo.get("revision") or repo.get("head") or
+                   ((repo.get("defaultBranchRef") or {}).get("target") or {}).get("oid") or "")
+    cve = cve or repo.get("cve")
+    if not cve:
+        cves = source_artifacts.identity_cves(name)
+        cve = next(iter(cves)) if len(cves) == 1 else None
+    evidence = {"repo": repo.get("id", repo.get("databaseId")),
+                "created": repo.get("created_at", repo.get("createdAt")),
+                "pushed_at": repo.get("pushed_at", repo.get("pushedAt")),
+                "checked": observed_at or utcnow(), "head": revision}
+    evidence = {k: v for k, v in evidence.items() if v is not None}
+    if not cve or not re.fullmatch(r"[0-9a-f]{40}", revision) or not paths:
+        return {**evidence, "error": "Missing CVE, pinned HEAD or qualifying artifact paths"}
+    history = history or GitHubHistory(headers)
+    history.begin(budget_seconds=budget_seconds, max_requests=max_requests)
+    rules = [(r["cve"], r["path"], r["sha256"]) for r in history.reviews if r["repository"].lower() == name.lower()]
+    cache_key = hashlib.sha256(json.dumps([HISTORY_VERSION, name.lower(), revision, cve, sorted(paths), sorted(rules)]).encode()).hexdigest()
+    cache_dir = cache_dir or Path(tempfile.gettempdir()) / "pocindex-release-cache"
+    cache_path = cache_dir / f"{cache_key}.json"
+    try:
+        if cache_path.exists():
+            cached = json.loads(cache_path.read_text())
+            if cached.get("history_version") == HISTORY_VERSION and cached.get("commit_verified") is True:
+                return {**cached, **evidence}
+        entries = history.qualifying(name, revision, paths, cve)
+        if not entries:
+            raise ValueError("Selected paths contain no verified reproduction material")
+        entrypoints = [entry for entry in entries if source_artifacts.PAYLOAD_RE.search(Path(entry["path"]).name)
+                       or cve.lower() in Path(entry["path"]).name.lower()]
+        if entrypoints:
+            entries = entrypoints
+        identity = artifact_identity(entries, [entry["path"] for entry in entries])
+        introductions = [history.introduction(name, revision, entry["path"], cve) for entry in entries]
+        first = min(introductions, key=lambda row: timestamp(row["commit"]))
+        result = {**evidence, **identity, **first, "history_version": HISTORY_VERSION}
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        temporary = cache_path.with_suffix(".tmp")
+        temporary.write_text(json.dumps(result, sort_keys=True) + "\n")
+        temporary.replace(cache_path)
+        return result
+    except Exception as problem:
+        return {**evidence, "error": str(problem)}
+
+
+def record_collection_snapshot(previous: dict, current: dict, pairs: list[tuple[str, str]], *,
+                               directory: Path | None = None, observed_at: str | None = None,
+                               history: GitHubHistory | None = None, limit: int = 8) -> None:
+    """Record only applied source links, after every collection completed.
+
+    Existing undated imports belong to the historical backfill. New/pending
+    pairs get bounded evidence reads; skipped work remains pending for retry.
+    """
+    directory = directory or DIRECTORY
+    if not directory.exists():
+        return  # Establish the explicit historical baseline before enabling hooks.
+    observed_at = observed_at or utcnow()
+    ledger = load_ledger(directory)
+    old_sources, sources = previous.get("sources", {}), current.get("sources", {})
+    token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+    history = history or GitHubHistory({"Authorization": f"Bearer {token}"} if token else {})
+    present, attempted = set(), 0
+    ordered = sorted(set(pairs), key=lambda pair: (
+        timestamp(ledger.get(key(*pair), {}).get("checked")) or datetime.min.replace(tzinfo=timezone.utc), pair))
+    for cve, url in ordered:
+        identity = key(cve, url)
+        present.add(identity)
+        location = _repository_artifact(url)
+        if not location:
+            continue
+        repository = next((name for name in sources if name.lower() == location[0]), None)
+        if repository is None:
+            continue
+        snapshot, prior = sources[repository], old_sources.get(repository, {})
+        old = ledger.get(identity) or {}
+        # A newly indexed historical source or newly published CVE record is
+        # not evidence of an artifact appearing at this observation.
+        already_in_source = url in (prior.get("links", {}).get(cve) or [])
+        import_mode = bool(old.get("imported")) if old else not prior.get("observed_at") or already_in_source
+        evidence = None
+        if not old or old.get("basis") == "pending" or old.get("basis") == "unknown" and not old.get("imported"):
+            previous_head = old.get("previous_head") or prior.get("revision")
+            absent_at = old.get("absent") or prior.get("observed_at")
+            evidence = {"error": "Deferred by collection history budget"}
+            if previous_head and absent_at:
+                evidence.update(previous_head=previous_head, absent=absent_at)
+            if attempted < limit:
+                attempted += 1
+                evidence.update(checked=observed_at, head=snapshot["revision"])
+                try:
+                    history.begin(budget_seconds=15, max_requests=3)
+                    repo = history.get(f"/repos/{repository}")
+                    # The parser and these reads use exactly the same captured HEAD.
+                    path = unquote(url.split(f"/{snapshot['branch']}/", 1)[1])
+                    verified = current_repository_evidence({**repo, "revision": snapshot["revision"]}, [path], {},
+                               cve=cve, history=history, observed_at=observed_at, budget_seconds=30, max_requests=20)
+                    evidence.update(verified)
+                    if "error" not in verified:
+                        evidence.pop("error", None)
+                        if previous_head and absent_at and not import_mode:
+                            evidence["absence_verified"] = history.absent(repository, previous_head, verified["paths"])
+                except Exception as problem:
+                    evidence["error"] = str(problem)
+        record(ledger, cve, url, evidence, observed_at=observed_at, import_mode=import_mode)
+    previously_owned = {key(cve, url) for snapshot in old_sources.values()
+                        for cve, urls in snapshot.get("links", {}).items() for url in urls}
+    removed = previously_owned.intersection(ledger) - present
+    if removed:
+        published = {key(*pair) for pair in published_pairs()}
+        for identity in removed:
+            if identity in published:
+                ledger[identity].pop("gone", None)
+            else:
+                ledger[identity].setdefault("gone", observed_at)
+    resolve_copies(ledger, now=observed_at)
+    mark_bulk(ledger)
+    save_ledger(ledger, directory)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--seed", action="store_true", help="historical import; no observation or release dates")
+    mode.add_argument("--record", action="store_true", help="reconcile the complete published artifact set")
+    parser.add_argument("--directory", type=Path, default=DIRECTORY)
+    parser.add_argument("--input", type=Path, help="published CVE_list.json; otherwise parse the current corpus")
+    parser.add_argument("--evidence", type=Path, help="JSON mapping of artifact keys to verified evidence")
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+    entries = json.loads(args.input.read_text()) if args.input else None
+    evidence = json.loads(args.evidence.read_text()) if args.evidence else {}
+    ledger = reconcile(load_ledger(args.directory), published_pairs(entries), evidence=evidence, import_mode=args.seed)
+    if not args.dry_run:
+        save_ledger(ledger, args.directory)
+    print(f"releases: {len(ledger):,} pairs, {sum(bool(row.get('released')) for row in ledger.values()):,} dated")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/source_artifacts.py

@@ -0,0 +1,345 @@
+"""Pinned source evidence for PoCs that the repository metadata gate misses.
+
+Only locally reviewed bytes authorize a new link. Remote names and README text
+select files to inspect, never assert that those files have been reviewed.
+"""
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import re
+import threading
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import quote
+from urllib.error import HTTPError
+
+ROOT = Path(__file__).resolve().parents[1]
+REVIEWS = ROOT / "index/source_artifact_reviews.json"
+CANDIDATES = ROOT / "index/source_artifact_candidates.json"
+MAX_FILES = 8
+MAX_BYTES = 500_000
+MAX_TREE = 10_000
+CVE_RE = re.compile(r"(?<![A-Z0-9])CVE[-_ ](\d{4})[-_ ](\d{4,})(?![A-Z0-9])", re.I)
+SOURCE_RE = re.compile(r"\.(?:c|cc|cpp|cs|go|java|js|mjs|nse|php|pl|ps1|py|rb|rs|sh|ts|asm|s|swift|lua|cna)$", re.I)
+PAYLOAD_RE = re.compile(r"(?:^|[/_. -])(?:poc|pocs|exploit|exploits|expl|reproduce|reproducer|payload|trigger)(?:$|[/_. -])", re.I)
+SUPPORT = {".git", "node_modules", "vendor", "third_party", "__pycache__"}
+_LOCK = threading.Lock()
+
+
+@dataclass(frozen=True)
+class ArtifactEvidence:
+    cve: str
+    repository: str
+    path: str
+    revision: str
+    sha256: str
+    review_id: str
+    released_at: str | None = None
+
+    @property
+    def url(self) -> str:
+        return f"https://github.com/{self.repository}/blob/{self.revision}/{quote(self.path, safe='/')}"
+
+
+def identity_cves(repository: str) -> set[str]:
+    # Treat underscores as separators, including adjacent CVE identifiers.
+    return {f"CVE-{year}-{number}" for year, number in CVE_RE.findall(repository.replace("_", " "))}
+
+
+def valid_path(path: str) -> bool:
+    return bool(path) and not path.startswith("/") and all(p not in {"", ".", ".."} for p in path.split("/"))
+
+
+def load_reviews(path: Path | None = None) -> list[dict]:
+    payload = json.loads((path or REVIEWS).read_text(encoding="utf-8"))
+    if payload.get("version") != 1 or not isinstance(payload.get("reviews"), list):
+        raise ValueError("Invalid source artifact review registry")
+    for row in payload["reviews"]:
+        if (not re.fullmatch(r"CVE-\d{4}-\d{4,}", str(row.get("cve", "")))
+                or not re.fullmatch(r"[^/]+/[^/]+", str(row.get("repository", "")))
+                or not valid_path(str(row.get("path", "")))
+                or not re.fullmatch(r"[0-9a-f]{40}", str(row.get("revision", "")))
+                or not re.fullmatch(r"[0-9a-f]{64}", str(row.get("sha256", "")))
+                or row.get("official_state") != "PUBLISHED"
+                or not re.fullmatch(r"[0-9a-f]{40}", str(row.get("official_record_revision", "")))
+                or not re.fullmatch(r"[0-9a-f]{64}", str(row.get("official_record_sha256", "")))
+                or not row.get("review_basis") or not row.get("id")):
+            raise ValueError("Incomplete source artifact review")
+    return payload["reviews"]
+
+
+
+def reviewed_cves(repository: str) -> set[str]:
+    return {row["cve"] for row in load_reviews() if row["repository"].lower() == repository.lower()}
+
+
+def excluded_cves(repository: str) -> set[str]:
+    payload = json.loads(REVIEWS.read_text(encoding="utf-8"))
+    rows = payload.get("exclusions", [])
+    if not isinstance(rows, list):
+        raise ValueError("Invalid source artifact exclusions")
+    for row in rows:
+        if (not isinstance(row, dict) or not re.fullmatch(r"CVE-\d{4}-\d{4,}", str(row.get("cve", "")))
+                or not row.get("repository") or not row.get("reason")
+                or not re.fullmatch(r"[0-9a-f]{40}", str(row.get("revision", "")))):
+            raise ValueError("Incomplete source artifact exclusion")
+    return {row["cve"] for row in rows if row["repository"].lower() == repository.lower()}
+
+
+def approved_cves_for_artifact(repository: str, path: str, source: bytes, *,
+                               reviews: list[dict] | None = None) -> set[str]:
+    """Exact author/path/content exceptions for strict collection attribution."""
+    digest = hashlib.sha256(source).hexdigest()
+    return {row["cve"] for row in (load_reviews() if reviews is None else reviews)
+            if row["repository"].lower() == repository.lower() and row["path"] == path
+            and row["sha256"] == digest}
+
+
+def _evidence(repository: str, revision: str, artifacts: list[dict], cves: set[str], reviews: list[dict]) -> tuple[ArtifactEvidence, ...]:
+    approved: dict[tuple[str, str], list[dict]] = {}
+    for row in reviews:
+        if row["repository"].lower() == repository.lower():
+            approved.setdefault((row["cve"], row["sha256"]), []).append(row)
+    result = []
+    for artifact in artifacts:
+        if not valid_path(str(artifact.get("path", ""))):
+            continue
+        for cve in sorted(cves):
+            matches = approved.get((cve, artifact.get("sha256")), [])
+            if matches:
+                review = next((r for r in matches if r["repository"].lower() == repository.lower()
+                               and r["path"] == artifact["path"]), matches[0])
+                # Preserve the reviewed revision and its sibling context. A
+                # copied harness alone is not evidence for another repository.
+                result.append(ArtifactEvidence(cve, review["repository"], review["path"], review["revision"],
+                                               artifact["sha256"], review["id"], review.get("released_at")))
+    return tuple(result)
+
+
+def reviewed_artifacts(repo: dict) -> tuple[ArtifactEvidence, ...]:
+    """Only the local collector can attach typed evidence; JSON cannot forge it."""
+    name = str(repo.get("nameWithOwner") or "")
+    return tuple(item for item in (repo.get("_source_artifacts") or ())
+                 if isinstance(item, ArtifactEvidence) and item.repository.lower() == name.lower()
+                 and item.cve not in excluded_cves(name))
+
+
+def approved_artifact_links(repo: dict, cve: str) -> list[str]:
+    return sorted({item.url for item in reviewed_artifacts(repo) if item.cve == cve})
+
+
+def _load_candidates(path: Path) -> dict:
+    if not path.exists():
+        return {"version": 1, "repositories": {}}
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    if payload.get("version") != 1 or not isinstance(payload.get("repositories"), dict):
+        raise ValueError("Invalid source artifact candidate state")
+    return payload
+
+
+def _save_candidate(path: Path, name: str, row: dict) -> None:
+    with _LOCK:
+        payload = _load_candidates(path)
+        payload["repositories"][name.lower()] = row
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temporary = path.with_suffix(".tmp")
+        temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+        temporary.replace(path)
+
+
+class InspectionDeferred(Exception):
+    pass
+
+
+def pending_reviewed_names(limit: int = 16, *, cves: set[str] | None = None) -> list[str]:
+    known = {row["repository"].lower() for row in load_reviews()
+             if cves is None or row["cve"] in cves}
+    with _LOCK:
+        rows = _load_candidates(CANDIDATES)["repositories"]
+    pending = [(row.get("last_checked_at") or row.get("last_attempt_at") or "", name) for name, row in rows.items()
+               if name in known and row.get("scan_complete") is False]
+    return [name for _, name in sorted(pending)[:limit]]
+
+
+def mark_pending_checked(names: list[str]) -> None:
+    """Rotate inaccessible replay entries without pretending their scans finished."""
+    with _LOCK:
+        rows = _load_candidates(CANDIDATES)["repositories"]
+    now = datetime.now(timezone.utc).isoformat()
+    for name in names:
+        row = rows.get(name.lower())
+        if row and row.get("scan_complete") is False:
+            _save_candidate(CANDIDATES, name, {**row, "last_checked_at": now})
+
+
+class InspectionBudget:
+    def __init__(self, requests: int = 64, seconds: float = 120):
+        self.remaining = requests
+        self.exhausted = False
+        self.deadline = time.monotonic() + seconds
+
+    def fetch(self, fetch_json, name: str, url: str, **kwargs):
+        remaining_time = self.deadline - time.monotonic()
+        if self.exhausted or self.remaining <= 0 or remaining_time <= 0:
+            raise InspectionDeferred("budget_deferred")
+        self.remaining -= 1
+        kwargs["timeout"] = min(15, remaining_time)
+        try:
+            return fetch_json(url, **kwargs)
+        except HTTPError as exc:
+            if exc.code in {403, 429} and str((exc.headers or {}).get("X-RateLimit-Remaining")) == "0":
+                self.exhausted = True
+                raise InspectionDeferred("rate_limited") from exc
+            raise
+
+
+_AUTOMATIC_BUDGET: InspectionBudget | None = None
+
+
+def automatic_budget() -> InspectionBudget:
+    global _AUTOMATIC_BUDGET
+    if _AUTOMATIC_BUDGET is None:
+        _AUTOMATIC_BUDGET = InspectionBudget()
+    return _AUTOMATIC_BUDGET
+
+
+def inspect_repository(repo: dict, cves: set[str], readme: str, fetch_json, headers: dict,
+                       *, reviews_path: Path | None = None, candidates_path: Path | None = None,
+                       persist: bool = True, budget: InspectionBudget | None = None) -> tuple[ArtifactEvidence, ...]:
+    """Verify reviewed paths first; retain bounded deferred work without rejecting it."""
+    name = str(repo.get("nameWithOwner") or "")
+    reviews = [r for r in load_reviews(reviews_path) if r["repository"].lower() == name.lower()]
+    known = {row["cve"] for row in reviews}
+    cves = set(cves) & (identity_cves(name) | known)
+    if not cves or repo.get("isFork") or ("defaultBranchRef" in repo and repo["defaultBranchRef"] is None):
+        return ()
+    reviews = [r for r in reviews if r["cve"] in cves]
+    state_path = candidates_path or CANDIDATES
+    with _LOCK:
+        cached = _load_candidates(state_path)["repositories"].get(name.lower()) or {}
+    revision = str(((repo.get("defaultBranchRef") or {}).get("target") or {}).get("oid") or "")
+    review_set = hashlib.sha256(json.dumps(reviews, sort_keys=True).encode()).hexdigest()
+    artifacts, skipped_paths, known_checked = [], set(), set()
+    attempted = False
+    status, complete = "needs_review", False
+
+    def read(url, *, allow_not_found=False):
+        nonlocal attempted
+        prior = budget.remaining if budget else None
+        try:
+            if budget:
+                return budget.fetch(fetch_json, name, url, headers=headers, allow_not_found=allow_not_found)
+            attempted = True
+            return fetch_json(url, headers=headers, allow_not_found=allow_not_found)
+        finally:
+            if budget and budget.remaining != prior:
+                attempted = True
+
+    def accept_blob(path, size, oid, blob):
+        if (not isinstance(size, int) or not 0 < size <= MAX_BYTES
+                or not re.fullmatch(r"[0-9a-f]{40}", str(oid or ""))):
+            raise RuntimeError(f"Invalid blob identity for {name}/{path}")
+        if blob.get("encoding") != "base64" or not isinstance(blob.get("content"), str):
+            raise RuntimeError(f"Incomplete source blob for {name}/{path}")
+        raw = base64.b64decode("".join(blob["content"].split()), validate=True)
+        if len(raw) != size:
+            raise RuntimeError(f"Source blob size mismatch for {name}/{path}")
+        if hashlib.sha1(f"blob {len(raw)}\0".encode() + raw).hexdigest() != oid:
+            raise RuntimeError(f"Source blob identity mismatch for {name}/{path}")
+        if b"\0" in raw:
+            skipped_paths.add(path)
+        else:
+            artifacts.append({"path": path, "sha256": hashlib.sha256(raw).hexdigest(), "bytes": len(raw)})
+
+    try:
+        if not revision:
+            try:
+                commit = read(f"https://api.github.com/repos/{name}/commits/HEAD", allow_not_found=True)
+            except HTTPError as exc:
+                if exc.code != 409:
+                    raise
+                commit = None
+            if commit is None:
+                return ()
+            revision = str(commit.get("sha") or "")
+        if not re.fullmatch(r"[0-9a-f]{40}", revision):
+            raise RuntimeError(f"Invalid artifact revision for {name}")
+        same_revision = cached.get("revision") == revision
+        if same_revision:
+            artifacts = list(cached.get("artifacts", []))
+            skipped_paths = set(cached.get("skipped_paths", []))
+            known_checked = set(cached.get("known_checked", []))
+        same_rules = cached.get("review_set_sha256") == review_set
+        if same_revision and same_rules and cves <= set(cached.get("cves", [])):
+            if cached.get("scan_complete", True) or cached.get("status") == "tree_limited":
+                if persist and cached.get("status") == "tree_limited":
+                    _save_candidate(state_path, name, {**cached, "last_checked_at": datetime.now(timezone.utc).isoformat()})
+                return _evidence(name, revision, artifacts, cves, reviews)
+        known_paths = {r["path"] for r in reviews}
+        inspected = {a["path"] for a in artifacts} | skipped_paths
+        pending_paths = sorted(known_paths - known_checked - inspected)
+        for path in pending_paths[:MAX_FILES]:
+            blob = read(f"https://api.github.com/repos/{name}/contents/{quote(path, safe='/')}?ref={revision}", allow_not_found=True)
+            if blob is not None and not isinstance(blob, (dict, list)):
+                raise RuntimeError(f"Incomplete source contents for {name}/{path}")
+            if isinstance(blob, dict) and blob.get("type") not in {"file", "dir", "symlink", "submodule"}:
+                raise RuntimeError(f"Incomplete source contents for {name}/{path}")
+            if isinstance(blob, dict) and blob.get("type") == "file":
+                size = blob.get("size")
+                if isinstance(size, int) and 0 < size <= MAX_BYTES:
+                    accept_blob(path, size, blob.get("sha"), blob)
+            known_checked.add(path)
+        if len(pending_paths) > MAX_FILES:
+            raise InspectionDeferred("file_limited")
+        expected = {(r["cve"], r["sha256"]) for r in reviews}
+        matched = {(item.cve, item.sha256) for item in _evidence(name, revision, artifacts, cves, reviews)}
+        if expected and expected <= matched:
+            complete = True
+        else:
+            tree = read(f"https://api.github.com/repos/{name}/git/trees/{revision}?recursive=1")
+            if not isinstance(tree, dict) or not isinstance(tree.get("tree"), list) or not isinstance(tree.get("truncated"), bool):
+                raise RuntimeError(f"Incomplete artifact tree for {name}")
+            if tree["truncated"] or len(tree["tree"]) > MAX_TREE:
+                raise InspectionDeferred("tree_limited")
+            eligible = []
+            for entry in tree["tree"]:
+                path = str(entry.get("path") or "")
+                if entry.get("type") != "blob" or entry.get("mode") not in {"100644", "100755"} or not valid_path(path):
+                    continue
+                if any(part.lower() in SUPPORT for part in path.split("/")[:-1]):
+                    continue
+                size = entry.get("size")
+                if not isinstance(size, int) or not 0 < size <= MAX_BYTES:
+                    continue
+                named = bool(PAYLOAD_RE.search(path)) or bool(identity_cves(path) & cves)
+                mentioned = path.lower() in readme.lower()
+                if path not in known_paths and not ((SOURCE_RE.search(path) or (named and not Path(path).suffix)) and (named or mentioned)):
+                    continue
+                eligible.append((path not in known_paths, not mentioned, not named, path.count("/"), path, entry))
+            inspected = {a["path"] for a in artifacts} | skipped_paths
+            eligible = [item for item in eligible if item[-2] not in inspected]
+            for *_, path, entry in sorted(eligible)[:MAX_FILES]:
+                blob = read(f"https://api.github.com/repos/{name}/git/blobs/{entry.get('sha')}")
+                accept_blob(path, entry["size"], entry.get("sha"), blob)
+            complete = len(eligible) <= MAX_FILES
+            if not complete:
+                status = "file_limited"
+    except InspectionDeferred as deferred:
+        status = str(deferred)
+    evidence = _evidence(name, revision, artifacts, cves, reviews)
+    if complete:
+        status = "reviewed" if evidence else "needs_review" if artifacts else "no_candidate_artifact"
+    observed = datetime.now(timezone.utc).isoformat()
+    row = {"revision": revision, "cves": sorted(cves), "artifacts": artifacts,
+           "skipped_paths": sorted(skipped_paths), "known_checked": sorted(known_checked), "review_set_sha256": review_set,
+           "status": status, "first_observed_at": cached.get("first_observed_at") or observed,
+           "revision_observed_at": observed, "last_attempt_at": observed if attempted else cached.get("last_attempt_at"),
+           "last_checked_at": observed if attempted else cached.get("last_checked_at"),
+           "scan_limited": not complete, "scan_complete": complete}
+    if persist:
+        _save_candidate(state_path, name, row)
+    return evidence

scripts/sync_collections.py

@@ -4,7 +4,9 @@
 from __future__ import annotations
 
 import argparse
+import hashlib
 import io
+import json
 import os
 import re
 import tarfile
@@ -13,10 +15,15 @@ from collections import defaultdict
 from pathlib import Path
 from urllib.parse import quote
 
-from update_cves import CVES, ensure_cve_entries, http_json
+import source_artifacts
+import releases
+from update_cves import CVES, INDEX, ensure_cve_entries, http_json
 
 SECTION = "#### Collections"
 USER_AGENT = "0xMarcio-cve-collections"
+SOURCE_STATE_FILE = INDEX / "collection_source_state.json"
+# Bump when collection selection or the cached mapping format changes.
+SOURCE_CACHE_VERSION = 2
 CVE = re.compile(r"(?i)CVE[-_](\d{4})[-_](\d{4,7})")
 AFROG_PASSIVE = re.compile(
     r"(?im)^\s{2}name:\s*.*\b(?:version|protocol|service)\s+detection\b"
@@ -27,12 +34,19 @@ SOURCES = (
     ("vulnerability", "tzwlhack/Vulnerability", "main"),
     ("0day", "helloexp/0day", "master"),
     ("xray", "chaitin/xray", "master"),
+    ("tenable", "tenable/poc", "master"),
+    ("pedrib", "pedrib/PoC", "master"),
 )
 TREE_SOURCES = (
     ("google-research", "google/security-research", "master", "pocs/"),
     ("github-securitylab", "github/securitylab", "main", "SecurityExploits/"),
 )
 CODE_SUFFIXES = {".c", ".cc", ".cpp", ".py", ".js", ".html", ".sh", ".rb", ".go", ".rs"}
+CODE_CONTENT = re.compile(
+    r"(?m)^\s*(?:#include\s*[<\"]|(?:from\s+\S+\s+)?import\s+\S+|require\s+['\"]|"
+    r"(?:def|class|function)\s+\w+|(?:int|void)\s+\w+\s*\(|<(?:form|script)\b|"
+    r"(?:curl|python[23]?|ruby|perl|java|msfconsole)\s)"
+)
 SUPPORT_DIRS = {"docs", "tests", "test", "include", "vendor", "third_party", "images", "fuzzer", "fuzzers"}
 SUPPORT_FILE = re.compile(r"^(?:test|build|setup|util|helper|clean|install|init)[-_.a-z0-9]*$", re.I)
 # Reviewed patches that turn upstream clients/servers into the reproduction.
@@ -52,6 +66,59 @@ def download_tree(repo: str, branch: str) -> dict:
     )
 
 
+def source_head(repo: str, branch: str) -> str:
+    token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+    headers = {"Authorization": f"Bearer {token}"} if token else {}
+    payload = http_json(
+        f"https://api.github.com/repos/{repo}/git/ref/heads/{quote(branch, safe='')}",
+        headers=headers,
+    )
+    obj = payload.get("object") if isinstance(payload, dict) else None
+    if (
+        not isinstance(obj, dict) or payload.get("ref") != f"refs/heads/{branch}"
+        or obj.get("type") != "commit" or not re.fullmatch(r"[0-9a-f]{40}", str(obj.get("sha") or ""))
+    ):
+        raise ValueError(f"{repo}: missing or invalid source HEAD")
+    return obj["sha"]
+
+
+def load_source_state() -> dict:
+    try:
+        state = json.loads(SOURCE_STATE_FILE.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    return state if isinstance(state, dict) else {}
+
+
+def valid_source_links(links: object, repo: str, branch: str) -> bool:
+    return isinstance(links, dict) and bool(links) and all(
+        isinstance(cve, str) and CVE.fullmatch(cve) and cve_ids(cve) == {cve}
+        and isinstance(urls, list) and bool(urls) and all(
+            isinstance(url, str) and url.startswith((
+                f"https://github.com/{repo}/blob/{branch}/",
+                f"https://github.com/{repo}/tree/{branch}/",
+            )) for url in urls
+        ) for cve, urls in links.items()
+    )
+
+
+def write_source_state(state: dict) -> None:
+    if state == load_source_state():
+        return
+    SOURCE_STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
+    temporary = SOURCE_STATE_FILE.with_suffix(".tmp")
+    temporary.write_text(json.dumps(state, separators=(",", ":"), sort_keys=True) + "\n", encoding="utf-8")
+    temporary.replace(SOURCE_STATE_FILE)
+
+
+def reviewed_rule_fingerprint(repo: str, reviews: list[dict]) -> str | None:
+    if repo not in {"tenable/poc", "pedrib/PoC"}:
+        return None
+    rules = sorted({(row["cve"], row["path"], row["sha256"]) for row in reviews
+                    if row["repository"].lower() == repo.lower()})
+    return hashlib.sha256(json.dumps(rules, separators=(",", ":")).encode()).hexdigest() if rules else None
+
+
 def collect_research_tree(repo: str, branch: str, prefix: str, payload: dict) -> dict[str, list[str]]:
     """CVE directories with reproduction artifacts in two research collections."""
     if not isinstance(payload, dict) or payload.get("truncated") is not False or not isinstance(payload.get("tree"), list):
@@ -84,8 +151,8 @@ def collect_research_tree(repo: str, branch: str, prefix: str, payload: dict) ->
     return {cve: sorted(urls) for cve, urls in rows.items()}
 
 
-def download(repo: str, branch: str) -> bytes:
-    url = f"https://codeload.github.com/{repo}/tar.gz/refs/heads/{branch}"
+def download(repo: str, revision: str) -> bytes:
+    url = f"https://codeload.github.com/{repo}/tar.gz/{quote(revision, safe='')}"
     request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
     with urllib.request.urlopen(request, timeout=900) as response:
         return response.read()
@@ -104,8 +171,55 @@ def github_link(repo: str, branch: str, kind: str, path: str) -> str:
     return f"https://github.com/{repo}/{kind}/{branch}/{encoded}"
 
 
-def collect(repo: str, branch: str, archive: bytes) -> dict[str, list[str]]:
+def author_artifact_ids(repo: str, path: str, text: str) -> set[str]:
+    named, mentioned = cve_ids(path), cve_ids(text)
+    parts = Path(path).parts
+    if not text.strip() or len(named) > 1:
+        return set()
+    if any(part.lower() in SUPPORT_DIRS for part in parts[:-1]) or SUPPORT_FILE.fullmatch(parts[-1]):
+        return set()
+    if Path(path).suffix.lower() in CODE_SUFFIXES:
+        if not CODE_CONTENT.search(text) or (repo == "pedrib/PoC" and not path.startswith("exploits/")):
+            return set()
+        ids = named or mentioned
+        return ids if len(ids) == 1 and (not mentioned or mentioned == ids) else set()
+    if repo != "pedrib/PoC" or not path.startswith("advisories/") or not path.endswith(".md"):
+        return set()
+
+    # Keep the CVE attribution local to an embedded reproducer. A writeup may
+    # also discuss historical vulnerabilities that its code does not reproduce.
+    found: set[str] = set()
+    headings: list[tuple[int, str]] = []
+    block: list[str] | None = None
+    for line in text.splitlines():
+        if line.startswith("```"):
+            if block is None:
+                block = []
+                continue
+            code = "\n".join(block)
+            block = None
+            if not CODE_CONTENT.search(code):
+                continue
+            ids = cve_ids(code)
+            if len(ids) == 1 and re.search(r"\b(?:exploit(?:ing)?|poc|proof.of.concept|reproduc(?:er|tion))\b", code, re.I):
+                if not named or ids == named:
+                    found.update(ids)
+            elif not ids and len(mentioned) == 1 and (not named or mentioned == named):
+                if any(re.search(r"\b(?:exploit(?:ation)?|proof.of.concept|steps to reproduce)\b", title, re.I) for _, title in headings):
+                    found.update(mentioned)
+        elif block is not None:
+            block.append(line)
+        elif heading := re.match(r"^(#{1,6})\s+(.+)", line):
+            level = len(heading[1])
+            headings = [(depth, title) for depth, title in headings if depth < level]
+            headings.append((level, heading[2]))
+    return found
+
+
+def collect(repo: str, branch: str, archive: bytes, *, reviews: list[dict] | None = None) -> dict[str, list[str]]:
     rows: dict[str, set[str]] = defaultdict(set)
+    if reviews is None and repo in {"tenable/poc", "pedrib/PoC"}:
+        reviews = source_artifacts.load_reviews()
     with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar:
         for member in tar:
             if not member.isfile():
@@ -116,7 +230,17 @@ def collect(repo: str, branch: str, archive: bytes) -> dict[str, list[str]]:
             relative = "/".join(parts[1:])
             suffix = parts[-1].lower()
 
-            if repo == "zan8in/afrog":
+            if repo in {"tenable/poc", "pedrib/PoC"}:
+                if Path(relative).suffix.lower() not in CODE_SUFFIXES | {".md"}:
+                    continue
+                handle = tar.extractfile(member)
+                source = handle.read() if handle else b""
+                text = source.decode("utf-8", "replace")
+                ids = source_artifacts.approved_cves_for_artifact(repo, relative, source, reviews=reviews)
+                if not ids:
+                    ids = author_artifact_ids(repo, relative, text)
+                target = github_link(repo, branch, "blob", relative)
+            elif repo == "zan8in/afrog":
                 if "pocs/afrog-pocs/CVE/" not in relative or not suffix.endswith((".yaml", ".yml")):
                     continue
                 handle = tar.extractfile(member)
@@ -202,28 +326,48 @@ def main() -> int:
     parser = argparse.ArgumentParser(description="Sync path-level CVE PoC collections")
     parser.add_argument("--dry-run", action="store_true", help="report without writing")
     parser.add_argument("--cvelist-dir", type=Path, help="local CVE List V5 root for a large backfill")
+    parser.add_argument("--release-history-limit", type=int, default=8, help="maximum new artifact history checks per run")
     args = parser.parse_args()
 
+    previous = load_source_state()
+    cached = previous.get("sources", {}) if previous.get("version") == SOURCE_CACHE_VERSION else {}
+    if not isinstance(cached, dict):
+        cached = {}
+    next_state = {"version": SOURCE_CACHE_VERSION, "sources": {}}
+    reviews = source_artifacts.load_reviews()
     combined: dict[str, set[str]] = defaultdict(set)
-    for name, repo, branch in SOURCES:
+    sources = [(name, repo, branch, None) for name, repo, branch in SOURCES] + list(TREE_SOURCES)
+    for name, repo, branch, prefix in sources:
         try:
-            found = collect(repo, branch, download(repo, branch))
+            revision = source_head(repo, branch)
+            review_sha256 = reviewed_rule_fingerprint(repo, reviews)
+            prior = cached.get(repo) or {}
+            reusable = (
+                isinstance(prior, dict) and prior.get("revision") == revision
+                and prior.get("branch") == branch and prior.get("prefix") == prefix
+                and prior.get("review_sha256") == review_sha256
+                and valid_source_links(prior.get("links"), repo, branch)
+            )
+            if reusable:
+                found = prior["links"]
+            elif prefix is None:
+                found = collect(repo, branch, download(repo, revision), reviews=reviews)
+            else:
+                found = collect_research_tree(repo, branch, prefix, download_tree(repo, revision))
+            if not valid_source_links(found, repo, branch):
+                raise ValueError(f"{repo}: empty or invalid collection mapping")
         except Exception as problem:
             print(f"{repo}: unavailable ({problem})")
             return 1
+        next_state["sources"][repo] = {"revision": revision, "branch": branch, "prefix": prefix, "links": found}
+        next_state["sources"][repo]["observed_at"] = (
+            prior.get("observed_at") if prior.get("revision") == revision else None
+        ) or releases.utcnow()
+        if review_sha256 is not None:
+            next_state["sources"][repo]["review_sha256"] = review_sha256
         for cve, urls in found.items():
             combined[cve].update(urls)
-        print(f"{name}: {len(found):,} CVEs, {sum(map(len, found.values())):,} path links")
-
-    for name, repo, branch, prefix in TREE_SOURCES:
-        try:
-            found = collect_research_tree(repo, branch, prefix, download_tree(repo, branch))
-        except Exception as problem:
-            print(f"{repo}: unavailable ({problem})")
-            return 1
-        for cve, urls in found.items():
-            combined[cve].update(urls)
-        print(f"{name}: {len(found):,} CVEs, {sum(map(len, found.values())):,} path links")
+        print(f"{name}: {len(found):,} CVEs, {sum(map(len, found.values())):,} path links" + (" (cached)" if reusable else ""))
 
     links = {cve: sorted(urls) for cve, urls in combined.items()}
     created, unavailable = ensure_cve_entries(
@@ -236,16 +380,25 @@ def main() -> int:
         print(f"base entries: {len(created):,} {verb}, {len(unavailable):,} unpublished or unavailable")
 
     tally = {"written": 0, "unchanged": 0, "absent": 0}
+    published = []
     for cve, urls in sorted(links.items()):
-        tally[apply_links(cve, urls, dry_run=args.dry_run)] += 1
+        outcome = apply_links(cve, urls, dry_run=args.dry_run)
+        tally[outcome] += 1
+        if outcome != "absent":
+            published.extend((cve, url) for url in urls)
 
     stale = 0
-    for path in CVES.glob("[12][0-9][0-9][0-9]/CVE-*.md"):
+    paths = CVES.glob("[12][0-9][0-9][0-9]/CVE-*.md") if next_state != previous else ()
+    for path in paths:
         if path.stem not in links and ("\n" + SECTION) in path.read_text(
             encoding="utf-8", errors="replace"
         ):
             stale += drop_section(path, dry_run=args.dry_run)
 
+    if not args.dry_run:
+        releases.record_collection_snapshot(previous, next_state, published, limit=max(0, args.release_history_limit))
+        write_source_state(next_state)
+
     print(
         f"collections: {len(links):,} CVEs | {tally['written']:,} written, "
         f"{tally['unchanged']:,} unchanged, {tally['absent']:,} absent, {stale:,} stale removed"

scripts/sync_exploits.py

@@ -16,8 +16,11 @@ import json
 import re
 import urllib.request
 from collections import defaultdict
+from datetime import date, datetime, timezone
 from pathlib import Path
+from urllib.parse import urlsplit
 
+import releases
 from update_cves import ensure_cve_entries
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -56,18 +59,31 @@ def fetch(url: str) -> bytes:
     return gzip.decompress(raw) if url.endswith(".gz") else raw
 
 
-def exploitdb() -> dict[str, list[str]]:
+def publication_date(value: str, *, today: date | None = None) -> str | None:
+    """ExploitDB supplies a calendar publication date, never an observation date."""
+    if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
+        return None
+    try:
+        published = date.fromisoformat(value)
+    except ValueError:
+        return None
+    return value if published <= (today or datetime.now(timezone.utc).date()) else None
+
+
+def exploitdb(publication_dates: dict[str, str] | None = None) -> dict[str, list[str]]:
     """CVE to ExploitDB entry links, verified entries first, newest first."""
     text = fetch(EDB_CSV).decode("utf-8", "replace")
     rows: dict[str, list[tuple[int, int]]] = defaultdict(list)
     for row in csv.DictReader(io.StringIO(text)):
         codes = row.get("codes") or ""
-        if "CVE-" not in codes.upper():
-            continue
         try:
             entry = int(row["id"])
         except (KeyError, TypeError, ValueError):
             continue
+        if publication_dates is not None and (published := publication_date(str(row.get("date_published") or "").strip())):
+            publication_dates[f"{EDB_ENTRY}{entry}"] = published
+        if "CVE-" not in codes.upper():
+            continue
         verified = 1 if str(row.get("verified") or "").strip() == "1" else 0
         for cve in {c.upper() for c in CVE.findall(codes)}:
             rows[cve].append((verified, entry))
@@ -181,13 +197,41 @@ def drop_section(cve: str, header: str, dry_run: bool) -> bool:
     return True
 
 
+def artifact_publication_date(url: str, publication_dates: dict[str, str]) -> str | None:
+    parsed = urlsplit(url)
+    if parsed.hostname not in {"exploit-db.com", "www.exploit-db.com"}:
+        return None
+    match = re.fullmatch(r"/exploits/(\d+)/?", parsed.path)
+    return publication_dates.get(EDB_ENTRY + match[1]) if match else None
+
+
+def sync_release_dates(publication_dates: dict[str, str], *, import_mode: bool = False) -> dict:
+    pairs = releases.published_pairs()
+    evidence = {releases.key(cve, url): {"source": published}
+                for cve, url in pairs if (published := artifact_publication_date(url, publication_dates))}
+    before = releases.load_ledger()
+    ledger = releases.reconcile(before, pairs, evidence=evidence, import_mode=import_mode)
+    releases.save_ledger(ledger)
+    dated = [ledger[identity] for identity in evidence if ledger[identity].get("released")]
+    return {"published_pairs": len(pairs), "edb_pairs_with_valid_source_date": len(evidence),
+            "dated_edb_pairs": len(dated), "dated_edb_cves": len({row["cve"] for row in dated}),
+            "new_release_rows": len(ledger) - len(before), "import_mode": import_mode}
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description="Sync ExploitDB, Metasploit and Vulhub into the index")
     parser.add_argument("--dry-run", action="store_true", help="report without writing")
+    parser.add_argument("--backfill-release-dates", action="store_true", help="Date existing ExploitDB links only; preserve historical import provenance")
     parser.add_argument("--cvelist-dir", type=Path, help="local CVE List V5 root for a large backfill")
     args = parser.parse_args()
 
-    collected = [collect() for collect in (exploitdb, metasploit, vulhub)]
+    publication_dates: dict[str, str] = {}
+    edb = exploitdb(publication_dates)
+    if args.backfill_release_dates:
+        if not args.dry_run:
+            print(json.dumps(sync_release_dates(publication_dates, import_mode=True), sort_keys=True))
+        return 0
+    collected = [edb, metasploit(), vulhub()]
     created, unavailable = ensure_cve_entries(
         {cve for found in collected for cve in found},
         dry_run=args.dry_run,
@@ -215,6 +259,8 @@ def main() -> int:
         print(f"{name}: {len(found):,} CVEs, {links:,} links | "
               f"{tally['added']:,} entries written, {tally['unchanged']:,} unchanged, "
               f"{tally['absent']:,} for CVEs not in the index, {stale:,} stale sections dropped")
+    if not args.dry_run:
+        print(json.dumps(sync_release_dates(publication_dates), sort_keys=True))
     return 0
 
 

scripts/sync_nuclei.py

@@ -17,6 +17,7 @@ import tarfile
 import urllib.request
 from pathlib import Path
 
+import releases
 from update_cves import ensure_cve_entries
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -253,6 +254,9 @@ def main() -> int:
             json.dump(payload, handle, indent=0, sort_keys=True)
             handle.write("\n")
 
+    if not args.dry_run:
+        releases.save_ledger(releases.reconcile(releases.load_ledger(), releases.published_pairs()))
+
     rated = sum(1 for e in payload.values() if "cvss" in e)
     print(f"markdown: {tally['added']:,} links added, {tally['unchanged']:,} already there, "
           f"{tally['absent']:,} for CVEs not in the index, {stale:,} sections withdrawn")

scripts/sync_pocingithub.py

@@ -26,6 +26,8 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 
+import source_artifacts
+
 from update_cves import (
     GITHUB_LIST,
     GITHUB_GRAPHQL_URL,
@@ -38,6 +40,7 @@ from update_cves import (
     load_blacklist,
     ensure_cve_entries,
     qualifying_repo_cves,
+    attach_source_artifacts,
     reconcile_inventory,
     replace_section,
     section_links,
@@ -51,6 +54,7 @@ BATCH = 10
 USER_AGENT = "0xMarcio-cve-poc-in-github"
 
 FIELDS = """
+  defaultBranchRef { target { oid } }
   nameWithOwner url description isFork isArchived
   repositoryTopics(first: 20) { nodes { topic { name } } }
   readmeMd: object(expression: "HEAD:README.md") { ... on Blob { text } }
@@ -185,10 +189,12 @@ def main() -> int:
                 repo = described.get(full_name)
                 if repo is None:
                     continue
+                attach_source_artifacts(repo, client.headers, blacklist, cves=pending[full_name], persist=not args.dry_run)
                 for cve in pending[full_name]:
                     year = int(cve.split("-")[1])
                     if cve in qualifying_repo_cves(repo, year, blacklist):
-                        accepted.setdefault(cve, set()).add(str(repo.get("url") or "").rstrip("/"))
+                        links = source_artifacts.approved_artifact_links(repo, cve) or [str(repo.get("url") or "").rstrip("/")]
+                        accepted.setdefault(cve, set()).update(links)
                         kept += 1
             if judged % 2000 < BATCH:
                 print(f"  judged {judged:,} of {len(names):,}")

scripts/sync_reference_pocs.py

@@ -0,0 +1,412 @@
+#!/usr/bin/env python3
+"""Discover NVD reference candidates and publish only reviewed reproductions."""
+from __future__ import annotations
+
+import argparse
+import gzip
+import hashlib
+import http.client
+import json
+import os
+import re
+import signal
+import tempfile
+import time
+from datetime import datetime, timedelta, timezone
+from html.parser import HTMLParser
+from pathlib import Path
+from urllib import error, parse, request
+
+import update_cves
+
+INDEX = update_cves.INDEX
+REVIEWS = INDEX / "reference_poc_reviews.json"
+QUEUE = INDEX / "reference_poc_queue.json"
+VERIFIED = INDEX / "reference_poc_verified.json"
+CACHE = update_cves.ROOT / "data" / "reference-pocs"
+FEED_ROOT = "https://nvd.nist.gov/feeds/json/cve/2.0"
+MAX_BODY = 2_000_000
+
+
+class BudgetExpired(Exception):
+    pass
+
+
+class Budget:
+    def __init__(self, seconds: float):
+        self.deadline = time.monotonic() + seconds
+        self.failures = 0
+
+    def remaining(self) -> float:
+        seconds = self.deadline - time.monotonic()
+        if seconds <= 0:
+            raise BudgetExpired
+        return seconds
+
+    def read(self, req, *, opener=None, limit=None) -> bytes:
+        remaining = self.remaining()
+        timeout = min(10, remaining)
+
+        def expired(signum, frame):
+            if remaining <= 10:
+                raise BudgetExpired
+            self.remaining()
+            raise TimeoutError("Reference request exceeded 10 seconds")
+
+        handler = signal.signal(signal.SIGALRM, expired)
+        signal.setitimer(signal.ITIMER_REAL, timeout)
+        try:
+            open_url = opener.open if opener is not None else request.urlopen
+            with open_url(req, timeout=timeout) as response:
+                body = response.read() if limit is None else response.read(limit)
+            self.remaining()
+            return body
+        finally:
+            signal.setitimer(signal.ITIMER_REAL, 0)
+            signal.signal(signal.SIGALRM, handler)
+
+
+def supported(url: str) -> bool:
+    parsed = parse.urlsplit(url)
+    host = (parsed.hostname or "").lower().removeprefix("www.")
+    if parsed.scheme not in {"http", "https"} or parsed.username or parsed.password:
+        return False
+    if host == "gist.github.com":
+        return bool(re.fullmatch(r"/(?:[^/]+/)?[a-fA-F0-9]{20,40}/?", parsed.path))
+    if host == "openwall.com":
+        return bool(re.fullmatch(r"/lists/(?:oss-security|fulldisclosure)/\d{4}/\d{2}/\d{2}/\d+/?", parsed.path))
+    if host == "seclists.org":
+        return bool(re.fullmatch(r"/(?:(?:bugtraq|fulldisclosure)/\d{4}/[A-Za-z]+|oss-sec/\d{4}/q[1-4])/\d+(?:\.html)?/?", parsed.path))
+    return False
+
+
+class Preformatted(HTMLParser):
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.blocks: list[str] = []
+        self.parts: list[str] | None = None
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "pre":
+            self.parts = []
+        elif tag == "br" and self.parts is not None:
+            self.parts.append("\n")
+
+    def handle_data(self, data):
+        if self.parts is not None:
+            self.parts.append(data)
+
+    def handle_endtag(self, tag):
+        if tag == "pre" and self.parts is not None:
+            self.blocks.append("".join(self.parts).replace("\r\n", "\n").strip())
+            self.parts = None
+
+
+def html_artifacts(body: bytes) -> dict[str, str]:
+    parser = Preformatted()
+    parser.feed(body.decode("utf-8", "replace"))
+    return {f"pre:{index}": text for index, text in enumerate(parser.blocks)}
+
+
+class MailText(HTMLParser):
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.parts = []
+
+    def handle_data(self, data):
+        self.parts.append(data)
+
+    def handle_endtag(self, tag):
+        if tag in {"pre", "tt", "p", "div"}:
+            self.parts.append("\n")
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "br":
+            self.parts.append("\n")
+
+
+def mail_artifacts(url: str, body: bytes) -> dict[str, str]:
+    artifacts = html_artifacts(body)
+    host = parse.urlsplit(url).hostname.removeprefix("www.")
+    if host == "openwall.com":
+        complete = any(all(re.search(rf"^{field}: .+", text, re.M) for field in ("Date", "From", "Subject")) for text in artifacts.values())
+    else:
+        complete = all(marker in body for marker in (b"<!--X-Body-of-Message-->", b"<!--X-Body-of-Message-End-->"))
+    if not complete or not any(artifacts.values()):
+        raise ValueError("Unrecognized or incomplete mail archive response")
+    if host == "seclists.org":
+        message = body.split(b"<!--X-Body-of-Message-->", 1)[1].split(b"<!--X-Body-of-Message-End-->", 1)[0]
+        if not any(html_artifacts(message).values()):
+            raise ValueError("Missing mail archive message body")
+        parser = MailText()
+        parser.feed(message.decode("utf-8", "replace"))
+        artifacts["mail:body"] = "".join(parser.parts).replace("\r\n", "\n").strip()
+    return artifacts
+
+
+def gist_artifacts(payload: dict) -> dict[str, str]:
+    if not isinstance(payload, dict) or not isinstance(payload.get("files"), dict) or payload.get("truncated"):
+        raise ValueError("Incomplete Gist response")
+    artifacts = {}
+    for name, file in payload["files"].items():
+        if not isinstance(file, dict) or file.get("truncated") or not isinstance(file.get("content"), str):
+            raise ValueError("Incomplete Gist file")
+        artifacts[f"file:{name}"] = file["content"]
+    return artifacts
+
+
+class SourceRedirect(request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        if not supported(newurl) or parse.urlsplit(newurl).hostname.removeprefix("www.") != parse.urlsplit(req.full_url).hostname.removeprefix("www."):
+            raise ValueError("Unexpected reference redirect")
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+def fetch_artifacts(url: str, *, budget: Budget | None = None) -> dict[str, str]:
+    if not supported(url):
+        raise ValueError("Unsupported reference host/path")
+    headers = {"User-Agent": "pocindex-reference-pocs"}
+    is_gist = parse.urlsplit(url).hostname == "gist.github.com"
+    if is_gist:
+        gist = parse.urlsplit(url).path.rstrip("/").split("/")[-1]
+        endpoint = "https://api.github.com/gists/" + gist
+        token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+    else:
+        endpoint = re.sub(r"^http:", "https:", url)
+    opener = request.build_opener(SourceRedirect())
+    budget = budget or Budget(10)
+    body = budget.read(request.Request(endpoint, headers=headers), opener=opener, limit=MAX_BODY + 1)
+    if len(body) > MAX_BODY:
+        raise ValueError("Reference response exceeds content limit")
+    return gist_artifacts(json.loads(body)) if is_gist else mail_artifacts(url, body)
+
+
+def load_json(path: Path, default):
+    return json.loads(path.read_text(encoding="utf-8")) if path.exists() else default
+
+
+def save_json(path: Path, value) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(".tmp")
+    temporary.write_text(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8")
+    temporary.replace(path)
+
+
+def feed_candidates(payload: dict) -> dict[str, list[str]]:
+    rows = payload.get("vulnerabilities") if isinstance(payload, dict) else None
+    if not isinstance(payload, dict) or payload.get("format") != "NVD_CVE" or payload.get("version") != "2.0" or not isinstance(rows, list) or payload.get("totalResults") != len(rows):
+        raise ValueError("Incomplete NVD feed")
+    found = {}
+    for wrapper in rows:
+        cve = wrapper.get("cve") or {}
+        cid = str(cve.get("id") or "")
+        if not update_cves.is_valid_cve(cid) or cid in found or not isinstance(cve.get("references", []), list):
+            raise ValueError("Invalid NVD CVE entry")
+        found[cid] = []
+        if cve.get("vulnStatus") == "Rejected":
+            continue
+        for reference in cve.get("references", []):
+            url = str(reference.get("url") or "")
+            if "Exploit" in reference.get("tags", []) and supported(url):
+                found[cid].append(url)
+        found[cid] = sorted(set(found[cid]))
+    return found
+
+
+def load_feeds(directory: Path | None, *, budget: Budget | None = None) -> dict[str, list[str]]:
+    found = {}
+    budget = budget or Budget(180)
+    if directory is not None:
+        paths = sorted(directory.glob("nvd-*.json.gz"))
+        if not paths:
+            raise ValueError("No local NVD feeds")
+        for path in paths:
+            budget.remaining()
+            with gzip.open(path, "rt", encoding="utf-8") as handle:
+                found.update(feed_candidates(json.load(handle)))
+    else:
+        for name in ("recent", "modified"):
+            req = request.Request(f"{FEED_ROOT}/nvdcve-2.0-{name}.json.gz", headers={"User-Agent": "pocindex-reference-pocs"})
+            found.update(feed_candidates(json.loads(gzip.decompress(budget.read(req)))))
+    return found
+
+
+def load_reviews(path: Path = REVIEWS) -> list[dict]:
+    payload = load_json(path, {})
+    rows = payload.get("reviews") if isinstance(payload, dict) else None
+    if not isinstance(rows, list) or payload.get("version") != 1:
+        raise ValueError("Invalid reviewed reference manifest")
+    for row in rows:
+        if not isinstance(row, dict) or not update_cves.is_valid_cve(row.get("cve", "")) or not supported(row.get("url", "")) or not re.fullmatch(r"[a-f0-9]{64}", row.get("sha256", "")) or not row.get("artifact") or not row.get("evidence"):
+            raise ValueError("Invalid reviewed reference evidence")
+    return rows
+
+
+def load_exclusions(path: Path | None = None) -> set[tuple[str, str]]:
+    payload = load_json(path or REVIEWS, {})
+    rows = payload.get("exclusions") if isinstance(payload, dict) else None
+    if not isinstance(rows, list) or payload.get("version") != 1:
+        raise ValueError("Invalid reference exclusion registry")
+    for row in rows:
+        if not isinstance(row, dict) or not update_cves.is_valid_cve(row.get("cve", "")) or not supported(row.get("url", "")) or not row.get("reason"):
+            raise ValueError("Invalid reference exclusion")
+    return {(row["cve"], update_cves.poc_link_key(row["url"])) for row in rows}
+
+
+def approvals(reviews: list[dict], candidates: dict, checks: dict, previous: dict) -> dict[str, list[str]]:
+    found: dict[str, set[str]] = {}
+    for row in reviews:
+        cve, url = row["cve"], row["url"]
+        if url not in candidates.get(cve, []):
+            continue
+        check = checks.get(url, {})
+        if check.get("error"):
+            accepted = url in previous.get(cve, [])
+        else:
+            accepted = check.get("digests", {}).get(row["artifact"]) == row["sha256"]
+        if accepted:
+            found.setdefault(cve, set()).add(url)
+    return {cve: sorted(urls) for cve, urls in found.items()}
+
+
+def refresh_checks(state: dict, reviews: list[dict], *, limit: int, days: int, cache: Path, dry_run: bool, budget: Budget | None = None) -> int:
+    now = datetime.now(timezone.utc)
+    today = now.date().isoformat()
+    cutoff = (now.date() - timedelta(days=days)).isoformat()
+    urls = {url for values in state["candidates"].values() for url in values}
+    reviewed = {row["url"] for row in reviews}
+    checks = state["checks"]
+    for url in set(checks) - urls:
+        del checks[url]
+    due = [url for url in urls if checks.get(url, {}).get("checked", "") <= (today if checks.get(url, {}).get("error") else cutoff)]
+    due.sort(key=lambda url: (url not in reviewed, checks.get(url, {}).get("checked", ""), url))
+    failures = 0
+    for url in due[:limit]:
+        try:
+            if budget is not None:
+                budget.remaining()
+            artifacts = fetch_artifacts(url, budget=budget) if budget is not None else fetch_artifacts(url)
+            digests = {name: hashlib.sha256(text.encode()).hexdigest() for name, text in artifacts.items()}
+            if not dry_run:
+                cache.mkdir(parents=True, exist_ok=True)
+                for name, text in artifacts.items():
+                    (cache / (digests[name] + ".txt")).write_text(text, encoding="utf-8")
+            checks[url] = {"checked": today, "digests": digests}
+        except error.HTTPError as problem:
+            if problem.code in {404, 410}:
+                checks[url] = {"checked": today, "digests": {}, "missing": problem.code}
+            else:
+                failures += 1
+                checks[url] = {**checks.get(url, {}), "checked": today, "error": f"HTTP {problem.code}"}
+        except (OSError, ValueError, http.client.HTTPException) as problem:
+            failures += 1
+            checks[url] = {**checks.get(url, {}), "checked": today, "error": str(problem)[:160]}
+        if budget is not None:
+            budget.failures = failures
+        if not dry_run:
+            save_json(QUEUE, state)
+    return failures
+
+
+def retain_previous(accepted: dict, previous: dict, *, dry_run: bool) -> None:
+    if not dry_run:
+        retained = {cve: [url for url in urls if url in accepted.get(cve, [])] for cve, urls in previous.items()}
+        save_json(VERIFIED, {cve: urls for cve, urls in retained.items() if urls})
+
+
+def published_entries(accepted: dict, previous: dict, args, budget: Budget) -> dict:
+    statuses = load_json(args.record_states, {}) if args.record_states else {}
+    if args.record_states and (not isinstance(statuses, dict) or any(cve not in statuses for cve in accepted)):
+        raise ValueError("Incomplete official CVE state snapshot")
+    with tempfile.TemporaryDirectory(prefix="pocindex-reference-records-") as directory:
+        local = Path(directory)
+        for cve in accepted:
+            budget.remaining()
+            path = update_cves.CVES / cve.split("-")[1] / f"{cve}.md"
+            if not args.record_states and cve in previous and path.exists():
+                # Previously admitted records retain their official publication
+                # evidence; metadata sync handles later rejections.
+                statuses[cve] = "PUBLISHED"
+            if statuses.get(cve) in {"RESERVED", "REJECTED"} or (cve in statuses and path.exists()):
+                continue
+            record = None
+            if args.cvelist_dir:
+                record = update_cves.read_record(update_cves.cvelist_record_path(args.cvelist_dir, cve))
+            if record is None:
+                req = request.Request(update_cves.CVE_API_URL.format(cve_id=cve), headers={"User-Agent": "pocindex-reference-pocs"})
+                try:
+                    record = json.loads(budget.read(req))
+                except error.HTTPError as problem:
+                    if problem.code != 404:
+                        raise
+                    statuses[cve] = "UNAVAILABLE"
+                    continue
+            status = (record.get("cveMetadata") or {}).get("state")
+            if update_cves.record_cve_id(record) != cve or status not in {"PUBLISHED", "RESERVED", "REJECTED"} or (status == "PUBLISHED" and update_cves.details_from_record(record) is None):
+                raise ValueError(f"Incomplete or mismatched official record: {cve}")
+            statuses[cve] = status
+            if not path.exists():
+                save_json(update_cves.cvelist_record_path(local, cve), record)
+        accepted = {cve: urls for cve, urls in accepted.items() if statuses.get(cve) == "PUBLISHED"}
+        # Every missing record is now local: the shared helper cannot start
+        # unbounded network retries after this command's budget expires.
+        _, unavailable = update_cves.ensure_cve_entries(accepted, dry_run=args.dry_run, cvelist_dir=local)
+    return {cve: urls for cve, urls in accepted.items() if cve not in unavailable}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--feed-dir", type=Path, help="Local complete nvd-YEAR.json.gz feeds")
+    parser.add_argument("--record-states", type=Path, help="Local official CVE ID to record-state snapshot")
+    parser.add_argument("--cvelist-dir", type=Path)
+    parser.add_argument("--cache-dir", type=Path, default=CACHE)
+    parser.add_argument("--limit", type=int, default=100)
+    parser.add_argument("--recheck-days", type=int, default=7)
+    parser.add_argument("--max-seconds", type=float, default=180)
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+    if args.limit < 1 or args.recheck_days < 1 or args.max_seconds <= 0:
+        parser.error("Limits must be positive")
+    budget = Budget(args.max_seconds)
+    excluded = load_exclusions()
+    reviews = [row for row in load_reviews() if (row["cve"], update_cves.poc_link_key(row["url"])) not in excluded]
+    state = load_json(QUEUE, {"candidates": {}, "checks": {}})
+    previous = load_json(VERIFIED, {})
+    try:
+        candidates = load_feeds(args.feed_dir, budget=budget)
+        for cve, urls in candidates.items():
+            if urls:
+                state["candidates"][cve] = urls
+            else:
+                state["candidates"].pop(cve, None)
+        failures = refresh_checks(state, reviews, limit=args.limit, days=args.recheck_days,
+                                  cache=args.cache_dir, dry_run=args.dry_run, budget=budget)
+        accepted = approvals(reviews, state["candidates"], state["checks"], previous)
+        retain_previous(accepted, previous, dry_run=args.dry_run)
+        accepted = published_entries(accepted, previous, args, budget)
+        for cve, urls in accepted.items():
+            path = update_cves.CVES / cve.split("-")[1] / f"{cve}.md"
+            if path.exists():
+                update_cves.update_existing_markdown(path, [], urls, dry_run=args.dry_run)
+        update_cves.append_inventory(update_cves.REFERENCE_LIST, accepted, dry_run=args.dry_run)
+        if not args.dry_run:
+            save_json(QUEUE, state)
+            save_json(VERIFIED, accepted)
+        pairs = sum(len(urls) for urls in accepted.values())
+        print(f"Reference PoCs: {pairs} reviewed links for {len(accepted)} published CVEs; {len(state['checks'])} URLs checked")
+        if failures:
+            raise RuntimeError(f"{failures} reference fetches failed; prior approvals and retry work preserved")
+    except BudgetExpired:
+        if not args.dry_run:
+            save_json(QUEUE, state)
+            retain_previous(approvals(reviews, state["candidates"], state["checks"], previous), previous, dry_run=False)
+        print(f"Reference PoCs: time budget reached; {len(state['checks'])} URLs checked, remaining work pending")
+        if budget.failures:
+            raise RuntimeError(f"{budget.failures} reference fetches failed before the budget expired") from None
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/update_cves.py

@@ -5,9 +5,11 @@ from __future__ import annotations
 
 import argparse
 import http.client
+import hashlib
 import json
 import os
 import re
+import subprocess
 import sys
 import time
 from collections import defaultdict
@@ -19,6 +21,8 @@ from typing import Any, Callable, Collection, Iterable, Iterator
 from urllib import error, request
 from urllib.parse import quote, urlparse
 
+import source_artifacts
+
 ROOT = Path(__file__).resolve().parents[1]
 CVES = ROOT / "cves"
 INDEX = ROOT / "index"
@@ -31,6 +35,7 @@ STATE_FILE = ROOT / ".github" / "cve_sync_state.json"
 KEV_FILE = INDEX / "kev.json"
 DATES_FILE = INDEX / "cve_dates.json"
 RECORD_STATE_FILE = INDEX / "cve_record_state.json"
+DISCOVERY_STATE_FILE = INDEX / "github_discovery_state.json"
 RECORD_RETRY_DAYS = 1
 
 GITHUB_GRAPHQL_URL = "https://api.github.com/graphql"
@@ -87,6 +92,7 @@ query($query: String!, $cursor: String) {
         isArchived
         isFork
         pushedAt
+        defaultBranchRef { target { oid } }
         repositoryTopics(first: 20) { nodes { topic { name } } }
       }
     }
@@ -378,8 +384,10 @@ def qualifying_repo_cves(
     description = str(repo.get("description") or "")
     topics = " ".join(topic_names(repo))
     readme = readme_text(repo)
+    reviewed = {item.cve for item in source_artifacts.reviewed_artifacts(repo)
+                if int(item.cve.split("-")[1]) == year}
     if repo_is_bare(repo, readme):
-        return set()
+        return reviewed
     all_name_cves = extract_cves(full_name) | extract_cves(normalise_identity(full_name))
     all_identity_cves = (
         all_name_cves
@@ -407,10 +415,10 @@ def qualifying_repo_cves(
     identity_is_non_poc = bool(NON_POC_RE.search(identity_text))
     has_code = root_has_code(repo)
     if identity_is_non_poc and not (name_cves and identity_has_poc and has_code):
-        return set()
+        return reviewed
     if len(readme_cves) > 5 and not identity_cves:
         readme_cves = set()
-    accepted: set[str] = set()
+    accepted: set[str] = set(reviewed)
     for cve_id in name_cves:
         has_context = readme_has_poc_context(readme, cve_id, full_name)
         has_artifact = root_has_poc_artifact(repo, cve_id)
@@ -446,7 +454,7 @@ def qualifying_repo_cves(
             and (has_artifact or identity_has_poc)
         ):
             accepted.add(cve_id)
-    return accepted
+    return accepted - source_artifacts.excluded_cves(full_name)
 
 
 def http_json(
@@ -526,6 +534,15 @@ class GitHubClient:
         if payload.get("errors") or not search:
             messages = "; ".join(str(item.get("message")) for item in payload.get("errors") or [])
             raise RuntimeError(f"GitHub search failed: {messages or 'missing response data'}")
+        page = search.get("pageInfo") or {}
+        if (
+            not isinstance(search.get("repositoryCount"), int)
+            or not isinstance(search.get("nodes"), list)
+            or not isinstance(page.get("hasNextPage"), bool)
+            or (page["hasNextPage"] and not page.get("endCursor"))
+            or any(not isinstance(node, dict) for node in search["nodes"])
+        ):
+            raise RuntimeError("GitHub search returned incomplete response data")
         rate = data.get("rateLimit") or {}
         if int(rate.get("remaining") or 0) < 5:
             raise RuntimeError(f"GitHub GraphQL quota is nearly exhausted; resets at {rate.get('resetAt')}")
@@ -618,14 +635,23 @@ def search_range(
 
     print(f"GitHub {qualifier} {start}..{end}: {count} repositories")
     page = first_page
+    received = 0
+    cursors: set[str] = set()
     while True:
         for repo in page.get("nodes") or []:
             if repo:
+                received += 1
                 yield repo
         page_info = page.get("pageInfo") or {}
         if not page_info.get("hasNextPage"):
             break
-        page = client.search_page(query_text, str(page_info.get("endCursor")))
+        cursor = str(page_info.get("endCursor") or "")
+        if not cursor or cursor in cursors:
+            raise RuntimeError("GitHub search returned an invalid pagination cursor")
+        cursors.add(cursor)
+        page = client.search_page(query_text, cursor)
+    if received < count:
+        raise RuntimeError(f"GitHub search returned only {received} of {count} repositories")
 
 
 def search_without_range(client: GitHubClient, search_terms: str) -> Iterator[dict[str, Any]]:
@@ -651,6 +677,9 @@ def discover_github_pocs(
     lookback_days: int,
     backfill: bool,
     cve_filter: set[str],
+    window_end: date | None = None,
+    checkpoints: dict[str, str] | None = None,
+    artifact_cache_write: bool = True,
 ) -> dict[str, list[str]]:
     client = GitHubClient(token)
     blacklist = load_blacklist()
@@ -672,8 +701,10 @@ def discover_github_pocs(
         for cve_id in qualifying_repo_cves(repo, year, blacklist):
             if cve_filter and cve_id not in cve_filter:
                 continue
-            if url not in discovered[cve_id]:
-                discovered[cve_id].append(url)
+            links = source_artifacts.approved_artifact_links(repo, cve_id) or [url]
+            for link in links:
+                if link not in discovered[cve_id]:
+                    discovered[cve_id].append(link)
 
     def search_terms(identifier: str) -> list[str]:
         return [
@@ -690,18 +721,45 @@ def discover_github_pocs(
                 for repo in search_without_range(client, terms):
                     remember(repo)
     else:
-        end = datetime.now(timezone.utc).date()
+        end = window_end or datetime.now(timezone.utc).date()
         for year in target_years:
             if backfill:
                 qualifier = "created"
                 start = date(2008, 1, 1)
             else:
                 qualifier = "pushed"
-                start = end - timedelta(days=max(1, lookback_days))
+                previous = (checkpoints or {}).get(str(year))
+                anchor = min(end, date.fromisoformat(previous)) if previous else end
+                start = anchor - timedelta(days=max(1, lookback_days))
             for terms in search_terms(f"CVE-{year}"):
                 for repo in search_range(client, terms, qualifier, start, end):
                     remember(repo)
 
+    # Deferred reviewed work survives the moving search window. Refresh its HEAD
+    # before resuming so a stale candidate cache cannot approve changed bytes.
+    pending_cves = {row["cve"] for row in source_artifacts.load_reviews()
+                    if int(row["cve"].split("-")[1]) in target_years
+                    and (not cve_filter or row["cve"] in cve_filter)}
+    pending_names = source_artifacts.pending_reviewed_names(cves=pending_cves)
+    missing = [name for name in pending_names if name.lower() not in {n.lower() for n in repositories}
+               and any(int(c.split("-")[1]) in target_years and (not cve_filter or c in cve_filter)
+                       for c in source_artifacts.reviewed_cves(name))
+               and not is_blacklisted_repo(name, blacklist)]
+    if missing:
+        aliases = []
+        for index, full_name in enumerate(missing):
+            owner, name = full_name.split("/", 1)
+            aliases.append(f"repo{index}: repository(owner: {json.dumps(owner)}, name: {json.dumps(name)}) "
+                           "{ nameWithOwner url description isFork defaultBranchRef { target { oid } } }")
+        payload = http_json(GITHUB_GRAPHQL_URL, headers=client.headers,
+                            data={"query": "query { " + " ".join(aliases) + " rateLimit { remaining resetAt } }"})
+        replay = github_repository_data(payload, {f"repo{index}" for index in range(len(missing))})
+        for index in range(len(missing)):
+            if replay.get(f"repo{index}") is not None:
+                remember(replay[f"repo{index}"])
+        if artifact_cache_write:
+            source_artifacts.mark_pending_checked(missing)
+
     names = sorted(repositories)
     batches = [names[offset : offset + 10] for offset in range(0, len(names), 10)]
     completed = 0
@@ -731,12 +789,42 @@ def discover_github_pocs(
     if removed:
         print(f"Discarded {len(removed)} repositories removed during discovery")
 
-    for repo in repositories.values():
+    pending_order = {name: index for index, name in enumerate(pending_names)}
+    for repo in sorted(repositories.values(), key=lambda r: pending_order.get(str(r.get("nameWithOwner", "")).lower(), len(pending_order))):
+        wanted = {c for c in (source_artifacts.identity_cves(str(repo.get("nameWithOwner") or ""))
+                            | source_artifacts.reviewed_cves(str(repo.get("nameWithOwner") or "")))
+                  if int(c.split("-")[1]) in target_years and (not cve_filter or c in cve_filter)}
+        attach_source_artifacts(repo, client.headers, blacklist, cves=wanted, persist=artifact_cache_write)
         for year in target_years:
             collect(repo, year)
     return discovered
 
 
+
+def attach_source_artifacts(
+    repo: dict[str, Any], headers: dict[str, str], blacklist: Collection[str],
+    *, cves: set[str] | None = None, persist: bool = True,
+) -> None:
+    """Add locally verified nested evidence without weakening the ordinary gate."""
+    name = str(repo.get("nameWithOwner") or "")
+    if not name or repo.get("isFork") or is_blacklisted_repo(name, blacklist):
+        return
+    if "defaultBranchRef" in repo and repo["defaultBranchRef"] is None:
+        return
+    known = source_artifacts.reviewed_cves(name)
+    # Unreviewed bytes cannot pass the same-repository approval registry.
+    # Do not spend the Actions REST quota harvesting candidates here.
+    wanted = known if cves is None else known & cves
+    if not wanted:
+        return
+    repo["_source_artifacts"] = source_artifacts.inspect_repository(
+        # A caller selecting one CVE (such as Trending) must not overwrite
+        # unfinished inspection of the other reviewed CVEs in this repository.
+        repo, known, readme_text(repo), http_json, headers, persist=persist,
+        budget=source_artifacts.automatic_budget(),
+    )
+
+
 def cna_container(record: dict[str, Any]) -> dict[str, Any]:
     return (record.get("containers") or {}).get("cna") or {}
 
@@ -788,9 +876,11 @@ def details_from_record(record: dict[str, Any]) -> CVEDetails | None:
         vendor = str(affected.get("vendor") or "").strip()
         if vendor and vendor.lower() not in ("n/a", "unknown"):
             vendors.append(vendor)
-        product = str(affected.get("product") or "").strip()
-        if product:
-            products.append(product)
+        for field in ("product", "packageName"):
+            product = str(affected.get(field) or "").strip()
+            if product and product.lower() not in {"n/a", "unknown"}:
+                products.append(product)
+                break
         for version in version_rows:
             status = str(version.get("status") or "affected").lower()
             value = str(version.get("version") or "").strip()
@@ -959,28 +1049,55 @@ def load_backfill_records(
     return records, references, paths
 
 
-def load_state(path: Path = STATE_FILE) -> str | None:
+def load_state(path: Path | None = None, *, allow_missing: bool = False) -> str | None:
+    path = path or STATE_FILE
     try:
         data = json.loads(path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError):
-        return None
-    value = data.get("cvelist_fetch_time")
-    return str(value) if value else None
+    except FileNotFoundError as exc:
+        if allow_missing:
+            return None
+        raise RuntimeError("Missing CVE delta checkpoint; use --bootstrap-delta explicitly to start from retained history") from exc
+    except (OSError, json.JSONDecodeError) as exc:
+        raise RuntimeError(f"Unreadable or corrupt CVE delta checkpoint: {path}") from exc
+    value = data.get("cvelist_fetch_time") if isinstance(data, dict) else None
+    if not isinstance(value, str) or not value.strip():
+        raise RuntimeError(f"Invalid CVE delta checkpoint: {path}")
+    parse_delta_time(value)
+    return value
 
 
-def delta_changes(last_fetch_time: str | None) -> tuple[dict[str, str], str]:
+def parse_delta_time(value: str) -> datetime:
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError as exc:
+        raise RuntimeError(f"Invalid CVE delta timestamp: {value!r}") from exc
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc)
+
+
+def delta_changes(last_fetch_time: str | None, *, bootstrap: bool = False) -> tuple[dict[str, str], str]:
+    if not last_fetch_time and not bootstrap:
+        raise RuntimeError("CVE delta discovery requires a checkpoint or explicit bootstrap")
     log = http_json(CVELIST_DELTA_LOG_URL)
     if not isinstance(log, list) or not log:
         raise RuntimeError("CVE List V5 delta log is empty")
+    times = [parse_delta_time(str(batch.get("fetchTime") or "")) for batch in log]
+    if times != sorted(times, reverse=True):
+        raise RuntimeError("CVE List V5 delta log is not ordered newest first")
+    if last_fetch_time and parse_delta_time(last_fetch_time) < times[-1]:
+        raise RuntimeError(
+            f"CVE List V5 delta retention gap: checkpoint {last_fetch_time} precedes "
+            f"oldest retained batch {log[-1]['fetchTime']}. "
+            "Recover with --catch-up-dir pointing to a complete pinned CVE List V5 snapshot."
+        )
     newest = str(log[0].get("fetchTime") or "")
     selected: list[dict[str, Any]] = []
     for batch in log:
         fetch_time = str(batch.get("fetchTime") or "")
-        if last_fetch_time and fetch_time <= last_fetch_time:
+        if last_fetch_time and parse_delta_time(fetch_time) <= parse_delta_time(last_fetch_time):
             break
         selected.append(batch)
-        if not last_fetch_time:
-            break
 
     changes: dict[str, str] = {}
     for batch in reversed(selected):
@@ -994,13 +1111,111 @@ def delta_changes(last_fetch_time: str | None) -> tuple[dict[str, str], str]:
     return changes, newest
 
 
+def catch_up_snapshot_files(snapshot: Path) -> tuple[str, dict[str, str], str]:
+    """Use the committed tree as the completeness manifest, never a file count."""
+    def git(*arguments: str) -> bytes:
+        try:
+            return subprocess.run(["git", "-C", str(snapshot), *arguments], check=True,
+                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout
+        except (OSError, subprocess.CalledProcessError) as exc:
+            raise RuntimeError("Catch-up requires a pinned CVE List V5 Git checkout; extracted directories lack a trusted completeness manifest") from exc
+
+    if Path(git("rev-parse", "--show-toplevel").decode().strip()).resolve() != snapshot.resolve():
+        raise RuntimeError("--catch-up-dir must point to the CVE List V5 Git checkout root")
+    revision = git("rev-parse", "--verify", "HEAD^{commit}").decode().strip()
+    algorithm = git("rev-parse", "--show-object-format").decode().strip()
+    if algorithm not in {"sha1", "sha256"}:
+        raise RuntimeError("Unsupported CVE snapshot object format")
+    files = {}
+    for entry in git("ls-tree", "-rz", "--full-tree", revision, "--", "cves").split(b"\0"):
+        if not entry:
+            continue
+        metadata, raw_path = entry.split(b"\t", 1)
+        path = raw_path.decode("utf-8")
+        if path != "cves/deltaLog.json" and not re.fullmatch(r"cves/[12][0-9]{3}/[^/]+/CVE-[0-9]{4}-[0-9]{4,}\.json", path):
+            continue
+        mode, kind, oid = metadata.decode().split()
+        if kind != "blob" or mode not in {"100644", "100755"}:
+            raise RuntimeError(f"Invalid catch-up snapshot entry: {path}")
+        files[path] = oid
+    if "cves/deltaLog.json" not in files or len(files) < 2:
+        raise RuntimeError("CVE List V5 HEAD must track deltaLog.json and CVE records")
+    return revision, files, algorithm
+
+
+def read_snapshot_json(path: Path, oid: str, algorithm: str) -> dict | list:
+    try:
+        if path.is_symlink():
+            raise ValueError("symlink")
+        raw = path.read_bytes()
+        digest = hashlib.new(algorithm, f"blob {len(raw)}\0".encode() + raw).hexdigest()
+        if digest != oid:
+            raise ValueError("blob differs from pinned HEAD")
+        return json.loads(raw)
+    except (OSError, ValueError) as exc:
+        raise RuntimeError(f"Invalid catch-up snapshot file (missing, dirty or sparse): {path}") from exc
+
+
+def load_catch_up_records(snapshot: Path) -> tuple[dict[str, dict[str, Any]], dict[str, list[str]], str]:
+    """Recover a retention gap from verified files in a pinned CVE List checkout."""
+    revision, files, algorithm = catch_up_snapshot_files(snapshot)
+    log = read_snapshot_json(snapshot / "cves/deltaLog.json", files["cves/deltaLog.json"], algorithm)
+    if not isinstance(log, list) or not log:
+        raise RuntimeError("Snapshot CVE delta log is empty")
+    checkpoint = str(log[0].get("fetchTime") or "")
+    snapshot_time = parse_delta_time(checkpoint)
+    previous = load_state()
+    if not previous:
+        raise RuntimeError("Catch-up requires an existing CVE delta checkpoint")
+    previous_time = parse_delta_time(previous)
+    if snapshot_time <= previous_time:
+        raise RuntimeError("Catch-up snapshot does not advance the existing CVE delta checkpoint")
+    records: dict[str, dict[str, Any]] = {}
+    references: dict[str, list[str]] = {}
+    blacklist = load_blacklist()
+    scanned = 0
+    for relative, oid in files.items():
+        if relative == "cves/deltaLog.json":
+            continue
+        path = snapshot / relative
+        record = read_snapshot_json(path, oid, algorithm)
+        if not isinstance(record, dict) or record_cve_id(record) != path.stem:
+            raise RuntimeError(f"Invalid catch-up CVE record: {path}")
+        scanned += 1
+        meta = record.get("cveMetadata") or {}
+        modified = str(meta.get("dateUpdated") or meta.get("datePublished") or "")
+        if modified and parse_delta_time(modified) <= previous_time:
+            continue
+        state = str(meta.get("state") or "").upper()
+        if state not in {"PUBLISHED", "REJECTED", "RESERVED"} or (state == "PUBLISHED" and details_from_record(record) is None):
+            raise RuntimeError(f"Incomplete catch-up CVE record: {path}")
+        cve_id = record_cve_id(record)
+        records[cve_id] = record
+        if selected := poc_references(record, blacklist):
+            references[cve_id] = selected
+    if not scanned:
+        raise RuntimeError("Catch-up snapshot contains no CVE records")
+    print(f"CVE List V5 catch-up ({revision[:12]}): {scanned} records scanned, {len(records)} changed since {previous}")
+    return records, references, checkpoint
+
+
 def fetch_records(urls: dict[str, str], *, allow_not_found: bool) -> dict[str, dict[str, Any]]:
     records: dict[str, dict[str, Any]] = {}
     failures: list[str] = []
 
     def fetch(cve_id: str, url: str) -> tuple[str, dict[str, Any] | None]:
         result = http_json(url, allow_not_found=allow_not_found)
-        return cve_id, result if isinstance(result, dict) else None
+        if result is None and allow_not_found:
+            return cve_id, None
+        if (
+            not isinstance(result, dict)
+            or record_cve_id(result) != cve_id
+            or str((result.get("cveMetadata") or {}).get("state") or "").upper()
+            not in {"PUBLISHED", "RESERVED", "REJECTED"}
+            or (record_is_published(result) and details_from_record(result) is None)
+        ):
+            raise RuntimeError("Incomplete or mismatched CVE record")
+        return cve_id, result
 
     with ThreadPoolExecutor(max_workers=12) as executor:
         futures = {executor.submit(fetch, cve_id, url): cve_id for cve_id, url in urls.items()}
@@ -1014,17 +1229,19 @@ def fetch_records(urls: dict[str, str], *, allow_not_found: bool) -> dict[str, d
             if record:
                 records[fetched_cve] = record
 
-    if failures and not allow_not_found:
-        raise RuntimeError("Failed to fetch CVE records: " + "; ".join(failures[:10]))
     if failures:
-        print(f"Skipped {len(failures)} CVE API failures", file=sys.stderr)
+        raise RuntimeError("Failed to fetch CVE records: " + "; ".join(failures[:10]))
     return records
 
 
 def load_incremental_records(
     cve_filter: set[str],
+    *, bootstrap: bool = False,
 ) -> tuple[dict[str, dict[str, Any]], dict[str, list[str]], str]:
-    changes, newest = delta_changes(load_state())
+    checkpoint = load_state(allow_missing=bootstrap)
+    if bootstrap and checkpoint:
+        raise RuntimeError("--bootstrap-delta requires a missing checkpoint; existing state cannot be reset")
+    changes, newest = delta_changes(checkpoint, bootstrap=bootstrap)
     if cve_filter:
         changes = {cve_id: url for cve_id, url in changes.items() if cve_id in cve_filter}
     records = fetch_records(changes, allow_not_found=False) if changes else {}
@@ -1139,7 +1356,7 @@ def ensure_cve_entries(
     if cvelist_dir is not None and not cvelist_dir.is_dir():
         raise ValueError(f"CVE List directory does not exist: {cvelist_dir}")
 
-    today = date.today()
+    today = datetime.now(timezone.utc).date()
     due = wanted if cvelist_dir is not None else {
         cve_id for cve_id in wanted if retry_record(state.get(cve_id, {}), today)
     }
@@ -1414,8 +1631,10 @@ def sync_markdown(
     for cve_id in cve_ids:
         year = cve_id.split("-")[1]
         path = CVES / year / f"{cve_id}.md"
-        github_links = stable_unique(github.get(cve_id, []))
-        reference_links = stable_unique(references.get(cve_id, []))
+        github_links = [url for url in stable_unique(github.get(cve_id, []))
+                        if cve_id not in source_artifacts.excluded_cves(github_repo_from_url(url))]
+        reference_links = [url for url in stable_unique(references.get(cve_id, []))
+                           if cve_id not in source_artifacts.excluded_cves(github_repo_from_url(url))]
         record = records.get(cve_id)
 
         if record and not record_is_published(record):
@@ -1514,6 +1733,32 @@ def write_state(fetch_time: str, *, dry_run: bool) -> None:
     )
 
 
+def load_discovery_state() -> dict[str, Any]:
+    if not DISCOVERY_STATE_FILE.exists():
+        return {"years": {}, "pending": {}}
+    state = json.loads(DISCOVERY_STATE_FILE.read_text(encoding="utf-8"))
+    if not isinstance(state, dict) or not all(isinstance(state.get(key), dict) for key in ("years", "pending")):
+        raise ValueError("Invalid GitHub discovery state")
+    for year, endpoint in state["years"].items():
+        if not re.fullmatch(r"\d{4}", year):
+            raise ValueError("Invalid GitHub discovery year")
+        date.fromisoformat(endpoint)
+    for cve_id, entry in state["pending"].items():
+        if not is_valid_cve(cve_id) or not isinstance(entry, dict) or any(
+            not isinstance(entry.get(key), list) or any(not isinstance(url, str) for url in entry[key])
+            for key in ("github", "references")
+        ):
+            raise ValueError("Invalid pending GitHub discovery")
+    return state
+
+
+def write_discovery_state(state: dict[str, Any]) -> None:
+    DISCOVERY_STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
+    temporary = DISCOVERY_STATE_FILE.with_suffix(".tmp")
+    temporary.write_text(json.dumps(state, separators=(",", ":"), sort_keys=True) + "\n", encoding="utf-8")
+    temporary.replace(DISCOVERY_STATE_FILE)
+
+
 def newest_delta_time() -> str:
     log = http_json(CVELIST_DELTA_LOG_URL)
     if not isinstance(log, list) or not log:
@@ -1523,6 +1768,8 @@ def newest_delta_time() -> str:
 
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Discover and ingest CVE PoC links")
+    parser.add_argument("--bootstrap-delta", action="store_true", help="Explicitly initialize a missing checkpoint from all retained CVE delta batches; earlier history needs a separate backfill")
+    parser.add_argument("--catch-up-dir", type=Path, help="CVE List V5 Git checkout root; verifies every tracked CVE JSON against pinned HEAD before recovery")
     parser.add_argument("--backfill-dir", type=Path, help="Local CVE List V5 year directory")
     parser.add_argument("--year", type=int, help="Year to backfill")
     parser.add_argument("--lookback-days", type=int, default=3, help="GitHub pushed-date overlap")
@@ -1539,8 +1786,12 @@ def parse_args() -> argparse.Namespace:
         help="Rewrite the title and badges of every record from --cvelist-dir, and do nothing else",
     )
     args = parser.parse_args()
+    if args.bootstrap_delta and (args.catch_up_dir or args.backfill_dir or args.cve or args.skip_cvelist or args.refresh_headers):
+        parser.error("--bootstrap-delta cannot be filtered, skipped, or combined with catch-up/backfill/header refresh")
     if bool(args.backfill_dir) != bool(args.year):
         parser.error("--backfill-dir and --year must be used together")
+    if args.catch_up_dir and (args.backfill_dir or args.cve or args.skip_cvelist or args.refresh_headers):
+        parser.error("--catch-up-dir cannot be filtered, skipped, or combined with backfill/header refresh")
     if args.lookback_days < 1 or args.years < 1:
         parser.error("--lookback-days and --years must be positive")
     return args
@@ -1564,18 +1815,23 @@ def main() -> int:
     references: dict[str, list[str]] = {}
     record_paths: dict[str, Path] = {}
     checkpoint = ""
+    discovery_end = datetime.now(timezone.utc).date()
+    discovery_state = load_discovery_state() if not args.skip_github and not cve_filter else None
+    years: list[int] = []
 
     if not args.skip_cvelist:
-        if backfill:
+        if getattr(args, "catch_up_dir", None):
+            records, references, checkpoint = load_catch_up_records(args.catch_up_dir)
+            references = filter_live_references(references)
+        elif backfill:
             records, references, record_paths = load_backfill_records(
                 args.backfill_dir,
                 args.year,
                 cve_filter,
             )
-            checkpoint = newest_delta_time()
             print(f"CVE List V5 backfill: {len(references)} CVEs with PoC references")
         else:
-            records, references, checkpoint = load_incremental_records(cve_filter)
+            records, references, checkpoint = load_incremental_records(cve_filter, bootstrap=getattr(args, "bootstrap_delta", False))
             references = filter_live_references(references)
 
     github: dict[str, list[str]] = {}
@@ -1594,6 +1850,9 @@ def main() -> int:
             lookback_days=args.lookback_days,
             backfill=backfill,
             cve_filter=cve_filter,
+            window_end=discovery_end,
+            checkpoints=discovery_state["years"] if discovery_state else None,
+            artifact_cache_write=not args.dry_run,
         )
         print(f"GitHub discovery: {sum(len(urls) for urls in github.values())} links for {len(github)} CVEs")
 
@@ -1602,7 +1861,18 @@ def main() -> int:
             path = record_paths.get(cve_id)
             if path and (record := read_record(path)):
                 records[cve_id] = record
-    fetch_missing_records(set(github) | set(references), records)
+    deferred: set[str] = set()
+    if discovery_state is not None:
+        for cve_id, pending in discovery_state["pending"].items():
+            if backfill and int(cve_id.split("-")[1]) not in years:
+                continue
+            for mappings, field in ((github, "github"), (references, "references")):
+                if pending[field]:
+                    mappings[cve_id] = stable_unique([*mappings.get(cve_id, []), *pending[field]])
+            if not retry_record(pending, discovery_end):
+                deferred.add(cve_id)
+    candidates = set(github) | set(references)
+    fetch_missing_records(candidates - deferred, records)
 
     stats, accepted_github, accepted_references = sync_markdown(
         records,
@@ -1632,7 +1902,24 @@ def main() -> int:
     # list; the ones fetched this run are refreshed in place.
     refreshed = refresh_record_headers(records, dry_run=args.dry_run)
     dated = record_dates(records, dry_run=args.dry_run)
-    write_state(checkpoint, dry_run=args.dry_run)
+    if not backfill and not cve_filter:
+        write_state(checkpoint, dry_run=args.dry_run)
+    if discovery_state is not None and not args.dry_run:
+        for cve_id in candidates:
+            record = records.get(cve_id) or {}
+            status = str((record.get("cveMetadata") or {}).get("state") or "UNAVAILABLE").upper()
+            if cve_id in accepted_github or cve_id in accepted_references or status == "REJECTED":
+                discovery_state["pending"].pop(cve_id, None)
+            else:
+                previous = discovery_state["pending"].get(cve_id, {})
+                discovery_state["pending"][cve_id] = {
+                    "github": github.get(cve_id, []),
+                    "references": references.get(cve_id, []),
+                    "status": previous.get("status", status) if cve_id in deferred else status,
+                    "checked": previous["checked"] if cve_id in deferred else discovery_end.isoformat(),
+                }
+        discovery_state["years"].update({str(year): discovery_end.isoformat() for year in years})
+        write_discovery_state(discovery_state)
 
     print(
         f"Created: {len(stats.created)} | Updated: {len(stats.updated)} | "