Track PoC releases independently and expand verified sources

.github/getTrending.py

@@ -9,6 +9,8 @@ import re
 import sys
 import time
 from datetime import datetime, timedelta, timezone
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
 from urllib import error, parse, request
 
 SEARCH_URL = "https://api.github.com/search/repositories"
@@ -21,10 +23,6 @@ PAPERWORK = re.compile(
     re.IGNORECASE,
 )
 PATHS_PER_REPO = 12
-LANGUAGES = (
-    "Shell", "Go", "ASP", "WebAssembly", "R", "Lua", "Python", "C++", "C",
-    "JavaScript", "Perl", "PowerShell", "Ruby", "Rust", "Java", "PHP",
-)
 YEARS = 5
 PER_YEAR = 20
 SEARCH_PAGE = 100
@@ -41,6 +39,9 @@ LANDED_ROWS = 10
 # placeholders keeps the rows from filling.
 LANDED_BATCH = 20
 LANDED_LIMIT = 200
+HISTORY_LIMIT = 40
+HISTORY_REMAINING = HISTORY_LIMIT
+HISTORY_RETRY_HOURS = 4
 NVD_API = "https://services.nvd.nist.gov/rest/json/cves/2.0?cveId="
 # NVD allows five requests per thirty seconds unauthenticated, and only a
 # couple of rows per run ever need one.
@@ -50,7 +51,9 @@ ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir)
 sys.path.insert(0, os.path.join(ROOT, "scripts"))
 
 from brand import BRAND, DESCRIPTION, SEARCH_GUIDE, SITE, SLUG
-from update_cves import load_blacklist, qualifying_repo_cves
+from update_cves import load_blacklist, qualifying_repo_cves, attach_source_artifacts, ensure_cve_entries
+import releases
+from source_artifacts import reviewed_artifacts
 
 README = os.path.join(ROOT, "README.md")
 TRENDING = os.path.join(ROOT, "index", "trending.json")
@@ -163,75 +166,141 @@ def nvd_lookup(cve: str) -> str:
     return ""
 
 
-def just_landed(token: str) -> list[dict]:
-    """Fresh PoCs with no star floor and no language filter.
+def published_repositories(repositories: list[dict]) -> list[dict]:
+    from build_site import load_metadata
+    metadata = load_metadata()
+    repositories = [repo for repo in repositories if not (metadata.get(cve_of(repo)) or {}).get("rejected")]
+    _, unavailable = ensure_cve_entries({cve_of(repo) for repo in repositories if cve_of(repo)}, dry_run=False)
+    return [repo for repo in repositories if cve_of(repo) not in unavailable]
 
-    A repository published this morning usually has neither: GitHub has not
-    classified its language yet and nobody has starred it. Filtering on either
-    is what kept day-one exploits off the page. What replaces them is a check
-    that the repository actually carries code, since dropping the filters also
-    lets through the write-ups and empty placeholders that share the naming.
-    """
-    cutoff = (datetime.now(timezone.utc) - timedelta(days=LANDED_DAYS)).date()
-    since = cutoff.isoformat()
-    # Landed means created in the window. An old repository with a new commit
-    # has not landed; a new PoC for a CVE from any year has.
-    _, rows = search(f"CVE in:name created:>={since}")
 
+def release_dates(repositories: list[dict], token: str, paths: dict[str, list[str]],
+                  ledger: dict) -> None:
+    """Resolve first qualifying content once, retaining the evidence across runs."""
+    headers = {"Accept": "application/vnd.github+json", "User-Agent": USER_AGENT}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+
+    global HISTORY_REMAINING
+    now = releases.utcnow()
+    queued, reused = [], []
+    for repo in repositories:
+        cve = cve_of(repo)
+        url = repo.get("_artifact_url") or repo["html_url"]
+        old = ledger.get(releases.key(cve, url), {})
+        checked = releases.timestamp(old.get("checked"))
+        dated = (old.get("commit_verified") and old.get("released")
+                 and (old.get("history_version") or 0) >= releases.HISTORY_VERSION)
+        cooling = (not dated and HISTORY_RETRY_HOURS > 0 and old.get("head") == repo.get("revision")
+                   and checked and datetime.now(timezone.utc) - checked < timedelta(hours=HISTORY_RETRY_HOURS))
+        if dated or cooling:
+            reused.append((repo, url, None))
+        elif HISTORY_REMAINING <= 0:
+            reused.append((repo, url, {"error": "History inspection deferred", "deferred": True}))
+        else:
+            HISTORY_REMAINING -= 1
+            queued.append((repo, url))
+
+    def inspect(candidate):
+        repo, url = candidate
+        evidence = releases.current_repository_evidence(
+            repo, paths.get(repo["full_name"], []), headers, cve=cve_of(repo),
+        )
+        return repo, url, evidence
+
+    with ThreadPoolExecutor(max_workers=4) as pool:
+        inspected = list(pool.map(inspect, queued)) + reused
+    for repo, url, evidence in inspected:
+        identity = releases.key(cve_of(repo), url)
+        row = releases.record(ledger, cve_of(repo), url, evidence)
+        if evidence is not None and not evidence.get("deferred"):
+            row.update(checked=now, head=repo.get("revision"))
+    releases.resolve_copies(ledger)
+    releases.mark_bulk(ledger)
+    for repo, url, _ in inspected:
+        row = ledger[releases.key(cve_of(repo), url)]
+        repo.update(_released=row.get("released"), _basis=row.get("basis"),
+                    _copy=bool(row.get("copy")), _bulk=bool(row.get("bulk")))
+
+
+def just_landed(token: str, ledger: dict | None = None) -> list[dict]:
+    """First qualifying artifact releases, independently of stars or Trending."""
+    ledger = releases.load_ledger() if ledger is None else ledger
+    cutoff = datetime.now(timezone.utc) - timedelta(days=LANDED_DAYS)
+    since = cutoff.date().isoformat()
+    _, rows = search(f"CVE in:name pushed:>={since}")
     candidates, seen = [], set()
     for repo in rows:
         name = str(repo.get("full_name") or "")
-        try:
-            created = datetime.strptime(str(repo.get("created_at")), "%Y-%m-%dT%H:%M:%SZ").date()
-        except ValueError:
-            continue
-        # The query matches any name containing "cve"; a row needs a CVE id there.
-        if (created < cutoff or not name or name.lower() in seen
+        if (not name or name.lower() in seen
                 or not CVE_ID.search(str(repo.get("name") or ""))):
             continue
         seen.add(name.lower())
         candidates.append(repo)
     candidates.sort(key=lambda repo: str(repo.get("pushed_at") or ""), reverse=True)
-
     pool = candidates[:LANDED_LIMIT]
     fresh: list[dict] = []
-    judged = rejected = hollow = 0
+    judged = rejected = undated = 0
     for start in range(0, len(pool), LANDED_BATCH):
-        # An artifact date never follows its repository's last push, so once the
-        # last row is at least as new as the next push, nothing further down
-        # can place.
         if len(fresh) >= LANDED_ROWS and (
-            str(pool[start].get("pushed_at") or "") <= fresh[LANDED_ROWS - 1]["_shipped"]
+            str(pool[start].get("pushed_at") or "") <= fresh[LANDED_ROWS - 1]["_released"]
         ):
             break
-        batch = pool[start : start + LANDED_BATCH]
+        batch = pool[start:start + LANDED_BATCH]
         judged += len(batch)
         accepted, paths = qualifying_repositories(batch, token)
+        accepted = published_repositories(accepted)
         rejected += len(batch) - len(accepted)
-        shipped = code_pushed([str(repo["full_name"]) for repo in accepted], token, paths)
+        release_dates(accepted, token, paths, ledger)
         for repo in accepted:
-            pushed = shipped.get(str(repo["full_name"]))
-            if not pushed:
-                hollow += 1
-            elif pushed[:10] >= since:
-                repo["_shipped"] = pushed
+            released = releases.timestamp(repo.get("_released"))
+            if released is None:
+                undated += 1
+            elif cutoff <= released <= datetime.now(timezone.utc) and not repo.get("_copy") and not repo.get("_bulk"):
                 fresh.append(repo)
-        fresh.sort(key=lambda repo: repo["_shipped"], reverse=True)
-    print(
-        f"just landed: {judged} of {len(candidates)} candidates judged, "
-        f"{len(fresh)} qualified, {rejected} rejected by artifact gate, "
-        f"{hollow} without an artifact commit"
-    )
+        fresh.sort(key=lambda repo: repo["_released"], reverse=True)
+    print(f"just landed: {judged} of {len(candidates)} candidates judged, "
+          f"{len(fresh)} qualified, {rejected} rejected, {undated} awaiting release evidence")
     return fresh[:LANDED_ROWS]
 
 
+def ledger_landed(ledger: dict, candidates: list[dict]) -> list[dict]:
+    from build_site import build_cve_list, build_landed, REPO_META
+    entries, _ = build_cve_list(load_blacklist())
+    metadata = json.loads(REPO_META.read_text(encoding="utf-8"))
+    rows = {}
+    for item in build_landed(entries, metadata, known_exploited(), ledger):
+        rows[releases.key(item["cve"], item["url"])] = {
+            "cve": item["cve"], "name": item["name"], "html_url": item["url"],
+            "description": item["desc"], "stargazers_count": item["stars"],
+            "_released": item["released"], "_basis": item["basis"],
+        }
+    for repo in candidates:
+        identity = releases.key(cve_of(repo), repo.get("_artifact_url") or repo["html_url"])
+        current = ledger.get(identity, {})
+        if any(current.get(flag) for flag in ("copy", "gone", "bulk")):
+            continue
+        rows[identity] = repo
+    return sorted(rows.values(), key=lambda repo: repo["_released"], reverse=True)[:LANDED_ROWS]
+
+
+def star_cell(value: int | None) -> str:
+    return f"{value}\u2b50" if value is not None else ""
+
+
+def artifact_link(repo: dict) -> tuple[str, bool]:
+    url = repo.get("_artifact_url") or repo.get("html_url") or ""
+    parsed = parse.urlsplit(url)
+    artifact = parsed.hostname != "github.com" or len(parsed.path.strip("/").split("/")) != 2
+    return url, artifact
+
+
 def search_year(year: int, since: str) -> tuple[int, list[dict]]:
     """Return how many PoC repositories for one CVE year were pushed since a
     date, and every result GitHub exposes for the final artifact-date sort.
     """
     query = " ".join(
         [f'"CVE-{year}" in:name', f"stars:>{MIN_STARS}", f"pushed:>{since}"]
-        + [f"language:{language}" for language in LANGUAGES]
     )
     return search(query)
 
@@ -318,12 +387,13 @@ def qualifying_repositories(
     code: dict[str, list[str]] = {}
     blacklist = load_blacklist()
     fields = """
+      databaseId createdAt pushedAt defaultBranchRef { target { oid } }
       readmeMd: object(expression: \"HEAD:README.md\") { ... on Blob { text } }
       readmeUpper: object(expression: \"HEAD:README.MD\") { ... on Blob { text } }
       readmeRst: object(expression: \"HEAD:README.rst\") { ... on Blob { text } }
       readmeBare: object(expression: \"HEAD:README\") { ... on Blob { text } }
       root: object(expression: \"HEAD:\") {
-        ... on Tree { entries { name type } }
+        ... on Tree { entries { name type oid } }
       }
     """
     for start in range(0, len(repositories), 20):
@@ -353,16 +423,27 @@ def qualifying_repositories(
                 "repositoryTopics": {"nodes": topics},
             }
             cve = cve_of(repo)
+            if cve:
+                attach_source_artifacts(candidate, {"Authorization": f"Bearer {token}"}, blacklist, cves={cve})
             if not cve or cve not in qualifying_repo_cves(
                 candidate, int(cve.split("-")[1]), blacklist
             ):
                 continue
             entries = ((content.get("root") or {}).get("entries") or [])
-            code[full_name] = [
-                str(entry.get("name") or "")
-                for entry in entries
-                if entry.get("name") and not PAPERWORK.match(str(entry["name"]))
-            ][:PATHS_PER_REPO]
+            readme_path = next((path for field, path in (
+                ("readmeMd", "README.md"), ("readmeUpper", "README.MD"),
+                ("readmeRst", "README.rst"), ("readmeBare", "README"),
+            ) if (content.get(field) or {}).get("text")), None)
+            code[full_name] = releases.artifact_paths(
+                entries, cve, readme_path=readme_path, readme_qualified=True, limit=PATHS_PER_REPO,
+            )
+            if artifacts := reviewed_artifacts(candidate):
+                matching = [item for item in artifacts if item.cve == cve]
+                code[full_name] = sorted({item.path for item in matching})[:PATHS_PER_REPO]
+                if matching:
+                    repo["_artifact_url"] = matching[0].url
+            repo["revision"] = ((content.get("defaultBranchRef") or {}).get("target") or {}).get("oid")
+            repo["id"] = content.get("databaseId") or repo.get("id")
             accepted.append(repo)
     return accepted, code
 
@@ -406,7 +487,10 @@ def code_pushed(
             if dates:
                 # Commit clocks are user supplied; they cannot postdate the push.
                 now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-                latest[full_name] = min(max(dates), repository.get("pushedAt") or now, now)
+                ceiling = min(repository.get("pushedAt") or now, now)
+                valid = [value for value in dates if value <= ceiling]
+                if valid:
+                    latest[full_name] = max(valid)
 
     for full_name in full_names:
         if not paths.get(full_name):
@@ -419,9 +503,21 @@ def code_pushed(
     return latest
 
 
+def popularity(repo: dict) -> float:
+    stamp = releases.timestamp(repo.get("_released") or repo.get("_shipped"))
+    if stamp is None:
+        return 0.0
+    hours = max(0, (datetime.now(timezone.utc) - stamp).total_seconds() / 3600)
+    return int(repo.get("stargazers_count") or 0) / (hours + 24) ** 0.7
+
+
 def time_ago(timestamp: str) -> str:
-    moment = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
+    moment = releases.timestamp(timestamp)
+    if moment is None:
+        return ""
     delta = datetime.now(timezone.utc) - moment
+    if len(timestamp) == 10:
+        return f"{max(0, delta.days)}d ago"
     if delta.total_seconds() < 0:
         return "just now"
     for amount, unit in ((delta.days, "d"), (delta.seconds // 3600, "h"), (delta.seconds // 60, "m")):
@@ -447,7 +543,7 @@ def known_exploited() -> set[str]:
 
 
 def cve_of(repo: dict) -> str:
-    match = CVE_ID.search(f"{repo.get('name') or ''} {repo.get('description') or ''}")
+    match = CVE_ID.search(f"{repo.get('cve') or ''} {repo.get('name') or ''} {repo.get('description') or ''}")
     return f"CVE-{match.group(1)}-{match.group(2)}" if match else ""
 
 
@@ -633,7 +729,7 @@ jq -n --slurpfile kev kev.json --slurpfile epss epss.json \\
 | [`nuclei.json`](@SITE@/nuclei.json) | Template metadata for the CVEs covered by a runnable Nuclei check |
 | [`kev.json`](@SITE@/kev.json) | CISA known exploited, keyed by CVE id |
 | [`repo_meta.json`](@SITE@/repo_meta.json) | Stars and last push date per PoC repository, keys lowercased |
-| [`trending_poc.json`](@SITE@/trending_poc.json) | Trending repositories plus index totals |
+| [`trending_poc.json`](@SITE@/trending_poc.json) | Two independent lists in display order, `landed` by first release and `items` for Trending, plus index totals |
 | [`cves/2026/CVE-2026-68138.md`](cves/2026/CVE-2026-68138.md) | Markdown copy of one CVE, one directory per year |
 
 CVSS rows are `[version, score, severity, vector, source, assessment type]`.
@@ -649,7 +745,8 @@ Advisory rows are `[URL, NVD reference tags]`.
 | [ExploitDB](https://gitlab.com/exploit-database/exploitdb) | Archived exploits, mapped by their own CVE column |
 | [Metasploit](https://github.com/rapid7/metasploit-framework) | Modules, best ranked first |
 | [Vulhub](https://github.com/vulhub/vulhub) | Runnable vulnerable environments and reproduction steps |
-| [afrog](https://github.com/zan8in/afrog), [Vulnerability](https://github.com/tzwlhack/Vulnerability), [0day](https://github.com/helloexp/0day), [xray](https://github.com/chaitin/xray), [Google Security Research](https://github.com/google/security-research), [GitHub Security Lab](https://github.com/github/securitylab) | CVE-specific templates, code and reproduction guides inside multi-CVE repositories |
+| [afrog](https://github.com/zan8in/afrog), [Vulnerability](https://github.com/tzwlhack/Vulnerability), [0day](https://github.com/helloexp/0day), [xray](https://github.com/chaitin/xray), [Google Security Research](https://github.com/google/security-research), [GitHub Security Lab](https://github.com/github/securitylab), [Tenable](https://github.com/tenable/poc), [pedrib](https://github.com/pedrib/PoC) | CVE-specific templates, code and reproduction guides inside multi-CVE repositories |
+| [Openwall](https://www.openwall.com/lists/), [SecLists](https://seclists.org/) and [Gist](https://gist.github.com/) | Reproducers from disclosure posts and Gists that NVD references, linked only after review |
 | [EPSS](https://www.first.org/epss/) | Daily exploitation probability from FIRST |
 | [CISA KEV](https://www.cisa.gov/known-exploited-vulnerabilities-catalog) | What is being exploited in the wild |
 | [NVD](https://nvd.nist.gov/) | CVSS assessments and tagged vendor, third-party, patch and mitigation references |
@@ -661,12 +758,12 @@ Advisory rows are `[URL, NVD reference tags]`.
 | --- | --- | --- |
 | [Trending sweep](.github/workflows/hot_cves.yml) | hourly | Front-page repositories, also added to the searchable index |
 | [CVE sync](.github/workflows/sync_cve_pocs.yml) | daily | New CVEs, CNA references and recently pushed GitHub repositories for every CVE year |
-| [Metadata sync](.github/workflows/sync_metadata.yml) | daily plus weekly full pass | CVSS, advisories, rejected records and current CISA KEV status |
+| [Metadata sync](.github/workflows/sync_metadata.yml) | daily plus weekly full pass | CVSS, advisories, rejected records, current CISA KEV status and reviewed Gist and mailing list reproducers |
 | [Nuclei sync](.github/workflows/sync_nuclei.yml) | daily | New templates and rating changes |
 | [Exploit archives](.github/workflows/sync_exploits.yml) | daily | ExploitDB, Metasploit and Vulhub mappings |
-| [Historical GitHub sync](.github/workflows/sync_pocingithub.yml) | weekly | Older PoC repositories missed by the recent-push window |
-| [Path collection sync](.github/workflows/sync_collections.yml) | weekly | CVE-specific artifacts inside curated multi-CVE repositories |
-| [Link audit](.github/workflows/audit_poc_links.yml) | weekly | Repositories that went dead, dropped from the index |
+| [Historical GitHub sync](.github/workflows/sync_pocingithub.yml) | daily | Older PoC repositories missed by the recent-push window |
+| [Path collection sync](.github/workflows/sync_collections.yml) | every 6 hours | CVE-specific artifacts inside curated multi-CVE repositories |
+| [Link audit](.github/workflows/audit_poc_links.yml) | daily | Repositories that went dead, dropped from the index |
 
 ## Contributing
 
@@ -675,7 +772,10 @@ repository URL.
 """.replace("@SITE@", SITE)
 
 
-def main() -> int:
+def main(*, history_limit: int = HISTORY_LIMIT, retry_history: bool = False) -> int:
+    global HISTORY_REMAINING, HISTORY_RETRY_HOURS
+    HISTORY_REMAINING = history_limit
+    HISTORY_RETRY_HOURS = 0 if retry_history else 4
     now = datetime.now(timezone.utc)
     since = (now - timedelta(days=WINDOW_DAYS)).date().isoformat()
     current_year = now.year
@@ -684,10 +784,14 @@ def main() -> int:
     sections: list[list[str]] = []
 
     token = github_token()
+    ledger = releases.load_ledger()
+    fresh = just_landed(token, ledger)
     for year in range(current_year, current_year - YEARS, -1):
         total, repositories = search_year(year, since)
         searched = len(repositories)
         repositories, paths = qualifying_repositories(repositories, token)
+        repositories = published_repositories(repositories)
+        release_dates(repositories, token, paths, ledger)
         shipped = code_pushed(
             [str(r.get("full_name") or "") for r in repositories], token, paths
         )
@@ -695,11 +799,13 @@ def main() -> int:
             repo["_shipped"] = shipped.get(str(repo.get("full_name") or ""), "")
         qualified = len(repositories)
         repositories = [r for r in repositories if r["_shipped"][:10] >= since]
-        repositories.sort(key=lambda repo: repo["_shipped"], reverse=True)
+        repositories.sort(key=popularity, reverse=True)
+        recent = len(repositories)
         repositories = repositories[:PER_YEAR]
         print(f"CVE-{year}: {total} pushed since {since}, "
               f"{searched - qualified} rejected by artifact gate, "
-              f"{qualified - len(repositories)} without a recent artifact commit")
+              f"{qualified - recent} without a recent artifact commit, "
+              f"{len(repositories)} published")
         if not repositories:
             continue
         block = [
@@ -714,29 +820,37 @@ def main() -> int:
             # updated twenty-one hours ago.
             pushed = repo["_shipped"]
             cve = cve_of(repo)
+            url, artifact = artifact_link(repo)
+            stars = None if artifact else repo.get("stargazers_count")
             exploited = cve in kev
             # Same fallback the landed lane uses. These CVEs are old enough to
             # be in the index already, so it costs nothing to ask.
             summary = repository_summary(repo, cve)
             items.append({
                 "year": year,
-                "stars": int(repo.get("stargazers_count") or 0),
+                "stars": stars,
                 "name": cell(repo.get("name")),
-                "url": repo.get("html_url") or "",
+                "url": url,
+                "artifact": artifact,
+                "source": parse.urlsplit(url).hostname,
                 "desc": summary,
                 "pushed": pushed,
+                "released": repo.get("_released"),
+                "basis": repo.get("_basis"),
+                "score": popularity(repo),
+                "trending": True,
                 "created": str(repo.get("created_at") or ""),
                 "cve": cve,
                 "kev": exploited,
             })
             block.append(
-                f"| {repo.get('stargazers_count', 0)}\u2b50 | {time_ago(pushed)} "
-                f"| {KEV_MARK if exploited else ''}[{cell(repo.get('name'))}]({repo.get('html_url')}) "
+                f"| {star_cell(stars)} | {time_ago(pushed)} "
+                f"| {KEV_MARK if exploited else ''}[{cell(repo.get('name'))}]({url}) "
                 f"| {shorten(summary)} |"
             )
         sections.append(block)
 
-    landed = just_landed(token)
+    landed = ledger_landed(ledger, fresh)
 
     stamp = now.strftime("%Y%m%d%H%M")
     # No hyphens: they are the field separator in a shields badge path.
@@ -745,11 +859,13 @@ def main() -> int:
     if landed:
         lines.append("## Just landed")
         lines.append("")
-        lines.append("| Stars | Updated | Repository | Description |")
+        lines.append("| Stars | Released | PoC | Description |")
         lines.append("| --- | --- | --- | --- |")
         lookups = 0
         for repo in landed:
             cve = cve_of(repo)
+            url, artifact = artifact_link(repo)
+            stars = None if artifact else repo.get("stargazers_count")
             exploited = cve in kev
             # A day-old repository often ships without a description; the CVE's
             # own summary says more than an empty cell.
@@ -760,19 +876,25 @@ def main() -> int:
                 time.sleep(6)
             items.append({
                 "year": int(cve.split("-")[1]) if cve else current_year,
-                "stars": int(repo.get("stargazers_count") or 0),
+                "stars": stars,
                 "name": cell(repo.get("name")),
-                "url": repo.get("html_url") or "",
+                "url": url,
+                "artifact": artifact,
+                "source": parse.urlsplit(url).hostname,
                 "desc": summary,
-                "pushed": repo["_shipped"],
+                "pushed": repo.get("_shipped"),
+                "released": repo["_released"],
+                "basis": repo.get("_basis"),
+                "score": popularity(repo),
+                "trending": False,
                 "created": str(repo.get("created_at") or ""),
                 "cve": cve,
                 "kev": exploited,
                 "landed": True,
             })
             lines.append(
-                f"| {repo.get('stargazers_count', 0)}\u2b50 | {time_ago(repo['_shipped'])} "
-                f"| {KEV_MARK if exploited else ''}[{cell(repo.get('name'))}]({repo.get('html_url')}) "
+                f"| {star_cell(stars)} | {time_ago(repo['_released'])} "
+                f"| {KEV_MARK if exploited else ''}[{cell(repo.get('name'))}]({url}) "
                 f"| {shorten(summary)} |"
             )
         lines.append("")
@@ -791,10 +913,12 @@ def main() -> int:
     # written once and keeps its landed flag.
     unique: dict[str, dict] = {}
     for item in items:
-        kept = unique.setdefault(item["url"].lower(), item)
+        kept = unique.setdefault(releases.key(item["cve"], item["url"]), item)
         if item.get("landed"):
             kept["landed"] = True
-    items = list(unique.values())
+    items = sorted(unique.values(), key=lambda item: item["score"], reverse=True)
+    releases.resolve_copies(ledger)
+    releases.save_ledger(ledger)
     flagged = sum(1 for item in items if item["kev"])
 
     with open(README, "w", encoding="utf-8") as handle:
@@ -809,7 +933,11 @@ def main() -> int:
 if __name__ == "__main__":
     if sys.argv[1:] == ["--counts-only"]:
         raise SystemExit(refresh_count_header())
-    if sys.argv[1:]:
-        print("usage: getTrending.py [--counts-only]", file=sys.stderr)
-        raise SystemExit(2)
-    raise SystemExit(main())
+    import argparse
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--history-limit", type=int, default=HISTORY_LIMIT)
+    parser.add_argument("--retry-history", action="store_true")
+    args = parser.parse_args()
+    if args.history_limit < 0:
+        parser.error("--history-limit must not be negative")
+    raise SystemExit(main(history_limit=args.history_limit, retry_history=args.retry_history))

.github/workflows/audit_poc_links.yml

@@ -40,6 +40,9 @@ jobs:
       - name: Verify the generated index
         run: python3 scripts/build_site.py
 
+      - name: Record published artifact releases
+        run: python3 scripts/releases.py --record
+
       - name: Refresh published counts
         run: |
           python3 .github/hero.py
@@ -62,6 +65,7 @@ jobs:
             docs/social-card.svg \
             docs/stats.json \
             docs/kev.svg \
+            index/releases/ \
             cves/
 
           if git diff --cached --quiet; then

.github/workflows/hot_cves.yml

@@ -51,6 +51,9 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: python3 scripts/sync_pocingithub.py --input index/trending.json
 
+      - name: Record published artifact releases
+        run: python3 scripts/releases.py --record
+
       - name: Redraw the banner and recount the index
         run: |
           # Counts the corpus with the site build's own parser, so the banner can
@@ -74,10 +77,14 @@ jobs:
             index/reference_pocs.txt \
             index/cve_dates.json \
             index/cve_record_state.json \
+            index/github_discovery_state.json \
+            index/source_artifact_reviews.json \
+            index/source_artifact_candidates.json \
             docs/hero.svg \
             docs/social-card.svg \
             docs/stats.json \
             docs/kev.svg \
+            index/releases/ \
             cves/
           if git diff --cached --quiet; then
             echo "No changes to commit"

.github/workflows/sync_collections.yml

@@ -2,7 +2,7 @@ name: Sync PoC Collections
 
 on:
   schedule:
-    - cron: '40 5 * * 4'
+    - cron: '40 */6 * * *'
   workflow_dispatch:
 
 permissions:
@@ -57,6 +57,8 @@ jobs:
             docs/kev.svg \
             index/cve_dates.json \
             index/cve_record_state.json \
+            index/collection_source_state.json \
+            index/releases/ \
             cves/
 
           if git diff --cached --quiet; then

.github/workflows/sync_cve_pocs.yml

@@ -39,6 +39,9 @@ jobs:
           python3 -m unittest discover -s tests
           python3 scripts/build_site.py
 
+      - name: Record published artifact releases
+        run: python3 scripts/releases.py --record
+
       - name: Refresh published counts
         run: |
           python3 .github/hero.py
@@ -57,11 +60,15 @@ jobs:
             index/reference_pocs.txt \
             index/cve_dates.json \
             index/cve_record_state.json \
+            index/github_discovery_state.json \
+            index/source_artifact_reviews.json \
+            index/source_artifact_candidates.json \
             README.md \
             docs/hero.svg \
             docs/social-card.svg \
             docs/stats.json \
             docs/kev.svg \
+            index/releases/ \
             cves/
 
           if git diff --cached --quiet; then

.github/workflows/sync_exploits.yml

@@ -53,6 +53,7 @@ jobs:
             docs/social-card.svg \
             docs/stats.json \
             docs/kev.svg \
+            index/releases/ \
             index/cve_dates.json \
             index/cve_record_state.json \
             cves/

.github/workflows/sync_metadata.yml

@@ -49,11 +49,22 @@ jobs:
       - name: Refresh CISA known-exploited status
         run: python3 scripts/audit_poc_links.py --kev-only
 
+      - name: Refresh reviewed reference PoCs
+        id: references
+        continue-on-error: true
+        timeout-minutes: 5
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: python3 scripts/sync_reference_pocs.py --limit 40
+
       - name: Verify the generated index
         run: |
           python3 -m unittest discover -s tests
           python3 scripts/build_site.py
 
+      - name: Record published artifact releases
+        run: python3 scripts/releases.py --record
+
       - name: Refresh published counts
         run: |
           python3 .github/hero.py
@@ -66,6 +77,7 @@ jobs:
           git config user.email '[email protected]'
           git add -- \
             README.md \
+            index/releases/ \
             cves/ \
             docs/hero.svg \
             docs/social-card.svg \
@@ -76,7 +88,9 @@ jobs:
             index/github_advisories.json \
             index/kev.json \
             index/metadata/ \
-            index/references.txt
+            index/references.txt \
+            index/reference_poc_queue.json \
+            index/reference_poc_verified.json
 
           if git diff --cached --quiet; then
             echo "No changes to commit"
@@ -103,3 +117,7 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
         run: gh workflow run site.yml --ref main
+
+      - name: Report reference refresh failure
+        if: steps.references.outcome == 'failure'
+        run: exit 1

.github/workflows/sync_nuclei.yml

@@ -53,6 +53,7 @@ jobs:
             docs/social-card.svg \
             docs/stats.json \
             docs/kev.svg \
+            index/releases/ \
             index/cve_dates.json \
             index/cve_record_state.json \
             index/nuclei.json \

.github/workflows/sync_pocingithub.yml

@@ -44,6 +44,9 @@ jobs:
           python3 -m unittest discover -s tests
           python3 scripts/build_site.py
 
+      - name: Record published artifact releases
+        run: python3 scripts/releases.py --record
+
       - name: Refresh published counts
         run: |
           python3 .github/hero.py
@@ -63,6 +66,9 @@ jobs:
             index/github.txt \
             index/cve_dates.json \
             index/cve_record_state.json \
+            index/source_artifact_reviews.json \
+            index/source_artifact_candidates.json \
+            index/releases/ \
             cves/
 
           if git diff --cached --quiet; then