diff --git a/scripts/releases.py b/scripts/releases.py index a940f63ec0c..d1bd79b66de 100644 --- a/scripts/releases.py +++ b/scripts/releases.py @@ -41,6 +41,7 @@ SUPPORT_FILE = re.compile(r"^(?:setup|install|example|demo|test|conftest|helper| HISTORY_VERSION = 3 HISTORY_FIELDS = {"paths", "rev", "blobs", "commit", "sha", "commit_verified", "history_paths", "merged", "merged_verified", "history_version", "bulk"} +COMMIT_FIELDS = {"commit", "sha", "commit_verified", "history_version"} class Ledger(dict): @@ -221,6 +222,33 @@ def _same_artifact(row: dict, candidate: dict) -> bool: and left[1] == right[1] and left[0] != right[0]) +def _repository_rejected_shas(row: dict, candidates) -> set[str]: + rejected = set(row.get("future_shas", [])) + location = _repository_artifact(row["url"]) + if location: + for other in candidates: + if other["cve"] != row["cve"] or not other.get("future_shas"): + continue + peer = _repository_artifact(other["url"]) + same = (row["repo"] == other["repo"] if row.get("repo") and other.get("repo") + else peer and location[0] == peer[0]) + if peer and same: + rejected.update(other["future_shas"]) + return rejected + + +def _discard_rejected_commit(row: dict, rejected: set[str]) -> bool: + if rejected: + row["future_shas"] = sorted(rejected) + if row.get("sha") not in rejected: + return False + for name in COMMIT_FIELDS: + row.pop(name, None) + if row.get("basis") == "commit": + row.update(released=None, basis="unknown") + return True + + def record(ledger: dict[str, dict], cve: str, url: str, evidence: dict | None = None, *, observed_at: str | None = None, import_mode: bool = False) -> dict: observed_at = observed_at or utcnow() @@ -237,7 +265,9 @@ def record(ledger: dict[str, dict], cve: str, url: str, evidence: dict | None = if evidence.get("error") and row.get("released"): # A failed refresh may advance retry bookkeeping, never valid evidence. evidence = {name: value for name, value in evidence.items() if name in {"error", "checked", "head"}} - future_shas = set(row.get("future_shas", [])) + identities = ledger.by_cve.get(cve, ()) if isinstance(ledger, Ledger) else ledger.keys() + future_shas = _repository_rejected_shas( + {**row, "repo": evidence.get("repo") or row.get("repo")}, (ledger[item] for item in identities)) observed = min(timestamp(value) for value in ( observed_at, evidence.get("history_observed") or evidence.get("checked") ) if timestamp(value)) @@ -247,16 +277,9 @@ def record(ledger: dict[str, dict], cve: str, url: str, evidence: dict | None = clock = timestamp(proof.get("commit")) if proof.get("sha") and proof.get("commit_verified") is True and clock and checked and clock > checked: future_shas.add(proof["sha"]) - if future_shas: - row["future_shas"] = sorted(future_shas) - clock_fields = {"commit", "sha", "commit_verified", "history_version"} - if row.get("sha") in future_shas: - for name in clock_fields: - row.pop(name, None) - if row.get("basis") == "commit": - row.update(released=None, basis="unknown") - if evidence.get("sha") in future_shas: - evidence = {name: value for name, value in evidence.items() if name not in clock_fields} + _discard_rejected_commit(row, future_shas) + if evidence.get("sha") in future_shas: + evidence = {name: value for name, value in evidence.items() if name not in COMMIT_FIELDS} method = evidence.get("history_version", 0) method = method if type(method) is int and method > 0 else 0 old_method = row.get("history_version", 0) @@ -341,6 +364,16 @@ def resolve_copies(ledger: dict[str, dict], *, now: str | None = None) -> None: Entire selected nontrivial blob sets must match. One shared helper cannot collapse distinct variants. Tombstones still supply original provenance. """ + rejected = defaultdict(list) + for row in ledger.values(): + if row.get("future_shas"): + rejected[row["cve"]].append(dict(row)) + # Existing payloads can predate the peer that establishes clock rejection. + for row in ledger.values(): + if row["cve"] in rejected: + shas = _repository_rejected_shas(row, rejected[row["cve"]]) + if _discard_rejected_commit(row, shas): + row["released"], row["basis"] = date_release(row, now=now) groups, parents = defaultdict(list), {} for identity, row in ledger.items(): location = _repository_artifact(row["url"]) diff --git a/tests/test_releases.py b/tests/test_releases.py index 2a56ee581c6..884fb9ee4cc 100644 --- a/tests/test_releases.py +++ b/tests/test_releases.py @@ -191,6 +191,69 @@ class ReleaseLedgerTests(unittest.TestCase): self.assertTrue(row["imported"]) self.assertNotIn("seen", row) + def test_rejected_clock_survives_root_to_pinned_artifact_replacement(self): + first = "2026-10-01T02:00:00Z" + proof = {"repo": 12, "rev": "b" * 64, "blobs": ["c" * 40], "paths": ["exploit.py"], + "commit": "2026-10-03T00:00:00Z", "sha": "a" * 40, "commit_verified": True, + "history_version": releases.HISTORY_VERSION, "checked": first, "pushed_at": first} + artifact = URL + "/blob/" + "f" * 40 + "/exploit.py" + ledger = releases.Ledger() + releases.record(ledger, CVE, URL, {"error": "History pending"}, observed_at=first) + releases.record(ledger, CVE, URL, proof, observed_at=first) + ledger = releases.reconcile(ledger, [], observed_at=NOW) + row = releases.record(ledger, CVE, artifact, {**proof, "checked": NOW, "pushed_at": NOW}, observed_at=NOW) + self.assertNotIn("sha", row) + releases.resolve_copies(ledger, now=NOW) + self.assertIsNone(ledger[releases.key(CVE, artifact)]["released"]) + self.assertEqual(releases.landed(ledger, now=NOW), []) + # Rejecting one commit must not prevent a genuinely later introduction. + valid = {**proof, "commit": "2026-10-05T00:00:00Z", "sha": "d" * 40, + "checked": NOW, "pushed_at": NOW} + releases.record(ledger, CVE, artifact, valid, observed_at=NOW) + releases.resolve_copies(ledger, now=NOW) + self.assertEqual(ledger[releases.key(CVE, artifact)]["released"], valid["commit"]) + self.assertEqual(ledger[releases.key(CVE, URL)]["seen"], first) + self.assertIn("gone", ledger[releases.key(CVE, URL)]) + + def test_peer_clock_rejection_clears_existing_payload_in_either_batch_order(self): + first = "2026-10-01T02:00:00Z" + artifact = URL + "/blob/" + "f" * 40 + "/exploit.py" + proof = {"repo": 12, "rev": "b" * 64, "blobs": ["c" * 40], + "commit": "2026-10-03T00:00:00Z", "sha": "a" * 40, "commit_verified": True, + "history_version": releases.HISTORY_VERSION, "pushed_at": NOW} + for order in ((URL, artifact), (artifact, URL)): + with self.subTest(order=order): + ledger = releases.Ledger() + releases.record(ledger, CVE, URL, {"error": "History pending"}, observed_at=first) + for url in order: + checked = first if url == URL else NOW + releases.record(ledger, CVE, url, {**proof, "history_observed": checked}, observed_at=NOW) + ledger[releases.key(CVE, URL)]["gone"] = NOW + releases.resolve_copies(ledger, now=NOW) + row = ledger[releases.key(CVE, artifact)] + self.assertIsNone(row["released"]) + self.assertFalse(any(field in row for field in ("commit", "sha", "commit_verified", "history_version"))) + self.assertEqual(releases.landed(ledger, now=NOW), []) + + def test_clock_rejection_uses_repository_identity_without_merging_variants(self): + proof = {"repo": 12, "commit": "2026-10-03T00:00:00Z", "sha": "a" * 40, + "commit_verified": True, "history_version": releases.HISTORY_VERSION} + artifact = URL + "/blob/main/variant.py" + cases = ((CVE, artifact, 12, True), + (CVE, artifact.replace("researcher", "new-owner"), 12, True), + (CVE, artifact.replace("github.com/researcher", "www.github.com/RESEARCHER"), None, True), + (CVE, artifact.replace("researcher", "unrelated"), 13, False), + (CVE, artifact, 13, False), + ("CVE-2026-54321", artifact, 12, False)) + for cve, url, repo, rejected in cases: + with self.subTest(cve=cve, url=url, repo=repo): + ledger = releases.Ledger() + releases.record(ledger, CVE, URL, proof, observed_at="2026-10-01T00:00:00Z") + row = releases.record(ledger, cve, url, {**proof, "repo": repo}, observed_at=NOW) + releases.resolve_copies(ledger, now=NOW) + self.assertEqual(row["released"], None if rejected else proof["commit"]) + self.assertNotIn("copy", row) + def test_stale_push_date_can_catch_up_without_rejecting_observed_commit(self): ledger = releases.Ledger() proof = {"created": "2026-10-01T00:00:00Z", "pushed_at": "2026-10-01T01:00:00Z",