Give every entry its publication dates

The design asked for published and modified in the result column and the
frontend has been rendering them since the reskin, against a field nothing
supplied. Seeded from the CVE Program's daily release: 154,160 of 154,166
CVEs, so coverage on the indexed set is total.

The 590 MB archive is a one-off. Every record the delta log hands the daily
sync afterwards already carries its own dates, so update_cves keeps the file
current from work it was doing anyway.

Costs 0.86 MB gzipped on the index.
This commit is contained in:
0xMarcio committed 2026-09-01 13:44:47 +02:00
1 parent a2c376e77d
commit a150d3d1b9
4 files changed
+50 -1

No files matched your search

+15
View File
@@ -21,6 +21,7 @@ TRENDING_OUTPUT = DOCS_DIR / "trending_poc.json"
REPO_META = ROOT / "repo_meta.json"
REPO_META_OUTPUT = DOCS_DIR / "repo_meta.json"
KEV_INPUT = ROOT / "kev.json"
DATES_INPUT = ROOT / "cve_dates.json"
KEV_OUTPUT = DOCS_DIR / "kev.json"
NUCLEI_INPUT = ROOT / "nuclei.json"
NUCLEI_OUTPUT = DOCS_DIR / "nuclei.json"
@@ -134,6 +135,7 @@ def collect_links(block: str, *, blacklist: Optional[Collection[str]] = None) ->
def build_cve_list(blacklist: Collection[str]) -> tuple[List[Dict[str, object]], int]:
cve_entries = []
dates = load_dates()
total = 0
for md_path in sorted(ROOT.glob("[12][0-9][0-9][0-9]/CVE-*.md")):
@@ -168,6 +170,11 @@ def build_cve_list(blacklist: Collection[str]) -> tuple[List[Dict[str, object]],
"desc": description,
"poc": poc_entries,
}
published, modified = (dates.get(cve_id) or ["", ""])[:2]
if published:
entry["published"] = published
if modified and modified != published:
entry["modified"] = modified
entry.update({field: links for field, links in curated.items() if links})
cve_entries.append(entry)
@@ -189,6 +196,14 @@ def build_repo_meta(cve_entries: List[Dict[str, object]]) -> Dict[str, object]:
return {name: value for name, value in stored.items() if name in linked}
def load_dates() -> Dict[str, list]:
"""When the CVE Program first published each record and last changed it."""
try:
return json.loads(DATES_INPUT.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}
def build_kev(cve_entries: List[Dict[str, object]]) -> Dict[str, object]:
"""CISA's known-exploited entries, limited to CVEs the index carries."""
try:
+33 -1
View File
@@ -25,6 +25,7 @@ REFERENCE_LIST = ROOT / "references.txt"
BLACKLIST_FILE = ROOT / "blacklist.txt"
STATE_FILE = ROOT / ".github" / "cve_sync_state.json"
KEV_FILE = ROOT / "kev.json"
DATES_FILE = ROOT / "cve_dates.json"
GITHUB_GRAPHQL_URL = "https://api.github.com/graphql"
CVE_API_URL = "https://cveawg.mitre.org/api/cve/{cve_id}"
@@ -908,6 +909,36 @@ def fetch_missing_records(
records.update(fetch_records(urls, allow_not_found=True))
def record_dates(records: dict[str, dict[str, Any]], *, dry_run: bool) -> int:
"""Keep cve_dates.json current from the records this run already fetched.
The file was seeded from the CVE Program's daily 590 MB release; every
record the delta log hands us afterwards carries its own dates, so the
archive never has to be downloaded again.
"""
try:
stored = json.loads(DATES_FILE.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
stored = {}
changed = 0
for cve_id, record in records.items():
meta = record.get("cveMetadata") or {}
published = str(meta.get("datePublished") or "")[:10]
modified = str(meta.get("dateUpdated") or meta.get("dateReserved") or "")[:10]
if not published and not modified:
continue
value = [published, modified]
if stored.get(cve_id) != value:
stored[cve_id] = value
changed += 1
if changed and not dry_run:
DATES_FILE.write_text(
json.dumps(stored, separators=(",", ":"), sort_keys=True) + "\n",
encoding="utf-8",
)
return changed
def load_kev(path: Path = KEV_FILE) -> dict[str, list]:
"""CISA's known-exploited catalogue, as refreshed by the weekly audit."""
try:
@@ -1215,6 +1246,7 @@ def main() -> int:
github_additions = append_inventory(GITHUB_LIST, accepted_github, dry_run=args.dry_run)
reference_additions = append_inventory(REFERENCE_LIST, accepted_references, dry_run=args.dry_run)
dated = record_dates(records, dry_run=args.dry_run)
write_state(checkpoint, dry_run=args.dry_run)
print(
@@ -1223,7 +1255,7 @@ def main() -> int:
)
print(
f"Inventory additions: {github_additions} GitHub | "
f"{reference_additions} references"
f"{reference_additions} references | {dated} publication dates"
)
if stats.skipped:
for entry in stats.skipped[:20]: