mirror of
https://github.com/0xMarcio/cve.git
synced 2026-10-27 16:51:03 +00:00
Give every entry its publication dates
The design asked for published and modified in the result column and the frontend has been rendering them since the reskin, against a field nothing supplied. Seeded from the CVE Program's daily release: 154,160 of 154,166 CVEs, so coverage on the indexed set is total. The 590 MB archive is a one-off. Every record the delta log hands the daily sync afterwards already carries its own dates, so update_cves keeps the file current from work it was doing anyway. Costs 0.86 MB gzipped on the index.
This commit is contained in:
1 parent
a2c376e77d
commit
a150d3d1b9
4 files changed
+50
-1
No files matched your search
@@ -21,6 +21,7 @@ TRENDING_OUTPUT = DOCS_DIR / "trending_poc.json"
|
||||
REPO_META = ROOT / "repo_meta.json"
|
||||
REPO_META_OUTPUT = DOCS_DIR / "repo_meta.json"
|
||||
KEV_INPUT = ROOT / "kev.json"
|
||||
DATES_INPUT = ROOT / "cve_dates.json"
|
||||
KEV_OUTPUT = DOCS_DIR / "kev.json"
|
||||
NUCLEI_INPUT = ROOT / "nuclei.json"
|
||||
NUCLEI_OUTPUT = DOCS_DIR / "nuclei.json"
|
||||
@@ -134,6 +135,7 @@ def collect_links(block: str, *, blacklist: Optional[Collection[str]] = None) ->
|
||||
|
||||
def build_cve_list(blacklist: Collection[str]) -> tuple[List[Dict[str, object]], int]:
|
||||
cve_entries = []
|
||||
dates = load_dates()
|
||||
total = 0
|
||||
|
||||
for md_path in sorted(ROOT.glob("[12][0-9][0-9][0-9]/CVE-*.md")):
|
||||
@@ -168,6 +170,11 @@ def build_cve_list(blacklist: Collection[str]) -> tuple[List[Dict[str, object]],
|
||||
"desc": description,
|
||||
"poc": poc_entries,
|
||||
}
|
||||
published, modified = (dates.get(cve_id) or ["", ""])[:2]
|
||||
if published:
|
||||
entry["published"] = published
|
||||
if modified and modified != published:
|
||||
entry["modified"] = modified
|
||||
entry.update({field: links for field, links in curated.items() if links})
|
||||
cve_entries.append(entry)
|
||||
|
||||
@@ -189,6 +196,14 @@ def build_repo_meta(cve_entries: List[Dict[str, object]]) -> Dict[str, object]:
|
||||
return {name: value for name, value in stored.items() if name in linked}
|
||||
|
||||
|
||||
def load_dates() -> Dict[str, list]:
|
||||
"""When the CVE Program first published each record and last changed it."""
|
||||
try:
|
||||
return json.loads(DATES_INPUT.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}
|
||||
|
||||
|
||||
def build_kev(cve_entries: List[Dict[str, object]]) -> Dict[str, object]:
|
||||
"""CISA's known-exploited entries, limited to CVEs the index carries."""
|
||||
try:
|
||||
|
||||
+33
-1
@@ -25,6 +25,7 @@ REFERENCE_LIST = ROOT / "references.txt"
|
||||
BLACKLIST_FILE = ROOT / "blacklist.txt"
|
||||
STATE_FILE = ROOT / ".github" / "cve_sync_state.json"
|
||||
KEV_FILE = ROOT / "kev.json"
|
||||
DATES_FILE = ROOT / "cve_dates.json"
|
||||
|
||||
GITHUB_GRAPHQL_URL = "https://api.github.com/graphql"
|
||||
CVE_API_URL = "https://cveawg.mitre.org/api/cve/{cve_id}"
|
||||
@@ -908,6 +909,36 @@ def fetch_missing_records(
|
||||
records.update(fetch_records(urls, allow_not_found=True))
|
||||
|
||||
|
||||
def record_dates(records: dict[str, dict[str, Any]], *, dry_run: bool) -> int:
|
||||
"""Keep cve_dates.json current from the records this run already fetched.
|
||||
|
||||
The file was seeded from the CVE Program's daily 590 MB release; every
|
||||
record the delta log hands us afterwards carries its own dates, so the
|
||||
archive never has to be downloaded again.
|
||||
"""
|
||||
try:
|
||||
stored = json.loads(DATES_FILE.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
stored = {}
|
||||
changed = 0
|
||||
for cve_id, record in records.items():
|
||||
meta = record.get("cveMetadata") or {}
|
||||
published = str(meta.get("datePublished") or "")[:10]
|
||||
modified = str(meta.get("dateUpdated") or meta.get("dateReserved") or "")[:10]
|
||||
if not published and not modified:
|
||||
continue
|
||||
value = [published, modified]
|
||||
if stored.get(cve_id) != value:
|
||||
stored[cve_id] = value
|
||||
changed += 1
|
||||
if changed and not dry_run:
|
||||
DATES_FILE.write_text(
|
||||
json.dumps(stored, separators=(",", ":"), sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return changed
|
||||
|
||||
|
||||
def load_kev(path: Path = KEV_FILE) -> dict[str, list]:
|
||||
"""CISA's known-exploited catalogue, as refreshed by the weekly audit."""
|
||||
try:
|
||||
@@ -1215,6 +1246,7 @@ def main() -> int:
|
||||
github_additions = append_inventory(GITHUB_LIST, accepted_github, dry_run=args.dry_run)
|
||||
reference_additions = append_inventory(REFERENCE_LIST, accepted_references, dry_run=args.dry_run)
|
||||
|
||||
dated = record_dates(records, dry_run=args.dry_run)
|
||||
write_state(checkpoint, dry_run=args.dry_run)
|
||||
|
||||
print(
|
||||
@@ -1223,7 +1255,7 @@ def main() -> int:
|
||||
)
|
||||
print(
|
||||
f"Inventory additions: {github_additions} GitHub | "
|
||||
f"{reference_additions} references"
|
||||
f"{reference_additions} references | {dated} publication dates"
|
||||
)
|
||||
if stats.skipped:
|
||||
for entry in stats.skipped[:20]:
|
||||
|
||||
Reference in new issue
Block a user