cve/scripts/build_site.py

from __future__ import annotations

import argparse
import csv
import gzip
import html
import io
import json
import re
import urllib.request
from datetime import datetime, timezone
from functools import lru_cache
from pathlib import Path
from typing import Collection, Dict, Iterable, List, Optional
from urllib.parse import unquote, urlparse

from update_cves import markdown_record_status
import link_health

ROOT = Path(__file__).resolve().parents[1]
CVES = ROOT / "cves"
INDEX = ROOT / "index"
DOCS_DIR = ROOT / "docs"
BLACKLIST = INDEX / "blacklist.txt"
TRENDING_INPUT = INDEX / "trending.json"
CVE_OUTPUT = DOCS_DIR / "CVE_list.json"
TRENDING_OUTPUT = DOCS_DIR / "trending_poc.json"
REPO_META = INDEX / "repo_meta.json"
REPO_META_OUTPUT = DOCS_DIR / "repo_meta.json"
KEV_INPUT = INDEX / "kev.json"
DATES_INPUT = INDEX / "cve_dates.json"
KEV_OUTPUT = DOCS_DIR / "kev.json"
NUCLEI_INPUT = INDEX / "nuclei.json"
NUCLEI_OUTPUT = DOCS_DIR / "nuclei.json"
METADATA_INPUT = INDEX / "metadata"
METADATA_OUTPUT = DOCS_DIR / "cve_metadata.json"
# The same metadata in the two halves the page loads apart: the CVSS rows the
# severity filter needs before anything can be searched, and the advisories
# only a rendered row shows. cve_metadata.json stays whole for API readers.
CVSS_OUTPUT = DOCS_DIR / "cvss.json"
ADVISORIES_OUTPUT = DOCS_DIR / "advisories.json"
VERIFIED_REFERENCES = INDEX / "reference_pocs.txt"
REVIEWED_REFERENCES = INDEX / "reference_poc_verified.json"
EPSS_OUTPUT = DOCS_DIR / "epss.json"
EPSS_FEED = "https://epss.empiricalsecurity.com/epss_scores-current.csv.gz"
GITHUB_NON_REPOS = {
    "advisories", "apps", "collections", "marketplace", "notifications",
    "orgs", "security", "settings", "sponsors", "topics", "user-attachments",
}
CODE_SUFFIXES = {
    ".c", ".cc", ".cpp", ".cs", ".go", ".java", ".js", ".md", ".nse",
    ".php", ".pl", ".ps1", ".py", ".rb", ".rs", ".sh", ".ts", ".txt",
    ".yaml", ".yml", ".zip",
}
MEDIA_SUFFIXES = {
    ".bmp", ".gif", ".jpeg", ".jpg", ".mov", ".mp4", ".pdf", ".png",
    ".svg", ".webm", ".webp",
}


def load_blacklist() -> set[str]:
    if not BLACKLIST.exists():
        return set()
    return {
        line.strip().lower()
        for line in BLACKLIST.read_text(encoding="utf-8").splitlines()
        if line.strip() and not line.lstrip().startswith("#")
    }


def normalise_block(text: str) -> str:
    # CNA descriptions sometimes contain formatting HTML. Keep their text,
    # including encoded payload examples, rather than displaying markup.
    if re.match(r"\s*<(?:p|div|ul|ol|h[1-6])(?:\s|>)", text, flags=re.I):
        text = re.sub(r"</?(?:p|div|br|ul|ol|li|h[1-6])(?:\s[^>]*)?/?>", "\n", text, flags=re.I)
        text = re.sub(r"</?(?:a|strong|em|b|i|code|span)(?:\s[^>]*)?>", "", text, flags=re.I)
        text = html.unescape(text)
    text = text.replace("\r\n", "\n")
    text = re.sub(r"\n{2,}", "\n", text.strip())
    lines = [
        re.sub(r"^#{1,6}\s+", "", line.lstrip("- ")).rstrip()
        for line in text.split("\n")
    ]
    return "\n".join(line for line in lines if line)


SECTION_HEADERS = ("### Description", "### POC", "#### Reference", "#### Github",
                   "#### Nuclei", "#### ExploitDB", "#### Metasploit", "#### Vulhub",
                   "#### Collections")
# Sections whose links are kept apart from the repository list, with the
# field each becomes in the published index.
CURATED = (
    ("#### Nuclei", "nuclei"),
    ("#### ExploitDB", "edb"),
    ("#### Metasploit", "msf"),
    ("#### Vulhub", "vulhub"),
    ("#### Collections", "collections"),
)


@lru_cache(maxsize=1)
def load_metadata() -> Dict[str, object]:
    """Canonical NVD metadata assembled from the tracked year shards."""
    entries: Dict[str, object] = {}
    if not METADATA_INPUT.exists():
        return entries
    for path in sorted(METADATA_INPUT.glob("[12][0-9][0-9][0-9].json")):
        payload = json.loads(path.read_text(encoding="utf-8"))
        if not isinstance(payload, dict):
            raise ValueError(f"{path} is not a CVE metadata object")
        entries.update(payload)
    return entries


@lru_cache(maxsize=1)
def load_verified_references() -> set[tuple[str, str]]:
    """Independent CVE Program and reviewed-content reference evidence."""
    try:
        lines = VERIFIED_REFERENCES.read_text(encoding="utf-8").splitlines()
    except OSError:
        lines = []
    verified: set[tuple[str, str]] = set()
    for line in lines:
        if " - " not in line:
            continue
        cve_id, url = line.split(" - ", 1)
        if cve_id and url:
            verified.add((cve_id, link_key(url)))
    if REVIEWED_REFERENCES.exists():
        reviewed = json.loads(REVIEWED_REFERENCES.read_text(encoding="utf-8"))
        if not isinstance(reviewed, dict) or any(not isinstance(urls, list) for urls in reviewed.values()):
            raise ValueError("Invalid reviewed reference ledger")
        verified.update((cve_id, link_key(url)) for cve_id, urls in reviewed.items() for url in urls)
    return verified


BADGE = re.compile(r"!\[\]\(https://img\.shields\.io/static/v1\?label=(Vendor|Product)&message=([^&)]*)&")


def parse_preamble(content: str) -> tuple[str, List[str], List[str]]:
    """The record's title and its vendor and product badges, from above the description."""
    title = ""
    vendors: List[str] = []
    products: List[str] = []
    for line in content.splitlines():
        stripped = line.strip()
        if stripped == "### Description":
            break
        if not title and len(stripped) > 4 and stripped.startswith("**") and stripped.endswith("**"):
            title = " ".join(stripped[2:-2].split())
            continue
        match = BADGE.match(stripped)
        if not match:
            continue
        value = " ".join(unquote(match.group(2)).split())
        if value and value.lower() != "n/a":
            (vendors if match.group(1) == "Vendor" else products).append(value)
    return title, vendors, products


def parse_sections(content: str) -> Dict[str, str]:
    """Split an entry on its own headings only.

    CVE text lifted from a GitHub advisory often opens with "### Impact" or
    "### Summary"; treating any heading as a boundary let a description end
    itself and vanish from the index.
    """
    sections: Dict[str, str] = {}
    current_header: Optional[str] = None
    buffer: List[str] = []

    for line in content.splitlines():
        header = line.strip()
        if header in SECTION_HEADERS:
            if current_header is not None:
                sections[current_header] = "\n".join(buffer).strip()
            current_header = header
            buffer = []
        else:
            buffer.append(line)

    if current_header is not None:
        sections[current_header] = "\n".join(buffer).strip()

    return sections


GITHUB_HOSTS = {"github.com", "www.github.com"}


def repo_from_url(url: str) -> str:
    """owner/repo for a GitHub URL, matching repoFromUrl in logic.js exactly.

    Two disagreements with the browser lived here. "github" as a substring of
    the host accepted notgithub.example.com as a repository, and a clone URL
    kept its .git suffix, so poc-rebar3.git and poc-rebar3 counted as two
    sources on the page and one in search. 273 CVEs disagreed across the two.
    """
    try:
        parsed = urlparse(url)
        host = (parsed.hostname or "").lower()
        if host and host not in GITHUB_HOSTS:
            return ""
        path = parsed.path or url
    except Exception:
        path = url
    parts = path.strip("/").split("/")
    if (len(parts) < 2 or parts[0].lower() in GITHUB_NON_REPOS
            or [part.lower() for part in parts[2:4]] == ["security", "advisories"]):
        return ""
    owner, repo = parts[0].lower(), re.sub(r"\.git$", "", parts[1], flags=re.I).lower()
    return f"{owner}/{repo}" if repo else ""


def is_blacklisted(url: str, blacklist: Collection[str]) -> bool:
    full_name = repo_from_url(url)
    if not full_name:
        return False
    return full_name in blacklist


def link_key(url: str) -> str:
    parsed = urlparse(url)
    if (parsed.hostname or "").lower() not in {"github.com", "www.github.com"}:
        return url.rstrip("/")
    parts = [part for part in parsed.path.split("/") if part]
    if len(parts) < 2:
        return url
    repo = f"{parts[0]}/{parts[1]}".lower()
    suffix = "/" + "/".join(parts[2:]) if len(parts) > 2 else ""
    return f"github:{repo}{suffix}?{parsed.query}#{parsed.fragment}"


def source_key(url: str) -> str:
    """Identity of one source, independent of a path inside its repository."""
    repo = repo_from_url(url)
    if repo and "/" in repo:
        return f"github:{repo}"
    return url.rstrip("/")


def poc_link_rank(url: str, cve_id: str) -> tuple[int, int]:
    """Choose the most useful path when one repository supplied several."""
    if not repo_from_url(url):
        return 0, len(url)
    parsed = urlparse(url)
    parts = [part for part in parsed.path.split("/") if part]
    suffix = "/".join(parts[2:]).lower()
    if not suffix:
        return 0, len(url)
    compact_cve = re.sub(r"[^a-z0-9]", "", cve_id.lower())
    compact_path = re.sub(r"[^a-z0-9]", "", suffix)
    if compact_cve and compact_cve in compact_path:
        return 1, len(url)
    extension = Path(parts[-1]).suffix.lower()
    if extension in CODE_SUFFIXES or "/security/advisories/" in parsed.path.lower():
        return 2, len(url)
    if re.search(r"(?:^|/)(?:poc|exploit|reproducer)(?:/|$)", suffix):
        return 3, len(url)
    if extension in MEDIA_SUFFIXES:
        return 6, len(url)
    return 4, len(url)


@lru_cache(maxsize=1)
def reviewed_artifact_keys() -> Dict[str, set[str]]:
    from source_artifacts import ArtifactEvidence, load_reviews
    result: Dict[str, set[str]] = {}
    for row in load_reviews():
        artifact = ArtifactEvidence(row["cve"], row["repository"], row["path"], row["revision"],
                                    row["sha256"], row["id"])
        result.setdefault(row["cve"], set()).add(link_key(artifact.url))
    return result


@lru_cache(maxsize=1)
def load_source_exclusions() -> set[tuple[str, str]]:
    from source_artifacts import excluded_pairs
    return excluded_pairs()


def dedupe_source_links(urls: Iterable[str], cve_id: str, *, preserve_paths: bool = False) -> List[str]:
    selected: Dict[str, tuple[int, str]] = {}
    order: Dict[str, int] = {}
    reviewed = reviewed_artifact_keys().get(cve_id, set()) if not preserve_paths else set()

    def rank(url: str) -> tuple:
        if link_key(url) in reviewed:
            return (-1, poc_link_rank(url, cve_id), link_key(url))
        return (0, poc_link_rank(url, cve_id), "")

    for index, url in enumerate(urls):
        if (cve_id, repo_from_url(url)) in load_source_exclusions():
            continue
        key = link_key(url) if preserve_paths else source_key(url)
        order.setdefault(key, index)
        current = selected.get(key)
        if current is None or rank(url) < rank(current[1]):
            selected[key] = (index, url)
    return [selected[key][1] for key in sorted(selected, key=order.get)]


def advisory_rank(row: list, cve_id: str) -> tuple[int, int, int]:
    url = str(row[0])
    tags = set(row[1] if len(row) > 1 and isinstance(row[1], list) else [])
    if "GitHub Advisory" in tags or "/security/advisories/" in url.lower():
        tag_rank = 0
    elif "Exploit" in tags:
        tag_rank = 1
    elif "Vendor Advisory" in tags:
        tag_rank = 2
    elif "Third Party Advisory" in tags or "Advisory" in tags:
        tag_rank = 3
    elif "Issue Tracking" in tags:
        tag_rank = 4
    elif "Patch" in tags:
        tag_rank = 5
    else:
        tag_rank = 6
    path_rank, length = poc_link_rank(url, cve_id)
    return tag_rank, path_rank, length


def dedupe_advisories(rows: Iterable[list], cve_id: str) -> List[list]:
    selected: Dict[str, list] = {}
    order: Dict[str, int] = {}
    for index, row in enumerate(rows):
        if not isinstance(row, list) or not row:
            continue
        if (cve_id, link_key(str(row[0]))) in load_reference_exclusions():
            continue
        key = source_key(str(row[0]))
        order.setdefault(key, index)
        current = selected.get(key)
        if current is None or advisory_rank(row, cve_id) < advisory_rank(current, cve_id):
            selected[key] = row
    return [selected[key] for key in sorted(selected, key=order.get)]


def reference_is_verified_poc(
    cve_id: str,
    url: str,
    metadata: Dict[str, object],
    verified: Collection[tuple[str, str]],
) -> bool:
    key = link_key(url)
    if (cve_id, key) in load_reference_exclusions():
        return False
    if (cve_id, key) in verified:
        return True
    if (cve_id, key) in load_managed_references():
        return False
    for row in (metadata.get(cve_id) or {}).get("advisories", []):
        if not isinstance(row, list) or not row:
            continue
        tags = row[1] if len(row) > 1 and isinstance(row[1], list) else []
        if "Exploit" in tags and link_key(str(row[0])) == key:
            return True
    return False


@lru_cache(maxsize=1)
def load_reference_exclusions() -> set[tuple[str, str]]:
    import sync_reference_pocs
    return sync_reference_pocs.load_exclusions()


@lru_cache(maxsize=1)
def load_managed_references() -> set[tuple[str, str]]:
    import sync_reference_pocs
    return {(row["cve"], link_key(row["url"])) for row in sync_reference_pocs.load_reviews()}


def collect_links(block: str, *, blacklist: Optional[Collection[str]] = None) -> List[str]:
    links: List[str] = []
    blacklist = blacklist or set()
    for raw in block.splitlines():
        entry = raw.strip()
        if not entry or "No PoCs" in entry:
            continue
        if entry.startswith("- "):
            entry = entry[2:].strip()
        if not entry:
            continue
        try:
            parsed = urlparse(entry)
            if parsed.scheme not in {"http", "https"} or not parsed.hostname:
                continue
        except ValueError:
            continue
        if is_blacklisted(entry, blacklist) or link_health.unavailable(entry):
            continue
        if entry not in links:
            links.append(entry)
    return links


def build_cve_list(blacklist: Collection[str]) -> tuple[List[Dict[str, object]], int]:
    import archive_pocs
    archived = archive_pocs.archived_fallbacks(blacklist=blacklist, exclusions=load_source_exclusions())
    cve_entries = []
    dates = load_dates()
    metadata = load_metadata()
    verified_references = load_verified_references()
    total = 0

    for md_path in sorted(CVES.glob("[12][0-9][0-9][0-9]/CVE-*.md")):
        cve_id = md_path.stem
        if (metadata.get(cve_id) or {}).get("rejected"):
            continue
        total += 1
        content = md_path.read_text(encoding="utf-8")
        record_status = markdown_record_status(content)
        if record_status in {"REJECTED", "ABSENT"}:
            total -= 1
            continue
        sections = parse_sections(content)
        description = normalise_block(sections.get("### Description", ""))
        references = collect_links(sections.get("#### Reference", ""), blacklist=blacklist)
        github_links = collect_links(sections.get("#### Github", ""), blacklist=blacklist)
        # Keep artifact paths and their source attribution. Repository metadata
        # describes the hosting project, not the individual artifact.
        curated = {
            field: dedupe_source_links(
                collect_links(sections.get(header, ""), blacklist=blacklist),
                cve_id,
                preserve_paths=True,
            )
            for header, field in CURATED
        }
        curated_keys = {
            source_key(url)
            for links in curated.values()
            for url in links
        }
        poc_candidates: List[str] = []
        for link in references:
            key = source_key(link)
            if (
                key not in curated_keys
                and reference_is_verified_poc(cve_id, link, metadata, verified_references)
            ):
                poc_candidates.append(link)
        for link in github_links:
            if source_key(link) not in curated_keys:
                poc_candidates.append(link)
        poc_entries = dedupe_source_links(poc_candidates, cve_id)
        for field, url in archived.get(cve_id, []):
            original = original_artifact(url)
            if original in poc_entries or any(original in links for links in curated.values()):
                continue
            destination = curated.get(field, poc_entries)
            if url not in destination:
                destination.append(url)

        if not poc_entries and not any(curated.values()):
            continue

        entry = {
            "cve": cve_id,
            "desc": description,
            "poc": poc_entries,
        }
        if record_status != "PUBLISHED":
            entry["metadata_pending"] = True
        title, vendors, products = parse_preamble(content)
        if title:
            entry["title"] = title
        if vendors:
            entry["vendor"] = vendors
        if products:
            entry["product"] = products
        published, modified = (dates.get(cve_id) or ["", ""])[:2]
        if published:
            entry["published"] = published
        if modified and modified != published:
            entry["modified"] = modified
        entry.update({field: links for field, links in curated.items() if links})
        cve_entries.append(entry)

    return cve_entries, total


def add_observed_dates(cve_entries: list[dict], ledger: dict) -> None:
    """Date undated CVEs by their earliest retained PoC observation, explicitly."""
    import releases
    for entry in cve_entries:
        if entry.get("published"):
            continue
        stamps = [
            stamp for field in ["poc", *(field for _, field in CURATED)]
            for url in entry.get(field, [])
            if (stamp := releases.timestamp((ledger.get(releases.key(entry["cve"], original_artifact(url)))
                                            or ledger.get(releases.key(entry["cve"], url)) or {}).get("seen")))
        ]
        if stamps:
            entry["poc_observed"] = min(stamps).strftime("%Y-%m-%d")


def original_artifact(url: str) -> str:
    match = re.fullmatch(r'https://web\.archive\.org/web/\d{14}/(https?://.+)', url)
    return match[1] if match else url


def build_repo_meta(cve_entries: List[Dict[str, object]]) -> Dict[str, object]:
    """Star count and last-push date for the repositories the index actually links."""
    stored = json.loads(REPO_META.read_text(encoding="utf-8"))
    if not isinstance(stored, dict) or any(
        not isinstance(row, list) or len(row) < 2 or type(row[0]) is not int or row[0] < 0
        or not isinstance(row[1], str) for row in stored.values()
    ):
        raise ValueError("Invalid repository metadata; refusing to publish missing star columns")
    linked = {
        repo_from_url(original_artifact(url))
        for entry in cve_entries
        for field in ["poc", *(field for _, field in CURATED)]
        for url in entry.get(field, [])
        if repo_from_url(original_artifact(url))
    }
    return {name: value for name, value in stored.items() if name in linked}


def load_dates() -> Dict[str, list]:
    """When the CVE Program first published each record and last changed it."""
    return json.loads(DATES_INPUT.read_text(encoding="utf-8"))


def build_kev(cve_entries: List[Dict[str, object]]) -> Dict[str, object]:
    """CISA's known-exploited entries, limited to CVEs the index carries."""
    stored = json.loads(KEV_INPUT.read_text(encoding="utf-8"))
    have = {str(entry["cve"]) for entry in cve_entries}
    return {cve: value for cve, value in stored.items() if cve in have}


def build_nuclei(cve_entries: List[Dict[str, object]]) -> Dict[str, object]:
    """Severity, CVSS and CWE per CVE, limited to the ones indexed."""
    stored = json.loads(NUCLEI_INPUT.read_text(encoding="utf-8"))
    have = {str(entry["cve"]) for entry in cve_entries}
    keep = ("severity", "cvss", "cvss_vector", "cwe")
    return {
        cve: {field: value[field] for field in keep if field in value}
        for cve, value in stored.items()
        if cve in have
    }


def build_metadata(cve_entries: List[Dict[str, object]]) -> Dict[str, object]:
    """CVSS generations and vetted advisories for CVEs published on the site."""
    have = {str(entry["cve"]) for entry in cve_entries}
    published: Dict[str, object] = {}
    for cve, value in load_metadata().items():
        if cve not in have or not isinstance(value, dict):
            continue
        entry: Dict[str, object] = {}
        if value.get("cvss"):
            entry["cvss"] = value["cvss"]
        advisories = [row for row in dedupe_advisories(value.get("advisories", []), cve)
                      if not link_health.unavailable(row[0])]
        if advisories:
            entry["advisories"] = advisories
        if entry:
            published[cve] = entry
    return published


def build_epss(cve_entries: List[Dict[str, object]]) -> Dict[str, object]:
    """FIRST's exploitation probability, limited to the CVEs indexed.

    EPSS answers the question the PoC count cannot: of everything with exploit
    code published, which is actually being used. The feed covers essentially
    the whole index, so it is fetched here rather than tracked in the
    repository, where a daily rescore of 366,000 rows would be pure churn.

    A failure raises rather than returning {}. Swallowing it published an empty
    epss.json, which is not "EPSS is missing" to a reader: every score silently
    disappears from 82,000 pages and the site looks like it never had them. A
    build that cannot get the feed should not replace the one that did.
    """
    try:
        with urllib.request.urlopen(EPSS_FEED, timeout=180) as response:
            raw = gzip.decompress(response.read()).decode("utf-8", "replace")
    except Exception as problem:
        raise RuntimeError(f"EPSS feed unavailable: {problem}") from problem
    have = {str(entry["cve"]) for entry in cve_entries}
    text = io.StringIO(raw)
    text.readline()  # the feed opens with a model version comment, not a header
    scores: Dict[str, object] = {}
    for row in csv.DictReader(text):
        cve = (row.get("cve") or "").strip().upper()
        if cve not in have:
            continue
        try:
            scores[cve] = [round(float(row["epss"]), 5), round(float(row["percentile"]), 4)]
        except (KeyError, TypeError, ValueError):
            continue
    # The feed normally covers the whole index. A truncated or reshaped one
    # parses cleanly and yields a handful of rows, which would ship as a real
    # build; anything below nine in ten CVEs is a broken feed, not news.
    if have and len(scores) < 0.9 * len(have):
        raise RuntimeError(
            f"EPSS feed covered {len(scores):,} of {len(have):,} indexed CVEs; "
            "refusing to publish a partial rescore"
        )
    return scores


def build_trending(blacklist: Collection[str]) -> List[Dict[str, object]]:
    """The trending job's own output, minus anything the blacklist covers."""
    rows = json.loads(TRENDING_INPUT.read_text(encoding="utf-8"))
    return [
        row for row in rows
        if row.get("url") and not is_blacklisted(str(row["url"]), blacklist)
        and not link_health.unavailable(str(row["url"]))
        and (row.get("cve"), repo_from_url(str(row["url"]))) not in load_source_exclusions()
    ]


def trending_score(row: dict, *, now: datetime | None = None) -> float:
    import releases
    stamp = releases.timestamp(row.get("released") or row.get("pushed"))
    if stamp is None or row.get("shared_repository"):
        return 0.0
    hours = max(0, ((now or datetime.now(timezone.utc)) - stamp).total_seconds() / 3600)
    return int(row.get("stars") or 0) / (hours + 24) ** 0.7


def enrich_trending(rows: list[dict], repo_meta: dict, kev: dict, indexed: set[str]) -> list[dict]:
    items = []
    now = datetime.now(timezone.utc)
    for original in rows:
        if not original.get("trending", True) or original.get("cve") not in indexed:
            continue
        item = dict(original)
        item["page"] = f"/{item['cve']}"
        item["kev"] = item["cve"] in kev
        repository = repo_from_url(item["url"])
        if repository in repo_meta:
            item["stars"] = repo_meta[repository][0]
        item["score"] = trending_score(item, now=now)
        items.append(item)
    return sorted(items, key=lambda row: (-row["score"], row["url"].lower()))


def build_landed(cve_payload: list[dict], repo_meta: dict, kev: dict,
                 ledger: dict | None = None, *, now: str | None = None, candidates: list[dict] | None = None) -> list[dict]:
    import releases
    ledger = releases.load_ledger() if ledger is None else ledger
    entries = {entry["cve"]: entry for entry in cve_payload}
    published = {releases.key(cve, url) for cve, url in releases.published_pairs(cve_payload)}
    candidates = {releases.key(row["cve"], row["url"]): row for row in candidates or []}
    rows = []
    for artifact in releases.landed(ledger, now=now):
        cve, url = artifact["cve"], artifact["url"]
        if releases.key(cve, url) not in published:
            continue
        parsed = urlparse(url)
        parts = unquote(parsed.path).strip("/").split("/")
        repository = repo_from_url(url)
        root_link = parsed.hostname == "github.com" and len(parts) == 2
        meta = repo_meta.get(repository or "")
        current = candidates.get(releases.key(cve, url), {})
        stars = meta[0] if meta else current.get("stars") if repository else None
        rows.append({
            "year": int(cve.split("-")[1]), "cve": cve, "url": url,
            "name": parts[-1] or cve, "stars": stars,
            "desc": current.get("desc") or entries[cve].get("desc", ""), "released": artifact["released"],
            "basis": artifact["basis"], "kev": cve in kev, "page": f"/{cve}",
            "landed": True, "source": parsed.hostname, "artifact": not root_link,
        })
    return rows[:100]


def write_json(path: Path, data, *, indent: Optional[int] = None) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open("w", encoding="utf-8") as handle:
        json.dump(data, handle, ensure_ascii=False, indent=indent,
                  separators=None if indent else (",", ":"))


def main() -> int:
    parser = argparse.ArgumentParser(description="Build the CVE PoC site data files")
    parser.add_argument("--offline", action="store_true", help="Validate tracked inputs without network or output writes")
    args = parser.parse_args()

    if args.offline:
        entries, total = build_cve_list(load_blacklist())
        build_repo_meta(entries)
        build_kev(entries)
        build_nuclei(entries)
        build_metadata(entries)
        build_trending(load_blacklist())
        print(f"Verified tracked data: {len(entries)} of {total} CVEs have public links")
        return 0

    DOCS_DIR.mkdir(parents=True, exist_ok=True)
    blacklist = load_blacklist()

    import releases
    ledger = releases.load_ledger()
    cve_payload, total_cves = build_cve_list(blacklist)
    add_observed_dates(cve_payload, ledger)
    write_json(CVE_OUTPUT, cve_payload)

    import archive_pocs
    archive_rows = archive_pocs.load_records()
    allowed = {archive_pocs.identity(url) for entry in cve_payload
               for field in ['poc', *(f for _, f in CURATED)] for url in entry.get(field, [])}
    # Fallback entries point into Wayback; retain the original-to-capture map.
    allowed.update(key for key, row in archive_rows.items()
                   if archive_pocs.valid_snapshot(row) and archive_pocs.identity(row['snapshot']['url']) in allowed)
    snapshots = archive_pocs.published_snapshots(archive_rows, allowed)
    aliases = {archive_pocs.identity(url): value for url, value in snapshots.items()}
    for entry in cve_payload:
        for field in ['poc', *(field for _, field in CURATED)]:
            for url in entry.get(field, []):
                if value := aliases.get(archive_pocs.identity(url)):
                    snapshots[url] = value
    write_json(DOCS_DIR / 'archives.json', snapshots)

    repo_meta = build_repo_meta(cve_payload)
    write_json(REPO_META_OUTPUT, repo_meta)

    kev = build_kev(cve_payload)
    write_json(KEV_OUTPUT, kev)

    nuclei = build_nuclei(cve_payload)
    write_json(NUCLEI_OUTPUT, nuclei)

    metadata = build_metadata(cve_payload)
    write_json(METADATA_OUTPUT, metadata)
    write_json(CVSS_OUTPUT, {cve: value["cvss"] for cve, value in metadata.items() if value.get("cvss")})
    write_json(ADVISORIES_OUTPUT, {cve: value["advisories"] for cve, value in metadata.items() if value.get("advisories")})

    epss = build_epss(cve_payload)
    write_json(EPSS_OUTPUT, epss)

    indexed = {entry["cve"] for entry in cve_payload}
    frontpage_candidates = build_trending(blacklist)
    trending_items = enrich_trending(frontpage_candidates, repo_meta, kev, indexed)
    landed_items = build_landed(cve_payload, repo_meta, kev, ledger, candidates=frontpage_candidates)
    write_json(
        TRENDING_OUTPUT,
        {
            "generated": datetime.now(timezone.utc).isoformat(),
            "total_cves": total_cves,
            "with_pocs": len(cve_payload),
            "items": trending_items,
            "landed": landed_items,
            "ranking": "stars_weighted_by_artifact_release_age",
        },
        indent=2,
    )

    print(
        f"Wrote {CVE_OUTPUT.name} ({len(cve_payload)} of {total_cves} CVEs), "
        f"{REPO_META_OUTPUT.name} ({len(repo_meta)} repositories), "
        f"{KEV_OUTPUT.name} ({len(kev)} known-exploited), "
        f"{NUCLEI_OUTPUT.name} ({len(nuclei)} rated), "
        f"{METADATA_OUTPUT.name} ({len(metadata)} enriched), "
        f"{EPSS_OUTPUT.name} ({len(epss)} scored) and {TRENDING_OUTPUT.name}"
    )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())