diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml index 8e72d1aa676..fd900a90601 100644 --- a/.github/workflows/site.yml +++ b/.github/workflows/site.yml @@ -32,7 +32,10 @@ jobs: - name: Build site data run: python3 scripts/build_site.py - - name: Build robots.txt and sitemap.xml + - name: Build per-CVE pages + run: python3 scripts/build_pages.py + + - name: Build robots.txt and sitemaps run: python3 scripts/build_seo.py - name: Verify published counts diff --git a/.gitignore b/.gitignore index 11fbe576a69..affb24033d1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ docs/epss.json docs/api/v1/cve/ docs/robots.txt docs/sitemap.xml +docs/sitemap-*.xml +docs/CVE-*.html __pycache__/ # Assistant tooling. None of it belongs in the repository. diff --git a/docs/_urltest.html b/docs/_urltest.html deleted file mode 100644 index 1e1d36df03d..00000000000 --- a/docs/_urltest.html +++ /dev/null @@ -1 +0,0 @@ -urltest

extensionless-ok

diff --git a/docs/logic.js b/docs/logic.js index a81ea9d101f..cdc2042ee18 100644 --- a/docs/logic.js +++ b/docs/logic.js @@ -578,10 +578,11 @@ function resultRow(entry) { return `
- ${escapeHTML(id)} + ${escapeHTML(id)}
${formatCount(links.length)} linked PoC${links.length === 1 ? '' : 's'}
${dateHtml}
${kevChip}${severityChip}${epssChip} + NVD ↗ CVE.ORG ↗
diff --git a/docs/style.css b/docs/style.css index 9550093e6e1..f0a8406f629 100644 --- a/docs/style.css +++ b/docs/style.css @@ -548,3 +548,62 @@ a.chip:hover { color: var(--fg); border-color: var(--fg-muted); } .filter-popover { max-width: calc(100vw - 32px); } .filter-menu[data-filter-menu="source"] .filter-popover { right: 0; left: auto; } } + +/* ---- per-CVE page ------------------------------------------------------- */ + +.cve-bar { + display: flex; align-items: center; gap: 24px; + padding-top: 20px; padding-bottom: 20px; + border-bottom: 1px solid var(--border-muted); +} +.cve-home { font: 700 19px/1 var(--mono); color: var(--fg); letter-spacing: .04em; } +.cve-home:hover { color: var(--accent); } +.cve-search { flex: 1; max-width: 520px; } +.cve-search input { + width: 100%; padding: 9px 14px; border-radius: 8px; + background: var(--canvas-subtle); border: 1px solid var(--border); + color: var(--fg); font: 400 13px/1.4 var(--mono); +} +.cve-search input::placeholder { color: var(--neutral); } +.cve-search input:focus { outline: none; border-color: var(--accent-emphasis); } + +.cve-page { flex: 1; padding-top: 32px; padding-bottom: 56px; max-width: 900px; } +.cve-page h1 { margin: 0 0 14px; font: 700 32px/1.1 var(--mono); color: var(--fg); } +.cve-page h2 { + margin: 34px 0 12px; font: 500 11px/1 var(--mono); letter-spacing: .1em; + text-transform: uppercase; color: var(--fg-muted); +} +.cve-desc { margin: 16px 0 0; font: 400 15px/1.65 var(--sans); color: var(--fg-subtle); } + +.cve-facts { + display: grid; grid-template-columns: max-content 1fr; gap: 8px 20px; + margin: 24px 0 0; padding: 18px 20px; + background: var(--canvas-subtle); border: 1px solid var(--border-muted); border-radius: 10px; +} +.cve-facts dt { font: 500 11px/1.5 var(--mono); letter-spacing: .06em; color: var(--fg-muted); text-transform: uppercase; } +.cve-facts dd { margin: 0; font: 400 13px/1.5 var(--mono); color: var(--fg-subtle); } +.cve-facts dd code { display: block; margin-top: 3px; font-size: 11px; color: var(--neutral); word-break: break-all; } + +.cve-links { list-style: none; margin: 0; padding: 0; } +.cve-links li { + display: flex; justify-content: space-between; align-items: baseline; gap: 16px; + padding: 8px 0; border-bottom: 1px solid var(--border-muted); + font: 400 13px/1.5 var(--mono); +} +.cve-links li span { flex: none; font-size: 11px; color: var(--neutral); white-space: nowrap; } +.cve-links a { word-break: break-all; } + +.cve-related { list-style: none; display: flex; flex-wrap: wrap; gap: 8px; margin: 0; padding: 0; } +.cve-related a { + display: inline-block; padding: 5px 11px; border-radius: 6px; + background: var(--canvas-subtle); border: 1px solid var(--border-muted); + font: 400 12px/1 var(--mono); +} + +@media (max-width: 700px) { + .cve-bar { flex-wrap: wrap; gap: 14px; } + .cve-page h1 { font-size: 25px; } + .cve-facts { grid-template-columns: 1fr; gap: 3px 0; } + .cve-facts dd { margin-bottom: 10px; } + .cve-links li { flex-direction: column; gap: 2px; } +} diff --git a/index/blacklist.txt b/index/blacklist.txt index d5023de6808..993848499e4 100644 --- a/index/blacklist.txt +++ b/index/blacklist.txt @@ -121,3 +121,4 @@ yiwen-ming/fpauthanalysis yogeshshe1ke/CVE zecool/cve zulloper/cve-poc +0xMarcio/pocindex diff --git a/scripts/build_pages.py b/scripts/build_pages.py new file mode 100644 index 00000000000..ec5e048d3c4 --- /dev/null +++ b/scripts/build_pages.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Render one static page per CVE, at the site root. + +The index held 82,000 CVEs behind a single URL. Every one of them is a query +somebody types ("CVE-2026-72898 poc"), and the pages that answered those queries +belonged to other people. These are flat files at /CVE-2026-72898, no prefix and +no extension: GitHub Pages resolves an extensionless request to the .html beside +it, verified against the live site rather than assumed. + +Nothing here is committed. The payloads this reads are already gitignored and +built in CI, so the pages go straight into the Pages artifact and git never sees +them. Rendering all of them costs about fifteen seconds. +""" + +from __future__ import annotations + +import html +import json +import os +import sys +from collections import defaultdict + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir) +sys.path.insert(0, os.path.join(ROOT, "scripts")) + +from brand import BRAND, SITE + +DOCS = os.path.join(ROOT, "docs") +RELATED = 6 +GITHUB = "https://github.com/" + +SOURCES = ( + ("nuclei", "Nuclei templates", "projectdiscovery/nuclei-templates"), + ("msf", "Metasploit modules", "rapid7/metasploit-framework"), + ("edb", "ExploitDB entries", "exploit-db.com"), + ("vulhub", "Vulhub environments", "vulhub/vulhub"), + ("collections", "Exploit collections", ""), +) + + +def load(name: str) -> dict | list: + with open(os.path.join(DOCS, name), encoding="utf-8") as handle: + return json.load(handle) + + +def esc(value) -> str: + return html.escape(str(value or ""), quote=True) + + +def short(text: str, limit: int) -> str: + text = " ".join((text or "").split()) + if len(text) <= limit: + return text + return text[: limit - 1].rsplit(" ", 1)[0] + "…" + + +def best_cvss(rows: list) -> tuple | None: + """The highest-scoring assessment, which is the one a reader wants first.""" + scored = [r for r in rows or [] if isinstance(r[1], (int, float))] + return max(scored, key=lambda r: r[1]) if scored else None + + +def owner_of(url: str) -> str | None: + if not url.startswith(GITHUB): + return None + parts = url[len(GITHUB):].split("/") + return parts[0].lower() if parts and parts[0] else None + + +def build_related(cves: list) -> dict: + """A crawlable graph. A sitemap gets pages discovered; links get them crawled + sooner and pass context. Sharing a PoC author is a real relation, so it beats + padding the page with neighbouring CVE numbers, which is used only as filler.""" + by_owner = defaultdict(list) + for entry in cves: + for url in entry.get("poc") or []: + owner = owner_of(url) + if owner: + by_owner[owner].append(entry["cve"]) + # An author with hundreds of PoCs relates nothing in particular. + by_owner = {o: v for o, v in by_owner.items() if 1 < len(v) <= 40} + + by_year = defaultdict(list) + for entry in cves: + by_year[entry["cve"].split("-")[1]].append(entry["cve"]) + position = {} + for year, ids in by_year.items(): + ids.sort() + for i, cid in enumerate(ids): + position[cid] = (year, i) + + related = {} + for entry in cves: + cid = entry["cve"] + seen = {cid} + picks = [] + for url in entry.get("poc") or []: + owner = owner_of(url) + for other in by_owner.get(owner, ()): + if other not in seen: + seen.add(other) + picks.append(other) + if len(picks) >= RELATED: + break + if len(picks) >= RELATED: + break + if len(picks) < RELATED: + year, i = position[cid] + siblings = by_year[year] + for j in range(1, RELATED + 1): + for k in (i - j, i + j): + if 0 <= k < len(siblings) and siblings[k] not in seen: + seen.add(siblings[k]) + picks.append(siblings[k]) + if len(picks) >= RELATED: + break + if len(picks) >= RELATED: + break + related[cid] = picks[:RELATED] + return related + + +def link_rows(urls: list, repo_meta: dict) -> str: + rows = [] + for url in urls: + label = url[len(GITHUB):] if url.startswith(GITHUB) else url + meta = repo_meta.get(label.lower()) if url.startswith(GITHUB) else None + note = f"{meta[0]}★ · {esc(meta[1])}" if meta else "" + rows.append( + f'
  • ' + f"{esc(short(label, 90))}{note}
  • " + ) + return "".join(rows) + + +def page(entry: dict, data: dict) -> str: + cid = entry["cve"] + desc = " ".join((entry.get("desc") or "").split()) + pocs = entry.get("poc") or [] + cvss = best_cvss((data["meta"].get(cid) or {}).get("cvss")) + ep = data["epss"].get(cid) + flagged = data["kev"].get(cid) + tpl = data["nuclei"].get(cid) + + count = f"{len(pocs):,} public PoC exploit" + ("" if len(pocs) == 1 else "s") + title = f"{cid}: {count} | {BRAND}" + meta_desc = short(f"{cid}. {count}. {desc}", 155) + url = f"{SITE}/{cid}" + + chips = [] + if flagged: + ransom = " RANSOMWARE" if flagged[1] else "" + chips.append(f'KEV{ransom}') + if cvss: + chips.append( + f'' + f"{esc(cvss[2])} {esc(cvss[1])}" + ) + if ep: + hot = " is-hot" if ep[0] >= 0.1 else "" + chips.append(f'EPSS {ep[0] * 100:.1f}%') + + facts = [] + for row in ((data["meta"].get(cid) or {}).get("cvss") or []): + if isinstance(row[1], (int, float)): + facts.append( + f"
    CVSS v{esc(row[0])}
    {esc(row[1])} {esc(row[2])}" + f'{esc(row[3])}
    ' + ) + if ep: + facts.append( + f"
    EPSS
    {ep[0] * 100:.2f}% chance of exploitation in the next " + f"30 days, {ep[1] * 100:.0f}th percentile
    " + ) + if flagged: + used = ", used in ransomware campaigns" if flagged[1] else "" + facts.append(f"
    CISA KEV
    added {esc(flagged[0])}{used}
    ") + if tpl: + cwe = f" · {esc(tpl.get('cwe'))}" if tpl.get("cwe") else "" + facts.append(f"
    Nuclei
    {esc(tpl.get('severity'))}{cwe}
    ") + if entry.get("published"): + facts.append(f"
    Published
    {esc(entry['published'])}
    ") + if entry.get("modified"): + facts.append(f"
    Updated
    {esc(entry['modified'])}
    ") + + blocks = [] + if pocs: + blocks.append( + f"

    Proof-of-concept exploits ({len(pocs):,})

    " + f'' + ) + for key, heading, _ in SOURCES: + urls = entry.get(key) or [] + if urls: + blocks.append( + f"

    {heading} ({len(urls)})

    " + f'' + ) + + siblings = "".join( + f'
  • {other}
  • ' for other in data["related"].get(cid, ()) + ) + + ld = json.dumps( + { + "@context": "https://schema.org", + "@type": "TechArticle", + "headline": f"{cid} proof-of-concept exploits", + "url": url, + "description": meta_desc, + "datePublished": entry.get("published"), + "dateModified": entry.get("modified") or entry.get("published"), + "isPartOf": {"@type": "WebSite", "name": BRAND, "url": f"{SITE}/"}, + }, + separators=(",", ":"), + ) + + return f""" + + + + +{esc(title)} + + + + + + + + + + + + + + + + + + +
    {BRAND} +
    +
    +

    {cid}

    +
    {''.join(chips)}
    +

    {esc(desc)}

    +
    {''.join(facts)}
    +{''.join(blocks)} +

    References

    + +{f'

    Related

    ' if siblings else ''} +
    + + + +""" + + +def main() -> int: + cves = load("CVE_list.json") + data = { + "cves": cves, + "meta": load("cve_metadata.json"), + "epss": load("epss.json"), + "kev": load("kev.json"), + "nuclei": load("nuclei.json"), + "repo_meta": load("repo_meta.json"), + } + data["related"] = build_related(cves) + + written = 0 + for entry in cves: + path = os.path.join(DOCS, f"{entry['cve']}.html") + with open(path, "w", encoding="utf-8") as handle: + handle.write(page(entry, data)) + written += 1 + + print(f"Wrote {written:,} CVE pages into {DOCS}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_seo.py b/scripts/build_seo.py index f4b0367bd23..961f3ab6319 100644 --- a/scripts/build_seo.py +++ b/scripts/build_seo.py @@ -8,6 +8,7 @@ domain move is one edit rather than a hunt through the tree. from __future__ import annotations +import json import os import sys from datetime import datetime, timezone @@ -45,7 +46,7 @@ def robots() -> str: return "\n".join(lines) -def sitemap(urls: list[tuple[str, str]]) -> str: +def urlset(urls: list[tuple[str, str]]) -> str: body = "".join( f" \n {loc}\n {lastmod}\n \n" for loc, lastmod in urls @@ -53,22 +54,62 @@ def sitemap(urls: list[tuple[str, str]]) -> str: return ( '\n' '\n' - f"{body}" - "\n" + f"{body}\n" + ) + + +def sitemap_index(names: list[str], today: str) -> str: + body = "".join( + f" \n {SITE}/{name}\n" + f" {today}\n \n" + for name in names + ) + return ( + '\n' + '\n' + f"{body}\n" ) def main() -> int: today = datetime.now(timezone.utc).strftime("%Y-%m-%d") - urls = [(f"{SITE}/", today)] - os.makedirs(DOCS, exist_ok=True) + with open(ROBOTS, "w", encoding="utf-8") as handle: handle.write(robots()) - with open(SITEMAP, "w", encoding="utf-8") as handle: - handle.write(sitemap(urls)) - print(f"Wrote {ROBOTS} and {SITEMAP} ({len(urls)} URL) for {SITE}") + # One sitemap per CVE year. The 50,000 URL cap makes some split necessary and + # the year is the split the data already has, which also means a crawler only + # refetches the shard that moved. + try: + with open(os.path.join(DOCS, "CVE_list.json"), encoding="utf-8") as handle: + cves = json.load(handle) + except FileNotFoundError: + cves = [] + + years: dict[str, list[tuple[str, str]]] = {} + for entry in cves: + cid = entry["cve"] + year = cid.split("-")[1] + lastmod = entry.get("modified") or entry.get("published") or today + years.setdefault(year, []).append((f"{SITE}/{cid}", lastmod)) + + names = [] + for year in sorted(years): + name = f"sitemap-{year}.xml" + with open(os.path.join(DOCS, name), "w", encoding="utf-8") as handle: + handle.write(urlset(years[year])) + names.append(name) + + with open(os.path.join(DOCS, "sitemap-home.xml"), "w", encoding="utf-8") as handle: + handle.write(urlset([(f"{SITE}/", today)])) + names.insert(0, "sitemap-home.xml") + + with open(SITEMAP, "w", encoding="utf-8") as handle: + handle.write(sitemap_index(names, today)) + + total = sum(len(v) for v in years.values()) + 1 + print(f"Wrote {ROBOTS}, {SITEMAP} and {len(names)} shards ({total:,} URLs) for {SITE}") return 0