From 9b9957ea7f44919df64b2d50c6ac9f7d61cacdaf Mon Sep 17 00:00:00 2001 From: 0xMarcio Date: Wed, 2 Sep 2026 04:00:28 +0200 Subject: [PATCH] Give every CVE its own page at the site root 82,000 CVEs sat behind one URL. Each of them is a query somebody types, and the pages answering those queries belonged to other people: searching "CVE-2026-72898 poc" returns SentinelOne, Horizon3, Wiz and BishopFox, all of them serving one page per CVE. Pages are flat files at /CVE-2026-72898. No /cve/ prefix, no extension, no trailing slash. GitHub Pages resolves an extensionless request to the .html beside it, checked against the live site rather than assumed. Each page carries what the index already knows and nothing invented: every CVSS assessment with its vector, EPSS with percentile, KEV date and ransomware flag, Nuclei severity and CWE, and the PoC links grouped by source with stars and last push. 94% of them carry three or more of those signals. Discovery works two ways. sitemap.xml is now an index over one shard per year, which fits the 50,000 URL cap on the split the data already has and lets a crawler refetch only the year that moved. Search results link to the CVE page instead of straight out to NVD, which moves to a chip beside CVE.ORG, and each page links six related CVEs picked by shared PoC author. Rendering all of them takes 20s and adds nothing to the repository: the payloads they read are already gitignored and built in CI, so the pages go into the Pages artifact and git never sees them. --- .github/workflows/site.yml | 5 +- .gitignore | 2 + docs/_urltest.html | 1 - docs/logic.js | 3 +- docs/style.css | 59 ++++++++ index/blacklist.txt | 1 + scripts/build_pages.py | 292 +++++++++++++++++++++++++++++++++++++ scripts/build_seo.py | 57 +++++++- 8 files changed, 409 insertions(+), 11 deletions(-) delete mode 100644 docs/_urltest.html create mode 100644 scripts/build_pages.py diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml index 8e72d1aa676..fd900a90601 100644 --- a/.github/workflows/site.yml +++ b/.github/workflows/site.yml @@ -32,7 +32,10 @@ jobs: - name: Build site data run: python3 scripts/build_site.py - - name: Build robots.txt and sitemap.xml + - name: Build per-CVE pages + run: python3 scripts/build_pages.py + + - name: Build robots.txt and sitemaps run: python3 scripts/build_seo.py - name: Verify published counts diff --git a/.gitignore b/.gitignore index 11fbe576a69..affb24033d1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ docs/epss.json docs/api/v1/cve/ docs/robots.txt docs/sitemap.xml +docs/sitemap-*.xml +docs/CVE-*.html __pycache__/ # Assistant tooling. None of it belongs in the repository. diff --git a/docs/_urltest.html b/docs/_urltest.html deleted file mode 100644 index 1e1d36df03d..00000000000 --- a/docs/_urltest.html +++ /dev/null @@ -1 +0,0 @@ -urltest

extensionless-ok

diff --git a/docs/logic.js b/docs/logic.js index a81ea9d101f..cdc2042ee18 100644 --- a/docs/logic.js +++ b/docs/logic.js @@ -578,10 +578,11 @@ function resultRow(entry) { return `
- ${escapeHTML(id)} + ${escapeHTML(id)}
${formatCount(links.length)} linked PoC${links.length === 1 ? '' : 's'}
${dateHtml}
${kevChip}${severityChip}${epssChip} + NVD ↗ CVE.ORG ↗
diff --git a/docs/style.css b/docs/style.css index 9550093e6e1..f0a8406f629 100644 --- a/docs/style.css +++ b/docs/style.css @@ -548,3 +548,62 @@ a.chip:hover { color: var(--fg); border-color: var(--fg-muted); } .filter-popover { max-width: calc(100vw - 32px); } .filter-menu[data-filter-menu="source"] .filter-popover { right: 0; left: auto; } } + +/* ---- per-CVE page ------------------------------------------------------- */ + +.cve-bar { + display: flex; align-items: center; gap: 24px; + padding-top: 20px; padding-bottom: 20px; + border-bottom: 1px solid var(--border-muted); +} +.cve-home { font: 700 19px/1 var(--mono); color: var(--fg); letter-spacing: .04em; } +.cve-home:hover { color: var(--accent); } +.cve-search { flex: 1; max-width: 520px; } +.cve-search input { + width: 100%; padding: 9px 14px; border-radius: 8px; + background: var(--canvas-subtle); border: 1px solid var(--border); + color: var(--fg); font: 400 13px/1.4 var(--mono); +} +.cve-search input::placeholder { color: var(--neutral); } +.cve-search input:focus { outline: none; border-color: var(--accent-emphasis); } + +.cve-page { flex: 1; padding-top: 32px; padding-bottom: 56px; max-width: 900px; } +.cve-page h1 { margin: 0 0 14px; font: 700 32px/1.1 var(--mono); color: var(--fg); } +.cve-page h2 { + margin: 34px 0 12px; font: 500 11px/1 var(--mono); letter-spacing: .1em; + text-transform: uppercase; color: var(--fg-muted); +} +.cve-desc { margin: 16px 0 0; font: 400 15px/1.65 var(--sans); color: var(--fg-subtle); } + +.cve-facts { + display: grid; grid-template-columns: max-content 1fr; gap: 8px 20px; + margin: 24px 0 0; padding: 18px 20px; + background: var(--canvas-subtle); border: 1px solid var(--border-muted); border-radius: 10px; +} +.cve-facts dt { font: 500 11px/1.5 var(--mono); letter-spacing: .06em; color: var(--fg-muted); text-transform: uppercase; } +.cve-facts dd { margin: 0; font: 400 13px/1.5 var(--mono); color: var(--fg-subtle); } +.cve-facts dd code { display: block; margin-top: 3px; font-size: 11px; color: var(--neutral); word-break: break-all; } + +.cve-links { list-style: none; margin: 0; padding: 0; } +.cve-links li { + display: flex; justify-content: space-between; align-items: baseline; gap: 16px; + padding: 8px 0; border-bottom: 1px solid var(--border-muted); + font: 400 13px/1.5 var(--mono); +} +.cve-links li span { flex: none; font-size: 11px; color: var(--neutral); white-space: nowrap; } +.cve-links a { word-break: break-all; } + +.cve-related { list-style: none; display: flex; flex-wrap: wrap; gap: 8px; margin: 0; padding: 0; } +.cve-related a { + display: inline-block; padding: 5px 11px; border-radius: 6px; + background: var(--canvas-subtle); border: 1px solid var(--border-muted); + font: 400 12px/1 var(--mono); +} + +@media (max-width: 700px) { + .cve-bar { flex-wrap: wrap; gap: 14px; } + .cve-page h1 { font-size: 25px; } + .cve-facts { grid-template-columns: 1fr; gap: 3px 0; } + .cve-facts dd { margin-bottom: 10px; } + .cve-links li { flex-direction: column; gap: 2px; } +} diff --git a/index/blacklist.txt b/index/blacklist.txt index d5023de6808..993848499e4 100644 --- a/index/blacklist.txt +++ b/index/blacklist.txt @@ -121,3 +121,4 @@ yiwen-ming/fpauthanalysis yogeshshe1ke/CVE zecool/cve zulloper/cve-poc +0xMarcio/pocindex diff --git a/scripts/build_pages.py b/scripts/build_pages.py new file mode 100644 index 00000000000..ec5e048d3c4 --- /dev/null +++ b/scripts/build_pages.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Render one static page per CVE, at the site root. + +The index held 82,000 CVEs behind a single URL. Every one of them is a query +somebody types ("CVE-2026-72898 poc"), and the pages that answered those queries +belonged to other people. These are flat files at /CVE-2026-72898, no prefix and +no extension: GitHub Pages resolves an extensionless request to the .html beside +it, verified against the live site rather than assumed. + +Nothing here is committed. The payloads this reads are already gitignored and +built in CI, so the pages go straight into the Pages artifact and git never sees +them. Rendering all of them costs about fifteen seconds. +""" + +from __future__ import annotations + +import html +import json +import os +import sys +from collections import defaultdict + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir) +sys.path.insert(0, os.path.join(ROOT, "scripts")) + +from brand import BRAND, SITE + +DOCS = os.path.join(ROOT, "docs") +RELATED = 6 +GITHUB = "https://github.com/" + +SOURCES = ( + ("nuclei", "Nuclei templates", "projectdiscovery/nuclei-templates"), + ("msf", "Metasploit modules", "rapid7/metasploit-framework"), + ("edb", "ExploitDB entries", "exploit-db.com"), + ("vulhub", "Vulhub environments", "vulhub/vulhub"), + ("collections", "Exploit collections", ""), +) + + +def load(name: str) -> dict | list: + with open(os.path.join(DOCS, name), encoding="utf-8") as handle: + return json.load(handle) + + +def esc(value) -> str: + return html.escape(str(value or ""), quote=True) + + +def short(text: str, limit: int) -> str: + text = " ".join((text or "").split()) + if len(text) <= limit: + return text + return text[: limit - 1].rsplit(" ", 1)[0] + "…" + + +def best_cvss(rows: list) -> tuple | None: + """The highest-scoring assessment, which is the one a reader wants first.""" + scored = [r for r in rows or [] if isinstance(r[1], (int, float))] + return max(scored, key=lambda r: r[1]) if scored else None + + +def owner_of(url: str) -> str | None: + if not url.startswith(GITHUB): + return None + parts = url[len(GITHUB):].split("/") + return parts[0].lower() if parts and parts[0] else None + + +def build_related(cves: list) -> dict: + """A crawlable graph. A sitemap gets pages discovered; links get them crawled + sooner and pass context. Sharing a PoC author is a real relation, so it beats + padding the page with neighbouring CVE numbers, which is used only as filler.""" + by_owner = defaultdict(list) + for entry in cves: + for url in entry.get("poc") or []: + owner = owner_of(url) + if owner: + by_owner[owner].append(entry["cve"]) + # An author with hundreds of PoCs relates nothing in particular. + by_owner = {o: v for o, v in by_owner.items() if 1 < len(v) <= 40} + + by_year = defaultdict(list) + for entry in cves: + by_year[entry["cve"].split("-")[1]].append(entry["cve"]) + position = {} + for year, ids in by_year.items(): + ids.sort() + for i, cid in enumerate(ids): + position[cid] = (year, i) + + related = {} + for entry in cves: + cid = entry["cve"] + seen = {cid} + picks = [] + for url in entry.get("poc") or []: + owner = owner_of(url) + for other in by_owner.get(owner, ()): + if other not in seen: + seen.add(other) + picks.append(other) + if len(picks) >= RELATED: + break + if len(picks) >= RELATED: + break + if len(picks) < RELATED: + year, i = position[cid] + siblings = by_year[year] + for j in range(1, RELATED + 1): + for k in (i - j, i + j): + if 0 <= k < len(siblings) and siblings[k] not in seen: + seen.add(siblings[k]) + picks.append(siblings[k]) + if len(picks) >= RELATED: + break + if len(picks) >= RELATED: + break + related[cid] = picks[:RELATED] + return related + + +def link_rows(urls: list, repo_meta: dict) -> str: + rows = [] + for url in urls: + label = url[len(GITHUB):] if url.startswith(GITHUB) else url + meta = repo_meta.get(label.lower()) if url.startswith(GITHUB) else None + note = f"{meta[0]}★ · {esc(meta[1])}" if meta else "" + rows.append( + f'
  • ' + f"{esc(short(label, 90))}{note}
    • " + ) + return "".join(rows) + + +def page(entry: dict, data: dict) -> str: + cid = entry["cve"] + desc = " ".join((entry.get("desc") or "").split()) + pocs = entry.get("poc") or [] + cvss = best_cvss((data["meta"].get(cid) or {}).get("cvss")) + ep = data["epss"].get(cid) + flagged = data["kev"].get(cid) + tpl = data["nuclei"].get(cid) + + count = f"{len(pocs):,} public PoC exploit" + ("" if len(pocs) == 1 else "s") + title = f"{cid}: {count} | {BRAND}" + meta_desc = short(f"{cid}. {count}. {desc}", 155) + url = f"{SITE}/{cid}" + + chips = [] + if flagged: + ransom = " RANSOMWARE" if flagged[1] else "" + chips.append(f'KEV{ransom}') + if cvss: + chips.append( + f'' + f"{esc(cvss[2])} {esc(cvss[1])}" + ) + if ep: + hot = " is-hot" if ep[0] >= 0.1 else "" + chips.append(f'EPSS {ep[0] * 100:.1f}%') + + facts = [] + for row in ((data["meta"].get(cid) or {}).get("cvss") or []): + if isinstance(row[1], (int, float)): + facts.append( + f"
    CVSS v{esc(row[0])}
    {esc(row[1])} {esc(row[2])}" + f'{esc(row[3])}
    ' + ) + if ep: + facts.append( + f"
    EPSS
    {ep[0] * 100:.2f}% chance of exploitation in the next " + f"30 days, {ep[1] * 100:.0f}th percentile
    " + ) + if flagged: + used = ", used in ransomware campaigns" if flagged[1] else "" + facts.append(f"
    CISA KEV
    added {esc(flagged[0])}{used}
    ") + if tpl: + cwe = f" · {esc(tpl.get('cwe'))}" if tpl.get("cwe") else "" + facts.append(f"
    Nuclei
    {esc(tpl.get('severity'))}{cwe}
    ") + if entry.get("published"): + facts.append(f"
    Published
    {esc(entry['published'])}
    ") + if entry.get("modified"): + facts.append(f"
    Updated
    {esc(entry['modified'])}
    ") + + blocks = [] + if pocs: + blocks.append( + f"

    Proof-of-concept exploits ({len(pocs):,})

    " + f'' + ) + for key, heading, _ in SOURCES: + urls = entry.get(key) or [] + if urls: + blocks.append( + f"

    {heading} ({len(urls)})

    " + f'' + ) + + siblings = "".join( + f'
  • {other}
    • ' for other in data["related"].get(cid, ()) + ) + + ld = json.dumps( + { + "@context": "https://schema.org", + "@type": "TechArticle", + "headline": f"{cid} proof-of-concept exploits", + "url": url, + "description": meta_desc, + "datePublished": entry.get("published"), + "dateModified": entry.get("modified") or entry.get("published"), + "isPartOf": {"@type": "WebSite", "name": BRAND, "url": f"{SITE}/"}, + }, + separators=(",", ":"), + ) + + return f""" + + + + +{esc(title)} + + + + + + + + + + + + + + + + + + +
    {BRAND} +
    +
    +

    {cid}

    +
    {''.join(chips)}
    +

    {esc(desc)}

    +
    {''.join(facts)}
    +{''.join(blocks)} +

    References

    + +{f'

    Related

      {siblings}
    ' if siblings else ''} +
    + + + +""" + + +def main() -> int: + cves = load("CVE_list.json") + data = { + "cves": cves, + "meta": load("cve_metadata.json"), + "epss": load("epss.json"), + "kev": load("kev.json"), + "nuclei": load("nuclei.json"), + "repo_meta": load("repo_meta.json"), + } + data["related"] = build_related(cves) + + written = 0 + for entry in cves: + path = os.path.join(DOCS, f"{entry['cve']}.html") + with open(path, "w", encoding="utf-8") as handle: + handle.write(page(entry, data)) + written += 1 + + print(f"Wrote {written:,} CVE pages into {DOCS}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_seo.py b/scripts/build_seo.py index f4b0367bd23..961f3ab6319 100644 --- a/scripts/build_seo.py +++ b/scripts/build_seo.py @@ -8,6 +8,7 @@ domain move is one edit rather than a hunt through the tree. from __future__ import annotations +import json import os import sys from datetime import datetime, timezone @@ -45,7 +46,7 @@ def robots() -> str: return "\n".join(lines) -def sitemap(urls: list[tuple[str, str]]) -> str: +def urlset(urls: list[tuple[str, str]]) -> str: body = "".join( f" \n {loc}\n {lastmod}\n \n" for loc, lastmod in urls @@ -53,22 +54,62 @@ def sitemap(urls: list[tuple[str, str]]) -> str: return ( '\n' '\n' - f"{body}" - "\n" + f"{body}\n" + ) + + +def sitemap_index(names: list[str], today: str) -> str: + body = "".join( + f" \n {SITE}/{name}\n" + f" {today}\n \n" + for name in names + ) + return ( + '\n' + '\n' + f"{body}\n" ) def main() -> int: today = datetime.now(timezone.utc).strftime("%Y-%m-%d") - urls = [(f"{SITE}/", today)] - os.makedirs(DOCS, exist_ok=True) + with open(ROBOTS, "w", encoding="utf-8") as handle: handle.write(robots()) - with open(SITEMAP, "w", encoding="utf-8") as handle: - handle.write(sitemap(urls)) - print(f"Wrote {ROBOTS} and {SITEMAP} ({len(urls)} URL) for {SITE}") + # One sitemap per CVE year. The 50,000 URL cap makes some split necessary and + # the year is the split the data already has, which also means a crawler only + # refetches the shard that moved. + try: + with open(os.path.join(DOCS, "CVE_list.json"), encoding="utf-8") as handle: + cves = json.load(handle) + except FileNotFoundError: + cves = [] + + years: dict[str, list[tuple[str, str]]] = {} + for entry in cves: + cid = entry["cve"] + year = cid.split("-")[1] + lastmod = entry.get("modified") or entry.get("published") or today + years.setdefault(year, []).append((f"{SITE}/{cid}", lastmod)) + + names = [] + for year in sorted(years): + name = f"sitemap-{year}.xml" + with open(os.path.join(DOCS, name), "w", encoding="utf-8") as handle: + handle.write(urlset(years[year])) + names.append(name) + + with open(os.path.join(DOCS, "sitemap-home.xml"), "w", encoding="utf-8") as handle: + handle.write(urlset([(f"{SITE}/", today)])) + names.insert(0, "sitemap-home.xml") + + with open(SITEMAP, "w", encoding="utf-8") as handle: + handle.write(sitemap_index(names, today)) + + total = sum(len(v) for v in years.values()) + 1 + print(f"Wrote {ROBOTS}, {SITEMAP} and {len(names)} shards ({total:,} URLs) for {SITE}") return 0