mirror of
https://github.com/0xMarcio/cve.git
synced 2026-10-27 16:51:03 +00:00
0xMarcio
9b9957ea7f
82,000 CVEs sat behind one URL. Each of them is a query somebody types, and the pages answering those queries belonged to other people: searching "CVE-2026-72898 poc" returns SentinelOne, Horizon3, Wiz and BishopFox, all of them serving one page per CVE. Pages are flat files at /CVE-2026-72898. No /cve/ prefix, no extension, no trailing slash. GitHub Pages resolves an extensionless request to the .html beside it, checked against the live site rather than assumed. Each page carries what the index already knows and nothing invented: every CVSS assessment with its vector, EPSS with percentile, KEV date and ransomware flag, Nuclei severity and CWE, and the PoC links grouped by source with stars and last push. 94% of them carry three or more of those signals. Discovery works two ways. sitemap.xml is now an index over one shard per year, which fits the 50,000 URL cap on the split the data already has and lets a crawler refetch only the year that moved. Search results link to the CVE page instead of straight out to NVD, which moves to a chip beside CVE.ORG, and each page links six related CVEs picked by shared PoC author. Rendering all of them takes 20s and adds nothing to the repository: the payloads they read are already gitignored and built in CI, so the pages go into the Pages artifact and git never sees them.
118 lines
3.8 KiB
Python
118 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Write docs/robots.txt and docs/sitemap.xml.
|
|
|
|
Both files were 404 before this, so nothing told a crawler where the site ends
|
|
or how often it changes. Everything here is derived from scripts/brand.py, so a
|
|
domain move is one edit rather than a hunt through the tree.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir)
|
|
sys.path.insert(0, os.path.join(ROOT, "scripts"))
|
|
|
|
from brand import SITE
|
|
|
|
DOCS = os.path.join(ROOT, "docs")
|
|
ROBOTS = os.path.join(DOCS, "robots.txt")
|
|
SITEMAP = os.path.join(DOCS, "sitemap.xml")
|
|
|
|
# Googlebot renders the page before it indexes it, so anything the first paint
|
|
# needs has to stay crawlable. These three are small and carry the only server
|
|
# rendered content on the page, the trending table and the header counts.
|
|
RENDER_CRITICAL = ("trending_poc.json", "kev.json", "stats.json")
|
|
|
|
# These only feed the search box, which a crawler never types into. Together they
|
|
# are over 13 MB on the wire, which is crawl budget spent on nothing. Blocking
|
|
# them here does not touch curl: robots.txt binds crawlers, not clients.
|
|
BULK_PAYLOADS = (
|
|
"CVE_list.json",
|
|
"cve_metadata.json",
|
|
"epss.json",
|
|
"repo_meta.json",
|
|
"nuclei.json",
|
|
)
|
|
|
|
|
|
def robots() -> str:
|
|
lines = ["User-agent: *", "Allow: /"]
|
|
lines += [f"Disallow: /{name}" for name in BULK_PAYLOADS]
|
|
lines += ["", f"Sitemap: {SITE}/sitemap.xml", ""]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def urlset(urls: list[tuple[str, str]]) -> str:
|
|
body = "".join(
|
|
f" <url>\n <loc>{loc}</loc>\n <lastmod>{lastmod}</lastmod>\n </url>\n"
|
|
for loc, lastmod in urls
|
|
)
|
|
return (
|
|
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
|
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
|
|
f"{body}</urlset>\n"
|
|
)
|
|
|
|
|
|
def sitemap_index(names: list[str], today: str) -> str:
|
|
body = "".join(
|
|
f" <sitemap>\n <loc>{SITE}/{name}</loc>\n"
|
|
f" <lastmod>{today}</lastmod>\n </sitemap>\n"
|
|
for name in names
|
|
)
|
|
return (
|
|
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
|
'<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
|
|
f"{body}</sitemapindex>\n"
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
|
os.makedirs(DOCS, exist_ok=True)
|
|
|
|
with open(ROBOTS, "w", encoding="utf-8") as handle:
|
|
handle.write(robots())
|
|
|
|
# One sitemap per CVE year. The 50,000 URL cap makes some split necessary and
|
|
# the year is the split the data already has, which also means a crawler only
|
|
# refetches the shard that moved.
|
|
try:
|
|
with open(os.path.join(DOCS, "CVE_list.json"), encoding="utf-8") as handle:
|
|
cves = json.load(handle)
|
|
except FileNotFoundError:
|
|
cves = []
|
|
|
|
years: dict[str, list[tuple[str, str]]] = {}
|
|
for entry in cves:
|
|
cid = entry["cve"]
|
|
year = cid.split("-")[1]
|
|
lastmod = entry.get("modified") or entry.get("published") or today
|
|
years.setdefault(year, []).append((f"{SITE}/{cid}", lastmod))
|
|
|
|
names = []
|
|
for year in sorted(years):
|
|
name = f"sitemap-{year}.xml"
|
|
with open(os.path.join(DOCS, name), "w", encoding="utf-8") as handle:
|
|
handle.write(urlset(years[year]))
|
|
names.append(name)
|
|
|
|
with open(os.path.join(DOCS, "sitemap-home.xml"), "w", encoding="utf-8") as handle:
|
|
handle.write(urlset([(f"{SITE}/", today)]))
|
|
names.insert(0, "sitemap-home.xml")
|
|
|
|
with open(SITEMAP, "w", encoding="utf-8") as handle:
|
|
handle.write(sitemap_index(names, today))
|
|
|
|
total = sum(len(v) for v in years.values()) + 1
|
|
print(f"Wrote {ROBOTS}, {SITEMAP} and {len(names)} shards ({total:,} URLs) for {SITE}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|