mirror of
https://github.com/0xMarcio/cve.git
synced 2026-10-27 16:51:03 +00:00
Rebrand to PoC Index and move to pocindex.io
The site called itself three different things: CVE PoC Hub in the title, CVE Radar in the h1 and the social metadata, and cve on GitHub. Google had no consistent site name to settle on, so none of the mentions compounded. CVE Radar was taken anyway, by SOCRadar's free tool and by cveradar.com. Every user-facing name and URL now comes from scripts/brand.py, so the next move is one edit rather than a sweep through the tree. Also repairs the parts of the site a crawler could not use: - robots.txt and sitemap.xml were both 404. scripts/build_seo.py writes them, wired into the Pages job. trending_poc.json, kev.json and stats.json stay crawlable because the first paint needs them; the five bulk payloads are disallowed, which is 13.2 MB per crawl spent on data no crawler queries. - Search state lives in ?q= now. Results were unlinkable, so nobody could cite one and there was no route for a SearchAction to declare. replaceState rather than pushState, so typing does not bury the back button. - A WebSite JSON-LD block carries the name Google prints as the site name. - The footer carries the MITRE trademark notice.
This commit is contained in:
1 parent
e58a5a26cc
commit
31da22381c
22 files changed
+270
-75
No files matched your search
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Write docs/robots.txt and docs/sitemap.xml.
|
||||
|
||||
Both files were 404 before this, so nothing told a crawler where the site ends
|
||||
or how often it changes. Everything here is derived from scripts/brand.py, so a
|
||||
domain move is one edit rather than a hunt through the tree.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir)
|
||||
sys.path.insert(0, os.path.join(ROOT, "scripts"))
|
||||
|
||||
from brand import SITE
|
||||
|
||||
DOCS = os.path.join(ROOT, "docs")
|
||||
ROBOTS = os.path.join(DOCS, "robots.txt")
|
||||
SITEMAP = os.path.join(DOCS, "sitemap.xml")
|
||||
|
||||
# Googlebot renders the page before it indexes it, so anything the first paint
|
||||
# needs has to stay crawlable. These three are small and carry the only server
|
||||
# rendered content on the page, the trending table and the header counts.
|
||||
RENDER_CRITICAL = ("trending_poc.json", "kev.json", "stats.json")
|
||||
|
||||
# These only feed the search box, which a crawler never types into. Together they
|
||||
# are over 13 MB on the wire, which is crawl budget spent on nothing. Blocking
|
||||
# them here does not touch curl: robots.txt binds crawlers, not clients.
|
||||
BULK_PAYLOADS = (
|
||||
"CVE_list.json",
|
||||
"cve_metadata.json",
|
||||
"epss.json",
|
||||
"repo_meta.json",
|
||||
"nuclei.json",
|
||||
)
|
||||
|
||||
|
||||
def robots() -> str:
|
||||
lines = ["User-agent: *", "Allow: /"]
|
||||
lines += [f"Disallow: /{name}" for name in BULK_PAYLOADS]
|
||||
lines += ["", f"Sitemap: {SITE}/sitemap.xml", ""]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def sitemap(urls: list[tuple[str, str]]) -> str:
|
||||
body = "".join(
|
||||
f" <url>\n <loc>{loc}</loc>\n <lastmod>{lastmod}</lastmod>\n </url>\n"
|
||||
for loc, lastmod in urls
|
||||
)
|
||||
return (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
||||
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
|
||||
f"{body}"
|
||||
"</urlset>\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
urls = [(f"{SITE}/", today)]
|
||||
|
||||
os.makedirs(DOCS, exist_ok=True)
|
||||
with open(ROBOTS, "w", encoding="utf-8") as handle:
|
||||
handle.write(robots())
|
||||
with open(SITEMAP, "w", encoding="utf-8") as handle:
|
||||
handle.write(sitemap(urls))
|
||||
|
||||
print(f"Wrote {ROBOTS} and {SITEMAP} ({len(urls)} URL) for {SITE}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user