Refine footer links and fix search and index edge cases

scripts/audit_poc_links.py

@@ -260,7 +260,7 @@ def dead_references(urls: list[str], *, workers: int, timeout: int) -> set[str]:
                 code = exc.code
                 exc.close()
                 # a server that dislikes HEAD is not a server without the page
-                if method == "GET" or code not in {403, 405, 501}:
+                if method == "GET" or code not in {403, 404, 405, 410, 501}:
                     return url, code in {404, 410}
             except Exception:
                 # a timeout or a refused connection keeps the link, and asking

scripts/brand.py

@@ -44,6 +44,12 @@ FONTS = (
 # One footer credit line. The homepage and the generated pages listed different
 # sources, so a reader met two different claims about where the data comes from.
 SOURCES_LINE = (
-    "sources: nvd · cve program · github · github advisories "
-    "· cisa vulnrichment · cisa kev · first epss"
+    'sources: <a href="https://nvd.nist.gov/">nvd</a>&nbsp;&middot; '
+    '<a href="https://www.cve.org/">cve program</a>&nbsp;&middot; '
+    '<a href="https://github.com/search?q=cve+poc&amp;type=repositories">github</a>&nbsp;&middot; '
+    '<a href="https://github.com/advisories">github advisories</a>&nbsp;&middot; '
+    '<a href="https://github.com/cisagov/vulnrichment">cisa vulnrichment</a>&nbsp;&middot; '
+    '<a href="https://www.cisa.gov/known-exploited-vulnerabilities-catalog">cisa kev</a>&nbsp;&middot; '
+    '<a href="https://www.first.org/epss/">first epss</a>'
 )
+REPO_LINK = f'<a href="https://github.com/{SLUG}">github.com/{SLUG}</a>'

scripts/build_pages.py

@@ -28,7 +28,7 @@ from urllib import error, request
 ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir)
 sys.path.insert(0, os.path.join(ROOT, "scripts"))
 
-from brand import BRAND, DESCRIPTION, FONTS, SEARCH_GUIDE, SITE, SLUG, SOURCES_LINE, SUBTITLE, TITLE, host
+from brand import BRAND, DESCRIPTION, FONTS, REPO_LINK, SITE, SLUG, SOURCES_LINE, SUBTITLE, TITLE, host
 
 DOCS = os.path.join(ROOT, "docs")
 RELATED = 6
@@ -52,7 +52,7 @@ def load(name: str) -> dict | list:
 
 
 def esc(value) -> str:
-    return html.escape(str(value or ""), quote=True)
+    return html.escape(str("" if value is None else value), quote=True)
 
 
 def json_ld(payload: dict) -> str:
@@ -114,6 +114,7 @@ ASSESSORS = {
     "[email protected]": "NVD",
     "[email protected]": "MITRE",
     "github advisory database": "GitHub",
+    "[email protected]": "IBM",
 }
 
 
@@ -121,7 +122,7 @@ def assessor(row: list) -> str:
     """Who scored it, from row[4].
 
     row[5] is the assessment type, Primary or Secondary, and labelling with it
-    left 129 pages showing two rows both marked "Secondary" — the repetition
+    left 129 pages showing two rows both marked "Secondary", the repetition
     the label was added to explain. row[4] names the party, which is the thing
     that actually differs. Scoring CNAs identify themselves by contact address
     or by UUID, so an address becomes its domain and a UUID stays generic.
@@ -312,7 +313,7 @@ def header(count: int | None = None) -> str:
 
 def footer(tail: str = '<a href="/">Back to the index</a>') -> str:
     return (f'<footer class="site-footer"><div class="container">'
-            f"<span>{SOURCES_LINE}</span><span>{tail}</span></div></footer>")
+            f"<span>{SOURCES_LINE}</span><span>{tail}</span>{REPO_LINK}</div></footer>")
 
 
 def page(entry: dict, data: dict) -> str:
@@ -683,8 +684,8 @@ def homepage(cves: list, kev: dict, trending: dict) -> str:
         template = Template(handle.read())
     return template.substitute(
         brand=esc(BRAND), title=esc(TITLE), subtitle=esc(SUBTITLE),
-        description=esc(DESCRIPTION), search_guide=esc(SEARCH_GUIDE), site=esc(SITE),
-        fonts=FONTS, sources_line=SOURCES_LINE, website_schema=json_ld(website),
+        description=esc(DESCRIPTION), site=esc(SITE), fonts=FONTS,
+        sources_line=SOURCES_LINE, repo_link=REPO_LINK, website_schema=json_ld(website),
         total_cves=f"{trending['total_cves']:,}", with_pocs=f"{len(indexed):,}", kev=f"{len(kev):,}",
         trending_rows="\n".join(rows), generated=esc(generated.strftime("%Y-%m-%d %H:%M UTC")),
         year_links=" ".join(f'<a href="/{esc(year)}">{esc(year)}</a>' for year in years),

scripts/build_seo.py

@@ -23,9 +23,6 @@ DOCS = os.path.join(ROOT, "docs")
 ROBOTS = os.path.join(DOCS, "robots.txt")
 SITEMAP = os.path.join(DOCS, "sitemap.xml")
 
-# These small feeds enhance the prerendered homepage and remain crawlable.
-RENDER_CRITICAL = ("trending_poc.json", "kev.json", "stats.json")
-
 # These only feed the search box, which a crawler never types into. Together they
 # are over 13 MB on the wire, which is crawl budget spent on nothing. Blocking
 # them here does not touch curl: robots.txt binds crawlers, not clients.

scripts/build_site.py

@@ -321,6 +321,12 @@ def collect_links(block: str, *, blacklist: Optional[Collection[str]] = None) ->
             entry = entry[2:].strip()
         if not entry:
             continue
+        try:
+            parsed = urlparse(entry)
+            if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+                continue
+        except ValueError:
+            continue
         if is_blacklisted(entry, blacklist):
             continue
         if entry not in links: