Improve crawlable SEO and search reliability

tests/test_build_pages.py

@@ -1,14 +1,86 @@
 from __future__ import annotations
 
+import json
 import sys
 import unittest
+from html.parser import HTMLParser
 from pathlib import Path
+from unittest.mock import patch
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
 
 import build_pages
 
 
+class ParsedPage(HTMLParser):
+    def __init__(self, markup: str) -> None:
+        super().__init__()
+        self.tags = []
+        self.text = []
+        self.feed(markup)
+
+    def handle_starttag(self, tag, attrs) -> None:
+        self.tags.append((tag, dict(attrs)))
+
+    def handle_data(self, data) -> None:
+        self.text.append(data)
+
+
+class HomepageTests(unittest.TestCase):
+    def render(self, total: int = 123456) -> str:
+        return build_pages.homepage(
+            [{"cve": "CVE-2030-1234"}, {"cve": "CVE-2024-3400"}],
+            {"CVE-2024-3400": ["2024-04-12", False]},
+            {"total_cves": total, "with_pocs": 2, "generated": "2030-01-02T12:00:00Z", "items": [
+                {"cve": "CVE-2030-1234", "stars": 5, "pushed": "2030-01-02T09:00:00Z",
+                 "name": "Example PoC", "url": "https://github.com/owner/poc", "desc": "A widget exploit"},
+                {"cve": "CVE-2030-9999", "stars": 1, "pushed": "2030-01-01T00:00:00Z",
+                 "name": "New PoC", "url": "https://github.com/owner/new", "desc": "</script><script>alert(1)</script>"},
+            ]},
+        )
+
+    def test_initial_html_has_current_counts_and_crawlable_links(self) -> None:
+        for total in (123456, 123457):
+            page = ParsedPage(self.render(total))
+            text = " ".join(page.text)
+            self.assertIn(f"{total:,}", text)
+            self.assertIn("Example PoC", text)
+            self.assertIn("A widget exploit", text)
+            links = [attrs.get("href") for tag, attrs in page.tags if tag == "a"]
+            self.assertIn("/2030", links)
+            self.assertIn("/2024", links)
+            self.assertNotIn("/2026", links)
+            self.assertIn("/CVE-2030-1234", links)
+            self.assertNotIn("/CVE-2030-9999", links)
+
+    def test_metadata_is_consistent_and_does_not_bake_in_counts(self) -> None:
+        page = ParsedPage(self.render())
+        descriptions = [attrs["content"] for tag, attrs in page.tags if tag == "meta" and
+                        (attrs.get("name") in {"description", "twitter:description"} or
+                         attrs.get("property") == "og:description")]
+        self.assertEqual(descriptions, [build_pages.DESCRIPTION] * 3)
+        self.assertTrue(all("82,000" not in text and "123,456" not in text for text in descriptions))
+        schemas = [json.loads(text) for text in page.text if text.startswith('{"@context"')]
+        self.assertEqual(schemas[0]["description"], build_pages.DESCRIPTION)
+        self.assertIn("pocindex", schemas[0]["alternateName"])
+        self.assertTrue(any(tag == "section" and "data-trending" in attrs and
+                            "data-nosnippet" in attrs for tag, attrs in page.tags))
+        self.assertEqual(sum(tag == "div" and "data-nosnippet" in attrs for tag, attrs in page.tags), 3)
+
+    def test_untrusted_text_cannot_add_markup_or_end_structured_data(self) -> None:
+        unsafe = '</script><script>alert("injected")</script>'
+        with patch.object(build_pages, "DESCRIPTION", unsafe):
+            page = ParsedPage(self.render())
+        scripts = [attrs for tag, attrs in page.tags if tag == "script"]
+        self.assertEqual(len(scripts), 2)  # One JSON-LD block and logic.js.
+        schemas = [json.loads(text) for text in page.text if text.startswith('{"@context"')]
+        self.assertEqual(schemas[0]["description"], unsafe)
+
+    def test_inconsistent_feed_counts_fail_the_build(self) -> None:
+        with self.assertRaisesRegex(ValueError, "counts disagree"):
+            build_pages.homepage([], {}, {"total_cves": 1, "with_pocs": 1})
+
+
 class RepositoryMetadataTests(unittest.TestCase):
     def test_repository_metadata_is_shown_for_repository_paths(self) -> None:
         metadata = {"owner/repo": [123, "2026-10-06", "2025-01-01"]}

tests/test_search.js

@@ -9,6 +9,89 @@ const { MessageChannel } = require('node:worker_threads');
 
 const script = fs.readFileSync(path.join(__dirname, '../docs/logic.js'), 'utf8');
 
+function startPage(t, { search = '', trending, kev = {}, indexError = false }) {
+  const elements = new Map();
+  const calls = [];
+  const document = {
+    addEventListener() {},
+    querySelectorAll: () => [],
+    querySelector(selector) {
+      if (!elements.has(selector)) elements.set(selector, {
+        value: '',
+        innerHTML: selector === '[data-trend-rows]' ? '<a href="/CVE-2026-1234">Prerendered PoC</a>' : '',
+        addEventListener() {}, setAttribute() {}, querySelectorAll: () => []
+      });
+      return elements.get(selector);
+    }
+  };
+  const context = vm.createContext({
+    console: { warn() {} }, document, performance, MessageChannel, URLSearchParams,
+    location: { pathname: '/', search },
+    window: { matchMedia: () => ({ matches: false }) },
+    setTimeout() {}, clearTimeout() {},
+    async fetch(url) {
+      calls.push(url);
+      if (url === '/CVE_list.json' && indexError) throw new Error('Index unavailable');
+      const data = url === '/trending_poc.json' ? await trending
+        : url === '/kev.json' ? await kev
+        : url === '/CVE_list.json' ? [{ cve: 'CVE-2026-1234', desc: 'widget', poc: ['https://github.com/owner/poc'] }]
+          : {};
+      return { ok: true, json: async () => data };
+    }
+  });
+  vm.runInContext(script, context);
+  t.after(() => vm.runInContext('yieldPort.port1.close(); yieldPort.port2.close();', context));
+  return { elements, calls };
+}
+
+test('a stalled trending feed does not block a bookmarked search', async t => {
+  const { elements, calls } = startPage(t, { search: '?q=widget', trending: new Promise(() => {}) });
+  await new Promise(setImmediate);
+  assert.ok(calls.includes('/CVE_list.json'));
+  assert.equal(elements.get('[data-search]').value, 'widget');
+  assert.match(elements.get('[data-results]').innerHTML, /href="\/CVE-2026-1234"/);
+});
+
+test('a failed trending feed preserves the published homepage rows', async t => {
+  const { elements } = startPage(t, { trending: Promise.reject(new Error('Feed unavailable')) });
+  await new Promise(setImmediate);
+  assert.match(elements.get('[data-trend-rows]').innerHTML, /Prerendered PoC/);
+});
+
+test('an empty successful feed replaces stale homepage rows', async t => {
+  const { elements } = startPage(t, { trending: { items: [], total_cves: 1, with_pocs: 1, generated: '2026-10-06' } });
+  await new Promise(setImmediate);
+  assert.match(elements.get('[data-trend-rows]').innerHTML, /No recent PoCs/);
+});
+
+test('an index failure stays visible when the optional trending feed arrives later', async t => {
+  let finishTrending;
+  const trending = new Promise(resolve => { finishTrending = resolve; });
+  const { elements } = startPage(t, { search: '?q=widget', trending, indexError: true });
+  await new Promise(setImmediate);
+  assert.match(elements.get('[data-results]').innerHTML, /could not be loaded/);
+  assert.equal(elements.get('[data-results]').hidden, false);
+  finishTrending({ items: [], total_cves: 1, with_pocs: 1, generated: '2026-10-06' });
+  await new Promise(setImmediate);
+  assert.match(elements.get('[data-results]').innerHTML, /could not be loaded/);
+  assert.equal(elements.get('[data-results]').hidden, false);
+  assert.equal(elements.get('[data-status]').textContent, 'index unavailable');
+});
+
+test('late KEV data cannot hide a bookmarked search failure', async t => {
+  let finishKev;
+  const kev = new Promise(resolve => { finishKev = resolve; });
+  const { elements } = startPage(t, {
+    search: '?q=widget', trending: new Promise(() => {}), kev, indexError: true
+  });
+  await new Promise(setImmediate);
+  finishKev({ 'CVE-2026-1234': ['2026-10-06', false] });
+  await new Promise(setImmediate);
+  assert.match(elements.get('[data-results]').innerHTML, /could not be loaded/);
+  assert.equal(elements.get('[data-results]').hidden, false);
+  assert.equal(elements.get('[data-trending]').hidden, true);
+});
+
 for (const sort of ['relevance', 'newest']) {
   test(`late EPSS data refreshes results while respecting ${sort} sort`, async t => {
     const elements = new Map();

tests/test_update_cves.py

@@ -218,6 +218,54 @@ class ReferenceAndMarkdownTests(unittest.TestCase):
             ],
         )
 
+    def test_excludes_explicitly_unaffected_products_and_vendors(self) -> None:
+        for versions in ([], [{
+            "version": "All",
+            "status": "unaffected",
+            "changes": [{"at": "2", "status": "unaffected"}],
+        }]):
+            with self.subTest(versions=versions):
+                record = self.record()
+                record["containers"]["cna"]["affected"].append({
+                    "vendor": "Unaffected vendor",
+                    "product": "Unaffected product",
+                    "defaultStatus": "unaffected",
+                    "versions": versions,
+                })
+                details = update_cves.details_from_record(record)
+                self.assertEqual(details.products, ["Linux"])
+                self.assertEqual(details.vendors, [])
+                self.assertEqual(details.versions, ["4.14"])
+
+    def test_keeps_products_with_affected_or_incomplete_statuses(self) -> None:
+        cases = [
+            {"defaultStatus": "affected", "versions": [{"status": "unaffected"}]},
+            {"defaultStatus": "unaffected", "versions": [{"status": "affected"}]},
+            {"defaultStatus": "unaffected", "versions": [{
+                "status": "unaffected", "changes": [{"at": "2", "status": "affected"}],
+            }]},
+            {"defaultStatus": "unknown", "versions": [{"status": "unaffected"}]},
+            {"versions": [{"status": "unaffected"}]},
+            {"defaultStatus": "unaffected", "versions": [{"status": "unknown"}]},
+            {"defaultStatus": "unaffected", "versions": [{"version": "1"}]},
+            {"defaultStatus": "unaffected", "versions": [{
+                "status": "unaffected", "changes": [{"at": "2", "status": "unknown"}],
+            }]},
+            {"defaultStatus": "unaffected", "versions": [{
+                "status": "unaffected", "changes": [{"at": "2"}],
+            }]},
+            {},
+        ]
+        for statuses in cases:
+            with self.subTest(statuses=statuses):
+                record = self.record()
+                record["containers"]["cna"]["affected"] = [{
+                    "vendor": "Example vendor", "product": "Example product", **statuses,
+                }]
+                details = update_cves.details_from_record(record)
+                self.assertEqual(details.products, ["Example product"])
+                self.assertEqual(details.vendors, ["Example vendor"])
+
     def test_preserves_markdown_layout(self) -> None:
         details = update_cves.details_from_record(self.record())
         self.assertIsNotNone(details)