mirror of
https://github.com/0xMarcio/cve.git
synced 2026-10-23 16:35:19 +00:00
Preserve public PoCs with resumable Wayback captures
This commit is contained in:
0xMarcio committed
1 parent
03db7ffa99
commit
5480e9df70
29 files changed
+18454
-15780
No files matched your search
@@ -0,0 +1,336 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
from urllib.error import HTTPError
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
import archive_pocs as archive
|
||||
import build_site
|
||||
import releases
|
||||
|
||||
URL = 'https://github.com/owner/poc'
|
||||
CVE = 'CVE-2026-1234'
|
||||
|
||||
|
||||
def snapshot(url=URL, stamp=None):
|
||||
stamp = stamp or datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')
|
||||
return {'url': archive.snapshot_url(url, stamp), 'timestamp': stamp, 'verified_at': archive.now()}
|
||||
|
||||
|
||||
class ArchiveTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.target = {'original': URL, 'cves': [CVE], 'priority': archive.now()}
|
||||
self.rows = {}
|
||||
self.client = MagicMock()
|
||||
self.client.lookup.return_value = None
|
||||
self.client.submit.return_value = {'job_id': 'pending-123'}
|
||||
p = patch.object(archive.link_health, 'unavailable', return_value=False)
|
||||
self.dead = p.start()
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
def test_inventory_limits_cve_year_and_preserves_source_paths(self):
|
||||
ledger = {
|
||||
'old': {'url': URL + '-old', 'cve': 'CVE-2025-1234'},
|
||||
'new': {'url': URL, 'cve': CVE, 'head': 'b' * 40, 'sha': 'a' * 40,
|
||||
'commit_verified': True, 'paths': ['wrong.py'], 'history_paths': ['PoC x.py']},
|
||||
'future': {'url': URL, 'cve': 'CVE-2027-4567'},
|
||||
'bad': {'url': URL + '-blocked', 'cve': CVE},
|
||||
'excluded': {'url': URL + '-excluded', 'cve': CVE},
|
||||
'withdrawn': {'url': URL + '-withdrawn', 'cve': CVE, 'gone': '2026-01-01'},
|
||||
}
|
||||
with patch.object(releases, 'load_ledger', return_value=ledger), \
|
||||
patch.object(build_site, 'load_blacklist', return_value={'owner/poc-blocked'}), \
|
||||
patch.object(build_site, 'load_source_exclusions', return_value={(CVE, 'owner/poc-excluded')}):
|
||||
result = archive.inventory()
|
||||
self.assertEqual(len(result), 2)
|
||||
self.assertEqual(set(result[URL]['cves']), {CVE, 'CVE-2027-4567'})
|
||||
raw = next(v for v in result.values() if v.get('parent'))
|
||||
self.assertTrue(raw['original'].endswith('/' + 'a' * 40 + '/PoC%20x.py'))
|
||||
|
||||
def test_snapshot_validation_rejects_fake_hosts_invalid_dates_and_private_urls(self):
|
||||
good = snapshot()
|
||||
self.assertTrue(archive.valid_snapshot({'snapshot': good}))
|
||||
for url in ('http://localhost/a', 'http://127.0.0.1/a', 'http://169.254.169.254/a',
|
||||
'https://user:[email protected]/a', 'javascript:alert(1)', 'https://host.internal/a'):
|
||||
self.assertFalse(archive.public_url(url), url)
|
||||
for bad in ({**good, 'url': good['url'].replace('web.archive.org', 'web.archive.org.evil.example')},
|
||||
{**good, 'timestamp': '20260231000000'}, {**good, 'verified_at': ''}):
|
||||
self.assertFalse(archive.valid_snapshot({'snapshot': bad}))
|
||||
|
||||
def test_first_observation_does_not_disqualify_older_public_captures(self):
|
||||
ledger = [{'url': URL, 'cve': CVE, 'seen': '2026-10-21T12:00:00Z'},
|
||||
{'url': URL + '-dated', 'cve': CVE, 'seen': '2026-10-21T12:00:00Z',
|
||||
'source': '2026-05-13'}]
|
||||
with patch.object(archive, 'eligible_ledger', return_value=ledger):
|
||||
result = archive.inventory()
|
||||
self.assertNotIn('not_before', result[URL])
|
||||
self.assertEqual(result[URL + '-dated']['not_before'], '2026-05-13T00:00:00Z')
|
||||
|
||||
def test_pinned_file_urls_preserve_bytes_without_additional_github_queries(self):
|
||||
parent = URL + '/blob/' + 'b' * 40 + '/path/PoC%20file.py#example'
|
||||
with patch.object(archive, 'eligible_ledger', return_value=[{'url': parent, 'cve': CVE}]):
|
||||
result = archive.inventory()
|
||||
raw = 'https://raw.githubusercontent.com/owner/poc/' + 'b' * 40 + '/path/PoC%20file.py'
|
||||
self.assertEqual(result[raw]['parent'], archive.identity(parent))
|
||||
self.assertEqual(result[raw]['parent_original'], parent)
|
||||
|
||||
def test_lookup_rejects_capture_of_a_different_url(self):
|
||||
client = archive.Client()
|
||||
snap = snapshot(URL + '-other')
|
||||
response = {'archived_snapshots': {'closest': {**snap, 'status': '200', 'available': True}}}
|
||||
with patch.object(client, 'json', return_value=response):
|
||||
self.assertIsNone(client.lookup(URL))
|
||||
|
||||
def test_existing_capture_is_verified_without_resubmission(self):
|
||||
self.client.lookup.return_value = snapshot()
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=False)
|
||||
self.assertTrue(archive.valid_snapshot(self.rows[URL]))
|
||||
self.client.submit.assert_not_called()
|
||||
|
||||
def test_pending_job_is_checkpointed_and_polled_without_duplicate_submission(self):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertEqual(self.rows[URL]['job_id'], 'pending-123')
|
||||
self.client.status.return_value = {'status': 'pending'}
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.submit.assert_called_once()
|
||||
self.client.lookup.assert_not_called()
|
||||
self.assertEqual(self.rows[URL]['attempts'], 1)
|
||||
|
||||
def test_success_requires_replay_verification(self):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
snap = snapshot()
|
||||
self.client.status.return_value = {'status': 'success', 'timestamp': snap['timestamp'], 'original_url': URL}
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertEqual(self.rows[URL]['status'], 'available')
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
self.assertEqual(archive.published_snapshots(self.rows)[URL]['url'], snap['url'])
|
||||
|
||||
def test_expired_job_recovers_by_lookup(self):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.status.side_effect = archive.ArchiveError('http-404')
|
||||
self.client.lookup.return_value = snapshot()
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
self.client.submit.assert_called_once()
|
||||
|
||||
def test_refresh_error_keeps_previous_good_capture(self):
|
||||
old = snapshot(stamp='20260101000000')
|
||||
self.rows[URL] = {**self.target, 'snapshot': old, 'job_id': 'pending-123', 'attempts': 1}
|
||||
self.client.status.return_value = {'status': 'error', 'status_ext': 'error:read-timeout'}
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertEqual(self.rows[URL]['snapshot'], old)
|
||||
self.assertEqual(archive.published_snapshots(self.rows)[URL]['url'], old['url'])
|
||||
|
||||
def test_unverified_terminal_job_never_resubmits_in_a_loop(self):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.status.return_value = {'status': 'success', 'timestamp': snapshot()['timestamp'], 'original_url': URL}
|
||||
with patch.object(archive, 'verify_snapshot', return_value=False):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.submit.assert_called_once()
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
self.assertNotIn('candidate', self.rows[URL])
|
||||
self.assertGreater(self.rows[URL]['next_check_at'], archive.now())
|
||||
self.assertFalse(archive.valid_snapshot(self.rows[URL]))
|
||||
|
||||
def test_transient_replay_failure_retains_candidate_without_resubmission(self):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.status.return_value = {'status': 'success', 'timestamp': snapshot()['timestamp'], 'original_url': URL}
|
||||
with patch.object(archive, 'verify_snapshot', side_effect=archive.ArchiveError('replay-http-404')):
|
||||
with self.assertRaises(archive.ArchiveError):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertIn('candidate', self.rows[URL])
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.submit.assert_called_once()
|
||||
self.assertTrue(archive.valid_snapshot(self.rows[URL]))
|
||||
|
||||
def test_expired_jobs_do_not_reserve_a_capture_slot(self):
|
||||
self.rows[URL] = {**self.target, 'job_id': 'old-job', 'submitted_at': archive.after(-7200)}
|
||||
self.assertFalse(archive.active_job(self.rows[URL]))
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=False)
|
||||
self.client.status.assert_not_called()
|
||||
self.client.lookup.assert_called_once_with(URL)
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
|
||||
def test_dead_original_with_good_capture_clears_stale_work(self):
|
||||
self.rows[URL] = {**self.target, 'snapshot': snapshot(), 'job_id': 'stale',
|
||||
'candidate': snapshot(), 'submitted_at': archive.after(-7200)}
|
||||
self.dead.return_value = True
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
self.assertNotIn('candidate', self.rows[URL])
|
||||
self.assertEqual(self.rows[URL]['status'], 'available')
|
||||
|
||||
def test_fragments_never_enter_capture_identity_or_poison_a_shard(self):
|
||||
original = URL + '#poc'
|
||||
target = {**self.target, 'original': original}
|
||||
self.rows[URL] = {**target, 'job_id': 'job', 'submitted_at': archive.now()}
|
||||
self.client.status.return_value = {'status': 'success', 'timestamp': snapshot()['timestamp'], 'original_url': original}
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, target, self.rows, self.client, capture=True)
|
||||
self.assertNotIn('#', self.rows[URL]['snapshot']['url'])
|
||||
self.assertTrue(archive.valid_snapshot(self.rows[URL]))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
archive.save_records(self.rows, {URL}, Path(directory))
|
||||
self.assertEqual(archive.load_records(Path(directory)), self.rows)
|
||||
|
||||
def test_repeated_unverifiable_capture_gets_increasing_backoff(self):
|
||||
self.rows[URL] = {**self.target, 'attempts': 7, 'candidate': snapshot()}
|
||||
with patch.object(archive, 'verify_snapshot', return_value=False):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=False)
|
||||
self.assertGreater(self.rows[URL]['next_check_at'], archive.after(6 * 3600))
|
||||
|
||||
def test_redirect_homepage_and_capture_predating_poc_are_not_published(self):
|
||||
for target, captured in ((self.target, snapshot('https://github.com/')),
|
||||
({**self.target, 'not_before': archive.now()}, snapshot(stamp='20260101000000'))):
|
||||
rows = {URL: {**target, 'candidate': captured}}
|
||||
with patch.object(archive, 'verify_snapshot') as replay:
|
||||
archive.process(URL, target, rows, self.client, capture=False)
|
||||
replay.assert_not_called()
|
||||
self.assertFalse(archive.valid_snapshot(rows[URL]))
|
||||
|
||||
def test_account_errors_are_separate_from_target_errors(self):
|
||||
self.assertEqual(archive.service_error('error:max-daily-bandwidth').cooldown, 86400)
|
||||
for reason in ('error:filesize-limit', 'error:bandwidth-limit-exceeded',
|
||||
'error:too-many-daily-captures', 'error:soft-time-limit-exceeded'):
|
||||
self.assertIsNone(archive.service_error(reason), reason)
|
||||
|
||||
def test_daily_submission_budget_is_durable_and_first_capture_is_retained(self):
|
||||
self.rows[URL] = {**self.target, 'submissions': {'date': archive.now()[:10], 'count': 2}}
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.submit.assert_not_called()
|
||||
old = snapshot(stamp='20260101000000')
|
||||
self.rows[URL].update(snapshot=old, candidate=snapshot())
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertEqual(self.rows[URL]['snapshot'], old)
|
||||
|
||||
def test_source_snapshots_are_attached_to_their_published_parent(self):
|
||||
raw = 'https://raw.githubusercontent.com/owner/poc/' + 'a' * 40 + '/PoC.py'
|
||||
rows = {raw: {'original': raw, 'parent': URL, 'snapshot': snapshot(raw)}}
|
||||
result = archive.published_snapshots(rows, {URL})
|
||||
self.assertEqual(result[URL]['files'][0]['original'], raw)
|
||||
self.assertEqual(archive.published_snapshots(rows, set()), {})
|
||||
|
||||
def test_source_file_can_recover_a_dead_parent_without_a_page_capture(self):
|
||||
raw = 'https://raw.githubusercontent.com/owner/poc/' + 'a' * 40 + '/PoC.py'
|
||||
rows = {raw: {'original': raw, 'parent': URL, 'cves': [CVE], 'snapshot': snapshot(raw)}}
|
||||
self.dead.return_value = True
|
||||
with patch.object(archive, 'eligible_ledger', return_value=[{'cve': CVE, 'url': URL, 'field': 'nuclei'}]):
|
||||
self.assertEqual(archive.archived_fallbacks(rows)[CVE], [('nuclei', rows[raw]['snapshot']['url'])])
|
||||
|
||||
def test_recovered_original_keeps_archive_until_it_is_reimported(self):
|
||||
self.dead.return_value = False
|
||||
self.rows[URL] = {**self.target, 'snapshot': snapshot()}
|
||||
with patch.object(archive, 'eligible_ledger', return_value=[{'cve': CVE, 'url': URL, 'gone_reason': 'unavailable'}]):
|
||||
self.assertEqual(len(archive.archived_fallbacks(self.rows)[CVE]), 1)
|
||||
|
||||
def test_managed_fallback_does_not_redate_or_remove_legacy_archive_links(self):
|
||||
managed = snapshot()['url']
|
||||
legacy = archive.snapshot_url(URL + '-legacy', '20200101000000')
|
||||
with patch.object(archive, 'load_records', return_value={URL: {**self.target, 'snapshot': snapshot()}}):
|
||||
self.assertEqual(releases.published_pairs([{'cve': CVE, 'poc': [managed, legacy]}]), [(CVE, legacy)])
|
||||
old = releases.record({}, CVE, URL, observed_at='2026-01-01T00:00:00Z')
|
||||
self.dead.return_value = True
|
||||
ledger = releases.reconcile({releases.key(CVE, URL): old}, [], observed_at='2026-02-01T00:00:00Z')
|
||||
row = ledger[releases.key(CVE, URL)]
|
||||
self.assertEqual(row['seen'], '2026-01-01T00:00:00Z')
|
||||
self.assertEqual(row['gone_reason'], 'unavailable')
|
||||
|
||||
def test_dead_original_is_only_looked_up_and_blacklist_cannot_be_bypassed(self):
|
||||
self.dead.return_value = True
|
||||
self.client.lookup.return_value = snapshot()
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.submit.assert_not_called()
|
||||
with patch.object(archive, 'eligible_ledger', return_value=[{'cve': CVE, 'url': URL}]):
|
||||
self.assertEqual(len(archive.archived_fallbacks(self.rows)[CVE]), 1)
|
||||
self.assertFalse(archive.archived_fallbacks(self.rows, blacklist={'owner/poc'}))
|
||||
self.assertFalse(archive.archived_fallbacks(self.rows, exclusions={(CVE, 'owner/poc')}))
|
||||
self.rows[URL]['parent'] = URL + '-parent'
|
||||
self.assertFalse(archive.archived_fallbacks(self.rows))
|
||||
|
||||
def test_cooldown_and_checkpoints_survive_merge(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
source, destination = root / 'incoming', root / 'current'
|
||||
incoming = {URL: {**self.target, 'checked_at': archive.now(), 'status': 'retry'}}
|
||||
existing = {URL: {**self.target, 'checked_at': '2026-01-01', 'snapshot': snapshot()}}
|
||||
archive.save_records(incoming, {URL}, source)
|
||||
archive.save_records(existing, {URL}, destination)
|
||||
control = {'retry_after': archive.after(300), 'reason': 'http-429'}
|
||||
(source / 'control.json').write_text(json.dumps(control))
|
||||
with patch.object(sys, 'argv', ['archive', '--directory', str(destination), '--merge-from', str(source)]):
|
||||
self.assertEqual(archive.main(), 0)
|
||||
self.assertTrue(archive.valid_snapshot(archive.load_records(destination)[URL]))
|
||||
self.assertEqual(json.loads((destination / 'control.json').read_text()), control)
|
||||
|
||||
def test_quota_and_retry_after_are_respected(self):
|
||||
client = archive.Client('dummy', 'dummy')
|
||||
with patch.object(client, 'json', return_value={'daily_captures': 30000, 'daily_captures_limit': 30000, 'available': 3}):
|
||||
with self.assertRaises(archive.ArchiveError) as result:
|
||||
client.capacity()
|
||||
self.assertEqual(result.exception.cooldown, 86400)
|
||||
self.assertEqual(archive.retry_delay('1200'), 1200)
|
||||
self.assertGreaterEqual(archive.retry_delay('invalid'), 300)
|
||||
opener = MagicMock()
|
||||
opener.open.side_effect = HTTPError('https://web.archive.org/save', 429, 'wait', {'Retry-After': '600'}, None)
|
||||
with patch.object(archive.request, 'build_opener', return_value=opener):
|
||||
with self.assertRaises(archive.ArchiveError) as result:
|
||||
client.submit(URL)
|
||||
self.assertEqual(result.exception.cooldown, 600)
|
||||
|
||||
def test_capture_uses_only_explicit_url_and_never_archives_errors(self):
|
||||
client = archive.Client('dummy', 'dummy')
|
||||
with patch.object(client, 'json', return_value={'job_id': '1'}) as send:
|
||||
client.submit(URL)
|
||||
payload = send.call_args.args[1]
|
||||
self.assertEqual(payload['capture_all'], '0')
|
||||
self.assertEqual(payload['capture_outlinks'], '0')
|
||||
self.assertEqual(payload['if_not_archived_within'], '30d')
|
||||
|
||||
def test_replay_rejects_login_shells_errors_and_external_redirects(self):
|
||||
snap = snapshot()
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value = response
|
||||
response.status = 200
|
||||
response.url = snap['url'].replace(snap['timestamp'] + '/', snap['timestamp'] + 'id_/')
|
||||
response.headers = {'Content-Type': 'text/html'}
|
||||
opener = MagicMock()
|
||||
opener.open.return_value = response
|
||||
with patch.object(archive.request, 'build_opener', return_value=opener):
|
||||
for body in (b'', b'<title>404 Not Found</title>', b'<title>Sign in to GitHub</title>'):
|
||||
response.read.return_value = body
|
||||
self.assertFalse(archive.verify_snapshot(copy.deepcopy(snap), URL))
|
||||
response.read.return_value = b'<title>PoC for CVE-2026-4044</title><article>research</article>'
|
||||
self.assertTrue(archive.verify_snapshot(snap, URL))
|
||||
self.assertFalse(archive.verify_snapshot(snap, 'https://hackerone.com/reports/1234'))
|
||||
redirect = archive.ReplayRedirect()
|
||||
self.assertIsNone(redirect.redirect_request(None, None, 302, '', {}, 'http://127.0.0.1/private'))
|
||||
self.assertIsNone(archive.NoRedirect().redirect_request(None, None, 302, '', {}, 'https://archive.org/'))
|
||||
|
||||
def test_gzip_replay_is_decoded_before_content_and_hash_validation(self):
|
||||
response = io.BytesIO(gzip.compress(b'<title>Sign in to GitHub</title>'))
|
||||
response.headers = {'Content-Encoding': 'gzip'}
|
||||
self.assertEqual(archive.response_bytes(response, 1000), b'<title>Sign in to GitHub</title>')
|
||||
response = io.BytesIO(gzip.compress(b'x' * 1001))
|
||||
response.headers = {'Content-Encoding': 'gzip'}
|
||||
with self.assertRaisesRegex(archive.ArchiveError, 'too-large'):
|
||||
archive.response_bytes(response, 1000)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -87,10 +87,11 @@ process.stdout.write(JSON.stringify(markup));
|
||||
|
||||
|
||||
def page_blocks(entry: dict) -> dict[str, list[tuple[str, str, str]]]:
|
||||
"""(href, label, note) for each row under each counted heading of a CVE page."""
|
||||
"""(href, label, note) for each row under each counted heading of a CVE page,
|
||||
read without the Wayback control each PoC row ends with."""
|
||||
data = {"cves": [entry], "meta": {}, "epss": {}, "kev": {}, "nuclei": {}, "repo_meta": REPO_META,
|
||||
"related": {}, "lastmod": {entry["cve"]: "2026-10-07"}}
|
||||
markup = build_pages.page(entry, data)
|
||||
markup = re.sub(r'(?: · )?<(a|span) class="wayback[^"]*"[^>]*>[^<]*</\1>', "", build_pages.page(entry, data))
|
||||
return {
|
||||
heading: [tuple(html.unescape(part) for part in row) for row in re.findall(
|
||||
r'<li><a href="([^"]*)"[^>]*>([^<]*)</a>(?:<span>([^<]*)</span>)?</li>', body)]
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -16,6 +17,17 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
||||
import build_pages
|
||||
import build_seo
|
||||
|
||||
# A source file saved at a pinned commit, as archives.json lists it under its page.
|
||||
COMMIT = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
|
||||
def raw_file(path: str, repository: str = "owner/poc", commit: str = COMMIT) -> str:
|
||||
return f"https://raw.githubusercontent.com/{repository}/{commit}/{path}"
|
||||
|
||||
|
||||
def saved_file(original: str, stamp: str = "20261022161013") -> dict:
|
||||
return {"original": original, "url": f"https://web.archive.org/web/{stamp}/{original}", "timestamp": stamp}
|
||||
|
||||
|
||||
class ContentDateTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
@@ -70,6 +82,65 @@ class ContentDateTests(unittest.TestCase):
|
||||
changed = {**self.entry, "desc": "Widget flaw, now remote"}
|
||||
self.assertEqual(build_pages.content_state(changed, self.data, published)[1], self.data["today"])
|
||||
|
||||
def test_confirmed_snapshots_advance_the_date_and_other_archive_records_do_not(self) -> None:
|
||||
url, stamp = self.entry["poc"][0], "20251021093000"
|
||||
record = {"url": f"https://web.archive.org/web/{stamp}/{url}", "timestamp": stamp}
|
||||
original = build_pages.content_state(self.entry, self.data, {})
|
||||
previous = {self.entry["cve"]: original}
|
||||
# Records the page cannot render leave it as published.
|
||||
for archives in ({}, {"https://github.com/owner/other": record}, {url: {**record, "timestamp": "20251021093001"}},
|
||||
{url: {**record, "url": f"http://web.archive.org/web/{stamp}/{url}"}}):
|
||||
with self.subTest(archives=archives):
|
||||
self.assertEqual(build_pages.content_state(self.entry, {**self.data, "archives": archives}, previous),
|
||||
original)
|
||||
archived = build_pages.content_state(self.entry, {**self.data, "archives": {url: record}}, previous)
|
||||
self.assertNotEqual(archived[0], original[0])
|
||||
self.assertEqual(archived[1], self.data["today"])
|
||||
later = {**self.data, "today": "2026-10-07"}
|
||||
self.assertEqual(build_pages.content_state(self.entry, {**later, "archives": {url: {**record, "checked": "x"}}},
|
||||
{self.entry["cve"]: archived}), archived)
|
||||
self.assertEqual(build_pages.content_state(self.entry, later, {self.entry["cve"]: archived})[1], "2026-10-07")
|
||||
|
||||
def test_saved_files_advance_the_date_and_other_file_records_do_not(self) -> None:
|
||||
url, cid = self.entry["poc"][0], self.entry["cve"]
|
||||
|
||||
def archived(files: object, data: dict = self.data) -> dict:
|
||||
return {**data, "archives": {url: {"files": files}}}
|
||||
|
||||
original = build_pages.content_state(self.entry, self.data, {})
|
||||
previous = {cid: original}
|
||||
# Files the page cannot show leave it as published.
|
||||
for files in ([], [saved_file(raw_file("poc.py", "owner/other"))], [saved_file(raw_file("poc.py", commit="main"))],
|
||||
[{**saved_file(raw_file("poc.py")), "timestamp": "20261022161014"}],
|
||||
[saved_file("https://hackerone.com/reports/123456.json")]):
|
||||
with self.subTest(files=files):
|
||||
self.assertEqual(build_pages.content_state(self.entry, archived(files), previous), original)
|
||||
# A first saved file advances the page while its own snapshot is pending.
|
||||
first = build_pages.content_state(self.entry, archived([saved_file(raw_file("poc.py"))]), previous)
|
||||
self.assertNotEqual(first[0], original[0])
|
||||
self.assertEqual(first[1], self.data["today"])
|
||||
later = {**self.data, "today": "2026-10-07"}
|
||||
# Order, repeats and older captures of the same file change nothing the page shows.
|
||||
same = [saved_file(raw_file("poc.py"), "20261021120000"), saved_file(raw_file("poc.py")), saved_file(raw_file("poc.py"))]
|
||||
self.assertEqual(build_pages.content_state(self.entry, archived(same, later), {cid: first}), first)
|
||||
# A refreshed capture, or another file, does.
|
||||
for files in ([saved_file(raw_file("poc.py"), "20261023100000")],
|
||||
[saved_file(raw_file("poc.py")), saved_file(raw_file("lib.py"))]):
|
||||
with self.subTest(files=files):
|
||||
changed = build_pages.content_state(self.entry, archived(files, later), {cid: first})
|
||||
self.assertNotEqual(changed[0], first[0])
|
||||
self.assertEqual(changed[1], "2026-10-07")
|
||||
# A file the page lists in a row of its own is not disclosed, or counted, twice.
|
||||
listed = {**self.entry, "poc": [url, raw_file("poc.py")]}
|
||||
self.assertEqual(build_pages.content_state(listed, archived([saved_file(raw_file("poc.py"))]), {}),
|
||||
build_pages.content_state(listed, self.data, {}))
|
||||
# Pages with a snapshot and no files keep the fingerprint they were published with.
|
||||
snapshot = {"url": f"https://web.archive.org/web/20251021093000/{url}", "timestamp": "20251021093000"}
|
||||
payload = {"entry": self.entry, "cvss": None, "advisories": None, "nuclei": None, "kev": None,
|
||||
"archives": {url: [snapshot["url"], snapshot["timestamp"]]}}
|
||||
self.assertEqual(build_pages.content_state(self.entry, {**self.data, "archives": {url: snapshot}}, {})[0],
|
||||
build_pages.digest(payload))
|
||||
|
||||
def test_missing_manifest_bootstraps_but_outages_do_not_erase_state(self) -> None:
|
||||
with patch.object(build_pages.request, "urlopen", side_effect=HTTPError("url", 404, "missing", {}, None)):
|
||||
self.assertEqual(build_pages.previous_page_state(), {})
|
||||
@@ -253,5 +324,290 @@ class RepositoryMetadataTests(unittest.TestCase):
|
||||
self.assertNotIn("★", row)
|
||||
|
||||
|
||||
class WaybackTests(unittest.TestCase):
|
||||
URL = "https://github.com/owner/poc"
|
||||
STAMP = "20251021093000"
|
||||
SNAPSHOT = f"https://web.archive.org/web/{STAMP}/{URL}"
|
||||
FILES_TITLE = "Wayback Machine captures of individual files at pinned commits, not a copy of the whole repository"
|
||||
|
||||
def render(self, entry: dict, archives: dict, repo_meta: dict | None = None, meta: dict | None = None) -> str:
|
||||
data = {"cves": [entry], "meta": meta or {}, "epss": {}, "kev": {}, "nuclei": {}, "repo_meta": repo_meta or {},
|
||||
"related": {}, "lastmod": {entry["cve"]: "2026-10-21"}, "archives": archives}
|
||||
return build_pages.page(entry, data)
|
||||
|
||||
def test_cve_pages_link_confirmed_snapshots_and_label_lookups(self) -> None:
|
||||
other = "https://www.exploit-db.com/exploits/50592"
|
||||
advisory = "https://vendor.example/advisory"
|
||||
entry = {"cve": "CVE-2026-1234", "desc": "Widget flaw", "poc": [self.URL], "edb": [other]}
|
||||
markup = self.render(entry, {self.URL: {"url": self.SNAPSHOT, "timestamp": self.STAMP}},
|
||||
meta={entry["cve"]: {"advisories": [[advisory, ["Vendor Advisory"]]]}})
|
||||
self.assertIn(
|
||||
'Repo metadata unavailable · '
|
||||
f'<a class="wayback is-snapshot" href="{self.SNAPSHOT}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Wayback Machine page snapshot, captured 2025-10-21 09:30 UTC" '
|
||||
'aria-label="snapshot of owner/poc on the Wayback Machine, captured 2025-10-21 09:30 UTC">snapshot</a>', markup)
|
||||
self.assertIn(
|
||||
f'<span><a class="wayback is-lookup" href="https://web.archive.org/web/*/{other}" target="_blank" '
|
||||
'rel="nofollow noopener" title="Search the Wayback Machine for captures; none confirmed" '
|
||||
f'aria-label="find archive of {other} on the Wayback Machine">find archive</a></span>', markup)
|
||||
# The original stays the PoC link, counts are unchanged and advisories are not PoCs.
|
||||
self.assertIn(f'<li><a href="{self.URL}" rel="nofollow noopener" target="_blank">owner/poc</a>', markup)
|
||||
self.assertIn("<title>CVE-2026-1234: 2 public PoC exploits | PoC Index</title>", markup)
|
||||
advisories = markup.split("<h2>Security advisories</h2>", 1)[1].split("</ul>", 1)[0]
|
||||
self.assertIn(f'href="{advisory}"', advisories)
|
||||
self.assertNotIn("wayback", advisories)
|
||||
self.assertNotIn("web.archive.org/save", markup)
|
||||
|
||||
def test_untrusted_snapshot_records_fall_back_to_a_lookup(self) -> None:
|
||||
url, stamp, snapshot = self.URL, self.STAMP, self.SNAPSHOT
|
||||
for record in (
|
||||
{"url": f"http://web.archive.org/web/{stamp}/{url}", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org.example/web/{stamp}/{url}", "timestamp": stamp},
|
||||
{"url": f"https://example.com/web.archive.org/web/{stamp}/{url}", "timestamp": stamp},
|
||||
{"url": f"https://[email protected]/web/{stamp}/{url}", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org/save/{url}", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org/web/*/{url}", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org/web/{stamp}id_/{url}", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org/web/2025102109300/{url}", "timestamp": "2025102109300"},
|
||||
{"url": f"https://web.archive.org/web/20250230093000/{url}", "timestamp": "20250230093000"},
|
||||
{"url": f"https://web.archive.org/web/20251021240000/{url}", "timestamp": "20251021240000"},
|
||||
{"url": f"https://web.archive.org/web/19951021093000/{url}", "timestamp": "19951021093000"},
|
||||
{"url": snapshot, "timestamp": "20251021093001"},
|
||||
{"url": snapshot, "timestamp": int(stamp)},
|
||||
{"url": snapshot},
|
||||
{"url": f"https://web.archive.org/web/{stamp}/javascript:alert(1)", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org/web/{stamp}/https://example.com/poc", "timestamp": stamp},
|
||||
{"url": f'{snapshot}"><script>alert(1)</script>', "timestamp": stamp},
|
||||
{"url": f"{snapshot} onmouseover=alert(1)", "timestamp": stamp},
|
||||
snapshot, None, [snapshot, stamp],
|
||||
):
|
||||
with self.subTest(record=record):
|
||||
link = build_pages.archive_link(url, "owner/poc", {url: record})
|
||||
self.assertIn(f'<a class="wayback is-lookup" href="https://web.archive.org/web/*/{url}"', link)
|
||||
self.assertNotIn("is-snapshot", link)
|
||||
self.assertNotIn("<script", link)
|
||||
self.assertNotIn("onmouseover", link)
|
||||
# The Wayback Machine may record the scheme, www. or default port differently.
|
||||
canonical = {"url": f"https://web.archive.org/web/{stamp}/http://www.github.com:80/owner/poc", "timestamp": stamp}
|
||||
self.assertIn('class="wayback is-snapshot"', build_pages.archive_link(url, "owner/poc", {url: canonical}))
|
||||
|
||||
def test_archive_links_escape_urls_and_labels(self) -> None:
|
||||
url = "https://example.com/exploit?id=1&name='x'"
|
||||
snapshot = f"https://web.archive.org/web/{self.STAMP}/{url}"
|
||||
for archives, href, kind in (({}, f"https://web.archive.org/web/*/{url}", "is-lookup"),
|
||||
({url: {"url": snapshot, "timestamp": self.STAMP}}, snapshot, "is-snapshot")):
|
||||
with self.subTest(kind=kind):
|
||||
markup = build_pages.archive_link(url, '<b>"label"</b>', archives)
|
||||
(tag, attrs), = ParsedPage(markup).tags
|
||||
self.assertEqual((tag, attrs["class"], attrs["href"]), ("a", f"wayback {kind}", href))
|
||||
self.assertIn('<b>"label"</b>', attrs["aria-label"])
|
||||
self.assertNotIn("<b>", markup)
|
||||
|
||||
def test_archived_fallbacks_link_their_snapshot_under_the_original_name(self) -> None:
|
||||
original = "https://github.com/owner/gone-poc"
|
||||
fallback = f"https://web.archive.org/web/20240101120000/{original}"
|
||||
live = "https://github.com/owner/live"
|
||||
collection = "https://github.com/google/security-research/tree/master/pocs/linux/kernelctf/CVE-2024-26642_lts"
|
||||
entry = {"cve": "CVE-2026-1234", "desc": "Widget flaw", "poc": [live, fallback],
|
||||
"collections": [f"https://web.archive.org/web/20240101120000/{collection}"]}
|
||||
markup = self.render(entry, {original: {"url": fallback, "timestamp": "20240101120000"}},
|
||||
{"owner/gone-poc": [50, "2024-01-01"], "owner/live": [10, "2026-10-01"]})
|
||||
stamp = ('<span class="wayback is-archived" title="Wayback Machine page snapshot captured 2024-01-01 12:00 UTC, '
|
||||
'linked in place of the original" aria-hidden="true">ARCHIVED</span>')
|
||||
rows = re.findall(r'<li><a href="([^"]*)"([^>]*)>([^<]*)</a>(?:<span>(.*?)</span>)?</li>', markup)
|
||||
# Ranked, named and measured as the original, linked to the snapshot.
|
||||
self.assertEqual(rows[:3], [
|
||||
(fallback, ' rel="nofollow noopener" target="_blank" '
|
||||
'aria-label="owner/gone-poc, archived page snapshot captured 2024-01-01 12:00 UTC"',
|
||||
"owner/gone-poc", f"Repo stars 50★ · pushed 2024-01-01 · {stamp}"),
|
||||
(live, ' rel="nofollow noopener" target="_blank"', "owner/live",
|
||||
"Repo stars 10★ · pushed 2026-10-01 · "
|
||||
f'<a class="wayback is-lookup" href="https://web.archive.org/web/*/{live}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Search the Wayback Machine for captures; none confirmed" '
|
||||
'aria-label="find archive of owner/live on the Wayback Machine">find archive</a>'),
|
||||
(f"https://web.archive.org/web/20240101120000/{collection}",
|
||||
' rel="noopener" target="_blank" aria-label="linux/kernelctf/CVE-2024-26642_lts, archived page snapshot '
|
||||
'captured 2024-01-01 12:00 UTC"',
|
||||
"linux/kernelctf/CVE-2024-26642_lts", f"google/security-research · Repo metadata unavailable · {stamp}"),
|
||||
])
|
||||
self.assertNotIn(f'href="{original}"', markup)
|
||||
self.assertNotIn(f'href="{collection}"', markup)
|
||||
self.assertNotIn("/web/*/https://web.archive.org", markup)
|
||||
self.assertIn("3 public PoC exploits", markup)
|
||||
|
||||
def test_cve_pages_disclose_saved_files_beside_snapshots_and_lookups(self) -> None:
|
||||
pending = "https://github.com/Owner/Other-PoC"
|
||||
report = "https://hackerone.com/reports/123456"
|
||||
newest, spaced, report_json = (saved_file(raw_file("poc.py")), saved_file(raw_file("src/exploit%20one.py")),
|
||||
saved_file(f"{report}.json"))
|
||||
entry = {"cve": "CVE-2026-1234", "desc": "Widget flaw", "poc": [self.URL, pending, report]}
|
||||
markup = self.render(entry, {
|
||||
# One capture per file, the latest, however often the record lists it.
|
||||
self.URL: {"url": self.SNAPSHOT, "timestamp": self.STAMP,
|
||||
"files": [spaced, saved_file(raw_file("poc.py"), "20261021120000"), newest, newest]},
|
||||
# A page whose own snapshot is still pending.
|
||||
pending: {"files": [saved_file(raw_file("run.sh", "owner/other-poc"))]},
|
||||
report: {"files": [report_json]},
|
||||
# Each file is also listed under its own original, which is no row.
|
||||
newest["original"]: {"url": newest["url"], "timestamp": newest["timestamp"]},
|
||||
})
|
||||
self.assertIn(
|
||||
'aria-label="snapshot of owner/poc on the Wayback Machine, captured 2025-10-21 09:30 UTC">snapshot</a></span>'
|
||||
f'<details class="sources" style="--files:2" data-sources="{self.URL}"><summary title="{self.FILES_TITLE}" '
|
||||
'aria-label="2 files saved from owner/poc">2 files</summary><div class="sources-list">'
|
||||
f'<a href="{newest["url"]}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Wayback Machine capture of poc.py at commit 0123456, 2026-10-22 16:10 UTC" '
|
||||
'aria-label="poc.py at commit 0123456 of owner/poc, Wayback Machine capture 2026-10-22 16:10 UTC">poc.py</a>'
|
||||
f'<a href="{spaced["url"]}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Wayback Machine capture of src/exploit one.py at commit 0123456, 2026-10-22 16:10 UTC" '
|
||||
'aria-label="src/exploit one.py at commit 0123456 of owner/poc, Wayback Machine capture 2026-10-22 16:10 UTC">'
|
||||
"src/exploit one.py</a></div></details></li>", markup)
|
||||
self.assertNotIn("20261021120000", markup)
|
||||
self.assertIn(
|
||||
f'find archive</a></span><details class="sources" style="--files:1" data-sources="{pending}">'
|
||||
f'<summary title="{self.FILES_TITLE}" aria-label="1 file saved from Owner/Other-PoC">1 file</summary>', markup)
|
||||
self.assertIn(
|
||||
f'find archive</a></span><details class="sources" style="--files:1" data-sources="{report}">'
|
||||
f'<summary title="Wayback Machine capture of the report JSON" aria-label="1 file saved from {report}">1 file</summary>'
|
||||
f'<div class="sources-list"><a href="{report_json["url"]}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Wayback Machine capture of 123456.json, the report JSON, 2026-10-22 16:10 UTC" '
|
||||
f'aria-label="123456.json, the report JSON of {report}, Wayback Machine capture 2026-10-22 16:10 UTC">'
|
||||
"123456.json</a></div></details></li>", markup)
|
||||
# Saved files are no PoCs of their own, and nothing saved is looked up again.
|
||||
self.assertIn("<title>CVE-2026-1234: 3 public PoC exploits | PoC Index</title>", markup)
|
||||
self.assertEqual(markup.count('class="wayback '), 3)
|
||||
self.assertEqual(markup.count("<details "), 3)
|
||||
self.assertNotRegex(markup, r"/web/\*/https://(?:web\.archive\.org|raw\.githubusercontent\.com|hackerone\.com/reports/123456\.json)")
|
||||
|
||||
def test_saved_files_must_be_captures_of_the_linked_pages_own_files(self) -> None:
|
||||
good = saved_file(raw_file("poc.py"))
|
||||
stamp = good["timestamp"]
|
||||
|
||||
def disclosed(parent: str, files: object) -> str:
|
||||
return build_pages.source_files(parent, "label", {parent: {"files": files}})
|
||||
|
||||
# GitHub names repositories without regard to case or a .git suffix.
|
||||
for parent in (self.URL, "https://github.com/Owner/POC.git", "https://www.github.com/owner/poc/blob/main/README.md"):
|
||||
with self.subTest(parent=parent):
|
||||
self.assertIn("1 file</summary>", disclosed(parent, [good]))
|
||||
for file in (
|
||||
saved_file(raw_file("poc.py", "owner/other")),
|
||||
saved_file(raw_file("poc.py", "other/poc")),
|
||||
saved_file(raw_file("poc.py", commit="main")),
|
||||
saved_file(raw_file("poc.py", commit=COMMIT[1:])),
|
||||
saved_file(raw_file("poc.py", commit=COMMIT.upper())),
|
||||
# A dot segment would open another repository's file.
|
||||
saved_file(raw_file(f"../../other/poc/{COMMIT}/poc.py")),
|
||||
saved_file(raw_file("%2e%2E/poc.py")),
|
||||
saved_file(raw_file("./poc.py")),
|
||||
saved_file(raw_file("src//poc.py")),
|
||||
saved_file(raw_file("poc.py?token=1")),
|
||||
saved_file(raw_file("poc.py#L1")),
|
||||
saved_file(f"http://raw.githubusercontent.com/owner/poc/{COMMIT}/poc.py"),
|
||||
saved_file(f"https://raw.githubusercontent.com.example/owner/poc/{COMMIT}/poc.py"),
|
||||
saved_file(f"https://gist.githubusercontent.com/owner/poc/raw/{COMMIT}/poc.py"),
|
||||
saved_file(f"https://github.com/owner/poc/blob/{COMMIT}/poc.py"),
|
||||
saved_file(f"https://example.com/owner/poc/{COMMIT}/poc.py"),
|
||||
saved_file("https://hackerone.com/reports/123456.json"),
|
||||
# An archive link is never saved again.
|
||||
saved_file(good["url"]),
|
||||
{**good, "url": saved_file(raw_file("other.py"))["url"]},
|
||||
{**good, "url": f"https://web.archive.org/web/{stamp}/https://example.com/owner/poc/{COMMIT}/poc.py"},
|
||||
{**good, "timestamp": "20261022161014"},
|
||||
{**good, "timestamp": int(stamp)},
|
||||
{**good, "url": f"https://web.archive.org/web/{stamp}id_/{good['original']}"},
|
||||
{**good, "url": f"https://web.archive.org/web/*/{good['original']}"},
|
||||
{**good, "url": f"http://web.archive.org/web/{stamp}/{good['original']}"},
|
||||
{**good, "url": f"https://web.archive.org/web/20250230093000/{good['original']}", "timestamp": "20250230093000"},
|
||||
{"url": good["url"], "timestamp": stamp},
|
||||
good["url"], None, [good],
|
||||
):
|
||||
with self.subTest(file=file):
|
||||
self.assertEqual(disclosed(self.URL, [file]), "")
|
||||
for files in (good, "files", None, {}):
|
||||
with self.subTest(files=files):
|
||||
self.assertEqual(disclosed(self.URL, files), "")
|
||||
# Advisories, gists, other hosts and files themselves hold no repository files.
|
||||
for parent in ("https://github.com/owner/poc/security/advisories/GHSA-1234-5678-9abc",
|
||||
"https://gist.github.com/owner/0123", "https://example.com/owner/poc", good["original"]):
|
||||
with self.subTest(parent=parent):
|
||||
self.assertEqual(disclosed(parent, [good]), "")
|
||||
# A HackerOne report holds its own report JSON and nothing else.
|
||||
report_json = saved_file("https://hackerone.com/reports/123456.json")
|
||||
for parent in ("https://hackerone.com/reports/123456", "https://www.hackerone.com/reports/123456/",
|
||||
"https://hackerone.com/reports/123456?subject=x"):
|
||||
with self.subTest(parent=parent):
|
||||
self.assertIn("1 file</summary>", disclosed(parent, [report_json]))
|
||||
for parent, file in (
|
||||
("https://hackerone.com/reports/123456", saved_file("https://hackerone.com/reports/654321.json")),
|
||||
("https://hackerone.com/reports/123456", saved_file("https://hackerone.example/reports/123456.json")),
|
||||
("https://hackerone.com/reports/123456", saved_file("https://hackerone.com/reports/123456")),
|
||||
("https://hackerone.com/reports/123456", saved_file("https://hackerone.com/reports/123456.json?x=1")),
|
||||
("https://hackerone.com/reports/123456", good),
|
||||
("https://hackerone.com/reports/123456.json", report_json),
|
||||
("https://hackerone.com/bugs?report_id=123456", report_json),
|
||||
("https://example.com/reports/123456", report_json),
|
||||
):
|
||||
with self.subTest(parent=parent, file=file):
|
||||
self.assertEqual(disclosed(parent, [file]), "")
|
||||
|
||||
def test_saved_file_names_links_and_labels_are_escaped(self) -> None:
|
||||
odd = saved_file(raw_file("x%22%3E%3Cimg%20src%3Dx%20onerror%3Dalert(1)%3E/it's.py"))
|
||||
malformed = saved_file(raw_file("100%.py"))
|
||||
markup =build_pages.source_files(self.URL, '<b>"label"</b>', {self.URL: {"files": [odd, malformed]}})
|
||||
page = ParsedPage(markup)
|
||||
self.assertEqual([tag for tag, _ in page.tags], ["details", "summary", "div", "a", "a"])
|
||||
self.assertEqual(page.tags[1][1]["aria-label"], '2 files saved from <b>"label"</b>')
|
||||
links = [attrs for tag, attrs in page.tags if tag == "a"]
|
||||
self.assertEqual([attrs["href"] for attrs in links], [malformed["url"], odd["url"]])
|
||||
self.assertEqual(links[1]["aria-label"], 'x"><img src=x onerror=alert(1)>/it\'s.py at commit 0123456 of '
|
||||
'<b>"label"</b>, Wayback Machine capture 2026-10-22 16:10 UTC')
|
||||
# A malformed escape is shown as written.
|
||||
self.assertEqual(page.text[1:], ["100%.py", 'x"><img src=x onerror=alert(1)>/it\'s.py'])
|
||||
self.assertNotIn("<img", markup)
|
||||
self.assertNotIn("<b>", markup)
|
||||
|
||||
def test_archived_fallbacks_disclose_their_original_saved_files_once(self) -> None:
|
||||
original = "https://github.com/owner/gone-poc"
|
||||
fallback = f"https://web.archive.org/web/20240101120000/{original}"
|
||||
exploit = saved_file(raw_file("exploit.c", "owner/gone-poc"))
|
||||
listed, helper = saved_file(raw_file("poc.py")), saved_file(raw_file("helper.py"))
|
||||
entry = {"cve": "CVE-2026-1234", "desc": "Widget flaw", "poc": [fallback, self.URL, listed["original"]]}
|
||||
archives = {original: {"url": fallback, "timestamp": "20240101120000", "files": [exploit]},
|
||||
self.URL: {"files": [listed, helper]},
|
||||
listed["original"]: {"url": listed["url"], "timestamp": listed["timestamp"]}}
|
||||
markup = self.render(entry, archives)
|
||||
self.assertIn('linked in place of the original" aria-hidden="true">ARCHIVED</span></span>'
|
||||
f'<details class="sources" style="--files:1" data-sources="{fallback}"><summary', markup)
|
||||
self.assertIn(f'<a href="{exploit["url"]}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Wayback Machine capture of exploit.c at commit 0123456, 2026-10-22 16:10 UTC" '
|
||||
'aria-label="exploit.c at commit 0123456 of owner/gone-poc, Wayback Machine capture '
|
||||
'2026-10-22 16:10 UTC">exploit.c</a>', markup)
|
||||
# The lost original is never linked, and no capture is looked up again.
|
||||
self.assertNotIn(f'href="{original}"', markup)
|
||||
self.assertNotRegex(markup, r"/web/\*/https://(?:web\.archive\.org|raw\.githubusercontent\.com)")
|
||||
# poc.py has a row and a snapshot of its own, so its page discloses only helper.py.
|
||||
self.assertIn('aria-label="1 file saved from owner/poc">1 file</summary>', markup)
|
||||
self.assertIn(f'<a href="{helper["url"]}"', markup)
|
||||
self.assertEqual(markup.count(f'href="{listed["url"]}"'), 1)
|
||||
self.assertIn("3 public PoC exploits", markup)
|
||||
# The page's fingerprint follows the files it shows.
|
||||
data = {"today": "2026-10-21", "meta": {}, "nuclei": {}, "kev": {}, "repo_meta": {}}
|
||||
without = {**archives, original: {"url": fallback, "timestamp": "20240101120000"}}
|
||||
self.assertNotEqual(build_pages.content_state(entry, {**data, "archives": archives}, {})[0],
|
||||
build_pages.content_state(entry, {**data, "archives": without}, {})[0])
|
||||
|
||||
def test_a_missing_archive_index_is_empty_and_a_malformed_one_stops_the_build(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory, patch.object(build_pages, "DOCS", directory):
|
||||
self.assertEqual(build_pages.load_optional("archives.json"), {})
|
||||
index = Path(directory, "archives.json")
|
||||
records = {self.URL: {"url": self.SNAPSHOT, "timestamp": self.STAMP}}
|
||||
index.write_text(json.dumps(records), encoding="utf-8")
|
||||
self.assertEqual(build_pages.load_optional("archives.json"), records)
|
||||
index.write_text("[]", encoding="utf-8")
|
||||
with self.assertRaisesRegex(ValueError, "archives.json"):
|
||||
build_pages.load_optional("archives.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -77,8 +77,10 @@ const context = vm.createContext({
|
||||
location: { pathname: '/', search: '' },
|
||||
window: { matchMedia: () => ({ matches: false }) },
|
||||
setTimeout() {}, clearTimeout() {},
|
||||
fetch: url => url === '/trending_poc.json'
|
||||
? Promise.resolve({ ok: true, json: async () => input.feed })
|
||||
// With archives, an empty index lets the archive download that follows it run.
|
||||
fetch: url => url === '/trending_poc.json' ? Promise.resolve({ ok: true, json: async () => input.feed })
|
||||
: input.archives && url === '/CVE_list.json' ? Promise.resolve({ ok: true, json: async () => [] })
|
||||
: input.archives && url === '/archives.json' ? Promise.resolve({ ok: true, json: async () => input.archives })
|
||||
: new Promise(() => {})
|
||||
});
|
||||
vm.runInContext(fs.readFileSync(input.script, 'utf8'), context);
|
||||
@@ -100,12 +102,13 @@ setImmediate(() => {
|
||||
"""
|
||||
|
||||
|
||||
def browser(feed: dict) -> dict:
|
||||
"""What logic.js shows once the feed arrives, then for each switch value."""
|
||||
def browser(feed: dict, archives: dict | None = None) -> dict:
|
||||
"""What logic.js shows once the feed, and any archive index, arrives, then
|
||||
for each switch value."""
|
||||
node = shutil.which("node")
|
||||
if node is None:
|
||||
raise unittest.SkipTest("node renders the browser lists")
|
||||
payload = {"script": str(ROOT / "docs" / "logic.js"), "feed": feed, "now": NOW.isoformat()}
|
||||
payload = {"script": str(ROOT / "docs" / "logic.js"), "feed": feed, "now": NOW.isoformat(), "archives": archives}
|
||||
done = subprocess.run([node, "-e", HARNESS], input=json.dumps(payload), capture_output=True,
|
||||
text=True, timeout=60)
|
||||
if done.returncode:
|
||||
@@ -191,6 +194,64 @@ class HomepageListTests(unittest.TestCase):
|
||||
self.assertEqual([self.views[mode]["pressed"] for mode in ("initial", "TRENDING", "RECENT")],
|
||||
[["true", "false"], ["false", "true"], ["true", "false"]])
|
||||
|
||||
def test_wayback_snapshots_and_lookups_match_in_browser_and_prerender(self) -> None:
|
||||
stamp = "20261006160000"
|
||||
archives = {
|
||||
SHARED: {"url": f"https://web.archive.org/web/{stamp}/{SHARED}", "timestamp": stamp},
|
||||
# Recorded with another scheme, www. and port: still this page.
|
||||
LANDED[2]["url"]: {"url": f"https://web.archive.org/web/{stamp}/http://exploit-db.com:80/exploits/50592",
|
||||
"timestamp": stamp},
|
||||
# Another site's capture is not this link's snapshot.
|
||||
KERNELCTF + "mitigation": {"url": f"https://web.archive.org/web/{stamp}/https://example.com/", "timestamp": stamp},
|
||||
}
|
||||
views = browser(FEED, archives)
|
||||
landed = build_pages.trend_rows(LANDED, True, NOW, archives)
|
||||
self.assertEqual(views["initial"]["rows"], as_browser(landed))
|
||||
self.assertEqual(views["TRENDING"]["rows"], as_browser(build_pages.trend_rows(TRENDING, False, NOW, archives)))
|
||||
self.assertIn(landed, build_pages.homepage(INDEXED, {}, FEED, NOW, archives))
|
||||
self.assertEqual(landed.count('class="wayback is-snapshot"'), 2)
|
||||
self.assertIn(f'<a class="wayback is-lookup" href="https://web.archive.org/web/*/{KERNELCTF}mitigation"', landed)
|
||||
self.assertIn('aria-label="snapshot of CVE-2026-21589 on the Wayback Machine, captured 2026-10-06 16:00 UTC"', landed)
|
||||
# Everything else in the rows reads as it does without archives.
|
||||
self.assertEqual(cells(views["initial"]["rows"]), cells(self.views["LANDED"]["rows"]))
|
||||
|
||||
def test_saved_files_match_in_browser_and_prerender(self) -> None:
|
||||
stamp, commit = "20261006160000", "0123456789abcdef0123456789abcdef01234567"
|
||||
exploit = (f"https://raw.githubusercontent.com/google/security-research/{commit}/"
|
||||
"pocs/linux/kernelctf/CVE-2024-26642_mitigation/exploit.c")
|
||||
poc = f"https://raw.githubusercontent.com/owner/cve-2026-21589/{commit}/poc.py"
|
||||
|
||||
def saved(original: str) -> dict:
|
||||
return {"original": original, "url": f"https://web.archive.org/web/{stamp}/{original}", "timestamp": stamp}
|
||||
|
||||
archives = {
|
||||
SHARED: {"url": f"https://web.archive.org/web/{stamp}/{SHARED}", "timestamp": stamp, "files": [saved(poc)]},
|
||||
# Saved files while the page snapshot is pending.
|
||||
KERNELCTF + "mitigation": {"files": [saved(exploit)]},
|
||||
# Not this page's file.
|
||||
LANDED[2]["url"]: {"files": [saved(poc)]},
|
||||
}
|
||||
views = browser(FEED, archives)
|
||||
landed = build_pages.trend_rows(LANDED, True, NOW, archives)
|
||||
trending = build_pages.trend_rows(TRENDING, False, NOW, archives)
|
||||
self.assertEqual(views["initial"]["rows"], as_browser(landed))
|
||||
self.assertEqual(views["TRENDING"]["rows"], as_browser(trending))
|
||||
self.assertIn(landed, build_pages.homepage(INDEXED, {}, FEED, NOW, archives))
|
||||
self.assertEqual((landed.count("<details "), trending.count("<details ")), (2, 1))
|
||||
self.assertIn(
|
||||
f'find archive</a><details class="sources" style="--files:1" data-sources="{KERNELCTF}mitigation">'
|
||||
'<summary title="Wayback Machine captures of individual files at pinned commits, not a copy of the whole '
|
||||
'repository" aria-label="1 file saved from CVE-2024-26642_mitigation">1 file</summary>'
|
||||
f'<div class="sources-list"><a href="https://web.archive.org/web/{stamp}/{exploit}" target="_blank" '
|
||||
'rel="nofollow noopener" title="Wayback Machine capture of pocs/linux/kernelctf/CVE-2024-26642_mitigation/'
|
||||
'exploit.c at commit 0123456, 2026-10-06 16:00 UTC" aria-label="pocs/linux/kernelctf/CVE-2024-26642_mitigation/'
|
||||
'exploit.c at commit 0123456 of CVE-2024-26642_mitigation, Wayback Machine capture 2026-10-06 16:00 UTC">'
|
||||
'pocs/linux/kernelctf/CVE-2024-26642_mitigation/exploit.c</a></div></details></div></div>', landed)
|
||||
self.assertIn(f'snapshot</a><details class="sources" style="--files:1" data-sources="{SHARED}">', trending)
|
||||
# Everything else in the rows reads as it does without archives.
|
||||
self.assertEqual(cells(views["initial"]["rows"]), cells(self.views["LANDED"]["rows"]))
|
||||
self.assertEqual(cells(views["TRENDING"]["rows"]), cells(self.views["TRENDING"]["rows"]))
|
||||
|
||||
def test_an_empty_list_shows_the_empty_state_in_both(self) -> None:
|
||||
empty = build_pages.trend_rows([], True)
|
||||
self.assertIn("No recent PoCs.", empty)
|
||||
|
||||
+385
-2
@@ -9,7 +9,8 @@ const { MessageChannel } = require('node:worker_threads');
|
||||
|
||||
const script = fs.readFileSync(path.join(__dirname, '../docs/logic.js'), 'utf8');
|
||||
|
||||
function startPage(t, { search = '', trending, kev = {}, repositories = {}, indexError = false, cvssError = false, entries }) {
|
||||
function startPage(t, { search = '', trending, kev = {}, repositories = {}, archives = {}, published = [],
|
||||
indexError = false, cvssError = false, archivesError = false, entries }) {
|
||||
const elements = new Map();
|
||||
const calls = [];
|
||||
const document = {
|
||||
@@ -19,7 +20,9 @@ function startPage(t, { search = '', trending, kev = {}, repositories = {}, inde
|
||||
if (!elements.has(selector)) elements.set(selector, {
|
||||
value: '',
|
||||
innerHTML: selector === '[data-trend-rows]' ? '<a href="/CVE-2026-1234">Prerendered PoC</a>' : '',
|
||||
addEventListener() {}, setAttribute() {}, querySelectorAll: () => []
|
||||
addEventListener() {}, setAttribute() {},
|
||||
// The snapshot links a prerendered homepage carries.
|
||||
querySelectorAll: () => selector === '[data-trend-rows]' ? published : []
|
||||
});
|
||||
return elements.get(selector);
|
||||
}
|
||||
@@ -33,9 +36,11 @@ function startPage(t, { search = '', trending, kev = {}, repositories = {}, inde
|
||||
calls.push(url);
|
||||
if (url === '/CVE_list.json' && indexError) throw new Error('Index unavailable');
|
||||
if (url === '/cvss.json' && cvssError) throw new Error('CVSS unavailable');
|
||||
if (url === '/archives.json' && archivesError) throw new Error('Archive index unavailable');
|
||||
const data = url === '/trending_poc.json' ? await trending
|
||||
: url === '/kev.json' ? await kev
|
||||
: url === '/repo_meta.json' ? await repositories
|
||||
: url === '/archives.json' ? await archives
|
||||
: url === '/CVE_list.json' ? entries || [{ cve: 'CVE-2026-1234', desc: 'widget', poc: ['https://github.com/owner/poc'] }]
|
||||
: {};
|
||||
return { ok: true, json: async () => data };
|
||||
@@ -55,6 +60,13 @@ async function searchFixture(t, entries, options = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
// A source file saved at a pinned commit, as archives.json lists it under its page.
|
||||
const COMMIT = '0123456789abcdef0123456789abcdef01234567';
|
||||
const rawFile = (path, repository = 'owner/poc', commit = COMMIT) =>
|
||||
`https://raw.githubusercontent.com/${repository}/${commit}/${path}`;
|
||||
const savedFile = (original, stamp = '20261022161013') =>
|
||||
({ original, url: `https://web.archive.org/web/${stamp}/${original}`, timestamp: stamp });
|
||||
|
||||
test('repository name prefixes beat incidental prose without joining numbered filenames', async t => {
|
||||
const search = await searchFixture(t, [
|
||||
{ cve: 'CVE-2004-0660', desc: 'XSS in (1) show_archives.php, (2) show_news.php', edb: ['https://www.exploit-db.com/exploits/24238'] },
|
||||
@@ -366,6 +378,377 @@ test('curated artifact paths survive search rendering without duplicate reposito
|
||||
assert.equal(vm.runInContext(`normalizedLink(${JSON.stringify(variants[0])}) === normalizedLink(${JSON.stringify(base)})`, context), true);
|
||||
});
|
||||
|
||||
test('PoC rows link a confirmed Wayback snapshot and otherwise offer a labelled lookup', async t => {
|
||||
const archived = 'https://github.com/owner/archived-poc';
|
||||
const other = 'https://www.exploit-db.com/exploits/50592';
|
||||
const snapshot = `https://web.archive.org/web/20251021093000/${archived}`;
|
||||
const entries = [{ cve: 'CVE-2026-1234', desc: 'widget', poc: [archived], edb: [other] }];
|
||||
const { elements } = startPage(t, { search: '?q=widget', entries, trending: new Promise(() => {}),
|
||||
archives: { [archived]: { url: snapshot, timestamp: '20251021093000' } } });
|
||||
await new Promise(setImmediate);
|
||||
const html = elements.get('[data-results]').innerHTML;
|
||||
assert.ok(html.includes(`<a class="wayback is-snapshot" href="${snapshot}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Wayback Machine page snapshot, captured 2025-10-21 09:30 UTC" ' +
|
||||
'aria-label="snapshot of owner/archived-poc on the Wayback Machine, captured 2025-10-21 09:30 UTC">snapshot</a>'));
|
||||
assert.ok(html.includes(`<a class="wayback is-lookup" href="https://web.archive.org/web/*/${other}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Search the Wayback Machine for captures; none confirmed" ' +
|
||||
'aria-label="find archive of exploit-db.com/50592 on the Wayback Machine">find archive</a>'));
|
||||
assert.equal(html.match(/class="wayback /g).length, 2);
|
||||
// The original stays the PoC link and the count is unchanged.
|
||||
assert.ok(html.includes(`<a href="${archived}" target="_blank" rel="nofollow noopener">archived-poc</a>`));
|
||||
assert.match(html, /2 linked PoCs/);
|
||||
assert.doesNotMatch(html, /web\.archive\.org\/save/);
|
||||
});
|
||||
|
||||
test('malformed or untrusted snapshot records fall back to a lookup and links are escaped', async t => {
|
||||
const { context } = startPage(t, { trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
const url = 'https://github.com/owner/poc';
|
||||
const stamp = '20251021093000';
|
||||
const snapshot = `https://web.archive.org/web/${stamp}/${url}`;
|
||||
const untrusted = [
|
||||
{ url: `http://web.archive.org/web/${stamp}/${url}`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org.example/web/${stamp}/${url}`, timestamp: stamp },
|
||||
{ url: `https://example.com/web.archive.org/web/${stamp}/${url}`, timestamp: stamp },
|
||||
{ url: `https://[email protected]/web/${stamp}/${url}`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org/save/${url}`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org/web/*/${url}`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org/web/${stamp}id_/${url}`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org/web/2025102109300/${url}`, timestamp: '2025102109300' },
|
||||
{ url: `https://web.archive.org/web/20250230093000/${url}`, timestamp: '20250230093000' },
|
||||
{ url: `https://web.archive.org/web/20251021240000/${url}`, timestamp: '20251021240000' },
|
||||
{ url: `https://web.archive.org/web/19951021093000/${url}`, timestamp: '19951021093000' },
|
||||
{ url: snapshot, timestamp: '20251021093001' },
|
||||
{ url: snapshot, timestamp: Number(stamp) },
|
||||
{ url: snapshot },
|
||||
{ url: `https://web.archive.org/web/${stamp}/javascript:alert(1)`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org/web/${stamp}/https://example.com/poc`, timestamp: stamp },
|
||||
{ url: `${snapshot}"><script>alert(1)</script>`, timestamp: stamp },
|
||||
{ url: `${snapshot} onmouseover=alert(1)`, timestamp: stamp },
|
||||
snapshot, null, [snapshot, stamp]
|
||||
];
|
||||
for (const record of untrusted) {
|
||||
context.record = record;
|
||||
const row = vm.runInContext(`archives = { ${JSON.stringify(url)}: record }; pocRow(${JSON.stringify(url)})`, context);
|
||||
assert.ok(row.includes(`<a class="wayback is-lookup" href="https://web.archive.org/web/*/${url}"`), JSON.stringify(record));
|
||||
assert.doesNotMatch(row, /is-snapshot|<script|onmouseover/);
|
||||
}
|
||||
// The Wayback Machine may record the scheme, www. or default port differently.
|
||||
context.record = { url: `https://web.archive.org/web/${stamp}/http://www.github.com:80/owner/poc`, timestamp: stamp };
|
||||
assert.match(vm.runInContext(`archives = { ${JSON.stringify(url)}: record }; pocRow(${JSON.stringify(url)})`, context),
|
||||
/class="wayback is-snapshot"/);
|
||||
const query = "https://example.com/exploit?id=1&name='x'";
|
||||
context.record = { url: `https://web.archive.org/web/${stamp}/${query}`, timestamp: stamp };
|
||||
const linked = vm.runInContext(`archives = { ${JSON.stringify(query)}: record }; archiveLink(${JSON.stringify(query)}, '<b>"label"</b>')`, context);
|
||||
assert.ok(linked.includes(`href="https://web.archive.org/web/${stamp}/https://example.com/exploit?id=1&name='x'"`));
|
||||
assert.ok(linked.includes('aria-label="snapshot of <b>"label"</b> on the Wayback Machine, captured 2025-10-21 09:30 UTC"'));
|
||||
const lookup = vm.runInContext(`archives = {}; archiveLink(${JSON.stringify(query)}, 'label')`, context);
|
||||
assert.ok(lookup.includes('href="https://web.archive.org/web/*/https://example.com/exploit?id=1&name='x'"'));
|
||||
});
|
||||
|
||||
test('an archived fallback stands in for its original without a second archive link', async t => {
|
||||
const original = 'https://github.com/owner/php2shell-poc';
|
||||
const fallback = `https://web.archive.org/web/20240101120000/${original}`;
|
||||
const live = 'https://github.com/owner/live';
|
||||
const entries = [
|
||||
{ cve: 'CVE-2026-1000', desc: 'Pending details', poc: [live, fallback] },
|
||||
{ cve: 'CVE-2026-2000', desc: 'A report mentions php2shell in passing.', poc: ['https://github.com/owner/unrelated'] }
|
||||
];
|
||||
const { context, elements } = startPage(t, { search: '?q=php2shell', entries, trending: new Promise(() => {}),
|
||||
repositories: { 'owner/php2shell-poc': [50, '2024-01-01'], 'owner/live': [10, '2026-10-01'] },
|
||||
archives: { [original]: { url: fallback, timestamp: '20240101120000' } } });
|
||||
while (!vm.runInContext('wordIndex.ready', context)) await new Promise(setImmediate);
|
||||
await new Promise(setImmediate);
|
||||
const search = (query, scan = false) => {
|
||||
context.query = query;
|
||||
return Array.from(vm.runInContext(`runSearch(query, ${scan})`, context), row => row.cve);
|
||||
};
|
||||
// Matched, filtered and ranked as the original would be, not as web.archive.org.
|
||||
assert.deepEqual(search('php2shell'), ['CVE-2026-1000', 'CVE-2026-2000']);
|
||||
assert.deepEqual(search('php2shell'), search('php2shell', true));
|
||||
assert.deepEqual(search('archive'), []);
|
||||
assert.deepEqual([...vm.runInContext('entrySources(dataset[0])', context)], ['GITHUB']);
|
||||
assert.deepEqual(Array.from(vm.runInContext('rankedLinks(dataset[0])', context)), [fallback, live]);
|
||||
const html = elements.get('[data-results]').innerHTML;
|
||||
assert.match(html, /2 linked PoCs/);
|
||||
assert.ok(html.includes(`owner<span class="poc-sep"> / </span><a href="${fallback}" ` +
|
||||
'aria-label="php2shell-poc, archived page snapshot captured 2024-01-01 12:00 UTC" target="_blank" rel="nofollow noopener">php2shell-poc</a>'));
|
||||
assert.ok(html.includes('<span class="wayback is-archived" title="Wayback Machine page snapshot captured 2024-01-01 12:00 UTC, ' +
|
||||
'linked in place of the original" aria-hidden="true">ARCHIVED</span>'));
|
||||
// The lost original is never linked, and the snapshot gets no archive link of its own.
|
||||
assert.ok(!html.includes(`href="${original}"`));
|
||||
assert.ok(!html.includes(`/web/*/${fallback}`));
|
||||
assert.equal(html.match(/class="wayback is-lookup"/g).length, 2);
|
||||
const template = 'https://github.com/projectdiscovery/nuclei-templates/blob/main/http/cves/2021/CVE-2021-44228.yaml';
|
||||
assert.match(vm.runInContext(`pocRow('https://web.archive.org/web/20240101120000/${template}', true)`, context),
|
||||
/<span class="poc-tag is-nuclei"[^>]*>NUCLEI<\/span><a href="https:\/\/web\.archive\.org\/web\/20240101120000\/https:[^"]+" aria-label="CVE-2021-44228\.yaml, archived page snapshot captured 2024-01-01 12:00 UTC"[^>]*>CVE-2021-44228\.yaml<\/a>/);
|
||||
// Other archive links, readable or not, get no Wayback control at all.
|
||||
for (const url of ['https://web.archive.org/web/2011/https://example.com/old', 'https://archive.org/details/exploit-kit']) {
|
||||
assert.doesNotMatch(vm.runInContext(`pocRow(${JSON.stringify(url)})`, context), /class="wayback/);
|
||||
}
|
||||
});
|
||||
|
||||
test('saved files are disclosed beside a snapshot or a lookup without counting as PoCs', async t => {
|
||||
const repo = 'https://github.com/owner/poc';
|
||||
const pending = 'https://github.com/Owner/Other-PoC';
|
||||
const report = 'https://hackerone.com/reports/123456';
|
||||
const live = 'https://github.com/owner/live';
|
||||
const page = `https://web.archive.org/web/20261022160913/${repo}`;
|
||||
const newest = savedFile(rawFile('poc.py'));
|
||||
const spaced = savedFile(rawFile('src/exploit%20one.py'));
|
||||
const json = savedFile(`${report}.json`);
|
||||
const entries = [{ cve: 'CVE-2026-1234', desc: 'widget', poc: [repo, pending, report, live] }];
|
||||
const archives = {
|
||||
// One capture per file, the latest, however often the record lists it.
|
||||
[repo]: { url: page, timestamp: '20261022160913',
|
||||
files: [spaced, savedFile(rawFile('poc.py'), '20261021120000'), newest, newest] },
|
||||
// A page whose own snapshot is still pending.
|
||||
[pending]: { files: [savedFile(rawFile('run.sh', 'owner/other-poc'))] },
|
||||
[report]: { files: [json] },
|
||||
// Each file is also listed under its own original, which is no row.
|
||||
[newest.original]: { url: newest.url, timestamp: newest.timestamp }
|
||||
};
|
||||
const { elements } = startPage(t, { search: '?q=widget', entries, archives, trending: new Promise(() => {}),
|
||||
repositories: { 'owner/live': [90, '2026-10-01'], 'owner/poc': [50, '2026-10-01'], 'owner/other-poc': [10, '2026-10-01'] } });
|
||||
await new Promise(setImmediate);
|
||||
const html = elements.get('[data-results]').innerHTML;
|
||||
// Rows keep their star order and count, and each keeps one Wayback control.
|
||||
const order = [live, repo, pending, report].map(url => html.indexOf(`href="${url}"`));
|
||||
assert.ok(order[0] >= 0);
|
||||
assert.deepEqual([...order].sort((a, b) => a - b), order);
|
||||
assert.match(html, /4 linked PoCs/);
|
||||
assert.equal(html.match(/class="wayback /g).length, 4);
|
||||
assert.equal(html.match(/<details /g).length, 3);
|
||||
assert.ok(html.includes(`<div class="poc-archive"><a class="wayback is-snapshot" href="${page}"`));
|
||||
assert.ok(html.includes('snapshot</a><details class="sources" style="--files:2" data-sources="https://github.com/owner/poc">' +
|
||||
'<summary title="Wayback Machine captures of individual files at pinned commits, not a copy of the whole repository" ' +
|
||||
'aria-label="2 files saved from owner/poc">2 files</summary><div class="sources-list">' +
|
||||
`<a href="${newest.url}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Wayback Machine capture of poc.py at commit 0123456, 2026-10-22 16:10 UTC" ' +
|
||||
'aria-label="poc.py at commit 0123456 of owner/poc, Wayback Machine capture 2026-10-22 16:10 UTC">poc.py</a>' +
|
||||
`<a href="${spaced.url}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Wayback Machine capture of src/exploit one.py at commit 0123456, 2026-10-22 16:10 UTC" ' +
|
||||
'aria-label="src/exploit one.py at commit 0123456 of owner/poc, Wayback Machine capture 2026-10-22 16:10 UTC">' +
|
||||
'src/exploit one.py</a></div></details></div>'));
|
||||
assert.ok(!html.includes('20261021120000'));
|
||||
assert.ok(html.includes(`find archive</a><details class="sources" style="--files:1" data-sources="${pending}">` +
|
||||
'<summary title="Wayback Machine captures of individual files at pinned commits, not a copy of the whole repository" ' +
|
||||
'aria-label="1 file saved from Owner/Other-PoC">1 file</summary>'));
|
||||
assert.ok(html.includes(`find archive</a><details class="sources" style="--files:1" data-sources="${report}">` +
|
||||
'<summary title="Wayback Machine capture of the report JSON" aria-label="1 file saved from hackerone.com/reports/123456">' +
|
||||
`1 file</summary><div class="sources-list"><a href="${json.url}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Wayback Machine capture of 123456.json, the report JSON, 2026-10-22 16:10 UTC" ' +
|
||||
'aria-label="123456.json, the report JSON of hackerone.com/reports/123456, Wayback Machine capture 2026-10-22 16:10 UTC">' +
|
||||
'123456.json</a></div></details>'));
|
||||
// No saved file or snapshot is looked up again.
|
||||
assert.doesNotMatch(html, /\/web\/\*\/https:\/\/(?:web\.archive\.org|raw\.githubusercontent\.com|hackerone\.com\/reports\/123456\.json)/);
|
||||
});
|
||||
|
||||
test('saved files must be captures of the linked page\'s own files', async t => {
|
||||
const { context } = startPage(t, { trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
const disclosed = (parent, files) => {
|
||||
context.record = { files };
|
||||
return vm.runInContext(`archives = { ${JSON.stringify(parent)}: record }; sourceFiles(${JSON.stringify(parent)}, 'label')`, context);
|
||||
};
|
||||
const repo = 'https://github.com/owner/poc';
|
||||
const good = savedFile(rawFile('poc.py'));
|
||||
const stamp = good.timestamp;
|
||||
// GitHub names repositories without regard to case or a .git suffix.
|
||||
for (const parent of [repo, 'https://github.com/Owner/POC.git', 'https://www.github.com/owner/poc/blob/main/README.md']) {
|
||||
assert.match(disclosed(parent, [good]), /1 file<\/summary>/, parent);
|
||||
}
|
||||
for (const file of [
|
||||
savedFile(rawFile('poc.py', 'owner/other')),
|
||||
savedFile(rawFile('poc.py', 'other/poc')),
|
||||
savedFile(rawFile('poc.py', 'owner/poc', 'main')),
|
||||
savedFile(rawFile('poc.py', 'owner/poc', COMMIT.slice(1))),
|
||||
savedFile(rawFile('poc.py', 'owner/poc', COMMIT.toUpperCase())),
|
||||
// A dot segment would open another repository's file.
|
||||
savedFile(rawFile(`../../other/poc/${COMMIT}/poc.py`)),
|
||||
savedFile(rawFile('%2e%2E/poc.py')),
|
||||
savedFile(rawFile('./poc.py')),
|
||||
savedFile(rawFile('src//poc.py')),
|
||||
savedFile(rawFile('poc.py?token=1')),
|
||||
savedFile(rawFile('poc.py#L1')),
|
||||
savedFile(`http://raw.githubusercontent.com/owner/poc/${COMMIT}/poc.py`),
|
||||
savedFile(`https://raw.githubusercontent.com.example/owner/poc/${COMMIT}/poc.py`),
|
||||
savedFile(`https://gist.githubusercontent.com/owner/poc/raw/${COMMIT}/poc.py`),
|
||||
savedFile(`https://github.com/owner/poc/blob/${COMMIT}/poc.py`),
|
||||
savedFile(`https://example.com/owner/poc/${COMMIT}/poc.py`),
|
||||
savedFile('https://hackerone.com/reports/123456.json'),
|
||||
// An archive link is never saved again.
|
||||
savedFile(good.url),
|
||||
{ ...good, url: savedFile(rawFile('other.py')).url },
|
||||
{ ...good, url: `https://web.archive.org/web/${stamp}/https://example.com/owner/poc/${COMMIT}/poc.py` },
|
||||
{ ...good, timestamp: '20261022161014' },
|
||||
{ ...good, timestamp: Number(stamp) },
|
||||
{ ...good, url: `https://web.archive.org/web/${stamp}id_/${good.original}` },
|
||||
{ ...good, url: `https://web.archive.org/web/*/${good.original}` },
|
||||
{ ...good, url: `http://web.archive.org/web/${stamp}/${good.original}` },
|
||||
{ ...good, url: `https://web.archive.org/web/20250230093000/${good.original}`, timestamp: '20250230093000' },
|
||||
{ url: good.url, timestamp: stamp },
|
||||
good.url, null, [good]
|
||||
]) {
|
||||
assert.equal(disclosed(repo, [file]), '', JSON.stringify(file));
|
||||
}
|
||||
for (const files of [good, 'files', null, {}]) assert.equal(disclosed(repo, files), '', JSON.stringify(files));
|
||||
// Advisories, gists, other hosts and files themselves hold no repository files.
|
||||
for (const parent of ['https://github.com/owner/poc/security/advisories/GHSA-1234-5678-9abc', 'https://gist.github.com/owner/0123',
|
||||
'https://example.com/owner/poc', good.original]) {
|
||||
assert.equal(disclosed(parent, [good]), '', parent);
|
||||
}
|
||||
// A HackerOne report holds its own report JSON and nothing else.
|
||||
const json = savedFile('https://hackerone.com/reports/123456.json');
|
||||
for (const parent of ['https://hackerone.com/reports/123456', 'https://www.hackerone.com/reports/123456/',
|
||||
'https://hackerone.com/reports/123456?subject=x']) {
|
||||
assert.match(disclosed(parent, [json]), /1 file<\/summary>/, parent);
|
||||
}
|
||||
for (const [parent, file] of [
|
||||
['https://hackerone.com/reports/123456', savedFile('https://hackerone.com/reports/654321.json')],
|
||||
['https://hackerone.com/reports/123456', savedFile('https://hackerone.example/reports/123456.json')],
|
||||
['https://hackerone.com/reports/123456', savedFile('https://hackerone.com/reports/123456')],
|
||||
['https://hackerone.com/reports/123456', savedFile('https://hackerone.com/reports/123456.json?x=1')],
|
||||
['https://hackerone.com/reports/123456', good],
|
||||
['https://hackerone.com/reports/123456.json', json],
|
||||
['https://hackerone.com/bugs?report_id=123456', json],
|
||||
['https://example.com/reports/123456', json]
|
||||
]) {
|
||||
assert.equal(disclosed(parent, [file]), '', `${parent} ${JSON.stringify(file)}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('saved file names, links and labels are escaped', async t => {
|
||||
const { context } = startPage(t, { trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
const repo = 'https://github.com/owner/poc';
|
||||
const odd = savedFile(rawFile("x%22%3E%3Cimg%20src%3Dx%20onerror%3Dalert(1)%3E/it's.py"));
|
||||
context.record = { files: [odd, savedFile(rawFile('100%.py'))] };
|
||||
const html = vm.runInContext(`archives = { ${JSON.stringify(repo)}: record }; sourceFiles(${JSON.stringify(repo)}, '<b>"label"</b>')`, context);
|
||||
assert.ok(html.includes(`<a href="${odd.url.replace("'", ''')}" target="_blank"`));
|
||||
assert.ok(html.includes('aria-label="x"><img src=x onerror=alert(1)>/it's.py at commit 0123456 of ' +
|
||||
'<b>"label"</b>, Wayback Machine capture 2026-10-22 16:10 UTC">' +
|
||||
'x"><img src=x onerror=alert(1)>/it's.py</a>'));
|
||||
assert.ok(html.includes('aria-label="2 files saved from <b>"label"</b>"'));
|
||||
// A malformed escape is shown as written.
|
||||
assert.ok(html.includes('>100%.py</a>'));
|
||||
assert.doesNotMatch(html, /<img|<b>|"label"/);
|
||||
});
|
||||
|
||||
test('an archived fallback discloses its original\'s saved files and no file repeats a row', async t => {
|
||||
const original = 'https://github.com/owner/gone-poc';
|
||||
const fallback = `https://web.archive.org/web/20240101120000/${original}`;
|
||||
const exploit = savedFile(rawFile('exploit.c', 'owner/gone-poc'));
|
||||
const repo = 'https://github.com/owner/poc';
|
||||
const listed = savedFile(rawFile('poc.py'));
|
||||
const helper = savedFile(rawFile('helper.py'));
|
||||
const entries = [{ cve: 'CVE-2026-1000', desc: 'widget', poc: [fallback, repo, listed.original] }];
|
||||
const archives = {
|
||||
[original]: { url: fallback, timestamp: '20240101120000', files: [exploit] },
|
||||
[repo]: { files: [listed, helper] },
|
||||
[listed.original]: { url: listed.url, timestamp: listed.timestamp }
|
||||
};
|
||||
const { elements } = startPage(t, { search: '?q=widget', entries, archives, trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
const html = elements.get('[data-results]').innerHTML;
|
||||
assert.match(html, /3 linked PoCs/);
|
||||
assert.ok(html.includes('<div class="poc-archive"><span class="wayback is-archived" title="Wayback Machine page snapshot captured ' +
|
||||
'2024-01-01 12:00 UTC, linked in place of the original" aria-hidden="true">ARCHIVED</span>' +
|
||||
`<details class="sources" style="--files:1" data-sources="${fallback}"><summary`));
|
||||
assert.ok(html.includes(`<a href="${exploit.url}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Wayback Machine capture of exploit.c at commit 0123456, 2026-10-22 16:10 UTC" ' +
|
||||
'aria-label="exploit.c at commit 0123456 of owner/gone-poc, Wayback Machine capture 2026-10-22 16:10 UTC">exploit.c</a>'));
|
||||
// The lost original is never linked, and no capture is looked up again.
|
||||
assert.ok(!html.includes(`href="${original}"`));
|
||||
assert.doesNotMatch(html, /\/web\/\*\/https:\/\/(?:web\.archive\.org|raw\.githubusercontent\.com)/);
|
||||
// poc.py has a row and a snapshot of its own, so its page discloses only helper.py.
|
||||
assert.ok(html.includes('aria-label="1 file saved from owner/poc">1 file</summary>'));
|
||||
assert.ok(html.includes(`<a href="${helper.url}"`));
|
||||
assert.ok(html.includes(`<a class="wayback is-snapshot" href="${listed.url}"`));
|
||||
assert.equal(html.split(`href="${listed.url}"`).length, 2);
|
||||
});
|
||||
|
||||
test('an open saved-files list stays open while results re-render', async t => {
|
||||
const repo = 'https://github.com/owner/poc';
|
||||
const entries = [{ cve: 'CVE-2026-1234', desc: 'widget', poc: [repo] }];
|
||||
const { context, elements } = startPage(t, { search: '?q=widget', entries, trending: new Promise(() => {}),
|
||||
archives: { [repo]: { files: [savedFile(rawFile('poc.py'))] } } });
|
||||
await new Promise(setImmediate);
|
||||
const results = () => elements.get('[data-results]').innerHTML;
|
||||
const toggle = (open, matches = true) => vm.runInContext('rememberSources({ target: { matches: () => ' + matches +
|
||||
`, open: ${open}, dataset: { sources: ${JSON.stringify(repo)} } } }); renderResults(null);`, context);
|
||||
assert.ok(results().includes(`data-sources="${repo}"><summary`));
|
||||
toggle(true);
|
||||
assert.ok(results().includes(`data-sources="${repo}" open><summary`));
|
||||
// A toggle elsewhere changes nothing.
|
||||
toggle(false, false);
|
||||
assert.ok(results().includes(`data-sources="${repo}" open><summary`));
|
||||
toggle(false);
|
||||
assert.ok(results().includes(`data-sources="${repo}"><summary`));
|
||||
});
|
||||
|
||||
test('late or failed archive data never blocks search, reorders results or drops snapshots', async t => {
|
||||
let finishArchives;
|
||||
const archives = new Promise(resolve => { finishArchives = resolve; });
|
||||
const entries = ['CVE-2025-1234', 'CVE-2024-1234'].map(cve => ({ cve, desc: 'widget', poc: [`https://github.com/owner/${cve}`] }));
|
||||
const { context, elements } = startPage(t, { search: '?q=widget', entries, archives, trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
const results = () => elements.get('[data-results]').innerHTML;
|
||||
const ids = () => [...results().matchAll(/class="result-id" href="\/([^"]+)"/g)].map(match => match[1]);
|
||||
assert.deepEqual(ids(), ['CVE-2025-1234', 'CVE-2024-1234']);
|
||||
assert.equal(results().match(/class="wayback is-lookup"/g).length, 2);
|
||||
const status = elements.get('[data-status]').textContent;
|
||||
const original = 'https://github.com/owner/CVE-2024-1234';
|
||||
const snapshot = `https://web.archive.org/web/20251021093000/${original}`;
|
||||
const saved = savedFile(rawFile('poc.py', 'owner/CVE-2024-1234'));
|
||||
finishArchives({ [original]: { url: snapshot, timestamp: '20251021093000', files: [saved] } });
|
||||
await new Promise(setImmediate);
|
||||
assert.deepEqual(ids(), ['CVE-2025-1234', 'CVE-2024-1234']);
|
||||
assert.ok(results().includes(`class="wayback is-snapshot" href="${snapshot}"`));
|
||||
assert.ok(results().includes(`<a href="${saved.url}"`));
|
||||
assert.equal(elements.get('[data-status]').textContent, status);
|
||||
// A refresh whose archive download fails or is unusable keeps the snapshot and saved file shown.
|
||||
for (const [at, failure] of ['error', 'malformed'].entries()) {
|
||||
const feed = { generated: `2026-10-21T1${at}:00:00Z`, with_pocs: 2, total_cves: 2, items: [], landed: [] };
|
||||
context.fetch = async url => {
|
||||
const file = url.split('?')[0];
|
||||
if (file === '/archives.json' && failure === 'error') throw new Error('Unavailable');
|
||||
return { ok: true, json: async () => file === '/trending_poc.json' ? feed : file === '/CVE_list.json' ? entries
|
||||
: file === '/archives.json' ? [] : {} };
|
||||
};
|
||||
assert.equal(await vm.runInContext('refreshData(true)', context), true);
|
||||
await new Promise(setImmediate);
|
||||
assert.deepEqual(ids(), ['CVE-2025-1234', 'CVE-2024-1234']);
|
||||
assert.ok(results().includes(`class="wayback is-snapshot" href="${snapshot}"`), failure);
|
||||
assert.ok(results().includes(`<a href="${saved.url}"`), failure);
|
||||
assert.equal(elements.get('[data-notices]').hidden, true);
|
||||
}
|
||||
});
|
||||
|
||||
test('snapshots and saved files published with the homepage survive a failed archive download', async t => {
|
||||
const original = 'https://github.com/owner/poc';
|
||||
const snapshot = `https://web.archive.org/web/20251021093000/${original}`;
|
||||
const file = savedFile(rawFile('poc.py'));
|
||||
const name = { getAttribute: () => original };
|
||||
const row = { querySelector: () => name };
|
||||
const sources = { getAttribute: attribute => attribute === 'data-sources' ? original : null };
|
||||
const published = [
|
||||
{ getAttribute: () => snapshot, closest: selector => selector === '.trend-row' ? row : null },
|
||||
{ getAttribute: () => file.url, closest: selector => selector === 'details.sources' ? sources : row }
|
||||
];
|
||||
const feed = { generated: '2026-10-21T10:00:00Z', with_pocs: 1, total_cves: 1, items: [], landed: [
|
||||
{ cve: 'CVE-2026-1234', url: original, name: 'poc', stars: 1, released: '2026-10-21T09:00:00Z', page: '/CVE-2026-1234' }] };
|
||||
const { elements, calls } = startPage(t, { trending: feed, published, archivesError: true });
|
||||
await new Promise(setImmediate);
|
||||
assert.ok(calls.includes('/archives.json'));
|
||||
const rows = elements.get('[data-trend-rows]').innerHTML;
|
||||
assert.ok(rows.includes('<div class="trend-sub"><a class="trend-detail" href="/CVE-2026-1234">CVE-2026-1234 details</a>' +
|
||||
`<a class="wayback is-snapshot" href="${snapshot}"`));
|
||||
assert.ok(rows.includes(`<details class="sources" style="--files:1" data-sources="${original}"><summary`));
|
||||
assert.ok(rows.includes(`<a href="${file.url}"`));
|
||||
assert.doesNotMatch(rows, /is-lookup/);
|
||||
});
|
||||
|
||||
test('a stalled trending feed does not block a bookmarked search', async t => {
|
||||
const { elements, calls } = startPage(t, { search: '?q=widget', trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
|
||||
Reference in new issue
Block a user