mirror of
https://github.com/0xMarcio/cve.git
synced 2026-10-23 16:35:19 +00:00
Preserve public PoCs with resumable Wayback captures
This commit is contained in:
0xMarcio committed
1 parent
03db7ffa99
commit
5480e9df70
29 files changed
+18454
-15780
No files matched your search
@@ -0,0 +1,149 @@
|
||||
name: Archive Public PoCs
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
limit:
|
||||
description: Maximum new targets in this batch
|
||||
type: choice
|
||||
default: '100'
|
||||
options: ['100', '10', '3']
|
||||
max_seconds:
|
||||
description: Maximum worker time in seconds
|
||||
type: choice
|
||||
default: '2100'
|
||||
options: ['2100', '300']
|
||||
workflow_run:
|
||||
workflows: ['Hot CVEs List']
|
||||
types: [completed]
|
||||
schedule:
|
||||
- cron: '47 */6 * * *'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
|
||||
# One owner of the Archive account, including manual backfill runs. The hourly
|
||||
# discovery workflow triggers incremental batches; cron is a recovery trigger.
|
||||
concurrency:
|
||||
group: wayback-captures
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
capture:
|
||||
if: github.event_name != 'workflow_run' || github.event.workflow_run.head_branch == 'main'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 55
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: main
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- name: Find the last durable worker checkpoint
|
||||
id: previous
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
gh api "repos/$GITHUB_REPOSITORY/actions/artifacts?name=archive-checkpoints&per_page=100" \
|
||||
--jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | last | if . == null then empty else "artifact=\(.id)\nrun=\(.workflow_run.id)" end' >> "$GITHUB_OUTPUT"
|
||||
- name: Recover state even if the previous publish failed
|
||||
if: steps.previous.outputs.artifact != ''
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
artifact-ids: ${{ steps.previous.outputs.artifact }}
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ steps.previous.outputs.run }}
|
||||
path: /tmp/previous-archive-checkpoint
|
||||
merge-multiple: true
|
||||
- name: Merge the recovered checkpoint
|
||||
if: steps.previous.outputs.artifact != ''
|
||||
run: python3 scripts/archive_pocs.py --merge-from /tmp/previous-archive-checkpoint
|
||||
- name: Resume 2026-onward captures within service limits
|
||||
timeout-minutes: 42
|
||||
env:
|
||||
IA_ACCESS_KEY: ${{ secrets.IA_ACCESS_KEY }}
|
||||
IA_SECRET_KEY: ${{ secrets.IA_SECRET_KEY }}
|
||||
ARCHIVE_LIMIT: ${{ inputs.limit || '100' }}
|
||||
ARCHIVE_SECONDS: ${{ inputs.max_seconds || '2100' }}
|
||||
run: |
|
||||
case "$ARCHIVE_LIMIT" in 3|10|100) ;; *) exit 1 ;; esac
|
||||
case "$ARCHIVE_SECONDS" in 300|2100) ;; *) exit 1 ;; esac
|
||||
python3 scripts/archive_pocs.py --capture --min-year 2026 --limit "$ARCHIVE_LIMIT" --max-seconds "$ARCHIVE_SECONDS"
|
||||
- name: Preserve checkpoints even after interruption
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: archive-checkpoints
|
||||
path: index/archives/
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
overwrite: true
|
||||
|
||||
publish:
|
||||
if: always() && needs.capture.result != 'skipped'
|
||||
needs: capture
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
# Network captures never hold the lock needed by hourly discovery.
|
||||
concurrency:
|
||||
group: repository-updates
|
||||
queue: max
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: main
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: archive-checkpoints
|
||||
path: /tmp/archive-checkpoints
|
||||
- name: Merge durable capture results into current data
|
||||
id: snapshots
|
||||
run: |
|
||||
python3 -c 'import sys,json; sys.path.insert(0,"scripts"); import archive_pocs; print(json.dumps(archive_pocs.published_snapshots(),sort_keys=True))' > /tmp/archive-before.json
|
||||
python3 scripts/archive_pocs.py --merge-from /tmp/archive-checkpoints
|
||||
python3 scripts/archive_pocs.py --stats >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 -c 'import sys,json; sys.path.insert(0,"scripts"); import archive_pocs; print(json.dumps(archive_pocs.published_snapshots(),sort_keys=True))' > /tmp/archive-after.json
|
||||
if ! cmp -s /tmp/archive-before.json /tmp/archive-after.json; then
|
||||
echo 'changed=true' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- name: Refresh counts for preserved sources
|
||||
if: steps.snapshots.outputs.changed == 'true'
|
||||
run: |
|
||||
python3 .github/hero.py
|
||||
python3 .github/getTrending.py --counts-only
|
||||
- name: Publish verified archive links
|
||||
id: commit
|
||||
run: |
|
||||
git config user.name 'CVE-BOT'
|
||||
git config user.email '[email protected]'
|
||||
git add --all -- index/archives/ README.md docs/hero.svg docs/social-card.svg docs/stats.json
|
||||
if git diff --cached --quiet; then
|
||||
echo 'No archive updates'
|
||||
exit 0
|
||||
fi
|
||||
git commit -m "Preserve public PoC snapshots $(date -u +'%Y-%m-%d')"
|
||||
for attempt in 1 2 3; do
|
||||
if git push origin HEAD:main; then
|
||||
echo 'pushed=true' >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "Push rejected; replaying archive updates (attempt $attempt)"
|
||||
git fetch --unshallow origin main 2>/dev/null || git fetch origin main
|
||||
git rebase FETCH_HEAD || { git rebase --abort; exit 1; }
|
||||
done
|
||||
exit 1
|
||||
- name: Deploy archive links
|
||||
if: steps.commit.outputs.pushed == 'true' && steps.snapshots.outputs.changed == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: gh workflow run site.yml --ref main
|
||||
@@ -38,3 +38,4 @@ GEMINI.md
|
||||
.mcp.json
|
||||
docs/page_lastmod.json
|
||||
docs/page_state.json
|
||||
docs/archives.json
|
||||
@@ -0,0 +1,65 @@
|
||||
# Public PoC archives
|
||||
|
||||
The archive queue covers CVE-2026 and later, including new years automatically.
|
||||
It uses the durable release ledger so removing an unavailable original does not
|
||||
forget the opportunity to recover an older snapshot. Blacklisted repositories
|
||||
and excluded CVE/repository associations stay excluded.
|
||||
|
||||
`Archive Public PoCs` runs after hourly discovery, with a six-hour recovery
|
||||
schedule and manual dispatch. Each batch processes at most 100 URLs or 35 minutes,
|
||||
whichever comes first. New discoveries, including retries, take priority over
|
||||
older URLs and 30-day refreshes of already preserved pages.
|
||||
The same queue drains the initial backfill and handles future discoveries.
|
||||
Manual dispatch also offers 3- or 10-target batches and a five-minute budget for
|
||||
checking credentials, captures and checkpoint recovery before a regular run.
|
||||
|
||||
- One workflow at a time owns captures. Do not run a local capture worker while
|
||||
that workflow is active.
|
||||
- Submissions are spaced at least 15 seconds apart, with at most two pending
|
||||
captures and one API request per second. Account capacity and daily quotas are
|
||||
checked before a batch. Each URL is limited to two submissions per day.
|
||||
HTTP throttling, `Retry-After`, and account quota errors pause the queue;
|
||||
target failures only defer their URL or host. Cooldowns survive the next run.
|
||||
- Pending job IDs, attempts, retry dates and verified captures are checkpointed
|
||||
in `index/archives/`. A separate publishing job merges results into current main
|
||||
under the existing repository update lock. Capture requests do not hold that
|
||||
lock. The next worker automatically recovers the latest checkpoint artifact
|
||||
if publishing failed; artifacts are retained for 14 days.
|
||||
- Save Page Now reuses captures from the last 30 days. Unavailable originals and
|
||||
expired jobs use the availability API to recover older captures. A successful API
|
||||
response alone is insufficient: replay must return content, rather than a
|
||||
login/error page. The first verified snapshot is retained across refreshes.
|
||||
Captures must match the original URL and known publication window; GitHub
|
||||
repository identity and known source-file hashes are checked where available.
|
||||
- `/archives.json` contains only verified links. The UI distinguishes a dated
|
||||
snapshot from an unverified history search. Confirmed dead original links can
|
||||
be replaced by a verified archive without resurrecting the dead link.
|
||||
|
||||
The account uses Internet Archive API keys, not paid Amazon S3 storage. Configure
|
||||
`IA_ACCESS_KEY` and `IA_SECRET_KEY` as repository secrets. For a local run the same
|
||||
environment variables work, or a private mode-0600
|
||||
`~/.config/pocindex/internet-archive.json` containing `access` and `secret`.
|
||||
Never commit credentials.
|
||||
|
||||
```sh
|
||||
python3 scripts/archive_pocs.py --stats
|
||||
python3 scripts/archive_pocs.py --capture --min-year 2026 --limit 100 --max-seconds 2100
|
||||
```
|
||||
|
||||
At 100 new URLs per hourly batch, 16,000 targets would take about seven days in
|
||||
ideal conditions. Existing snapshots may reduce capture work; throttling, slow
|
||||
replays, blocked hosts and retries can extend that substantially. Workflow
|
||||
summaries report actual progress rather than an estimated completion promise.
|
||||
|
||||
## Preservation limits
|
||||
|
||||
A page snapshot is not a complete repository backup. The queue additionally
|
||||
captures raw source files at immutable Git revisions when the release ledger has
|
||||
identified those files. It does not crawl arbitrary outbound links or claim to
|
||||
save every repository file, attachment or dependency. Dynamic/login-only reports
|
||||
may not be capturable. A missing snapshot remains visibly unconfirmed, and
|
||||
Internet Archive can later remove or restrict access to a capture.
|
||||
|
||||
API references: [Wayback availability](https://archive.org/help/wayback_api.php),
|
||||
[Save Page Now](https://help.archive.org/help/save-pages-in-the-wayback-machine/),
|
||||
[Save Page Now API](https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA/edit).
|
||||
@@ -1,8 +1,8 @@
|
||||
<div align="center">
|
||||
|
||||
<a href="https://pocindex.io/"><img src="https://raw.githubusercontent.com/0xMarcio/pocindex/main/docs/hero.svg?v=175154-83880-1241" alt="PoC Index" width="100%"></a>
|
||||
<a href="https://pocindex.io/"><img src="https://raw.githubusercontent.com/0xMarcio/pocindex/main/docs/hero.svg?v=175154-83879-1241" alt="PoC Index" width="100%"></a>
|
||||
|
||||
[](https://github.com/0xMarcio/pocindex/commits/main) [](https://github.com/0xMarcio/pocindex/actions/workflows/hot_cves.yml) [](https://pocindex.io/) [](https://pocindex.io/) [](https://github.com/0xMarcio/pocindex/stargazers)
|
||||
[](https://github.com/0xMarcio/pocindex/commits/main) [](https://github.com/0xMarcio/pocindex/actions/workflows/hot_cves.yml) [](https://pocindex.io/) [](https://pocindex.io/) [](https://github.com/0xMarcio/pocindex/stargazers)
|
||||
|
||||
<a href="https://pocindex.io/"><img src="https://raw.githubusercontent.com/0xMarcio/pocindex/8dec6181e823d641132e0ee8a52a1612c2b5dd37/docs/search.svg" alt="Search PoC Index" width="100%"></a>
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ An issue has been discovered in GitLab CE/EE affecting all versions from 7.0 bef
|
||||
### POC
|
||||
|
||||
#### Reference
|
||||
- https://hackerone.com/reports/4093475
|
||||
No PoCs from references.
|
||||
|
||||
#### Github
|
||||
No PoCs found on GitHub currently.
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="180" viewBox="0 0 1200 180"
|
||||
role="img" aria-label="PoC Index, 83,880 CVEs with public proof-of-concept exploits">
|
||||
role="img" aria-label="PoC Index, 83,879 CVEs with public proof-of-concept exploits">
|
||||
<clipPath id="card"><rect x="1" y="1" width="1198" height="178" rx="12"/></clipPath>
|
||||
<rect x="0.5" y="0.5" width="1199" height="179" rx="12" fill="#0d1117" stroke="#30363d"/>
|
||||
<g clip-path="url(#card)" opacity="0.6"><g>
|
||||
@@ -22,7 +22,7 @@
|
||||
fill="#e6edf3" letter-spacing="-0.5">175,154</text>
|
||||
<text x="620" y="117" font-family="ui-monospace,SFMono-Regular,Menlo,Consolas,monospace" font-size="12"
|
||||
fill="#6e7681" letter-spacing="1.6">CVES INDEXED</text><text x="840" y="96" font-family="ui-monospace,SFMono-Regular,Menlo,Consolas,monospace" font-size="34" font-weight="700"
|
||||
fill="#2f81f7" letter-spacing="-0.5">83,880</text>
|
||||
fill="#2f81f7" letter-spacing="-0.5">83,879</text>
|
||||
<text x="840" y="117" font-family="ui-monospace,SFMono-Regular,Menlo,Consolas,monospace" font-size="12"
|
||||
fill="#6e7681" letter-spacing="1.6">WITH POCS</text><text x="1010" y="96" font-family="ui-monospace,SFMono-Regular,Menlo,Consolas,monospace" font-size="34" font-weight="700"
|
||||
fill="#f85149" letter-spacing="-0.5">1,241</text>
|
||||
|
||||
|
Before Width: | Height: | Size: 2.6 KiB After Width: | Height: | Size: 2.6 KiB |
+274
-38
@@ -27,6 +27,7 @@ const state = {
|
||||
descOpen: new Set(),
|
||||
pocOpen: new Set(),
|
||||
advisoryOpen: new Set(),
|
||||
sourcesOpen: new Set(),
|
||||
kevOnly: false,
|
||||
filters: {
|
||||
severity: new Set(FILTER_OPTIONS.severity.map(([value]) => value)),
|
||||
@@ -43,6 +44,8 @@ let kev = {};
|
||||
let ratings = {};
|
||||
let metadata = {};
|
||||
let epss = {};
|
||||
// Original PoC URL to its confirmed Wayback Machine capture, from archives.json.
|
||||
let archives = {};
|
||||
let indexMeta = null;
|
||||
let dataEpoch = 0;
|
||||
const enrichmentNotices = new Set();
|
||||
@@ -156,6 +159,172 @@ function plainLinkLabel(url) {
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- wayback machine --------------------------------------------------- */
|
||||
|
||||
const WAYBACK = 'https://web.archive.org/web/';
|
||||
// A page snapshot: https on web.archive.org, a 14-digit UTC capture time and
|
||||
// the http(s) page captured. A save request or a history search is not one.
|
||||
const CAPTURE = /^https:\/\/web\.archive\.org\/web\/([0-9]{14})\/(https?:\/\/[^\x00-\x20\x7f"<>\\]+)$/;
|
||||
|
||||
/** The capture a snapshot URL names, or null. Its time has to be a real one
|
||||
* from 1996 on, when the Wayback Machine began. build_pages.py reads it alike. */
|
||||
function waybackCapture(url) {
|
||||
const match = typeof url === 'string' ? CAPTURE.exec(url) : null;
|
||||
if (!match) return null;
|
||||
const stamp = match[1];
|
||||
const iso = stamp.replace(/^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$/, '$1-$2-$3T$4:$5:$6.000Z');
|
||||
const time = Date.parse(iso);
|
||||
if (stamp < '1996' || Number.isNaN(time) || new Date(time).toISOString() !== iso) return null;
|
||||
return { url, stamp, original: match[2] };
|
||||
}
|
||||
|
||||
/** Lower-cased host of an http(s) link, without www. or a port. Empty for
|
||||
* anything else, a link carrying credentials included. */
|
||||
function linkHost(url) {
|
||||
const match = /^https?:\/\/([^/?#@\x00-\x20\x7f\\]*)(?=[/?#]|$)/.exec(String(url || ''));
|
||||
return match ? match[1].toLowerCase().replace(/:[0-9]*$/, '').replace(/^www\./, '') : '';
|
||||
}
|
||||
|
||||
/** The page a link stands for. A snapshot published in place of a lost
|
||||
* original stands for that original. */
|
||||
function linkTarget(url) {
|
||||
const capture = waybackCapture(url);
|
||||
return capture ? capture.original : url;
|
||||
}
|
||||
|
||||
/** The capture archives.json confirms for this exact link: a valid snapshot,
|
||||
* stamped as recorded, of a page on the link's own host. */
|
||||
function confirmedSnapshot(url) {
|
||||
const record = archives[url];
|
||||
const capture = record && typeof record === 'object' ? waybackCapture(record.url) : null;
|
||||
if (!capture || capture.stamp !== record.timestamp) return null;
|
||||
const host = linkHost(url);
|
||||
return host && linkHost(capture.original) === host ? capture : null;
|
||||
}
|
||||
|
||||
function captureDate(stamp) {
|
||||
return `${stamp.slice(0, 4)}-${stamp.slice(4, 6)}-${stamp.slice(6, 8)} ${stamp.slice(8, 10)}:${stamp.slice(10, 12)} UTC`;
|
||||
}
|
||||
|
||||
/** The accessible name of a link that opens a snapshot in place of its original. */
|
||||
function archivedName(label, capture) {
|
||||
return capture
|
||||
? ` aria-label="${escapeHTML(`${label}, archived page snapshot captured ${captureDate(capture.stamp)}`)}"`
|
||||
: '';
|
||||
}
|
||||
|
||||
/** The Wayback Machine beside a PoC link: its confirmed snapshot, or a lookup
|
||||
* that claims nothing. A snapshot is one captured page, not a copy of the
|
||||
* whole source. A link that is itself a snapshot is stamped instead, and any
|
||||
* other link into the archive gets nothing. */
|
||||
function archiveLink(url, label) {
|
||||
const capture = waybackCapture(url);
|
||||
if (capture) {
|
||||
return `<span class="wayback is-archived" title="Wayback Machine page snapshot captured ${captureDate(capture.stamp)}, ` +
|
||||
'linked in place of the original" aria-hidden="true">ARCHIVED</span>';
|
||||
}
|
||||
const host = linkHost(url);
|
||||
if (!host || /(?:^|\.)archive\.org$/.test(host)) return '';
|
||||
const snapshot = confirmedSnapshot(url);
|
||||
if (snapshot) {
|
||||
const date = captureDate(snapshot.stamp);
|
||||
return `<a class="wayback is-snapshot" href="${escapeHTML(snapshot.url)}" target="_blank" rel="nofollow noopener" ` +
|
||||
`title="Wayback Machine page snapshot, captured ${date}" ` +
|
||||
`aria-label="${escapeHTML(`snapshot of ${label} on the Wayback Machine, captured ${date}`)}">snapshot</a>`;
|
||||
}
|
||||
return `<a class="wayback is-lookup" href="${escapeHTML(WAYBACK + '*/' + url)}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Search the Wayback Machine for captures; none confirmed" ' +
|
||||
`aria-label="${escapeHTML(`find archive of ${label} on the Wayback Machine`)}">find archive</a>`;
|
||||
}
|
||||
|
||||
/* ---- saved source files ------------------------------------------------ */
|
||||
|
||||
// archives.json may list, under a page, captures of files that page holds:
|
||||
// the raw bytes of one file at a pinned commit of the page's own GitHub
|
||||
// repository, or the JSON of the page's own HackerOne report. Nothing else is
|
||||
// that page's file. build_pages.py reads them alike.
|
||||
const RAW_FILE = /^https:\/\/raw\.githubusercontent\.com\/([^/?#]+)\/([^/?#]+)\/([0-9a-f]{40})\/([^?#]+)$/;
|
||||
const REPORT_PAGE = /^https?:\/\/[^/?#]*\/reports\/([0-9]+)\/?(?:[?#]|$)/;
|
||||
const REPORT_JSON = /^https:\/\/(?:www\.)?hackerone\.com\/reports\/([0-9]+)\.json$/;
|
||||
|
||||
/** What follows a link's host: its path, query and fragment. */
|
||||
function linkPath(url) {
|
||||
return url.replace(/^https?:\/\/[^/?#]*/, '');
|
||||
}
|
||||
|
||||
/** The page a link stands for, whatever scheme, www. or port it carries. */
|
||||
function linkIdentity(url) {
|
||||
const target = linkTarget(url);
|
||||
const host = linkHost(target);
|
||||
return host ? host + linkPath(target) : target;
|
||||
}
|
||||
|
||||
/** The file a saved original is, when it belongs to the page at parent. */
|
||||
function sourceFile(parent, original) {
|
||||
const repo = repoFromUrl(parent);
|
||||
const raw = repo && RAW_FILE.exec(original);
|
||||
if (raw) {
|
||||
// A dot segment would resolve into another repository's files.
|
||||
if (`${raw[1]}/${raw[2]}`.toLowerCase() !== `${repo.owner}/${repo.repo}`.toLowerCase() ||
|
||||
raw[4].split('/').some(part => !part || /^(?:\.|%2e){1,2}$/i.test(part))) return null;
|
||||
let name = raw[4];
|
||||
try {
|
||||
name = decodeURIComponent(name);
|
||||
} catch (err) {
|
||||
// A malformed escape is shown as written.
|
||||
}
|
||||
return { key: `${raw[3]}/${raw[4]}`, name, commit: raw[3] };
|
||||
}
|
||||
const report = linkHost(parent) === 'hackerone.com' ? REPORT_PAGE.exec(parent) : null;
|
||||
const json = report && REPORT_JSON.exec(original);
|
||||
return json && json[1] === report[1] ? { key: json[1], name: `${json[1]}.json`, commit: '' } : null;
|
||||
}
|
||||
|
||||
/** The saved files archives.json confirms for this exact link: valid
|
||||
* captures, stamped as recorded, of files that belong to it. One capture per
|
||||
* file, the latest, in a stable order. */
|
||||
function savedFiles(url) {
|
||||
const record = archives[url];
|
||||
const files = record && typeof record === 'object' && Array.isArray(record.files) ? record.files : [];
|
||||
const found = new Map();
|
||||
for (const file of files) {
|
||||
const capture = file && typeof file === 'object' ? waybackCapture(file.url) : null;
|
||||
const original = capture && typeof file.original === 'string' ? file.original : '';
|
||||
if (!original || capture.stamp !== file.timestamp || linkHost(original) !== linkHost(capture.original) ||
|
||||
linkPath(original) !== linkPath(capture.original)) continue;
|
||||
const source = sourceFile(url, original);
|
||||
const kept = source && found.get(source.key);
|
||||
if (source && (!kept || kept.capture.stamp < capture.stamp)) found.set(source.key, { ...source, original, capture });
|
||||
}
|
||||
return [...found.values()].sort((a, b) => (a.original < b.original ? -1 : a.original > b.original ? 1 : 0));
|
||||
}
|
||||
|
||||
/** A link's saved files, disclosed beside its Wayback control. Each opens the
|
||||
* capture of one file's bytes, so none stands for the whole source and none
|
||||
* gets an archive control of its own. A snapshot in place of a lost original
|
||||
* discloses that original's files; a file listed in a row of its own is not
|
||||
* repeated. */
|
||||
function sourceFiles(url, label, listed = new Set()) {
|
||||
const files = savedFiles(linkTarget(url)).filter(file => !listed.has(linkIdentity(file.original)));
|
||||
if (!files.length) return '';
|
||||
const count = `${files.length} file${files.length === 1 ? '' : 's'}`;
|
||||
const title = files[0].commit
|
||||
? 'Wayback Machine captures of individual files at pinned commits, not a copy of the whole repository'
|
||||
: 'Wayback Machine capture of the report JSON';
|
||||
const links = files.map(file => {
|
||||
const date = captureDate(file.capture.stamp);
|
||||
const what = file.commit ? `${file.name} at commit ${file.commit.slice(0, 7)}` : `${file.name}, the report JSON`;
|
||||
return `<a href="${escapeHTML(file.capture.url)}" target="_blank" rel="nofollow noopener" ` +
|
||||
`title="${escapeHTML(`Wayback Machine capture of ${what}, ${date}`)}" ` +
|
||||
`aria-label="${escapeHTML(`${what} of ${label}, Wayback Machine capture ${date}`)}">${escapeHTML(file.name)}</a>`;
|
||||
}).join('');
|
||||
// The count sizes the room the open list takes, so nothing beside it moves.
|
||||
return `<details class="sources" style="--files:${files.length}" data-sources="${escapeHTML(url)}"` +
|
||||
`${state.sourcesOpen.has(url) ? ' open' : ''}><summary title="${title}" ` +
|
||||
`aria-label="${escapeHTML(`${count} saved from ${label}`)}">${count}</summary>` +
|
||||
`<div class="sources-list">${links}</div></details>`;
|
||||
}
|
||||
|
||||
/* ---- matching ---------------------------------------------------------- */
|
||||
|
||||
function normalizeToSpaces(value) {
|
||||
@@ -210,7 +379,8 @@ function leadBonus(index) {
|
||||
|
||||
function pocText(entry) {
|
||||
if (entry._pocText === undefined) {
|
||||
entry._pocText = entryLinks(entry).join(' ').toLowerCase();
|
||||
// An archived source matches as its original would, not as web.archive.org.
|
||||
entry._pocText = entryLinks(entry).map(linkTarget).join(' ').toLowerCase();
|
||||
}
|
||||
return entry._pocText;
|
||||
}
|
||||
@@ -242,11 +412,12 @@ function identityText(entry) {
|
||||
if (entry._identityText === undefined) {
|
||||
const artifacts = new Set(POC_FIELDS.slice(1).flatMap(field => entry[field] || []));
|
||||
entry._identityText = entryLinks(entry).map(url => {
|
||||
const repo = repoFromUrl(url);
|
||||
const source = linkTarget(url);
|
||||
const repo = repoFromUrl(source);
|
||||
if (!repo) return '';
|
||||
// Repository and artifact names identify the PoC; its owner and host
|
||||
// and GitHub's routing/ref segments stay in the lower URL tier.
|
||||
const path = url.replace(/^https?:\/\/(?:www\.)?github\.com\/[^/]+\/[^/?#]+/i, '').split(/[?#]/)[0];
|
||||
const path = source.replace(/^https?:\/\/(?:www\.)?github\.com\/[^/]+\/[^/?#]+/i, '').split(/[?#]/)[0];
|
||||
const artifact = /^\/(?:blob|tree)\/[^/]+\/(.+)/i.exec(path);
|
||||
const leaf = artifact ? artifact[1].replace(/\/$/, '').split('/').pop() : '';
|
||||
// A collection's repository and category folders describe the catalogue,
|
||||
@@ -305,7 +476,7 @@ function scoreEntry(entry, matcher) {
|
||||
function countLinkHits(entry, matchers) {
|
||||
let hits = 0;
|
||||
for (const url of entryLinks(entry)) {
|
||||
const text = url.toLowerCase();
|
||||
const text = linkTarget(url).toLowerCase();
|
||||
if (matchers.every(m => matchAt(text, m) >= 0)) {
|
||||
hits += 1;
|
||||
if (hits >= 999) break;
|
||||
@@ -358,7 +529,7 @@ function entrySources(entry) {
|
||||
for (const [source, field] of Object.entries(fields)) {
|
||||
if ((entry[field] || []).length) sources.add(source);
|
||||
}
|
||||
for (const url of entry.poc || []) {
|
||||
for (const url of (entry.poc || []).map(linkTarget)) {
|
||||
if (/^https?:\/\/(?:www\.)?github\.com\/(?:advisories|[^/]+\/[^/]+\/security\/advisories)\/GHSA-/i.test(url)) {
|
||||
sources.add('GHSA');
|
||||
} else if (repoFromUrl(url)) {
|
||||
@@ -917,55 +1088,62 @@ function repositoryCells(url) {
|
||||
`<span class="poc-age" title="Repository last push${meta && meta[1] ? ': ' + escapeHTML(meta[1]) + ' UTC' : ' unavailable'}">${escapeHTML(age)}</span>`;
|
||||
}
|
||||
|
||||
function pocRow(url, artifact = false) {
|
||||
const curated = CURATED.find(source => url.includes(source.match));
|
||||
if (curated) {
|
||||
return '<div class="poc-row"><span class="poc-name">' +
|
||||
`<span class="poc-tag is-${curated.tag.toLowerCase()}" title="${escapeHTML(curated.hint)}">${curated.tag}</span>` +
|
||||
`<a href="${escapeHTML(url)}" target="_blank" rel="noopener">${escapeHTML(curated.label(url))}</a>` +
|
||||
'</span>' + repositoryCells(url) + '</div>';
|
||||
}
|
||||
const parsed = repoFromUrl(url);
|
||||
function pocRow(url, artifact = false, listed = new Set()) {
|
||||
// A snapshot published in place of a lost original is named, tagged and
|
||||
// measured as that original, and its link opens the snapshot.
|
||||
const capture = waybackCapture(url);
|
||||
const target = capture ? capture.original : url;
|
||||
const href = escapeHTML(url);
|
||||
const row = (name, label) => {
|
||||
// Saved files share the Wayback control's cell, so the columns hold.
|
||||
const files = sourceFiles(url, label, listed);
|
||||
const archive = files ? `<div class="poc-archive">${archiveLink(url, label)}${files}</div>` : archiveLink(url, label);
|
||||
return '<div class="poc-row"><span class="poc-name">' + name + '</span>' + archive + repositoryCells(target) + '</div>';
|
||||
};
|
||||
const curated = CURATED.find(source => target.includes(source.match));
|
||||
if (curated) {
|
||||
const label = curated.label(target);
|
||||
return row(`<span class="poc-tag is-${curated.tag.toLowerCase()}" title="${escapeHTML(curated.hint)}">${curated.tag}</span>` +
|
||||
`<a href="${href}"${archivedName(label, capture)} target="_blank" rel="noopener">${escapeHTML(label)}</a>`, label);
|
||||
}
|
||||
const parsed = repoFromUrl(target);
|
||||
// Gists and advisories do not inherit a project's repository metadata.
|
||||
if (!parsed) {
|
||||
return '<div class="poc-row"><span class="poc-name">' +
|
||||
`<a class="plain" href="${href}" target="_blank" rel="nofollow noopener">${escapeHTML(plainLinkLabel(url))}</a>` +
|
||||
'</span>' + repositoryCells(url) + '</div>';
|
||||
const label = plainLinkLabel(target);
|
||||
return row(`<a class="plain" href="${href}"${archivedName(label, capture)} target="_blank" rel="nofollow noopener">` +
|
||||
`${escapeHTML(label)}</a>`, label);
|
||||
}
|
||||
// Keep artifact identity while labelling stars and pushes as repository metadata.
|
||||
const path = artifact ? artifactPath(url) : '';
|
||||
const path = artifact ? artifactPath(target) : '';
|
||||
if (path) {
|
||||
const name = `${parsed.owner}/${parsed.repo}`;
|
||||
const source = COLLECTION_SOURCES[name.toLowerCase()] || { tag: parsed.owner.toUpperCase(), hint: name };
|
||||
return '<div class="poc-row"><span class="poc-name">' +
|
||||
`<span class="poc-tag is-${escapeHTML(source.tag.toLowerCase())}" title="${escapeHTML(source.hint)}">${escapeHTML(source.tag)}</span>` +
|
||||
`<a href="${href}" target="_blank" rel="noopener">${escapeHTML(path)}</a>` +
|
||||
'</span>' + repositoryCells(url) + '</div>';
|
||||
return row(`<span class="poc-tag is-${escapeHTML(source.tag.toLowerCase())}" title="${escapeHTML(source.hint)}">${escapeHTML(source.tag)}</span>` +
|
||||
`<a href="${href}"${archivedName(path, capture)} target="_blank" rel="noopener">${escapeHTML(path)}</a>`, path);
|
||||
}
|
||||
return '<div class="poc-row">' +
|
||||
`<span class="poc-name">${escapeHTML(parsed.owner)}<span class="poc-sep"> / </span>` +
|
||||
`<a href="${href}" target="_blank" rel="nofollow noopener">${escapeHTML(parsed.repo)}</a></span>` +
|
||||
repositoryCells(url) + '</div>';
|
||||
return row(`${escapeHTML(parsed.owner)}<span class="poc-sep"> / </span>` +
|
||||
`<a href="${href}"${archivedName(parsed.repo, capture)} target="_blank" rel="nofollow noopener">${escapeHTML(parsed.repo)}</a>`,
|
||||
`${parsed.owner}/${parsed.repo}`);
|
||||
}
|
||||
|
||||
/** Published links are already scoped to the CVE. Rank their repositories by
|
||||
* stars regardless of naming, with unknown readings after measured zeroes. */
|
||||
* stars regardless of naming, with unknown readings after measured zeroes.
|
||||
* An archived source ranks where its original would. */
|
||||
function rankedLinks(entry) {
|
||||
if (entry._ranked) return entry._ranked;
|
||||
const scored = entryLinks(entry).map((url, index) => {
|
||||
const parsed = repoFromUrl(url);
|
||||
const target = linkTarget(url);
|
||||
const parsed = repoFromUrl(target);
|
||||
const meta = parsed && repoMeta[(parsed.owner + '/' + parsed.repo).toLowerCase()];
|
||||
return {
|
||||
url,
|
||||
key: target.toLowerCase(),
|
||||
index,
|
||||
stars: meta ? meta[0] : -1
|
||||
};
|
||||
});
|
||||
scored.sort((a, b) => {
|
||||
const left = a.url.toLowerCase(), right = b.url.toLowerCase();
|
||||
return (b.stars - a.stars) || (left < right ? -1 : left > right ? 1 : 0) || (a.index - b.index);
|
||||
});
|
||||
scored.sort((a, b) =>
|
||||
(b.stars - a.stars) || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0) || (a.index - b.index));
|
||||
entry._ranked = scored.map(item => item.url);
|
||||
return entry._ranked;
|
||||
}
|
||||
@@ -1023,6 +1201,7 @@ function resultRow(entry) {
|
||||
const links = rankedLinks(entry);
|
||||
const visible = all ? links : links.slice(0, POC_PREVIEW);
|
||||
const artifacts = new Set(POC_FIELDS.slice(1).flatMap(field => entry[field] || []));
|
||||
const listed = new Set(links.map(linkIdentity));
|
||||
// Nothing to expand means no control: a line reading "all repositories
|
||||
// shown" under a list of three is chrome that answers a question nobody has.
|
||||
const moreButton = links.length > POC_PREVIEW
|
||||
@@ -1095,7 +1274,7 @@ function resultRow(entry) {
|
||||
<p class="result-desc${open ? ' is-open' : ''}">${escapeHTML(entry.desc || '')}</p>
|
||||
<button type="button" class="expander" data-toggle-desc="${escapeHTML(id)}">${open ? '↑ collapse' : '↓ full description'}</button>
|
||||
<div class="poc-list">
|
||||
${visible.map(url => pocRow(url, artifacts.has(url))).join('')}
|
||||
${visible.map(url => pocRow(url, artifacts.has(url), listed)).join('')}
|
||||
${moreButton}
|
||||
</div>
|
||||
${advisoryHtml}
|
||||
@@ -1166,8 +1345,9 @@ function renderResults(elapsed) {
|
||||
// newest-first value, still sent by cached pages.
|
||||
const TREND_LISTS = { LANDED: 'landed', TRENDING: 'items', RECENT: 'landed' };
|
||||
|
||||
function trendRow(item, landed) {
|
||||
const parsed = repoFromUrl(item.url);
|
||||
function trendRow(item, landed, listed = new Set()) {
|
||||
const url = String(item.url || '');
|
||||
const parsed = repoFromUrl(url);
|
||||
const repository = parsed ? (parsed.owner + '/' + parsed.repo).toLowerCase() : '';
|
||||
const stars = repository ? item.stars : null;
|
||||
const starCell = stars == null ? 'N/A' : `${formatStars(stars)} <span class="star">★</span>`;
|
||||
@@ -1181,9 +1361,12 @@ function trendRow(item, landed) {
|
||||
const detail = item.page
|
||||
? `<a class="trend-detail" href="${escapeHTML(item.page)}">${escapeHTML(item.cve)} details</a>`
|
||||
: '';
|
||||
const label = String(name || url);
|
||||
return `<div class="trend-row"><span class="trend-stars${stars >= 500 ? ' is-popular' : ''}" title="${escapeHTML(title)}">${starCell}</span>` +
|
||||
trendAge(landed ? item.released : item.pushed) +
|
||||
`<span class="trend-name-cell">${flagged}<a class="trend-name" href="${escapeHTML(item.url)}" target="_blank" rel="nofollow noopener">${escapeHTML(name)}</a>${detail}</span>` +
|
||||
`<div class="trend-name-cell">${flagged}<a class="trend-name" href="${escapeHTML(item.url)}" target="_blank" rel="nofollow noopener"` +
|
||||
`${archivedName(label, waybackCapture(url))}>${escapeHTML(name)}</a>` +
|
||||
`<div class="trend-sub">${detail}${archiveLink(url, label)}${sourceFiles(url, label, listed)}</div></div>` +
|
||||
`<span class="trend-desc">${escapeHTML(item.desc || 'Description unavailable')}</span></div>`;
|
||||
}
|
||||
|
||||
@@ -1198,8 +1381,9 @@ function renderTrending() {
|
||||
const list = TREND_LISTS[state.mode];
|
||||
const landed = list === 'landed';
|
||||
const rows = (Array.isArray(indexMeta[list]) ? indexMeta[list] : []).slice(0, TREND_ROWS);
|
||||
const listed = new Set(rows.map(item => linkIdentity(String(item.url || ''))));
|
||||
el.trendRows.innerHTML = rows.length
|
||||
? rows.map(item => trendRow(item, landed)).join('')
|
||||
? rows.map(item => trendRow(item, landed, listed)).join('')
|
||||
: '<div class="empty">No recent PoCs.</div>';
|
||||
if (el.trendDate) {
|
||||
el.trendDate.textContent = landed ? 'RELEASED' : 'UPDATED';
|
||||
@@ -1403,8 +1587,9 @@ document.addEventListener('keydown', event => {
|
||||
return;
|
||||
}
|
||||
const active = document.activeElement;
|
||||
// Space on a saved-files disclosure toggles it, as on a button.
|
||||
const editing = active && (active.isContentEditable
|
||||
|| ['INPUT', 'TEXTAREA', 'SELECT', 'BUTTON'].includes(active.tagName));
|
||||
|| ['INPUT', 'TEXTAREA', 'SELECT', 'BUTTON', 'SUMMARY'].includes(active.tagName));
|
||||
if (editing) return;
|
||||
// A lone slash is the shortcut, so it focuses without typing itself.
|
||||
if (event.key === '/') {
|
||||
@@ -1456,6 +1641,17 @@ el.results.addEventListener('click', event => {
|
||||
renderResults(null);
|
||||
});
|
||||
|
||||
// An open saved-files list stays open through the re-renders that late data
|
||||
// and other toggles cause. Toggle events do not bubble, so this listens early.
|
||||
function rememberSources(event) {
|
||||
const box = event.target;
|
||||
if (!box || typeof box.matches !== 'function' || !box.matches('details.sources')) return;
|
||||
if (box.open) state.sourcesOpen.add(box.dataset.sources);
|
||||
else state.sourcesOpen.delete(box.dataset.sources);
|
||||
}
|
||||
el.results.addEventListener('toggle', rememberSources, true);
|
||||
el.trendRows.addEventListener('toggle', rememberSources, true);
|
||||
|
||||
async function loadJSON(url, options) {
|
||||
const response = await fetch(url, options);
|
||||
if (!response.ok) throw new Error(`${url} (${response.status})`);
|
||||
@@ -1476,6 +1672,41 @@ function enrichmentUnavailable(kind, error) {
|
||||
render();
|
||||
}
|
||||
|
||||
// Snapshot links are display-only enrichment. They land whenever they arrive,
|
||||
// never hold up the index or reorder a result, and a failed or malformed
|
||||
// download keeps the snapshots already shown.
|
||||
function landArchives(request, epoch) {
|
||||
request.then(data => {
|
||||
if (dataEpoch !== epoch || !data || typeof data !== 'object' || Array.isArray(data)) return;
|
||||
archives = data;
|
||||
if (state.query.length < MIN_QUERY && !hasActiveFilters()) renderTrending();
|
||||
else if (state.ready) renderResults(null);
|
||||
}).catch(err => console.warn(`Archive snapshots unavailable: ${err.message}`));
|
||||
}
|
||||
|
||||
/** Snapshots and saved files the build confirmed for the rows it published.
|
||||
* They stand until archives.json lands, so a row never trades its snapshot
|
||||
* for a lookup or loses its files. */
|
||||
function publishedSnapshots() {
|
||||
const found = new Map();
|
||||
const record = url => found.get(url) || found.set(url, {}).get(url);
|
||||
for (const link of el.trendRows.querySelectorAll('.trend-row a.wayback.is-snapshot, .trend-row details.sources a')) {
|
||||
const capture = waybackCapture(link.getAttribute('href'));
|
||||
const sources = capture && link.closest('details.sources');
|
||||
if (sources) {
|
||||
// A saved file, filed under the original its row discloses it for.
|
||||
const parent = sources.getAttribute('data-sources');
|
||||
const saved = parent ? record(linkTarget(parent)) : null;
|
||||
if (saved) (saved.files || (saved.files = [])).push({ original: capture.original, url: capture.url, timestamp: capture.stamp });
|
||||
continue;
|
||||
}
|
||||
const row = capture && link.closest('.trend-row');
|
||||
const name = row && row.querySelector('.trend-name');
|
||||
if (name) Object.assign(record(name.getAttribute('href')), { url: capture.url, timestamp: capture.stamp });
|
||||
}
|
||||
return Object.fromEntries(found);
|
||||
}
|
||||
|
||||
function paintRefreshTime(meta) {
|
||||
const minutes = Math.max(0, Math.round((Date.now() - Date.parse(meta.generated)) / 60000));
|
||||
el.refreshed.textContent = minutes < 60
|
||||
@@ -1537,6 +1768,8 @@ async function refreshData(force = false) {
|
||||
paintHeroStats();
|
||||
paintRefreshTime(feed);
|
||||
render();
|
||||
// Optional: until it lands, or if it fails, the previous snapshots stand.
|
||||
landArchives(loadJSON(`/archives.json?v=${encodeURIComponent(feed.generated)}`, { cache: 'no-store' }), dataEpoch);
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.warn(`Index refresh deferred: ${err.message}`);
|
||||
@@ -1555,6 +1788,7 @@ document.addEventListener('visibilitychange', () => { if (!document.hidden) refr
|
||||
window.addEventListener?.('focus', () => { refreshData(); });
|
||||
|
||||
(async () => {
|
||||
archives = publishedSnapshots();
|
||||
readURLState();
|
||||
syncFilterControls();
|
||||
render();
|
||||
@@ -1630,4 +1864,6 @@ window.addEventListener?.('focus', () => { refreshData(); });
|
||||
render();
|
||||
}
|
||||
loadAdvisories();
|
||||
// One snapshot per linked PoC adds up, so it follows the index as well.
|
||||
landArchives(loadJSON('/archives.json', { cache: 'no-cache' }), 0);
|
||||
})();
|
||||
@@ -23,7 +23,7 @@
|
||||
fill="#e6edf3" letter-spacing="-0.5">175,154</text>
|
||||
<text x="96" y="459" font-family="ui-monospace,SFMono-Regular,Menlo,Consolas,monospace" font-size="16"
|
||||
fill="#6e7681" letter-spacing="1.6">CVES INDEXED</text><text x="500" y="430" font-family="ui-monospace,SFMono-Regular,Menlo,Consolas,monospace" font-size="46" font-weight="700"
|
||||
fill="#2f81f7" letter-spacing="-0.5">83,880</text>
|
||||
fill="#2f81f7" letter-spacing="-0.5">83,879</text>
|
||||
<text x="500" y="459" font-family="ui-monospace,SFMono-Regular,Menlo,Consolas,monospace" font-size="16"
|
||||
fill="#6e7681" letter-spacing="1.6">WITH POCS</text><text x="880" y="430" font-family="ui-monospace,SFMono-Regular,Menlo,Consolas,monospace" font-size="46" font-weight="700"
|
||||
fill="#f85149" letter-spacing="-0.5">1,241</text>
|
||||
|
||||
|
Before Width: | Height: | Size: 2.7 KiB After Width: | Height: | Size: 2.7 KiB |
+1
-1
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"kev": 1241,
|
||||
"total_cves": 175154,
|
||||
"with_pocs": 83880
|
||||
"with_pocs": 83879
|
||||
}
|
||||
+135
-4
@@ -287,7 +287,7 @@ a.chip:hover { color: var(--fg); border-color: var(--fg-muted); }
|
||||
.poc-list { margin-top: 14px; border-top: 1px solid var(--border-muted); }
|
||||
.poc-row {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) 78px 52px;
|
||||
grid-template-columns: minmax(0, 1fr) auto 78px 52px;
|
||||
gap: 14px;
|
||||
align-items: center;
|
||||
padding: 9px 0;
|
||||
@@ -324,10 +324,12 @@ a.chip:hover { color: var(--fg); border-color: var(--fg-muted); }
|
||||
.poc-tag.is-afrog, .poc-tag.is-xray, .poc-tag.is-0day, .poc-tag.is-poc { color: #a5d6ff; background: rgba(56, 139, 253, .1); border-color: rgba(56, 139, 253, .35); }
|
||||
.poc-tag.is-advisory { color: var(--fg-muted); background: rgba(110, 118, 129, .1); border-color: var(--border); }
|
||||
.poc-name a.plain:hover { color: var(--accent); }
|
||||
.poc-stars { text-align: right; font: 400 12px/1 var(--mono); color: var(--fg-muted); }
|
||||
/* Stars and age hold the last two tracks whether or not a Wayback control
|
||||
precedes them, so the columns line up on advisory and archive rows too. */
|
||||
.poc-stars { grid-column: -3; text-align: right; font: 400 12px/1 var(--mono); color: var(--fg-muted); }
|
||||
.poc-stars.is-popular { color: var(--attention); }
|
||||
.star { color: var(--neutral); }
|
||||
.poc-age { text-align: right; font: 400 11.5px/1 var(--mono); color: var(--fg-muted); }
|
||||
.poc-age { grid-column: -2; text-align: right; font: 400 11.5px/1 var(--mono); color: var(--fg-muted); }
|
||||
.poc-more {
|
||||
display: block;
|
||||
width: 100%;
|
||||
@@ -423,7 +425,9 @@ a.chip:hover { color: var(--fg); border-color: var(--fg-muted); }
|
||||
white-space: nowrap;
|
||||
}
|
||||
.trend-name { font: 500 12.5px/1 var(--mono); }
|
||||
.trend-detail { display: block; width: fit-content; margin-top: 5px; font: 400 10px/1.4 var(--mono); color: var(--fg-muted); }
|
||||
.trend-sub { display: flex; flex-wrap: wrap; align-items: center; gap: 4px 12px; margin-top: 5px; }
|
||||
.trend-sub:empty { display: none; }
|
||||
.trend-detail { font: 400 10px/1.4 var(--mono); color: var(--fg-muted); }
|
||||
.index-intro { margin: 0 0 14px; max-width: 960px; font: 400 13px/1.7 var(--sans); color: var(--fg-muted); }
|
||||
.trend-desc {
|
||||
font: 400 13px/1.4 var(--sans);
|
||||
@@ -433,6 +437,129 @@ a.chip:hover { color: var(--fg); border-color: var(--fg-muted); }
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* ---- wayback machine ---------------------------------------------------- */
|
||||
|
||||
/* Beside every PoC link. A filled dot in a frame is a confirmed page snapshot,
|
||||
an open ring a lookup that claims nothing, and the ARCHIVED stamp a row whose
|
||||
own link is the snapshot, published in place of a lost original. */
|
||||
.wayback {
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
padding: 2px 6px 2px 5px;
|
||||
border: 1px solid transparent;
|
||||
border-radius: 3px;
|
||||
font: 400 10.5px/1 var(--mono);
|
||||
color: var(--neutral);
|
||||
white-space: nowrap;
|
||||
}
|
||||
.wayback::before {
|
||||
content: '';
|
||||
display: inline-block;
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
margin-right: 5px;
|
||||
border: 1px solid currentColor;
|
||||
border-radius: 50%;
|
||||
}
|
||||
/* A fingertip-sized target without a bigger box; the focus ring stays inside
|
||||
cells that clip their overflow. */
|
||||
a.wayback::after { content: ''; position: absolute; inset: -5px -3px; }
|
||||
a.wayback:hover { color: var(--accent); }
|
||||
.wayback:focus-visible { outline-offset: -2px; }
|
||||
.wayback.is-snapshot { color: var(--fg-muted); border-color: var(--border); }
|
||||
.wayback.is-snapshot:hover { color: var(--fg); border-color: var(--fg-muted); }
|
||||
.wayback.is-snapshot::before, .wayback.is-archived::before { border-color: #3fb950; background: #3fb950; }
|
||||
.wayback.is-archived {
|
||||
padding: 2px 5px;
|
||||
font: 600 9px/1 var(--mono);
|
||||
letter-spacing: .08em;
|
||||
color: var(--fg-subtle);
|
||||
background: rgba(110, 118, 129, .12);
|
||||
border-color: var(--border);
|
||||
}
|
||||
.wayback.is-archived::before { width: 5px; height: 5px; margin-right: 4px; }
|
||||
.trend-sub a.wayback { font-size: 10px; }
|
||||
.cve-links .wayback { word-break: normal; }
|
||||
|
||||
/* Saved source files, disclosed beside the page's own control. Each line
|
||||
opens the capture of one file's bytes at a pinned commit, or of a report's
|
||||
JSON, and carries the dot of a confirmed capture. In the index the open
|
||||
list fills room the disclosure reserves under its row, aligned with the
|
||||
row, so nothing beside it moves. */
|
||||
.poc-row, .trend-name-cell { position: relative; }
|
||||
.poc-archive { display: flex; align-items: flex-start; gap: 6px; }
|
||||
.sources { flex: none; font: 400 10.5px/1 var(--mono); }
|
||||
.sources[open] { padding-bottom: calc(var(--files, 1) * 18px + 4px); }
|
||||
.sources > summary {
|
||||
display: inline-block;
|
||||
padding: 2px 6px 2px 5px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 3px;
|
||||
list-style: none;
|
||||
color: var(--fg-muted);
|
||||
white-space: nowrap;
|
||||
cursor: pointer;
|
||||
}
|
||||
.sources > summary::-webkit-details-marker { display: none; }
|
||||
/* A chevron where a snapshot has its dot, turning down while open. */
|
||||
.sources > summary::before {
|
||||
content: '';
|
||||
display: inline-block;
|
||||
width: 4px;
|
||||
height: 4px;
|
||||
margin: 0 6px 0 1px;
|
||||
border: solid currentColor;
|
||||
border-width: 0 1px 1px 0;
|
||||
vertical-align: 1px;
|
||||
transform: rotate(-45deg);
|
||||
transition: transform .15s ease;
|
||||
}
|
||||
.sources[open] > summary::before { transform: translateY(-1px) rotate(45deg); }
|
||||
.sources > summary:hover, .sources[open] > summary { color: var(--fg); border-color: var(--fg-muted); }
|
||||
.sources > summary:focus-visible { outline-offset: -2px; }
|
||||
.trend-sub .sources > summary { font-size: 10px; }
|
||||
.sources-list { position: absolute; left: 0; right: 0; margin-top: 4px; }
|
||||
.sources-list a {
|
||||
position: relative;
|
||||
display: block;
|
||||
height: 18px;
|
||||
padding-left: 16px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font: 400 11px/18px var(--mono);
|
||||
color: var(--fg-subtle);
|
||||
}
|
||||
.sources-list a::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: 6.5px;
|
||||
left: 5px;
|
||||
width: 5px;
|
||||
height: 5px;
|
||||
border-radius: 50%;
|
||||
background: #3fb950;
|
||||
}
|
||||
.sources-list a:hover { color: var(--accent); }
|
||||
.sources-list a:focus-visible { outline-offset: -2px; }
|
||||
/* While a list is open its row's cells keep to the first line, above it. */
|
||||
.poc-row:has(.sources[open]), .trend-row:has(.sources[open]) { align-items: start; }
|
||||
.trend-sub:has(> .sources[open]) { align-items: flex-start; }
|
||||
@supports not selector(:has(*)) {
|
||||
.poc-row, .trend-sub { align-items: start; }
|
||||
}
|
||||
/* On a CVE page the disclosure follows the note at the row's end. A label
|
||||
there may wrap, so the list opens in flow under the disclosure, which keeps
|
||||
its place at the end. */
|
||||
.cve-links li > a:first-child { margin-right: auto; }
|
||||
.cve-links li > .sources { margin-left: -8px; text-align: right; }
|
||||
.cve-links .sources[open] { padding-bottom: 0; }
|
||||
.cve-links .sources-list { position: static; text-align: left; }
|
||||
.cve-links .sources-list a { max-width: 48ch; }
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.sources > summary::before { transition: none; }
|
||||
}
|
||||
|
||||
/* ---- footer ------------------------------------------------------------- */
|
||||
|
||||
.site-footer { border-top: 1px solid var(--border-muted); padding: 14px 0; }
|
||||
@@ -474,6 +601,9 @@ a.chip:hover { color: var(--fg); border-color: var(--fg-muted); }
|
||||
.trending .col-head, .trend-row { grid-template-columns: 56px 96px minmax(0, 1fr); gap: 12px; }
|
||||
.trend-age { white-space: normal; line-height: 1.4; }
|
||||
.trend-name-cell { white-space: normal; overflow-wrap: anywhere; }
|
||||
/* The Wayback control takes a line under the name rather than its width. */
|
||||
.poc-row { grid-template-columns: minmax(0, 1fr) 78px 52px; row-gap: 6px; }
|
||||
.poc-row > .wayback, .poc-row > .poc-archive { grid-area: 2 / 1 / 3 / -1; justify-self: start; }
|
||||
/* Artifact names wrap while repository stars stay visible on small screens. */
|
||||
.poc-row:has(.plain) { grid-template-columns: minmax(0, 1fr); }
|
||||
.poc-row:has(.poc-tag, .plain) .poc-name { white-space: normal; overflow-wrap: anywhere; line-height: 1.5; }
|
||||
@@ -569,4 +699,5 @@ a.chip:hover { color: var(--fg); border-color: var(--fg-muted); }
|
||||
.cve-facts { grid-template-columns: 1fr; gap: 3px 0; }
|
||||
.cve-facts dd { margin-bottom: 10px; }
|
||||
.cve-links li { flex-direction: column; gap: 2px; }
|
||||
.cve-links li > .sources { margin-left: 0; text-align: left; }
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"entries":{"https://hackerone.com/reports/4093475":{"attempts":1,"checked_at":"2026-10-22T16:10:53Z","cves":["CVE-2026-110264"],"next_check_at":"2026-10-22T18:10:53Z","original":"https://hackerone.com/reports/4093475","priority":"2026-10-22T05:16:46Z","status":"not-found"}},"version":1}
|
||||
@@ -0,0 +1 @@
|
||||
{"entries":{"https://github.com/enactive-systems/cve-2026-109618-poc":{"attempts":3,"checked_at":"2026-10-22T16:09:29Z","cves":["CVE-2026-109618"],"next_check_at":"2026-11-21T16:09:37Z","original":"https://github.com/enactive-systems/CVE-2026-109618-poc","priority":"2026-10-21T15:09:55Z","snapshot":{"timestamp":"20261022160913","url":"https://web.archive.org/web/20261022160913/https://github.com/enactive-systems/CVE-2026-109618-poc","verified_at":"2026-10-22T16:09:37Z"},"status":"available","submitted_at":"2026-10-22T16:08:08Z"}},"version":1}
|
||||
@@ -0,0 +1 @@
|
||||
{"entries":{"https://raw.githubusercontent.com/enactive-systems/cve-2026-109618-poc/a33f383c0d42b3cacb1f1df7b24a64bce8a2c7ec/poc.php":{"attempts":0,"checked_at":"2026-10-22T16:36:57Z","cves":["CVE-2026-109618"],"expected_blobs":["6e21fdb4262681fe3f2ae59ac3d2582307ea6f2b","aa9b3cb1c534754810fa3b2cfe03631b38ed5dd3"],"next_check_at":"2026-11-21T16:36:58Z","original":"https://raw.githubusercontent.com/enactive-systems/cve-2026-109618-poc/a33f383c0d42b3cacb1f1df7b24a64bce8a2c7ec/poc.php","parent":"https://github.com/enactive-systems/cve-2026-109618-poc","parent_original":"https://github.com/enactive-systems/CVE-2026-109618-poc","priority":"2026-10-21T15:09:55Z","snapshot":{"timestamp":"20261022163647","url":"https://web.archive.org/web/20261022163647/https://raw.githubusercontent.com/enactive-systems/cve-2026-109618-poc/a33f383c0d42b3cacb1f1df7b24a64bce8a2c7ec/poc.php","verified_at":"2026-10-22T16:36:58Z"},"status":"available","submissions":{"count":1,"date":"2026-10-22"},"submitted_at":"2026-10-22T16:36:40Z"}},"version":1}
|
||||
@@ -0,0 +1 @@
|
||||
{"retry_after": "2026-10-22T16:07:09Z", "reason": "http-429"}
|
||||
@@ -0,0 +1 @@
|
||||
{"entries":{"https://raw.githubusercontent.com/enactive-systems/cve-2026-109618-poc/a33f383c0d42b3cacb1f1df7b24a64bce8a2c7ec/poc.py":{"attempts":1,"checked_at":"2026-10-22T16:11:06Z","cves":["CVE-2026-109618"],"next_check_at":"2026-11-21T16:11:12Z","original":"https://raw.githubusercontent.com/enactive-systems/cve-2026-109618-poc/a33f383c0d42b3cacb1f1df7b24a64bce8a2c7ec/poc.py","parent":"https://github.com/enactive-systems/cve-2026-109618-poc","priority":"2026-10-21T15:09:55Z","snapshot":{"timestamp":"20261022161107","url":"https://web.archive.org/web/20261022161107/https://raw.githubusercontent.com/enactive-systems/cve-2026-109618-poc/a33f383c0d42b3cacb1f1df7b24a64bce8a2c7ec/poc.py","verified_at":"2026-10-22T16:11:12Z"},"status":"available","submitted_at":"2026-10-22T16:11:06Z"}},"version":1}
|
||||
@@ -647,6 +647,7 @@
|
||||
"http://elegerov.blogspot.com/2006/10/here-is-lame-proof-of-concept-code-for.html": "2026-10-22T11:22:29.803975+00:00",
|
||||
"http://infosec42.blogspot.com/2014/01/exploit-asus-rt-n56u-remote-root-shell.html": "2026-10-22T11:43:14.927300+00:00",
|
||||
"http://ingehenriksen.blogspot.com/2006/04/office-2003-file-attachment-exploit.html": "2026-10-22T11:43:14.927300+00:00",
|
||||
"http://metasploit.blogspot.com/2006/04/exploit-development-groupwise_14.html": "2026-10-22T11:43:14.927300+00:00"
|
||||
"http://metasploit.blogspot.com/2006/04/exploit-development-groupwise_14.html": "2026-10-22T11:43:14.927300+00:00",
|
||||
"https://hackerone.com/reports/4093475": "2026-10-22T15:59:24.726913+00:00"
|
||||
}
|
||||
}
|
||||
@@ -38967,7 +38967,6 @@ CVE-2026-92997 - https://wpscan.com/vulnerability/2b7edbc9-9562-46a7-95f6-fa4336
|
||||
CVE-2026-92998 - https://wpscan.com/vulnerability/60b9bf53-a78a-4c5b-ab7d-4d118be99ab7/
|
||||
CVE-2026-92999 - https://wpscan.com/vulnerability/9f3723c9-40e3-4eb6-b0f9-139fb2c9df52/
|
||||
CVE-2026-96901 - https://wpscan.com/vulnerability/f00e95a2-a5fb-49da-aa6c-8c0279b6df2d/
|
||||
CVE-2026-110264 - https://hackerone.com/reports/4093475
|
||||
CVE-2025-60469 - https://github.com/sigdevel/pocs/blob/main/res/gpac/MP4Box/41/README.md
|
||||
CVE-2025-60476 - https://github.com/sigdevel/pocs/blob/main/res/gpac/MP4Box/42/README.md
|
||||
CVE-2025-60478 - https://github.com/sigdevel/pocs/blob/main/res/gpac/MP4Box/47/README.md
|
||||
|
||||
@@ -117303,7 +117303,6 @@ CVE-2026-113666 - https://github.com/advisories/GHSA-j5rq-xfvr-p969
|
||||
CVE-2026-113671 - https://github.com/advisories/GHSA-527g-ff96-x6wj
|
||||
CVE-2026-113712 - https://github.com/advisories/GHSA-x397-5hrv-j9m9
|
||||
CVE-2026-63209 - https://github.com/advisories/GHSA-259r-337f-4rfw
|
||||
CVE-2026-110264 - https://hackerone.com/reports/4093475
|
||||
CVE-2025-60469 - https://github.com/sigdevel/pocs/blob/main/res/gpac/MP4Box/41/README.md
|
||||
CVE-2025-60476 - https://github.com/sigdevel/pocs/blob/main/res/gpac/MP4Box/42/README.md
|
||||
CVE-2025-60478 - https://github.com/sigdevel/pocs/blob/main/res/gpac/MP4Box/47/README.md
|
||||
|
||||
+15686
-15686
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,659 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Resumable Wayback lookup/capture of published PoC pages and source artifacts."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import gzip
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
from http.client import HTTPException
|
||||
from pathlib import Path
|
||||
from urllib import error, parse, request
|
||||
|
||||
import link_health
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DIRECTORY = ROOT / 'index' / 'archives'
|
||||
API = 'https://web.archive.org/save'
|
||||
AVAILABILITY = 'https://archive.org/wayback/available'
|
||||
AGENT = 'pocindex/1.0 (+https://pocindex.io)'
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
|
||||
def public_url(url: str) -> bool:
|
||||
try:
|
||||
p = parse.urlsplit(url)
|
||||
if (p.scheme not in {'http', 'https'} or not p.hostname or p.username or p.password
|
||||
or p.port not in {None, 80, 443} or any(c.isspace() for c in url)):
|
||||
return False
|
||||
host = p.hostname.lower()
|
||||
if '.' not in host or host.endswith(('.localhost', '.local', '.internal')):
|
||||
return False
|
||||
try:
|
||||
return ipaddress.ip_address(host).is_global
|
||||
except ValueError:
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def identity(url: str) -> str:
|
||||
return link_health.url_key(url)
|
||||
|
||||
|
||||
def snapshot_url(url: str, stamp: str) -> str:
|
||||
if not re.fullmatch(r'\d{14}', stamp) or not public_url(url):
|
||||
raise ValueError('Invalid Wayback capture identity')
|
||||
datetime.strptime(stamp, '%Y%m%d%H%M%S')
|
||||
return f'https://web.archive.org/web/{stamp}/{parse.urldefrag(url).url}'
|
||||
|
||||
|
||||
def valid_snapshot(row: dict) -> bool:
|
||||
snap = row.get('snapshot') or {}
|
||||
try:
|
||||
p = parse.urlsplit(snap.get('url', ''))
|
||||
match = re.fullmatch(r'/web/(\d{14})/(https?://.+)', p.path + ('?' + p.query if p.query else ''))
|
||||
return bool(p.scheme == 'https' and p.netloc == 'web.archive.org' and match
|
||||
and snap.get('timestamp') == match[1]
|
||||
and snapshot_url(match[2], match[1]) == snap['url']
|
||||
and (not row.get('original') or equivalent(match[2], row['original']))
|
||||
and snap.get('verified_at'))
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def load_records(directory: Path = DIRECTORY) -> dict:
|
||||
rows = {}
|
||||
for path in sorted(directory.glob('[0-9a-f][0-9a-f].json')):
|
||||
payload = json.loads(path.read_text())
|
||||
if payload.get('version') != 1 or not isinstance(payload.get('entries'), dict):
|
||||
raise ValueError(f'Invalid archive shard: {path}')
|
||||
for key, row in payload['entries'].items():
|
||||
if not isinstance(row, dict) or identity(row.get('original', '')) != key or not public_url(key):
|
||||
raise ValueError(f'Invalid archive record: {path}')
|
||||
if row.get('snapshot') and not valid_snapshot(row):
|
||||
raise ValueError(f'Invalid archive snapshot: {path}')
|
||||
rows[key] = row
|
||||
return rows
|
||||
|
||||
|
||||
def save_records(rows: dict, changed: set[str], directory: Path = DIRECTORY) -> None:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
shards = {hashlib.sha256(key.encode()).hexdigest()[:2] for key in changed}
|
||||
for shard in sorted(shards):
|
||||
entries = {k: v for k, v in sorted(rows.items()) if hashlib.sha256(k.encode()).hexdigest()[:2] == shard}
|
||||
with tempfile.NamedTemporaryFile('w', dir=directory, delete=False) as handle:
|
||||
json.dump({'version': 1, 'entries': entries}, handle, sort_keys=True, separators=(',', ':'))
|
||||
handle.write('\n')
|
||||
temporary = Path(handle.name)
|
||||
temporary.replace(directory / (shard + '.json'))
|
||||
|
||||
|
||||
def published_snapshots(records: dict | None = None, allowed: set | None = None) -> dict:
|
||||
rows = load_records() if records is None else records
|
||||
result = {}
|
||||
for key, row in rows.items():
|
||||
if not valid_snapshot(row) or allowed is not None and key not in allowed and row.get('parent') not in allowed:
|
||||
continue
|
||||
snap = {k: row['snapshot'][k] for k in ('url', 'timestamp')}
|
||||
result.setdefault(row['original'], {}).update(snap)
|
||||
if row.get('parent'):
|
||||
parent = rows.get(row['parent'], {}).get('original') or row.get('parent_original') or row['parent']
|
||||
result.setdefault(parent, {}).setdefault('files', []).append({'original': row['original'], **snap})
|
||||
return result
|
||||
|
||||
|
||||
def archived_fallbacks(records: dict | None = None, blacklist=(), exclusions=()) -> dict:
|
||||
"""Recover the preserved artifact, never republish its unavailable original."""
|
||||
result = defaultdict(list)
|
||||
records = load_records() if records is None else records
|
||||
rows = [row for row in records.values() if valid_snapshot(row)
|
||||
and (not row.get('parent') or not valid_snapshot(records.get(row['parent'], {})))]
|
||||
if not rows:
|
||||
return result
|
||||
eligible = {(row['cve'], identity(row['url'])): row for row in eligible_ledger()}
|
||||
for row in rows:
|
||||
source = row.get('parent') or row['original']
|
||||
if link_health.repository(source) not in blacklist:
|
||||
for cve in row.get('cves', []):
|
||||
pair = eligible.get((cve, identity(source)))
|
||||
if (re.fullmatch(r'CVE-\d{4}-\d{4,}', cve)
|
||||
and (cve, link_health.repository(source)) not in exclusions
|
||||
and pair and (link_health.unavailable(source) or pair.get('gone_reason') == 'unavailable')):
|
||||
field = pair.get('field', 'poc')
|
||||
result[cve].append((field, row['snapshot']['url']))
|
||||
return result
|
||||
|
||||
|
||||
class ArchiveError(Exception):
|
||||
def __init__(self, reason: str, *, cooldown: int = 0, fatal: bool = False):
|
||||
super().__init__(reason)
|
||||
self.cooldown = cooldown
|
||||
self.fatal = fatal
|
||||
|
||||
|
||||
def service_error(reason: str) -> ArchiveError | None:
|
||||
if reason in {'error:max-daily-bandwidth', 'error:max-daily-bandwidth-from-ip'}:
|
||||
return ArchiveError(reason, cooldown=86400)
|
||||
if reason == 'error:user-session-limit':
|
||||
return ArchiveError(reason, cooldown=300)
|
||||
if reason == 'error:blocked-client-ip':
|
||||
return ArchiveError(reason, cooldown=3600)
|
||||
if reason in {'error:celery', 'error:job-failed', 'error:internal-server-error',
|
||||
'error:no-browsers-available', 'error:proxy-error', 'error:capture-location-error', 'error:cannot-fetch'}:
|
||||
return ArchiveError(reason, cooldown=600)
|
||||
return None
|
||||
|
||||
|
||||
class NoRedirect(request.HTTPRedirectHandler):
|
||||
# IA credentials are never forwarded to a redirected origin.
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
|
||||
def response_bytes(response, limit: int) -> bytes:
|
||||
encoding = response.headers.get('Content-Encoding', '').lower()
|
||||
if encoding in {'', 'identity'}:
|
||||
body = response.read(limit + 1)
|
||||
elif encoding == 'gzip':
|
||||
with gzip.GzipFile(fileobj=response) as stream:
|
||||
body = stream.read(limit + 1)
|
||||
else:
|
||||
raise ArchiveError('unsupported-response-encoding')
|
||||
if len(body) > limit:
|
||||
raise ArchiveError('response-too-large')
|
||||
return body
|
||||
|
||||
|
||||
class ReplayRedirect(request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
destination = parse.urlsplit(newurl)
|
||||
if destination.scheme != 'https' or destination.netloc != 'web.archive.org':
|
||||
return None
|
||||
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
||||
|
||||
|
||||
def after(seconds: float) -> str:
|
||||
return (datetime.now(timezone.utc) + timedelta(seconds=seconds)).strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
|
||||
def retry_delay(value: str, default: int = 300) -> int:
|
||||
try:
|
||||
seconds = int(value) if value.isdigit() else (parsedate_to_datetime(value) - datetime.now(timezone.utc)).total_seconds()
|
||||
return max(default, int(seconds))
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
return default
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, access: str = '', secret: str = '', interval: float = 15):
|
||||
self.auth = f'LOW {access}:{secret}' if access and secret else ''
|
||||
self.interval = max(10 if self.auth else 21, interval)
|
||||
self.last_submit = 0.0
|
||||
self.last_request = 0.0
|
||||
|
||||
def json(self, url: str, data: dict | None = None, *, auth: bool = False) -> dict:
|
||||
self.operation = ('capture' if data is not None else 'availability' if url.startswith(AVAILABILITY)
|
||||
else 'account' if '/status/user?' in url else 'capture-status')
|
||||
# Space lookups and status requests too, not just capture submissions.
|
||||
time.sleep(max(0, 1 - (time.monotonic() - self.last_request)))
|
||||
self.last_request = time.monotonic()
|
||||
headers = {'Accept': 'application/json', 'User-Agent': AGENT, 'Accept-Encoding': 'identity'}
|
||||
if auth and self.auth:
|
||||
if parse.urlsplit(url).netloc != 'web.archive.org':
|
||||
raise ValueError('Credentials may only be sent to Wayback')
|
||||
headers['Authorization'] = self.auth
|
||||
payload = None if data is None else parse.urlencode(data).encode()
|
||||
req = request.Request(url, data=payload, headers=headers)
|
||||
try:
|
||||
with request.build_opener(NoRedirect).open(req, timeout=30) as response:
|
||||
value = json.loads(response_bytes(response, 1_000_000))
|
||||
if not isinstance(value, dict):
|
||||
raise ArchiveError('invalid-response', cooldown=600)
|
||||
return value
|
||||
except error.HTTPError as exc:
|
||||
code = exc.code
|
||||
retry = exc.headers.get('Retry-After', '')
|
||||
exc.close()
|
||||
delay = retry_delay(retry) if code == 429 else 600 if code >= 500 else 0
|
||||
raise ArchiveError(f'http-{code}', cooldown=delay, fatal=code in {401, 403}) from None
|
||||
except (OSError, error.URLError, ValueError, HTTPException, EOFError) as exc:
|
||||
raise ArchiveError(type(exc).__name__, cooldown=600) from None
|
||||
|
||||
def lookup(self, url: str) -> dict | None:
|
||||
result = self.json(AVAILABILITY + '?' + parse.urlencode({'url': parse.urldefrag(url).url}))
|
||||
if not isinstance(result.get('archived_snapshots'), dict):
|
||||
raise ArchiveError('invalid-availability-response', cooldown=600)
|
||||
row = result['archived_snapshots'].get('closest') or {}
|
||||
if not row:
|
||||
return None
|
||||
if row.get('available') is not True or str(row.get('status')) != '200':
|
||||
return None
|
||||
replay = str(row.get('url', '')).replace('http://web.archive.org/', 'https://web.archive.org/', 1)
|
||||
candidate = {'url': replay, 'timestamp': row.get('timestamp'), 'verified_at': now()}
|
||||
if not valid_snapshot({'snapshot': candidate}):
|
||||
raise ArchiveError('invalid-snapshot-response', cooldown=600)
|
||||
captured = replay.split('/web/' + candidate['timestamp'] + '/', 1)[1]
|
||||
if not equivalent(captured, url):
|
||||
return None
|
||||
return candidate
|
||||
|
||||
def submit(self, url: str, *, retry: bool = False) -> dict:
|
||||
time.sleep(max(0, self.interval - (time.monotonic() - self.last_submit)))
|
||||
self.last_submit = time.monotonic()
|
||||
return self.json(API, {'url': parse.urldefrag(url).url, 'capture_all': '0', 'capture_outlinks': '0',
|
||||
'skip_first_archive': '1', 'if_not_archived_within': '1s' if retry else '30d'}, auth=True)
|
||||
|
||||
def status(self, job: str) -> dict:
|
||||
if not re.fullmatch(r'[a-zA-Z0-9-]+', job):
|
||||
raise ArchiveError('invalid-job-id')
|
||||
return self.json(API + '/status/' + job, auth=True)
|
||||
|
||||
def capacity(self) -> int:
|
||||
row = self.json(API + '/status/user?' + parse.urlencode({'_t': time.time_ns()}), auth=True)
|
||||
for field in ('daily_captures', 'daily_status'):
|
||||
used, limit = row.get(field), row.get(field + '_limit')
|
||||
if isinstance(used, int) and isinstance(limit, int) and used >= limit:
|
||||
raise ArchiveError('daily-quota-exhausted', cooldown=86400)
|
||||
available = row.get('available')
|
||||
if not isinstance(available, int) or available < 0:
|
||||
raise ArchiveError('invalid-capacity-response', cooldown=300)
|
||||
if available == 0:
|
||||
raise ArchiveError('no-capture-slots', cooldown=300)
|
||||
return min(2, available)
|
||||
|
||||
|
||||
def verify_snapshot(snap: dict, original: str, expected_blobs=(), repo_id=None) -> bool:
|
||||
"""Replay must load content, not a login page or an archive error wrapper."""
|
||||
replay = snap['url'].replace('/' + snap['timestamp'] + '/', '/' + snap['timestamp'] + 'id_/', 1)
|
||||
try:
|
||||
with request.build_opener(ReplayRedirect).open(request.Request(replay, headers={'User-Agent': AGENT}), timeout=30) as response:
|
||||
final = parse.urlsplit(response.url)
|
||||
if (response.status != 200 or final.hostname != 'web.archive.org'
|
||||
or not final.path.startswith('/web/' + snap['timestamp'] + 'id_/')):
|
||||
return False
|
||||
body = response_bytes(response, 2_000_000)
|
||||
mime = response.headers.get('Content-Type', '').lower()
|
||||
if not body:
|
||||
return False
|
||||
raw = parse.urlsplit(original).hostname == 'raw.githubusercontent.com'
|
||||
if raw and expected_blobs and len(body) >= 256:
|
||||
digest = hashlib.sha1(b'blob ' + str(len(body)).encode() + b'\0' + body).hexdigest()
|
||||
if digest not in expected_blobs:
|
||||
return False
|
||||
if not raw and ('html' in mime or body.lstrip().startswith(b'<!DOCTYPE html')):
|
||||
text = body.decode('utf-8', errors='replace')
|
||||
if repo_id and parse.urlsplit(original).hostname in {'github.com', 'www.github.com'}:
|
||||
found = re.search(r'<meta\s+name="octolytics-dimension-repository_id"\s+content="(\d+)"', text)
|
||||
if not found or found[1] != str(repo_id):
|
||||
return False
|
||||
title = re.search(r'<title[^>]*>(.*?)</title>', text, re.I | re.S)
|
||||
if title and re.search(r'^\s*(404\b|(?:page )?not found\b|page unavailable\b|access denied\b|sign in\b|log in\b|just a moment\b|Wayback Machine\s*$)', title[1], re.I):
|
||||
return False
|
||||
if parse.urlsplit(original).hostname in {'hackerone.com', 'www.hackerone.com'}:
|
||||
# A 200 HackerOne application shell is not a saved public report.
|
||||
if not re.search(r'"vulnerability_information"\s*:\s*"[^"\s]|report-content[^>]*>\s*\S', text):
|
||||
return False
|
||||
if parse.urlsplit(original).path.endswith('.json') and 'hackerone.com' == parse.urlsplit(original).hostname:
|
||||
report = json.loads(body)
|
||||
if not isinstance(report, dict) or not report.get('disclosed_at') or not report.get('vulnerability_information'):
|
||||
return False
|
||||
snap['verified_at'] = now()
|
||||
return True
|
||||
except error.HTTPError as exc:
|
||||
status = exc.code
|
||||
retry = exc.headers.get('Retry-After', '')
|
||||
exc.close()
|
||||
if status in {404, 429} or status >= 500:
|
||||
raise ArchiveError(f'replay-http-{status}', cooldown=retry_delay(retry) if status == 429 else 0) from None
|
||||
return False
|
||||
except (OSError, error.URLError, HTTPException, EOFError) as exc:
|
||||
raise ArchiveError('replay-' + type(exc).__name__) from None
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def eligible_ledger(min_year: int = 2026) -> list[dict]:
|
||||
import releases
|
||||
from build_site import load_blacklist, load_source_exclusions, load_reference_exclusions, link_key, METADATA_INPUT
|
||||
blacklist, exclusions, references = load_blacklist(), load_source_exclusions(), load_reference_exclusions()
|
||||
rejected = set()
|
||||
for path in METADATA_INPUT.glob('*.json'):
|
||||
if path.stem.isdigit() and int(path.stem) >= min_year:
|
||||
rejected.update(cve for cve, meta in json.loads(path.read_text()).items() if meta.get('rejected'))
|
||||
result = []
|
||||
for row in releases.load_ledger().values():
|
||||
url = row['url']
|
||||
cve = re.fullmatch(r'CVE-(\d{4})-\d{4,}', row.get('cve', ''))
|
||||
if (not cve or int(cve[1]) < min_year or not public_url(url) or row['cve'] in rejected
|
||||
or parse.urlsplit(url).hostname == 'web.archive.org'
|
||||
or link_health.repository(url) in blacklist
|
||||
or (row['cve'], link_health.repository(url)) in exclusions
|
||||
or (row['cve'], link_key(url)) in references
|
||||
or row.get('gone') and row.get('gone_reason') != 'unavailable'):
|
||||
continue
|
||||
result.append(row)
|
||||
return result
|
||||
|
||||
|
||||
def inventory(min_year: int = 2026) -> dict:
|
||||
import releases
|
||||
result = {}
|
||||
for row in eligible_ledger(min_year):
|
||||
url = row['url']
|
||||
key = identity(url)
|
||||
target = result.setdefault(key, {'original': url, 'cves': [], 'priority': ''})
|
||||
if row['cve'] not in target['cves']:
|
||||
target['cves'].append(row['cve'])
|
||||
target['priority'] = max(target['priority'], row.get('seen') or row.get('released') or '')
|
||||
# First observation is an upper bound, not proof the material did not
|
||||
# exist earlier. Accept older captures unless publication evidence rules
|
||||
# them out (verified history, repository creation or source date).
|
||||
released, _ = releases.date_release(row)
|
||||
earliest = releases.timestamp(released)
|
||||
if earliest:
|
||||
bound = earliest.strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
target['not_before'] = min(target.get('not_before', bound), bound)
|
||||
if row.get('repo'):
|
||||
target['repo_id'] = str(row['repo'])
|
||||
if re.fullmatch(r'/reports/\d+/?', parse.urlsplit(url).path) and parse.urlsplit(url).hostname in {'hackerone.com', 'www.hackerone.com'}:
|
||||
report_json = url.rstrip('/') + '.json'
|
||||
item = result.setdefault(identity(report_json), {'original': report_json, 'cves': [],
|
||||
'priority': target['priority'], 'parent': key, 'parent_original': url})
|
||||
if row['cve'] not in item['cves']:
|
||||
item['cves'].append(row['cve'])
|
||||
# Pin actual code bytes when source inspection already identified a revision.
|
||||
sha = row.get('sha') if row.get('commit_verified') else None
|
||||
repo = link_health.repository(url)
|
||||
files = set()
|
||||
if repo and re.fullmatch(r'[0-9a-f]{40}', sha or ''):
|
||||
files.update((sha, path) for path in row.get('history_paths', []))
|
||||
pinned = re.fullmatch(r'/[^/]+/[^/]+/(?:blob|raw)/([0-9a-f]{40})/(.+)', parse.urlsplit(url).path)
|
||||
if repo and pinned:
|
||||
files.add((pinned[1], parse.unquote(pinned[2])))
|
||||
for revision, path in sorted(files):
|
||||
raw = f'https://raw.githubusercontent.com/{repo}/{revision}/{parse.quote(path, safe="/")}'
|
||||
item = result.setdefault(identity(raw), {'original': raw, 'cves': [],
|
||||
'priority': target['priority'], 'parent': key,
|
||||
'parent_original': url})
|
||||
if row.get('head') == revision and path in row.get('history_paths', []) and row.get('blobs'):
|
||||
item['expected_blobs'] = row['blobs']
|
||||
if row['cve'] not in item['cves']:
|
||||
item['cves'].append(row['cve'])
|
||||
return result
|
||||
|
||||
|
||||
def equivalent(left: str, right: str) -> bool:
|
||||
def parts(url):
|
||||
parsed = parse.urlsplit(identity(url))
|
||||
return parsed.netloc.lower().removeprefix('www.'), parsed.path.rstrip('/'), parsed.query
|
||||
return public_url(left) and public_url(right) and parts(left) == parts(right)
|
||||
|
||||
|
||||
def active_job(row: dict) -> bool:
|
||||
return bool(row.get('job_id') and row.get('submitted_at', '') > after(-3600))
|
||||
|
||||
|
||||
def verify_candidate(row: dict) -> None:
|
||||
snap = row['candidate']
|
||||
captured = snap['url'].split('/web/' + snap['timestamp'] + '/', 1)[1]
|
||||
moment = datetime.strptime(snap['timestamp'], '%Y%m%d%H%M%S').replace(tzinfo=timezone.utc)
|
||||
lower = datetime.fromisoformat(row['not_before'].replace('Z', '+00:00')) if row.get('not_before') else None
|
||||
upper = datetime.fromisoformat(row['unavailable_since'].replace('Z', '+00:00')) if row.get('unavailable_since') else None
|
||||
row['next_check_at'] = after(60)
|
||||
row['verify_attempts'] = row.get('verify_attempts', 0) + 1
|
||||
if row['verify_attempts'] > 5:
|
||||
row['next_check_at'] = after(3600)
|
||||
valid = equivalent(captured, row['original']) and (lower is None or moment >= lower - timedelta(minutes=5))
|
||||
valid = valid and (upper is None or moment <= upper)
|
||||
if valid and verify_snapshot(snap, row['original'], row.get('expected_blobs', ()), row.get('repo_id')):
|
||||
snap['verified_at'] = now()
|
||||
if not valid_snapshot({'original': row['original'], 'snapshot': snap}):
|
||||
row.pop('candidate', None)
|
||||
row.update(error='invalid-capture-response', next_check_at=after(86400))
|
||||
return
|
||||
# Keep the first preserved version: subsequent owner edits must not
|
||||
# replace the research we captured when it was originally indexed.
|
||||
if not valid_snapshot(row):
|
||||
row['snapshot'] = snap
|
||||
else:
|
||||
row['latest_snapshot'] = snap
|
||||
row.update(status='available', attempts=0, next_check_at=after(30 * 86400))
|
||||
row.pop('error', None)
|
||||
row.pop('bad_capture', None)
|
||||
else:
|
||||
row.update(error='capture-content-unverified', bad_capture=True,
|
||||
next_check_at=after(min(7 * 86400, 3600 * 2 ** min(row.get('attempts', 0), 8))))
|
||||
row.pop('candidate', None)
|
||||
row.pop('verify_attempts', None)
|
||||
|
||||
|
||||
def capture_failure(row: dict, reason: str) -> None:
|
||||
row['error'] = reason
|
||||
if problem := service_error(reason):
|
||||
raise problem
|
||||
if reason == 'error:too-many-daily-captures':
|
||||
row['next_check_at'] = after(86400)
|
||||
if reason in {'error:too-many-requests', 'error:max-daily-bandwidth-host'}:
|
||||
row['host_retry_after'] = after(60 if reason == 'error:too-many-requests' else 86400)
|
||||
|
||||
|
||||
def process(key: str, target: dict, rows: dict, client: Client, *, capture: bool) -> None:
|
||||
row = rows.setdefault(key, {**target, 'attempts': 0})
|
||||
for field in ('not_before', 'repo_id', 'parent', 'parent_original', 'expected_blobs'):
|
||||
row.pop(field, None)
|
||||
row.update(target) # Drop corrected CVE associations; don't union stale ones.
|
||||
row.update(checked_at=now(), status='retry')
|
||||
unavailable = link_health.unavailable(row['original']) or bool(row.get('parent') and link_health.unavailable(row['parent']))
|
||||
if unavailable:
|
||||
row.setdefault('unavailable_since', link_health.unavailable_since(row['original'])
|
||||
or link_health.unavailable_since(row.get('parent', '')) or now())
|
||||
if valid_snapshot(row):
|
||||
row.pop('job_id', None)
|
||||
row.pop('candidate', None)
|
||||
row.update(status='available', next_check_at=after(30 * 86400))
|
||||
return
|
||||
else:
|
||||
row.pop('unavailable_since', None)
|
||||
if row.get('candidate'):
|
||||
verify_candidate(row)
|
||||
return
|
||||
recover_job = False
|
||||
if row.get('job_id'):
|
||||
row['next_check_at'] = after(30)
|
||||
try:
|
||||
status = client.status(row['job_id']) if active_job(row) else {}
|
||||
except ArchiveError as exc:
|
||||
if str(exc) not in {'http-404', 'http-410'}:
|
||||
raise
|
||||
status = {}
|
||||
if status.get('status') == 'pending':
|
||||
row.update(status='pending', next_check_at=after(10))
|
||||
return
|
||||
row.pop('job_id', None)
|
||||
if status.get('status') == 'success':
|
||||
try:
|
||||
row['candidate'] = {'url': snapshot_url(status.get('original_url', row['original']), status['timestamp']),
|
||||
'timestamp': status['timestamp']}
|
||||
except (ValueError, TypeError, KeyError):
|
||||
row.update(error='invalid-capture-response', next_check_at=after(3600))
|
||||
return
|
||||
verify_candidate(row)
|
||||
return # Never resubmit a terminal job in the same run.
|
||||
if status.get('status') == 'error':
|
||||
row['next_check_at'] = after(3600)
|
||||
capture_failure(row, str(status.get('status_ext') or 'capture-error'))
|
||||
return
|
||||
recover_job = True
|
||||
row['attempts'] = row.get('attempts', 0) + 1
|
||||
row['next_check_at'] = after(min(7 * 86400, 3600 * 2 ** min(row['attempts'], 8)))
|
||||
# Authenticated Save Page Now deduplicates live URLs itself; the public
|
||||
# availability API is needed for removed URLs and expired status jobs.
|
||||
if not capture or unavailable or recover_job:
|
||||
snap = client.lookup(row['original'])
|
||||
if snap:
|
||||
row['candidate'] = snap
|
||||
verify_candidate(row)
|
||||
return
|
||||
if not capture or unavailable:
|
||||
row['status'] = 'not-found'
|
||||
return
|
||||
day = now()[:10]
|
||||
budget = row.setdefault('submissions', {'date': day, 'count': 0})
|
||||
if budget['date'] != day:
|
||||
budget.update(date=day, count=0)
|
||||
if budget['count'] >= 2:
|
||||
row['next_check_at'] = after(86400)
|
||||
return
|
||||
budget['count'] += 1
|
||||
row['submitted_at'] = now()
|
||||
response = client.submit(row['original'], retry=bool(row.get('bad_capture')))
|
||||
job = response.get('job_id')
|
||||
if job and re.fullmatch(r'[a-zA-Z0-9-]+', str(job)):
|
||||
row.update(job_id=job, status='pending', next_check_at=after(10))
|
||||
row.pop('error', None)
|
||||
else:
|
||||
capture_failure(row, str(response.get('status_ext') or 'capture-not-accepted')[:160])
|
||||
|
||||
|
||||
def credentials() -> tuple[str, str]:
|
||||
access, secret = os.environ.get('IA_ACCESS_KEY', ''), os.environ.get('IA_SECRET_KEY', '')
|
||||
if bool(access) != bool(secret):
|
||||
raise ValueError('Both Internet Archive credentials must be set')
|
||||
if not access and not os.environ.get('GITHUB_ACTIONS'):
|
||||
path = Path.home() / '.config' / 'pocindex' / 'internet-archive.json'
|
||||
if path.exists():
|
||||
if path.stat().st_mode & 0o077:
|
||||
raise ValueError('Internet Archive credential file must be private (chmod 600)')
|
||||
stored = json.loads(path.read_text())
|
||||
access, secret = stored['access'], stored['secret']
|
||||
return access, secret
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--limit', type=int, default=100)
|
||||
parser.add_argument('--max-seconds', type=int, default=1200)
|
||||
parser.add_argument('--capture', action='store_true')
|
||||
parser.add_argument('--directory', type=Path, default=DIRECTORY)
|
||||
parser.add_argument('--url', action='append', help='Only process these known inventory URLs')
|
||||
parser.add_argument('--merge-from', type=Path)
|
||||
parser.add_argument('--stats', action='store_true')
|
||||
parser.add_argument('--min-year', type=int, default=2026)
|
||||
args = parser.parse_args()
|
||||
if args.limit < 1 or args.max_seconds < 1 or args.min_year < 1999:
|
||||
parser.error('Run limits must be positive')
|
||||
rows = load_records(args.directory)
|
||||
if args.merge_from:
|
||||
incoming = load_records(args.merge_from)
|
||||
for key, row in incoming.items():
|
||||
old = rows.get(key, {})
|
||||
if old.get('checked_at', '') > row.get('checked_at', ''):
|
||||
continue
|
||||
if valid_snapshot(old):
|
||||
row['snapshot'] = old['snapshot']
|
||||
rows[key] = row
|
||||
save_records(rows, set(incoming), args.directory)
|
||||
control = args.merge_from / 'control.json'
|
||||
current = args.directory / 'control.json'
|
||||
if control.exists():
|
||||
incoming_control = json.loads(control.read_text())
|
||||
existing = json.loads(current.read_text()) if current.exists() else {}
|
||||
if incoming_control.get('retry_after', '') > existing.get('retry_after', ''):
|
||||
current.write_text(json.dumps(incoming_control) + '\n')
|
||||
return 0
|
||||
targets = inventory(args.min_year)
|
||||
if args.stats:
|
||||
print(json.dumps({'min_year': args.min_year, 'targets': len(targets),
|
||||
'original_urls': sum(not r.get('parent') for r in targets.values()),
|
||||
'source_files': sum(bool(r.get('parent')) for r in targets.values()),
|
||||
'processed': sum(k in rows for k in targets),
|
||||
'snapshots': sum(valid_snapshot(rows.get(k, {})) for k in targets),
|
||||
'states': dict(Counter(rows[k].get('status') for k in targets if k in rows))}, indent=2))
|
||||
return 0
|
||||
if args.url:
|
||||
keys = {identity(url) for url in args.url}
|
||||
targets = {key: value for key, value in targets.items() if key in keys}
|
||||
if keys - targets.keys():
|
||||
parser.error('Requested URL is not in the published-artifact inventory')
|
||||
client = Client(*credentials())
|
||||
if args.capture and not client.auth:
|
||||
parser.error('Authenticated captures require IA_ACCESS_KEY and IA_SECRET_KEY')
|
||||
submitted = max((row.get('submitted_at', '') for row in rows.values()), default='')
|
||||
if submitted:
|
||||
elapsed = (datetime.now(timezone.utc) - datetime.fromisoformat(submitted.replace('Z', '+00:00'))).total_seconds()
|
||||
client.last_submit = time.monotonic() - max(0, elapsed)
|
||||
control = args.directory / 'control.json'
|
||||
if control.exists() and json.loads(control.read_text()).get('retry_after', '') > now():
|
||||
print('Archive service cooldown is still active')
|
||||
return 0
|
||||
slots = 1
|
||||
if args.capture:
|
||||
try:
|
||||
slots = client.capacity()
|
||||
except ArchiveError as exc:
|
||||
args.directory.mkdir(parents=True, exist_ok=True)
|
||||
control.write_text(json.dumps({'retry_after': after(exc.cooldown or 300), 'reason': str(exc)}) + '\n')
|
||||
print(f'Archive service requested backoff: {exc}', flush=True)
|
||||
return 1 if exc.fatal else 0
|
||||
selected = [key for key in targets if not rows.get(key, {}).get('job_id') and not rows.get(key, {}).get('candidate')
|
||||
and rows.get(key, {}).get('next_check_at', '') <= now()]
|
||||
# New discoveries first, then unsuccessful attempts, then refreshes of
|
||||
# already-preserved pages. Failed refreshes never discard a good snapshot.
|
||||
selected.sort(key=lambda key: (not valid_snapshot(rows.get(key, {})), targets[key]['priority'], key), reverse=True)
|
||||
start = time.monotonic()
|
||||
completed = 0
|
||||
while time.monotonic() - start < args.max_seconds:
|
||||
hosts = {parse.urlsplit(r['original']).hostname for r in rows.values() if r.get('host_retry_after', '') > now()}
|
||||
selected = [key for key in selected if parse.urlsplit(targets[key]['original']).hostname not in hosts]
|
||||
pending = [key for key in targets if rows.get(key, {}).get('job_id') or rows.get(key, {}).get('candidate')]
|
||||
due = [key for key in pending if rows[key].get('next_check_at', '') <= now()
|
||||
or rows[key].get('job_id') and not active_job(rows[key])]
|
||||
if due:
|
||||
key = min(due, key=lambda k: rows[k].get('next_check_at', ''))
|
||||
elif selected and completed < args.limit and sum(active_job(rows[k]) for k in pending) < slots:
|
||||
key = selected.pop(0)
|
||||
completed += 1
|
||||
elif pending:
|
||||
if all(rows[k].get('next_check_at', '') > after(args.max_seconds - (time.monotonic() - start)) for k in pending):
|
||||
break
|
||||
time.sleep(5)
|
||||
continue
|
||||
else:
|
||||
break
|
||||
attempts = rows.get(key, {}).get('attempts', 0)
|
||||
try:
|
||||
process(key, targets[key], rows, client, capture=args.capture)
|
||||
except ArchiveError as exc:
|
||||
rows[key]['error'] = str(exc)
|
||||
rows[key]['status'] = 'retry'
|
||||
if exc.fatal:
|
||||
print(f'Archive authentication failed: {exc}', flush=True)
|
||||
return 1
|
||||
if exc.cooldown:
|
||||
rows[key]['attempts'] = attempts
|
||||
rows[key]['next_check_at'] = (datetime.now(timezone.utc) + timedelta(seconds=exc.cooldown)).strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
save_records(rows, {key}, args.directory)
|
||||
control.write_text(json.dumps({'retry_after': rows[key]['next_check_at'], 'reason': str(exc)}) + '\n')
|
||||
print(f'Archive service requested backoff during {client.operation}: {exc}', flush=True)
|
||||
break
|
||||
finally:
|
||||
save_records(rows, {key}, args.directory)
|
||||
print(f'{completed}: {rows[key]["status"]} {targets[key]["original"]}', flush=True)
|
||||
print(json.dumps({'processed': completed, 'snapshots': sum(valid_snapshot(r) for r in rows.values())}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
+259
-25
@@ -54,6 +54,19 @@ SOURCES = (
|
||||
NAMED_BY_LEAF = {"zan8in/afrog", "chaitin/xray", "helloexp/0day", "tzwlhack/vulnerability"}
|
||||
COLLECTION_ROOTS = {"google/security-research": "pocs/", "github/securitylab": "SecurityExploits/"}
|
||||
|
||||
# As logic.js reads them. A page snapshot is https on web.archive.org, a 14-digit
|
||||
# UTC capture time and the http(s) page captured; a save request is not one.
|
||||
WAYBACK = "https://web.archive.org/web/"
|
||||
CAPTURE = re.compile(r'https://web\.archive\.org/web/([0-9]{14})/(https?://[^\x00-\x20\x7f"<>\\]+)')
|
||||
AUTHORITY = re.compile(r"https?://([^/?#@\x00-\x20\x7f\\]*)(?=[/?#]|\Z)")
|
||||
# A saved source file, as logic.js reads it: the raw bytes of one file at a
|
||||
# pinned commit of the page's own GitHub repository, or the JSON of the page's
|
||||
# own HackerOne report.
|
||||
RAW_FILE = re.compile(r"https://raw\.githubusercontent\.com/([^/?#]+)/([^/?#]+)/([0-9a-f]{40})/([^?#]+)")
|
||||
REPORT_PAGE = re.compile(r"https?://[^/?#]*/reports/([0-9]+)/?(?:[?#]|\Z)")
|
||||
REPORT_JSON = re.compile(r"https://(?:www\.)?hackerone\.com/reports/([0-9]+)\.json")
|
||||
DOT_SEGMENT = re.compile(r"(?:\.|%2e){1,2}", re.I)
|
||||
|
||||
|
||||
@lru_cache(maxsize=2)
|
||||
def asset_url(name: str) -> str:
|
||||
@@ -67,6 +80,17 @@ def load(name: str) -> dict | list:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def load_optional(name: str) -> dict:
|
||||
"""An enrichment the pages render without. A missing file is empty; one that
|
||||
is present but not an object stops the build like any other bad input."""
|
||||
if not os.path.exists(os.path.join(DOCS, name)):
|
||||
return {}
|
||||
payload = load(name)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"Invalid {name}: expected a JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
def esc(value) -> str:
|
||||
return html.escape(str("" if value is None else value), quote=True)
|
||||
|
||||
@@ -192,6 +216,171 @@ def artifact_label(url: str) -> str:
|
||||
return path[len(root):] if root and path.startswith(root) and len(path) > len(root) else path
|
||||
|
||||
|
||||
def wayback_capture(url: object) -> dict | None:
|
||||
"""The capture a snapshot URL names, as waybackCapture in logic.js reads it.
|
||||
Its time has to be a real one from 1996 on, when the Wayback Machine began."""
|
||||
match = CAPTURE.fullmatch(url) if isinstance(url, str) else None
|
||||
if not match:
|
||||
return None
|
||||
stamp, original = match.groups()
|
||||
try:
|
||||
moment = datetime(int(stamp[:4]), int(stamp[4:6]), int(stamp[6:8]),
|
||||
int(stamp[8:10]), int(stamp[10:12]), int(stamp[12:]))
|
||||
except ValueError:
|
||||
return None
|
||||
return {"url": url, "stamp": stamp, "original": original} if moment.year >= 1996 else None
|
||||
|
||||
|
||||
def link_host(url: str) -> str:
|
||||
"""Lower-cased host of an http(s) link, without www. or a port. Empty for
|
||||
anything else, a link carrying credentials included."""
|
||||
match = AUTHORITY.match(url)
|
||||
return re.sub(r"^www\.", "", re.sub(r":[0-9]*\Z", "", match[1].lower())) if match else ""
|
||||
|
||||
|
||||
def confirmed_snapshot(url: str, archives: dict) -> dict | None:
|
||||
"""The capture archives.json confirms for this exact link: a valid snapshot,
|
||||
stamped as recorded, of a page on the link's own host."""
|
||||
record = archives.get(url)
|
||||
capture = wayback_capture(record.get("url")) if isinstance(record, dict) else None
|
||||
if not capture or capture["stamp"] != record.get("timestamp"):
|
||||
return None
|
||||
host = link_host(url)
|
||||
return capture if host and link_host(capture["original"]) == host else None
|
||||
|
||||
|
||||
def capture_date(stamp: str) -> str:
|
||||
return f"{stamp[:4]}-{stamp[4:6]}-{stamp[6:8]} {stamp[8:10]}:{stamp[10:12]} UTC"
|
||||
|
||||
|
||||
def archived_name(label: str, capture: dict | None) -> str:
|
||||
"""The accessible name of a link that opens a snapshot in place of its original."""
|
||||
if not capture:
|
||||
return ""
|
||||
name = f"{label}, archived page snapshot captured {capture_date(capture['stamp'])}"
|
||||
return f' aria-label="{esc(name)}"'
|
||||
|
||||
|
||||
def archive_link(url: str, label: str, archives: dict) -> str:
|
||||
"""The Wayback Machine beside a PoC link, drawn as archiveLink in logic.js
|
||||
draws it: the confirmed snapshot, or a lookup that claims nothing. A link
|
||||
that is itself a snapshot is stamped, and any other archive link gets nothing."""
|
||||
capture = wayback_capture(url)
|
||||
if capture:
|
||||
return ('<span class="wayback is-archived" title="Wayback Machine page snapshot captured '
|
||||
f'{capture_date(capture["stamp"])}, linked in place of the original" aria-hidden="true">ARCHIVED</span>')
|
||||
host = link_host(url)
|
||||
if not host or re.search(r"(?:^|\.)archive\.org\Z", host):
|
||||
return ""
|
||||
snapshot = confirmed_snapshot(url, archives)
|
||||
if snapshot:
|
||||
date = capture_date(snapshot["stamp"])
|
||||
name = f"snapshot of {label} on the Wayback Machine, captured {date}"
|
||||
return (f'<a class="wayback is-snapshot" href="{esc(snapshot["url"])}" target="_blank" rel="nofollow noopener" '
|
||||
f'title="Wayback Machine page snapshot, captured {date}" aria-label="{esc(name)}">snapshot</a>')
|
||||
name = f"find archive of {label} on the Wayback Machine"
|
||||
return (f'<a class="wayback is-lookup" href="{esc(WAYBACK + "*/" + url)}" target="_blank" rel="nofollow noopener" '
|
||||
f'title="Search the Wayback Machine for captures; none confirmed" aria-label="{esc(name)}">find archive</a>')
|
||||
|
||||
|
||||
def link_path(url: str) -> str:
|
||||
"""What follows a link's host: its path, query and fragment."""
|
||||
return re.sub(r"\Ahttps?://[^/?#]*", "", url)
|
||||
|
||||
|
||||
def link_target(url: str) -> str:
|
||||
"""The page a link stands for. A snapshot published in place of a lost
|
||||
original stands for that original."""
|
||||
capture = wayback_capture(url)
|
||||
return capture["original"] if capture else url
|
||||
|
||||
|
||||
def link_identity(url: str) -> str:
|
||||
"""The page a link stands for, whatever scheme, www. or port it carries."""
|
||||
target = link_target(url)
|
||||
host = link_host(target)
|
||||
return host + link_path(target) if host else target
|
||||
|
||||
|
||||
def decoded(path: str) -> str:
|
||||
"""decodeURIComponent: a path with a malformed escape is shown as written."""
|
||||
if re.search(r"%(?![0-9A-Fa-f]{2})", path):
|
||||
return path
|
||||
try:
|
||||
return unquote(path, errors="strict")
|
||||
except UnicodeDecodeError:
|
||||
return path
|
||||
|
||||
|
||||
def source_file(parent: str, original: str) -> dict | None:
|
||||
"""The file a saved original is, when it belongs to the page at parent."""
|
||||
repo = repo_of(parent)
|
||||
raw = RAW_FILE.fullmatch(original) if repo else None
|
||||
if raw:
|
||||
owner, name, commit, path = raw.groups()
|
||||
# A dot segment would resolve into another repository's files.
|
||||
if (f"{owner}/{name}".lower() != repo
|
||||
or any(not part or DOT_SEGMENT.fullmatch(part) for part in path.split("/"))):
|
||||
return None
|
||||
return {"key": f"{commit}/{path}", "name": decoded(path), "commit": commit}
|
||||
report = REPORT_PAGE.match(parent) if link_host(parent) == "hackerone.com" else None
|
||||
saved = REPORT_JSON.fullmatch(original) if report else None
|
||||
return {"key": saved[1], "name": f"{saved[1]}.json", "commit": ""} if saved and saved[1] == report[1] else None
|
||||
|
||||
|
||||
def saved_files(url: str, archives: dict) -> list[dict]:
|
||||
"""The saved files archives.json confirms for this exact link, as savedFiles
|
||||
in logic.js reads them: valid captures, stamped as recorded, of files that
|
||||
belong to it. One capture per file, the latest, in a stable order."""
|
||||
record = archives.get(url)
|
||||
files = record.get("files") if isinstance(record, dict) else None
|
||||
found: dict = {}
|
||||
for file in files if isinstance(files, list) else ():
|
||||
capture = wayback_capture(file.get("url")) if isinstance(file, dict) else None
|
||||
original = file.get("original") if capture else None
|
||||
if (not isinstance(original, str) or not original or capture["stamp"] != file.get("timestamp")
|
||||
or link_host(original) != link_host(capture["original"])
|
||||
or link_path(original) != link_path(capture["original"])):
|
||||
continue
|
||||
source = source_file(url, original)
|
||||
kept = found.get(source["key"]) if source else None
|
||||
if source and (not kept or kept["capture"]["stamp"] < capture["stamp"]):
|
||||
found[source["key"]] = {**source, "original": original, "capture": capture}
|
||||
return sorted(found.values(), key=lambda saved: saved["original"])
|
||||
|
||||
|
||||
def shown_files(url: str, archives: dict, listed: frozenset | set = frozenset()) -> list[dict]:
|
||||
"""The saved files a link discloses. A snapshot in place of a lost original
|
||||
discloses that original's files; a file listed in a row of its own is not
|
||||
repeated."""
|
||||
return [saved for saved in saved_files(link_target(url), archives)
|
||||
if link_identity(saved["original"]) not in listed]
|
||||
|
||||
|
||||
def source_files(url: str, label: str, archives: dict, listed: frozenset | set = frozenset()) -> str:
|
||||
"""A link's saved files, disclosed beside its Wayback control as sourceFiles
|
||||
in logic.js discloses them. Each opens the capture of one file's bytes, so
|
||||
none stands for the whole source and none gets an archive control of its own."""
|
||||
files = shown_files(url, archives, listed)
|
||||
if not files:
|
||||
return ""
|
||||
count = f"{len(files)} file{'' if len(files) == 1 else 's'}"
|
||||
title = ("Wayback Machine captures of individual files at pinned commits, not a copy of the whole repository"
|
||||
if files[0]["commit"] else "Wayback Machine capture of the report JSON")
|
||||
links = []
|
||||
for saved in files:
|
||||
date = capture_date(saved["capture"]["stamp"])
|
||||
what = (f"{saved['name']} at commit {saved['commit'][:7]}" if saved["commit"]
|
||||
else f"{saved['name']}, the report JSON")
|
||||
links.append(f'<a href="{esc(saved["capture"]["url"])}" target="_blank" rel="nofollow noopener" '
|
||||
f'title="{esc(f"Wayback Machine capture of {what}, {date}")}" '
|
||||
f'aria-label="{esc(f"{what} of {label}, Wayback Machine capture {date}")}">{esc(saved["name"])}</a>')
|
||||
# The count sizes the room the open list takes, so nothing beside it moves.
|
||||
return (f'<details class="sources" style="--files:{len(files)}" data-sources="{esc(url)}">'
|
||||
f'<summary title="{title}" aria-label="{esc(f"{count} saved from {label}")}">{count}</summary>'
|
||||
f'<div class="sources-list">{"".join(links)}</div></details>')
|
||||
|
||||
|
||||
def page_lastmod(entry: dict, data: dict) -> str:
|
||||
"""Bootstrap date before a page has a published content fingerprint: the record's own."""
|
||||
stamps = (entry.get("modified") or "", entry.get("published") or "")
|
||||
@@ -239,7 +428,7 @@ def tracked_state(key: str, payload: object, today: str, fallback: str, previous
|
||||
|
||||
def content_state(entry: dict, data: dict, previous: dict) -> list[str]:
|
||||
"""Track significant content: the record, every link, CVSS, advisories,
|
||||
Nuclei and KEV.
|
||||
Nuclei, KEV and the Wayback snapshots and saved files the page links.
|
||||
|
||||
Daily EPSS fluctuations, star counts, repository pushes and site-wide totals
|
||||
are deliberately excluded; they are not a new exploit, assessment, or
|
||||
@@ -249,6 +438,20 @@ def content_state(entry: dict, data: dict, previous: dict) -> list[str]:
|
||||
meta = data["meta"].get(cid) or {}
|
||||
payload = {"entry": entry, "cvss": meta.get("cvss"), "advisories": meta.get("advisories"),
|
||||
"nuclei": data["nuclei"].get(cid), "kev": data["kev"].get(cid)}
|
||||
# Only what the page renders, and only on pages that link a snapshot or a
|
||||
# saved file, so every other page keeps the fingerprint and date it was
|
||||
# published with.
|
||||
archives = data.get("archives") or {}
|
||||
links = [url for field in ("poc", *(key for key, _, _ in SOURCES)) for url in entry.get(field) or []]
|
||||
snapshots = {url: [capture["url"], capture["stamp"]] for url in links
|
||||
if (capture := confirmed_snapshot(url, archives))}
|
||||
if snapshots:
|
||||
payload["archives"] = snapshots
|
||||
listed = frozenset(map(link_identity, links))
|
||||
files = {url: [[saved["original"], saved["capture"]["url"], saved["capture"]["stamp"]] for saved in shown]
|
||||
for url in links if (shown := shown_files(url, archives, listed))}
|
||||
if files:
|
||||
payload["files"] = files
|
||||
|
||||
def legacy() -> dict:
|
||||
# Manifests published while pushes were fingerprinted still match, so they keep their dates.
|
||||
@@ -299,7 +502,8 @@ def build_related(cves: list) -> dict:
|
||||
return related
|
||||
|
||||
|
||||
def link_rows(urls: list, repo_meta: dict, *, trusted: bool = False, collection: bool = False) -> str:
|
||||
def link_rows(urls: list, repo_meta: dict, *, trusted: bool = False, collection: bool = False,
|
||||
archives: dict | None = None, listed: frozenset = frozenset()) -> str:
|
||||
"""One row per source link.
|
||||
|
||||
The repository list is crowd-sourced: anyone can publish a repository named
|
||||
@@ -310,27 +514,43 @@ def link_rows(urls: list, repo_meta: dict, *, trusted: bool = False, collection:
|
||||
|
||||
Stars and push dates describe the repository, including for file links.
|
||||
Artifact release and update dates are kept separate on the homepage.
|
||||
|
||||
Given an archive index, each PoC row also offers the Wayback Machine, and
|
||||
discloses its saved files other than those in listed, which have rows of
|
||||
their own. A snapshot published in place of a lost original is named,
|
||||
ranked and measured as that original, and its link opens the snapshot.
|
||||
"""
|
||||
rel = "noopener" if trusted else "nofollow noopener"
|
||||
rows = []
|
||||
|
||||
def target(url: str) -> tuple[str, dict | None]:
|
||||
capture = wayback_capture(url) if archives is not None else None
|
||||
return (capture["original"] if capture else url), capture
|
||||
|
||||
def rank(url: str) -> tuple:
|
||||
meta = repo_meta.get(repo_of(url) or "")
|
||||
return -(meta[0] if meta else -1), url.lower()
|
||||
source = target(url)[0]
|
||||
meta = repo_meta.get(repo_of(source) or "")
|
||||
return -(meta[0] if meta else -1), source.lower()
|
||||
|
||||
for url in sorted(urls, key=rank):
|
||||
label = url[len(GITHUB):] if url.startswith(GITHUB) else url
|
||||
path = artifact_label(url) if trusted else ""
|
||||
repository = repo_of(url)
|
||||
source, capture = target(url)
|
||||
label = source[len(GITHUB):] if source.startswith(GITHUB) else source
|
||||
path = artifact_label(source) if trusted else ""
|
||||
repository = repo_of(source)
|
||||
meta = repo_meta.get(repository or "")
|
||||
detail = (f"Repo stars {meta[0]}★ · pushed {esc(meta[1]) or 'N/A'}" if meta
|
||||
else "Repo metadata unavailable" if repository else "")
|
||||
if path and collection:
|
||||
repository = "/".join(urlsplit(url).path.strip("/").split("/")[:2])
|
||||
repository = "/".join(urlsplit(source).path.strip("/").split("/")[:2])
|
||||
label, detail = path, f"{esc(repository)} · {detail}"
|
||||
note = f"<span>{detail}</span>" if detail else ""
|
||||
label = short(label, 90)
|
||||
archive = archive_link(url, label, archives) if archives is not None else ""
|
||||
note = " · ".join(part for part in (detail, archive) if part)
|
||||
note = f"<span>{note}</span>" if note else ""
|
||||
files = source_files(url, label, archives, listed) if archives is not None else ""
|
||||
rows.append(
|
||||
f'<li><a href="{esc(url)}" rel="{rel}" target="_blank">'
|
||||
f"{esc(short(label, 90))}</a>{note}</li>"
|
||||
f'<li><a href="{esc(url)}" rel="{rel}" target="_blank"{archived_name(label, capture)}>'
|
||||
f"{esc(label)}</a>{note}{files}</li>"
|
||||
)
|
||||
return "".join(rows)
|
||||
|
||||
@@ -446,19 +666,23 @@ def page(entry: dict, data: dict) -> str:
|
||||
if entry.get("modified"):
|
||||
facts.append(f"<dt>Updated</dt><dd>{esc(entry['modified'])}</dd>")
|
||||
|
||||
# Without an archive index every PoC still offers a Wayback lookup.
|
||||
archives = data.get("archives") or {}
|
||||
# Links with a row of their own here; no row's saved files repeat them.
|
||||
rowed =frozenset(link_identity(link) for link in
|
||||
[*pocs, *(link for key, _, _ in SOURCES for link in entry.get(key) or [])])
|
||||
blocks = []
|
||||
if pocs:
|
||||
blocks.append(
|
||||
f"<h2>Proof-of-concept exploits ({len(pocs):,})</h2>"
|
||||
f'<ul class="cve-links">{link_rows(pocs, data["repo_meta"])}</ul>'
|
||||
f'<ul class="cve-links">{link_rows(pocs, data["repo_meta"], archives=archives, listed=rowed)}</ul>'
|
||||
)
|
||||
for key, heading, _ in SOURCES:
|
||||
urls = entry.get(key) or []
|
||||
if urls:
|
||||
blocks.append(
|
||||
f"<h2>{heading} ({len(urls)})</h2>"
|
||||
f'<ul class="cve-links">{link_rows(urls, data["repo_meta"], trusted=True, collection=key == "collections")}</ul>'
|
||||
)
|
||||
listed_rows = link_rows(urls, data["repo_meta"], trusted=True, collection=key == "collections",
|
||||
archives=archives, listed=rowed)
|
||||
blocks.append(f'<h2>{heading} ({len(urls)})</h2><ul class="cve-links">{listed_rows}</ul>')
|
||||
|
||||
siblings = "".join(
|
||||
f'<li><a href="/{other}">{other}</a></li>' for other in data["related"].get(cid, ())
|
||||
@@ -736,13 +960,18 @@ def trend_age(value: object, now: datetime) -> str:
|
||||
return f'<time class="trend-age" datetime="{iso}" title="{title}">{label}</time>'
|
||||
|
||||
|
||||
def trend_rows(items: object, landed: bool, now: datetime | None = None) -> str:
|
||||
def trend_rows(items: object, landed: bool, now: datetime | None = None, archives: dict | None = None) -> str:
|
||||
"""A front-page list as logic.js renders it, in feed order. Just landed is
|
||||
dated by first release, Trending by latest artifact commit."""
|
||||
dated by first release, Trending by latest artifact commit. Each row offers
|
||||
its Wayback snapshot or a lookup, and its saved files."""
|
||||
now = now or datetime.now(timezone.utc)
|
||||
archives = archives or {}
|
||||
shown = (items if isinstance(items, list) else [])[:TREND_ROWS]
|
||||
listed = frozenset(link_identity(str(item.get("url") or "")) for item in shown)
|
||||
rows = []
|
||||
for item in (items if isinstance(items, list) else [])[:TREND_ROWS]:
|
||||
repository = repo_of(str(item.get("url") or ""))
|
||||
for item in shown:
|
||||
url = str(item.get("url") or "")
|
||||
repository = repo_of(url)
|
||||
stars = item.get("stars") if repository else None
|
||||
star_cell = "N/A" if stars is None else f'{star_label(stars)} <span class="star">★</span>'
|
||||
star_title = f"Repository stars: {repository}" if repository else "No GitHub repository"
|
||||
@@ -756,17 +985,21 @@ def trend_rows(items: object, landed: bool, now: datetime | None = None) -> str:
|
||||
flagged = '<span class="trend-kev" title="CISA known exploited">KEV</span>' if item.get("kev") else ""
|
||||
detail = (f'<a class="trend-detail" href="{esc(item["page"])}">{esc(item.get("cve"))} details</a>'
|
||||
if item.get("page") else "")
|
||||
label = str(name or url)
|
||||
rows.append(
|
||||
f'<div class="trend-row"><span class="trend-stars{popular}" title="{esc(star_title)}">{star_cell}</span>'
|
||||
f'{trend_age(item.get("released" if landed else "pushed"), now)}'
|
||||
f'<span class="trend-name-cell">{flagged}<a class="trend-name" href="{esc(item.get("url"))}" '
|
||||
f'target="_blank" rel="nofollow noopener">{esc(name)}</a>{detail}</span>'
|
||||
f'<div class="trend-name-cell">{flagged}<a class="trend-name" href="{esc(item.get("url"))}" '
|
||||
f'target="_blank" rel="nofollow noopener"{archived_name(label, wayback_capture(url))}>{esc(name)}</a>'
|
||||
f'<div class="trend-sub">{detail}{archive_link(url, label, archives)}'
|
||||
f'{source_files(url, label, archives, listed)}</div></div>'
|
||||
f'<span class="trend-desc">{esc(item.get("desc") or "Description unavailable")}</span></div>'
|
||||
)
|
||||
return "".join(rows) or '<div class="empty">No recent PoCs.</div>'
|
||||
|
||||
|
||||
def homepage(cves: list, kev: dict, trending: dict, now: datetime | None = None) -> str:
|
||||
def homepage(cves: list, kev: dict, trending: dict, now: datetime | None = None,
|
||||
archives: dict | None = None) -> str:
|
||||
"""Publish the same content before JavaScript or a crawler render runs."""
|
||||
indexed = {entry["cve"] for entry in cves}
|
||||
if trending["with_pocs"] != len(indexed) or trending["total_cves"] < len(indexed):
|
||||
@@ -792,7 +1025,7 @@ def homepage(cves: list, kev: dict, trending: dict, now: datetime | None = None)
|
||||
stylesheet_url=asset_url("style.css"), script_url=asset_url("logic.js"),
|
||||
sources_line=SOURCES_LINE, project_links=project_links(), website_schema=json_ld(website),
|
||||
total_cves=f"{trending['total_cves']:,}", with_pocs=f"{len(indexed):,}", kev=f"{len(kev):,}",
|
||||
landed_rows=trend_rows(trending.get("landed"), True, now),
|
||||
landed_rows=trend_rows(trending.get("landed"), True, now, archives),
|
||||
generated=esc(generated.strftime("%Y-%m-%d %H:%M UTC")),
|
||||
year_links=" ".join(f'<a href="/{esc(year)}">{esc(year)}</a>' for year in years),
|
||||
)
|
||||
@@ -839,6 +1072,7 @@ def main() -> int:
|
||||
"kev": load("kev.json"),
|
||||
"nuclei": load("nuclei.json"),
|
||||
"repo_meta": load("repo_meta.json"),
|
||||
"archives": load_optional("archives.json"),
|
||||
"today": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
|
||||
}
|
||||
data["related"] = build_related(cves)
|
||||
@@ -874,7 +1108,7 @@ def main() -> int:
|
||||
handle.write(not_found())
|
||||
|
||||
with open(os.path.join(DOCS, "index.html"), "w", encoding="utf-8") as handle:
|
||||
handle.write(homepage(cves, data["kev"], load("trending_poc.json")))
|
||||
handle.write(homepage(cves, data["kev"], load("trending_poc.json"), archives=data["archives"]))
|
||||
|
||||
related = sum(1 for cid in lastmod if data["related"].get(cid))
|
||||
print(f"Wrote {written:,} CVE pages and {len(hubs['pages']):,} hub pages into {DOCS} "
|
||||
|
||||
+34
-3
@@ -396,6 +396,8 @@ def collect_links(block: str, *, blacklist: Optional[Collection[str]] = None) ->
|
||||
|
||||
|
||||
def build_cve_list(blacklist: Collection[str]) -> tuple[List[Dict[str, object]], int]:
|
||||
import archive_pocs
|
||||
archived = archive_pocs.archived_fallbacks(blacklist=blacklist, exclusions=load_source_exclusions())
|
||||
cve_entries = []
|
||||
dates = load_dates()
|
||||
metadata = load_metadata()
|
||||
@@ -443,6 +445,13 @@ def build_cve_list(blacklist: Collection[str]) -> tuple[List[Dict[str, object]],
|
||||
if source_key(link) not in curated_keys:
|
||||
poc_candidates.append(link)
|
||||
poc_entries = dedupe_source_links(poc_candidates, cve_id)
|
||||
for field, url in archived.get(cve_id, []):
|
||||
original = original_artifact(url)
|
||||
if original in poc_entries or any(original in links for links in curated.values()):
|
||||
continue
|
||||
destination = curated.get(field, poc_entries)
|
||||
if url not in destination:
|
||||
destination.append(url)
|
||||
|
||||
if not poc_entries and not any(curated.values()):
|
||||
continue
|
||||
@@ -481,12 +490,18 @@ def add_observed_dates(cve_entries: list[dict], ledger: dict) -> None:
|
||||
stamps = [
|
||||
stamp for field in ["poc", *(field for _, field in CURATED)]
|
||||
for url in entry.get(field, [])
|
||||
if (stamp := releases.timestamp((ledger.get(releases.key(entry["cve"], url)) or {}).get("seen")))
|
||||
if (stamp := releases.timestamp((ledger.get(releases.key(entry["cve"], original_artifact(url)))
|
||||
or ledger.get(releases.key(entry["cve"], url)) or {}).get("seen")))
|
||||
]
|
||||
if stamps:
|
||||
entry["poc_observed"] = min(stamps).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def original_artifact(url: str) -> str:
|
||||
match = re.fullmatch(r'https://web\.archive\.org/web/\d{14}/(https?://.+)', url)
|
||||
return match[1] if match else url
|
||||
|
||||
|
||||
def build_repo_meta(cve_entries: List[Dict[str, object]]) -> Dict[str, object]:
|
||||
"""Star count and last-push date for the repositories the index actually links."""
|
||||
stored = json.loads(REPO_META.read_text(encoding="utf-8"))
|
||||
@@ -496,11 +511,11 @@ def build_repo_meta(cve_entries: List[Dict[str, object]]) -> Dict[str, object]:
|
||||
):
|
||||
raise ValueError("Invalid repository metadata; refusing to publish missing star columns")
|
||||
linked = {
|
||||
repo_from_url(url)
|
||||
repo_from_url(original_artifact(url))
|
||||
for entry in cve_entries
|
||||
for field in ["poc", *(field for _, field in CURATED)]
|
||||
for url in entry.get(field, [])
|
||||
if repo_from_url(url)
|
||||
if repo_from_url(original_artifact(url))
|
||||
}
|
||||
return {name: value for name, value in stored.items() if name in linked}
|
||||
|
||||
@@ -686,6 +701,22 @@ def main() -> int:
|
||||
add_observed_dates(cve_payload, ledger)
|
||||
write_json(CVE_OUTPUT, cve_payload)
|
||||
|
||||
import archive_pocs
|
||||
archive_rows = archive_pocs.load_records()
|
||||
allowed = {archive_pocs.identity(url) for entry in cve_payload
|
||||
for field in ['poc', *(f for _, f in CURATED)] for url in entry.get(field, [])}
|
||||
# Fallback entries point into Wayback; retain the original-to-capture map.
|
||||
allowed.update(key for key, row in archive_rows.items()
|
||||
if archive_pocs.valid_snapshot(row) and archive_pocs.identity(row['snapshot']['url']) in allowed)
|
||||
snapshots = archive_pocs.published_snapshots(archive_rows, allowed)
|
||||
aliases = {archive_pocs.identity(url): value for url, value in snapshots.items()}
|
||||
for entry in cve_payload:
|
||||
for field in ['poc', *(field for _, field in CURATED)]:
|
||||
for url in entry.get(field, []):
|
||||
if value := aliases.get(archive_pocs.identity(url)):
|
||||
snapshots[url] = value
|
||||
write_json(DOCS_DIR / 'archives.json', snapshots)
|
||||
|
||||
repo_meta = build_repo_meta(cve_payload)
|
||||
write_json(REPO_META_OUTPUT, repo_meta)
|
||||
|
||||
|
||||
@@ -72,6 +72,13 @@ def unavailable(url: str) -> bool:
|
||||
return bool(name and name in state["repositories"]) or url_key(url) in state["urls"]
|
||||
|
||||
|
||||
def unavailable_since(url: str) -> str:
|
||||
state = load()
|
||||
stamps = [value for value in (state['repositories'].get(repository(url)),
|
||||
state['urls'].get(url_key(url))) if value]
|
||||
return min(stamps, default='')
|
||||
|
||||
|
||||
def record(*, missing=(), healthy=(), dead_urls=(), live_urls=(), now: str | None = None) -> None:
|
||||
state = {field: dict(values) for field, values in load().items()}
|
||||
stamp = now or datetime.now(timezone.utc).isoformat()
|
||||
@@ -80,11 +87,11 @@ def record(*, missing=(), healthy=(), dead_urls=(), live_urls=(), now: str | Non
|
||||
state["repositories"].pop(key, None)
|
||||
for name in missing:
|
||||
if key := repository(f"https://github.com/{name}"):
|
||||
state["repositories"][key] = stamp
|
||||
state["repositories"].setdefault(key, stamp)
|
||||
for url in live_urls:
|
||||
state["urls"].pop(url_key(url), None)
|
||||
for url in dead_urls:
|
||||
state["urls"][url_key(url)] = stamp
|
||||
state["urls"].setdefault(url_key(url), stamp)
|
||||
if state == load():
|
||||
return
|
||||
STATE.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
+23
-3
@@ -256,6 +256,7 @@ def record(ledger: dict[str, dict], cve: str, url: str, evidence: dict | None =
|
||||
old = ledger.get(identity)
|
||||
row = copy.deepcopy(old) if old else {"cve": cve, "url": url, "basis": "unknown", "released": None}
|
||||
row.pop("gone", None)
|
||||
row.pop("gone_reason", None)
|
||||
if not old:
|
||||
if import_mode:
|
||||
row["imported"] = True
|
||||
@@ -431,7 +432,10 @@ def reconcile(ledger: dict[str, dict], pairs: list[tuple[str, str]], *, evidence
|
||||
if complete:
|
||||
for identity, row in result.items():
|
||||
if identity not in present:
|
||||
row.setdefault("gone", observed_at)
|
||||
if not row.get('gone'):
|
||||
import link_health
|
||||
row['gone_reason'] = 'unavailable' if link_health.unavailable(row['url']) else 'withdrawn'
|
||||
row['gone'] = observed_at
|
||||
resolve_copies(result, now=observed_at)
|
||||
mark_bulk(result)
|
||||
return result
|
||||
@@ -450,13 +454,15 @@ def landed(ledger: dict[str, dict], *, now: str | None = None, days: int = 10) -
|
||||
|
||||
|
||||
def published_pairs(entries: list[dict] | None = None) -> list[tuple[str, str]]:
|
||||
import archive_pocs
|
||||
managed = {row['snapshot']['url'] for row in archive_pocs.load_records().values() if archive_pocs.valid_snapshot(row)}
|
||||
if entries is None:
|
||||
# Lazy import also lets the site builder consume this ledger.
|
||||
from build_site import build_cve_list, load_blacklist
|
||||
entries, _ = build_cve_list(load_blacklist())
|
||||
return sorted({(row["cve"], url) for row in entries
|
||||
for field in ("poc", "nuclei", "edb", "msf", "vulhub", "collections")
|
||||
for url in row.get(field, [])})
|
||||
for url in row.get(field, []) if url not in managed})
|
||||
|
||||
|
||||
def qualifying_content(repository: str, cve: str, path: str, source: bytes, reviews: list[dict]) -> bool:
|
||||
@@ -755,8 +761,12 @@ def record_collection_snapshot(previous: dict, current: dict, pairs: list[tuple[
|
||||
for identity in removed:
|
||||
if identity in published:
|
||||
ledger[identity].pop("gone", None)
|
||||
ledger[identity].pop("gone_reason", None)
|
||||
else:
|
||||
ledger[identity].setdefault("gone", observed_at)
|
||||
if not ledger[identity].get('gone'):
|
||||
import link_health
|
||||
ledger[identity]['gone_reason'] = 'unavailable' if link_health.unavailable(ledger[identity]['url']) else 'withdrawn'
|
||||
ledger[identity]['gone'] = observed_at
|
||||
resolve_copies(ledger, now=observed_at)
|
||||
mark_bulk(ledger)
|
||||
save_ledger(ledger, directory)
|
||||
@@ -773,8 +783,18 @@ def main() -> int:
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
entries = json.loads(args.input.read_text()) if args.input else None
|
||||
if entries is None:
|
||||
from build_site import build_cve_list, load_blacklist
|
||||
entries, _ = build_cve_list(load_blacklist())
|
||||
evidence = json.loads(args.evidence.read_text()) if args.evidence else {}
|
||||
ledger = reconcile(load_ledger(args.directory), published_pairs(entries), evidence=evidence, import_mode=args.seed)
|
||||
for entry in entries:
|
||||
if int(entry['cve'].split('-')[1]) < 2026:
|
||||
continue
|
||||
for field in ('poc', 'nuclei', 'edb', 'msf', 'vulhub', 'collections'):
|
||||
for url in entry.get(field, []):
|
||||
if urlsplit(url).hostname != 'web.archive.org':
|
||||
ledger[key(entry['cve'], url)]['field'] = field
|
||||
if not args.dry_run:
|
||||
save_ledger(ledger, args.directory)
|
||||
print(f"releases: {len(ledger):,} pairs, {sum(bool(row.get('released')) for row in ledger.values()):,} dated")
|
||||
|
||||
@@ -0,0 +1,336 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
from urllib.error import HTTPError
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
import archive_pocs as archive
|
||||
import build_site
|
||||
import releases
|
||||
|
||||
URL = 'https://github.com/owner/poc'
|
||||
CVE = 'CVE-2026-1234'
|
||||
|
||||
|
||||
def snapshot(url=URL, stamp=None):
|
||||
stamp = stamp or datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')
|
||||
return {'url': archive.snapshot_url(url, stamp), 'timestamp': stamp, 'verified_at': archive.now()}
|
||||
|
||||
|
||||
class ArchiveTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.target = {'original': URL, 'cves': [CVE], 'priority': archive.now()}
|
||||
self.rows = {}
|
||||
self.client = MagicMock()
|
||||
self.client.lookup.return_value = None
|
||||
self.client.submit.return_value = {'job_id': 'pending-123'}
|
||||
p = patch.object(archive.link_health, 'unavailable', return_value=False)
|
||||
self.dead = p.start()
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
def test_inventory_limits_cve_year_and_preserves_source_paths(self):
|
||||
ledger = {
|
||||
'old': {'url': URL + '-old', 'cve': 'CVE-2025-1234'},
|
||||
'new': {'url': URL, 'cve': CVE, 'head': 'b' * 40, 'sha': 'a' * 40,
|
||||
'commit_verified': True, 'paths': ['wrong.py'], 'history_paths': ['PoC x.py']},
|
||||
'future': {'url': URL, 'cve': 'CVE-2027-4567'},
|
||||
'bad': {'url': URL + '-blocked', 'cve': CVE},
|
||||
'excluded': {'url': URL + '-excluded', 'cve': CVE},
|
||||
'withdrawn': {'url': URL + '-withdrawn', 'cve': CVE, 'gone': '2026-01-01'},
|
||||
}
|
||||
with patch.object(releases, 'load_ledger', return_value=ledger), \
|
||||
patch.object(build_site, 'load_blacklist', return_value={'owner/poc-blocked'}), \
|
||||
patch.object(build_site, 'load_source_exclusions', return_value={(CVE, 'owner/poc-excluded')}):
|
||||
result = archive.inventory()
|
||||
self.assertEqual(len(result), 2)
|
||||
self.assertEqual(set(result[URL]['cves']), {CVE, 'CVE-2027-4567'})
|
||||
raw = next(v for v in result.values() if v.get('parent'))
|
||||
self.assertTrue(raw['original'].endswith('/' + 'a' * 40 + '/PoC%20x.py'))
|
||||
|
||||
def test_snapshot_validation_rejects_fake_hosts_invalid_dates_and_private_urls(self):
|
||||
good = snapshot()
|
||||
self.assertTrue(archive.valid_snapshot({'snapshot': good}))
|
||||
for url in ('http://localhost/a', 'http://127.0.0.1/a', 'http://169.254.169.254/a',
|
||||
'https://user:[email protected]/a', 'javascript:alert(1)', 'https://host.internal/a'):
|
||||
self.assertFalse(archive.public_url(url), url)
|
||||
for bad in ({**good, 'url': good['url'].replace('web.archive.org', 'web.archive.org.evil.example')},
|
||||
{**good, 'timestamp': '20260231000000'}, {**good, 'verified_at': ''}):
|
||||
self.assertFalse(archive.valid_snapshot({'snapshot': bad}))
|
||||
|
||||
def test_first_observation_does_not_disqualify_older_public_captures(self):
|
||||
ledger = [{'url': URL, 'cve': CVE, 'seen': '2026-10-21T12:00:00Z'},
|
||||
{'url': URL + '-dated', 'cve': CVE, 'seen': '2026-10-21T12:00:00Z',
|
||||
'source': '2026-05-13'}]
|
||||
with patch.object(archive, 'eligible_ledger', return_value=ledger):
|
||||
result = archive.inventory()
|
||||
self.assertNotIn('not_before', result[URL])
|
||||
self.assertEqual(result[URL + '-dated']['not_before'], '2026-05-13T00:00:00Z')
|
||||
|
||||
def test_pinned_file_urls_preserve_bytes_without_additional_github_queries(self):
|
||||
parent = URL + '/blob/' + 'b' * 40 + '/path/PoC%20file.py#example'
|
||||
with patch.object(archive, 'eligible_ledger', return_value=[{'url': parent, 'cve': CVE}]):
|
||||
result = archive.inventory()
|
||||
raw = 'https://raw.githubusercontent.com/owner/poc/' + 'b' * 40 + '/path/PoC%20file.py'
|
||||
self.assertEqual(result[raw]['parent'], archive.identity(parent))
|
||||
self.assertEqual(result[raw]['parent_original'], parent)
|
||||
|
||||
def test_lookup_rejects_capture_of_a_different_url(self):
|
||||
client = archive.Client()
|
||||
snap = snapshot(URL + '-other')
|
||||
response = {'archived_snapshots': {'closest': {**snap, 'status': '200', 'available': True}}}
|
||||
with patch.object(client, 'json', return_value=response):
|
||||
self.assertIsNone(client.lookup(URL))
|
||||
|
||||
def test_existing_capture_is_verified_without_resubmission(self):
|
||||
self.client.lookup.return_value = snapshot()
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=False)
|
||||
self.assertTrue(archive.valid_snapshot(self.rows[URL]))
|
||||
self.client.submit.assert_not_called()
|
||||
|
||||
def test_pending_job_is_checkpointed_and_polled_without_duplicate_submission(self):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertEqual(self.rows[URL]['job_id'], 'pending-123')
|
||||
self.client.status.return_value = {'status': 'pending'}
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.submit.assert_called_once()
|
||||
self.client.lookup.assert_not_called()
|
||||
self.assertEqual(self.rows[URL]['attempts'], 1)
|
||||
|
||||
def test_success_requires_replay_verification(self):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
snap = snapshot()
|
||||
self.client.status.return_value = {'status': 'success', 'timestamp': snap['timestamp'], 'original_url': URL}
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertEqual(self.rows[URL]['status'], 'available')
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
self.assertEqual(archive.published_snapshots(self.rows)[URL]['url'], snap['url'])
|
||||
|
||||
def test_expired_job_recovers_by_lookup(self):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.status.side_effect = archive.ArchiveError('http-404')
|
||||
self.client.lookup.return_value = snapshot()
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
self.client.submit.assert_called_once()
|
||||
|
||||
def test_refresh_error_keeps_previous_good_capture(self):
|
||||
old = snapshot(stamp='20260101000000')
|
||||
self.rows[URL] = {**self.target, 'snapshot': old, 'job_id': 'pending-123', 'attempts': 1}
|
||||
self.client.status.return_value = {'status': 'error', 'status_ext': 'error:read-timeout'}
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertEqual(self.rows[URL]['snapshot'], old)
|
||||
self.assertEqual(archive.published_snapshots(self.rows)[URL]['url'], old['url'])
|
||||
|
||||
def test_unverified_terminal_job_never_resubmits_in_a_loop(self):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.status.return_value = {'status': 'success', 'timestamp': snapshot()['timestamp'], 'original_url': URL}
|
||||
with patch.object(archive, 'verify_snapshot', return_value=False):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.submit.assert_called_once()
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
self.assertNotIn('candidate', self.rows[URL])
|
||||
self.assertGreater(self.rows[URL]['next_check_at'], archive.now())
|
||||
self.assertFalse(archive.valid_snapshot(self.rows[URL]))
|
||||
|
||||
def test_transient_replay_failure_retains_candidate_without_resubmission(self):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.status.return_value = {'status': 'success', 'timestamp': snapshot()['timestamp'], 'original_url': URL}
|
||||
with patch.object(archive, 'verify_snapshot', side_effect=archive.ArchiveError('replay-http-404')):
|
||||
with self.assertRaises(archive.ArchiveError):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertIn('candidate', self.rows[URL])
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.submit.assert_called_once()
|
||||
self.assertTrue(archive.valid_snapshot(self.rows[URL]))
|
||||
|
||||
def test_expired_jobs_do_not_reserve_a_capture_slot(self):
|
||||
self.rows[URL] = {**self.target, 'job_id': 'old-job', 'submitted_at': archive.after(-7200)}
|
||||
self.assertFalse(archive.active_job(self.rows[URL]))
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=False)
|
||||
self.client.status.assert_not_called()
|
||||
self.client.lookup.assert_called_once_with(URL)
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
|
||||
def test_dead_original_with_good_capture_clears_stale_work(self):
|
||||
self.rows[URL] = {**self.target, 'snapshot': snapshot(), 'job_id': 'stale',
|
||||
'candidate': snapshot(), 'submitted_at': archive.after(-7200)}
|
||||
self.dead.return_value = True
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertNotIn('job_id', self.rows[URL])
|
||||
self.assertNotIn('candidate', self.rows[URL])
|
||||
self.assertEqual(self.rows[URL]['status'], 'available')
|
||||
|
||||
def test_fragments_never_enter_capture_identity_or_poison_a_shard(self):
|
||||
original = URL + '#poc'
|
||||
target = {**self.target, 'original': original}
|
||||
self.rows[URL] = {**target, 'job_id': 'job', 'submitted_at': archive.now()}
|
||||
self.client.status.return_value = {'status': 'success', 'timestamp': snapshot()['timestamp'], 'original_url': original}
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, target, self.rows, self.client, capture=True)
|
||||
self.assertNotIn('#', self.rows[URL]['snapshot']['url'])
|
||||
self.assertTrue(archive.valid_snapshot(self.rows[URL]))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
archive.save_records(self.rows, {URL}, Path(directory))
|
||||
self.assertEqual(archive.load_records(Path(directory)), self.rows)
|
||||
|
||||
def test_repeated_unverifiable_capture_gets_increasing_backoff(self):
|
||||
self.rows[URL] = {**self.target, 'attempts': 7, 'candidate': snapshot()}
|
||||
with patch.object(archive, 'verify_snapshot', return_value=False):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=False)
|
||||
self.assertGreater(self.rows[URL]['next_check_at'], archive.after(6 * 3600))
|
||||
|
||||
def test_redirect_homepage_and_capture_predating_poc_are_not_published(self):
|
||||
for target, captured in ((self.target, snapshot('https://github.com/')),
|
||||
({**self.target, 'not_before': archive.now()}, snapshot(stamp='20260101000000'))):
|
||||
rows = {URL: {**target, 'candidate': captured}}
|
||||
with patch.object(archive, 'verify_snapshot') as replay:
|
||||
archive.process(URL, target, rows, self.client, capture=False)
|
||||
replay.assert_not_called()
|
||||
self.assertFalse(archive.valid_snapshot(rows[URL]))
|
||||
|
||||
def test_account_errors_are_separate_from_target_errors(self):
|
||||
self.assertEqual(archive.service_error('error:max-daily-bandwidth').cooldown, 86400)
|
||||
for reason in ('error:filesize-limit', 'error:bandwidth-limit-exceeded',
|
||||
'error:too-many-daily-captures', 'error:soft-time-limit-exceeded'):
|
||||
self.assertIsNone(archive.service_error(reason), reason)
|
||||
|
||||
def test_daily_submission_budget_is_durable_and_first_capture_is_retained(self):
|
||||
self.rows[URL] = {**self.target, 'submissions': {'date': archive.now()[:10], 'count': 2}}
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.submit.assert_not_called()
|
||||
old = snapshot(stamp='20260101000000')
|
||||
self.rows[URL].update(snapshot=old, candidate=snapshot())
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.assertEqual(self.rows[URL]['snapshot'], old)
|
||||
|
||||
def test_source_snapshots_are_attached_to_their_published_parent(self):
|
||||
raw = 'https://raw.githubusercontent.com/owner/poc/' + 'a' * 40 + '/PoC.py'
|
||||
rows = {raw: {'original': raw, 'parent': URL, 'snapshot': snapshot(raw)}}
|
||||
result = archive.published_snapshots(rows, {URL})
|
||||
self.assertEqual(result[URL]['files'][0]['original'], raw)
|
||||
self.assertEqual(archive.published_snapshots(rows, set()), {})
|
||||
|
||||
def test_source_file_can_recover_a_dead_parent_without_a_page_capture(self):
|
||||
raw = 'https://raw.githubusercontent.com/owner/poc/' + 'a' * 40 + '/PoC.py'
|
||||
rows = {raw: {'original': raw, 'parent': URL, 'cves': [CVE], 'snapshot': snapshot(raw)}}
|
||||
self.dead.return_value = True
|
||||
with patch.object(archive, 'eligible_ledger', return_value=[{'cve': CVE, 'url': URL, 'field': 'nuclei'}]):
|
||||
self.assertEqual(archive.archived_fallbacks(rows)[CVE], [('nuclei', rows[raw]['snapshot']['url'])])
|
||||
|
||||
def test_recovered_original_keeps_archive_until_it_is_reimported(self):
|
||||
self.dead.return_value = False
|
||||
self.rows[URL] = {**self.target, 'snapshot': snapshot()}
|
||||
with patch.object(archive, 'eligible_ledger', return_value=[{'cve': CVE, 'url': URL, 'gone_reason': 'unavailable'}]):
|
||||
self.assertEqual(len(archive.archived_fallbacks(self.rows)[CVE]), 1)
|
||||
|
||||
def test_managed_fallback_does_not_redate_or_remove_legacy_archive_links(self):
|
||||
managed = snapshot()['url']
|
||||
legacy = archive.snapshot_url(URL + '-legacy', '20200101000000')
|
||||
with patch.object(archive, 'load_records', return_value={URL: {**self.target, 'snapshot': snapshot()}}):
|
||||
self.assertEqual(releases.published_pairs([{'cve': CVE, 'poc': [managed, legacy]}]), [(CVE, legacy)])
|
||||
old = releases.record({}, CVE, URL, observed_at='2026-01-01T00:00:00Z')
|
||||
self.dead.return_value = True
|
||||
ledger = releases.reconcile({releases.key(CVE, URL): old}, [], observed_at='2026-02-01T00:00:00Z')
|
||||
row = ledger[releases.key(CVE, URL)]
|
||||
self.assertEqual(row['seen'], '2026-01-01T00:00:00Z')
|
||||
self.assertEqual(row['gone_reason'], 'unavailable')
|
||||
|
||||
def test_dead_original_is_only_looked_up_and_blacklist_cannot_be_bypassed(self):
|
||||
self.dead.return_value = True
|
||||
self.client.lookup.return_value = snapshot()
|
||||
with patch.object(archive, 'verify_snapshot', return_value=True):
|
||||
archive.process(URL, self.target, self.rows, self.client, capture=True)
|
||||
self.client.submit.assert_not_called()
|
||||
with patch.object(archive, 'eligible_ledger', return_value=[{'cve': CVE, 'url': URL}]):
|
||||
self.assertEqual(len(archive.archived_fallbacks(self.rows)[CVE]), 1)
|
||||
self.assertFalse(archive.archived_fallbacks(self.rows, blacklist={'owner/poc'}))
|
||||
self.assertFalse(archive.archived_fallbacks(self.rows, exclusions={(CVE, 'owner/poc')}))
|
||||
self.rows[URL]['parent'] = URL + '-parent'
|
||||
self.assertFalse(archive.archived_fallbacks(self.rows))
|
||||
|
||||
def test_cooldown_and_checkpoints_survive_merge(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
source, destination = root / 'incoming', root / 'current'
|
||||
incoming = {URL: {**self.target, 'checked_at': archive.now(), 'status': 'retry'}}
|
||||
existing = {URL: {**self.target, 'checked_at': '2026-01-01', 'snapshot': snapshot()}}
|
||||
archive.save_records(incoming, {URL}, source)
|
||||
archive.save_records(existing, {URL}, destination)
|
||||
control = {'retry_after': archive.after(300), 'reason': 'http-429'}
|
||||
(source / 'control.json').write_text(json.dumps(control))
|
||||
with patch.object(sys, 'argv', ['archive', '--directory', str(destination), '--merge-from', str(source)]):
|
||||
self.assertEqual(archive.main(), 0)
|
||||
self.assertTrue(archive.valid_snapshot(archive.load_records(destination)[URL]))
|
||||
self.assertEqual(json.loads((destination / 'control.json').read_text()), control)
|
||||
|
||||
def test_quota_and_retry_after_are_respected(self):
|
||||
client = archive.Client('dummy', 'dummy')
|
||||
with patch.object(client, 'json', return_value={'daily_captures': 30000, 'daily_captures_limit': 30000, 'available': 3}):
|
||||
with self.assertRaises(archive.ArchiveError) as result:
|
||||
client.capacity()
|
||||
self.assertEqual(result.exception.cooldown, 86400)
|
||||
self.assertEqual(archive.retry_delay('1200'), 1200)
|
||||
self.assertGreaterEqual(archive.retry_delay('invalid'), 300)
|
||||
opener = MagicMock()
|
||||
opener.open.side_effect = HTTPError('https://web.archive.org/save', 429, 'wait', {'Retry-After': '600'}, None)
|
||||
with patch.object(archive.request, 'build_opener', return_value=opener):
|
||||
with self.assertRaises(archive.ArchiveError) as result:
|
||||
client.submit(URL)
|
||||
self.assertEqual(result.exception.cooldown, 600)
|
||||
|
||||
def test_capture_uses_only_explicit_url_and_never_archives_errors(self):
|
||||
client = archive.Client('dummy', 'dummy')
|
||||
with patch.object(client, 'json', return_value={'job_id': '1'}) as send:
|
||||
client.submit(URL)
|
||||
payload = send.call_args.args[1]
|
||||
self.assertEqual(payload['capture_all'], '0')
|
||||
self.assertEqual(payload['capture_outlinks'], '0')
|
||||
self.assertEqual(payload['if_not_archived_within'], '30d')
|
||||
|
||||
def test_replay_rejects_login_shells_errors_and_external_redirects(self):
|
||||
snap = snapshot()
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value = response
|
||||
response.status = 200
|
||||
response.url = snap['url'].replace(snap['timestamp'] + '/', snap['timestamp'] + 'id_/')
|
||||
response.headers = {'Content-Type': 'text/html'}
|
||||
opener = MagicMock()
|
||||
opener.open.return_value = response
|
||||
with patch.object(archive.request, 'build_opener', return_value=opener):
|
||||
for body in (b'', b'<title>404 Not Found</title>', b'<title>Sign in to GitHub</title>'):
|
||||
response.read.return_value = body
|
||||
self.assertFalse(archive.verify_snapshot(copy.deepcopy(snap), URL))
|
||||
response.read.return_value = b'<title>PoC for CVE-2026-4044</title><article>research</article>'
|
||||
self.assertTrue(archive.verify_snapshot(snap, URL))
|
||||
self.assertFalse(archive.verify_snapshot(snap, 'https://hackerone.com/reports/1234'))
|
||||
redirect = archive.ReplayRedirect()
|
||||
self.assertIsNone(redirect.redirect_request(None, None, 302, '', {}, 'http://127.0.0.1/private'))
|
||||
self.assertIsNone(archive.NoRedirect().redirect_request(None, None, 302, '', {}, 'https://archive.org/'))
|
||||
|
||||
def test_gzip_replay_is_decoded_before_content_and_hash_validation(self):
|
||||
response = io.BytesIO(gzip.compress(b'<title>Sign in to GitHub</title>'))
|
||||
response.headers = {'Content-Encoding': 'gzip'}
|
||||
self.assertEqual(archive.response_bytes(response, 1000), b'<title>Sign in to GitHub</title>')
|
||||
response = io.BytesIO(gzip.compress(b'x' * 1001))
|
||||
response.headers = {'Content-Encoding': 'gzip'}
|
||||
with self.assertRaisesRegex(archive.ArchiveError, 'too-large'):
|
||||
archive.response_bytes(response, 1000)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -87,10 +87,11 @@ process.stdout.write(JSON.stringify(markup));
|
||||
|
||||
|
||||
def page_blocks(entry: dict) -> dict[str, list[tuple[str, str, str]]]:
|
||||
"""(href, label, note) for each row under each counted heading of a CVE page."""
|
||||
"""(href, label, note) for each row under each counted heading of a CVE page,
|
||||
read without the Wayback control each PoC row ends with."""
|
||||
data = {"cves": [entry], "meta": {}, "epss": {}, "kev": {}, "nuclei": {}, "repo_meta": REPO_META,
|
||||
"related": {}, "lastmod": {entry["cve"]: "2026-10-07"}}
|
||||
markup = build_pages.page(entry, data)
|
||||
markup = re.sub(r'(?: · )?<(a|span) class="wayback[^"]*"[^>]*>[^<]*</\1>', "", build_pages.page(entry, data))
|
||||
return {
|
||||
heading: [tuple(html.unescape(part) for part in row) for row in re.findall(
|
||||
r'<li><a href="([^"]*)"[^>]*>([^<]*)</a>(?:<span>([^<]*)</span>)?</li>', body)]
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import copy
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -16,6 +17,17 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
||||
import build_pages
|
||||
import build_seo
|
||||
|
||||
# A source file saved at a pinned commit, as archives.json lists it under its page.
|
||||
COMMIT = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
|
||||
def raw_file(path: str, repository: str = "owner/poc", commit: str = COMMIT) -> str:
|
||||
return f"https://raw.githubusercontent.com/{repository}/{commit}/{path}"
|
||||
|
||||
|
||||
def saved_file(original: str, stamp: str = "20261022161013") -> dict:
|
||||
return {"original": original, "url": f"https://web.archive.org/web/{stamp}/{original}", "timestamp": stamp}
|
||||
|
||||
|
||||
class ContentDateTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
@@ -70,6 +82,65 @@ class ContentDateTests(unittest.TestCase):
|
||||
changed = {**self.entry, "desc": "Widget flaw, now remote"}
|
||||
self.assertEqual(build_pages.content_state(changed, self.data, published)[1], self.data["today"])
|
||||
|
||||
def test_confirmed_snapshots_advance_the_date_and_other_archive_records_do_not(self) -> None:
|
||||
url, stamp = self.entry["poc"][0], "20251021093000"
|
||||
record = {"url": f"https://web.archive.org/web/{stamp}/{url}", "timestamp": stamp}
|
||||
original = build_pages.content_state(self.entry, self.data, {})
|
||||
previous = {self.entry["cve"]: original}
|
||||
# Records the page cannot render leave it as published.
|
||||
for archives in ({}, {"https://github.com/owner/other": record}, {url: {**record, "timestamp": "20251021093001"}},
|
||||
{url: {**record, "url": f"http://web.archive.org/web/{stamp}/{url}"}}):
|
||||
with self.subTest(archives=archives):
|
||||
self.assertEqual(build_pages.content_state(self.entry, {**self.data, "archives": archives}, previous),
|
||||
original)
|
||||
archived = build_pages.content_state(self.entry, {**self.data, "archives": {url: record}}, previous)
|
||||
self.assertNotEqual(archived[0], original[0])
|
||||
self.assertEqual(archived[1], self.data["today"])
|
||||
later = {**self.data, "today": "2026-10-07"}
|
||||
self.assertEqual(build_pages.content_state(self.entry, {**later, "archives": {url: {**record, "checked": "x"}}},
|
||||
{self.entry["cve"]: archived}), archived)
|
||||
self.assertEqual(build_pages.content_state(self.entry, later, {self.entry["cve"]: archived})[1], "2026-10-07")
|
||||
|
||||
def test_saved_files_advance_the_date_and_other_file_records_do_not(self) -> None:
|
||||
url, cid = self.entry["poc"][0], self.entry["cve"]
|
||||
|
||||
def archived(files: object, data: dict = self.data) -> dict:
|
||||
return {**data, "archives": {url: {"files": files}}}
|
||||
|
||||
original = build_pages.content_state(self.entry, self.data, {})
|
||||
previous = {cid: original}
|
||||
# Files the page cannot show leave it as published.
|
||||
for files in ([], [saved_file(raw_file("poc.py", "owner/other"))], [saved_file(raw_file("poc.py", commit="main"))],
|
||||
[{**saved_file(raw_file("poc.py")), "timestamp": "20261022161014"}],
|
||||
[saved_file("https://hackerone.com/reports/123456.json")]):
|
||||
with self.subTest(files=files):
|
||||
self.assertEqual(build_pages.content_state(self.entry, archived(files), previous), original)
|
||||
# A first saved file advances the page while its own snapshot is pending.
|
||||
first = build_pages.content_state(self.entry, archived([saved_file(raw_file("poc.py"))]), previous)
|
||||
self.assertNotEqual(first[0], original[0])
|
||||
self.assertEqual(first[1], self.data["today"])
|
||||
later = {**self.data, "today": "2026-10-07"}
|
||||
# Order, repeats and older captures of the same file change nothing the page shows.
|
||||
same = [saved_file(raw_file("poc.py"), "20261021120000"), saved_file(raw_file("poc.py")), saved_file(raw_file("poc.py"))]
|
||||
self.assertEqual(build_pages.content_state(self.entry, archived(same, later), {cid: first}), first)
|
||||
# A refreshed capture, or another file, does.
|
||||
for files in ([saved_file(raw_file("poc.py"), "20261023100000")],
|
||||
[saved_file(raw_file("poc.py")), saved_file(raw_file("lib.py"))]):
|
||||
with self.subTest(files=files):
|
||||
changed = build_pages.content_state(self.entry, archived(files, later), {cid: first})
|
||||
self.assertNotEqual(changed[0], first[0])
|
||||
self.assertEqual(changed[1], "2026-10-07")
|
||||
# A file the page lists in a row of its own is not disclosed, or counted, twice.
|
||||
listed = {**self.entry, "poc": [url, raw_file("poc.py")]}
|
||||
self.assertEqual(build_pages.content_state(listed, archived([saved_file(raw_file("poc.py"))]), {}),
|
||||
build_pages.content_state(listed, self.data, {}))
|
||||
# Pages with a snapshot and no files keep the fingerprint they were published with.
|
||||
snapshot = {"url": f"https://web.archive.org/web/20251021093000/{url}", "timestamp": "20251021093000"}
|
||||
payload = {"entry": self.entry, "cvss": None, "advisories": None, "nuclei": None, "kev": None,
|
||||
"archives": {url: [snapshot["url"], snapshot["timestamp"]]}}
|
||||
self.assertEqual(build_pages.content_state(self.entry, {**self.data, "archives": {url: snapshot}}, {})[0],
|
||||
build_pages.digest(payload))
|
||||
|
||||
def test_missing_manifest_bootstraps_but_outages_do_not_erase_state(self) -> None:
|
||||
with patch.object(build_pages.request, "urlopen", side_effect=HTTPError("url", 404, "missing", {}, None)):
|
||||
self.assertEqual(build_pages.previous_page_state(), {})
|
||||
@@ -253,5 +324,290 @@ class RepositoryMetadataTests(unittest.TestCase):
|
||||
self.assertNotIn("★", row)
|
||||
|
||||
|
||||
class WaybackTests(unittest.TestCase):
|
||||
URL = "https://github.com/owner/poc"
|
||||
STAMP = "20251021093000"
|
||||
SNAPSHOT = f"https://web.archive.org/web/{STAMP}/{URL}"
|
||||
FILES_TITLE = "Wayback Machine captures of individual files at pinned commits, not a copy of the whole repository"
|
||||
|
||||
def render(self, entry: dict, archives: dict, repo_meta: dict | None = None, meta: dict | None = None) -> str:
|
||||
data = {"cves": [entry], "meta": meta or {}, "epss": {}, "kev": {}, "nuclei": {}, "repo_meta": repo_meta or {},
|
||||
"related": {}, "lastmod": {entry["cve"]: "2026-10-21"}, "archives": archives}
|
||||
return build_pages.page(entry, data)
|
||||
|
||||
def test_cve_pages_link_confirmed_snapshots_and_label_lookups(self) -> None:
|
||||
other = "https://www.exploit-db.com/exploits/50592"
|
||||
advisory = "https://vendor.example/advisory"
|
||||
entry = {"cve": "CVE-2026-1234", "desc": "Widget flaw", "poc": [self.URL], "edb": [other]}
|
||||
markup = self.render(entry, {self.URL: {"url": self.SNAPSHOT, "timestamp": self.STAMP}},
|
||||
meta={entry["cve"]: {"advisories": [[advisory, ["Vendor Advisory"]]]}})
|
||||
self.assertIn(
|
||||
'Repo metadata unavailable · '
|
||||
f'<a class="wayback is-snapshot" href="{self.SNAPSHOT}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Wayback Machine page snapshot, captured 2025-10-21 09:30 UTC" '
|
||||
'aria-label="snapshot of owner/poc on the Wayback Machine, captured 2025-10-21 09:30 UTC">snapshot</a>', markup)
|
||||
self.assertIn(
|
||||
f'<span><a class="wayback is-lookup" href="https://web.archive.org/web/*/{other}" target="_blank" '
|
||||
'rel="nofollow noopener" title="Search the Wayback Machine for captures; none confirmed" '
|
||||
f'aria-label="find archive of {other} on the Wayback Machine">find archive</a></span>', markup)
|
||||
# The original stays the PoC link, counts are unchanged and advisories are not PoCs.
|
||||
self.assertIn(f'<li><a href="{self.URL}" rel="nofollow noopener" target="_blank">owner/poc</a>', markup)
|
||||
self.assertIn("<title>CVE-2026-1234: 2 public PoC exploits | PoC Index</title>", markup)
|
||||
advisories = markup.split("<h2>Security advisories</h2>", 1)[1].split("</ul>", 1)[0]
|
||||
self.assertIn(f'href="{advisory}"', advisories)
|
||||
self.assertNotIn("wayback", advisories)
|
||||
self.assertNotIn("web.archive.org/save", markup)
|
||||
|
||||
def test_untrusted_snapshot_records_fall_back_to_a_lookup(self) -> None:
|
||||
url, stamp, snapshot = self.URL, self.STAMP, self.SNAPSHOT
|
||||
for record in (
|
||||
{"url": f"http://web.archive.org/web/{stamp}/{url}", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org.example/web/{stamp}/{url}", "timestamp": stamp},
|
||||
{"url": f"https://example.com/web.archive.org/web/{stamp}/{url}", "timestamp": stamp},
|
||||
{"url": f"https://[email protected]/web/{stamp}/{url}", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org/save/{url}", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org/web/*/{url}", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org/web/{stamp}id_/{url}", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org/web/2025102109300/{url}", "timestamp": "2025102109300"},
|
||||
{"url": f"https://web.archive.org/web/20250230093000/{url}", "timestamp": "20250230093000"},
|
||||
{"url": f"https://web.archive.org/web/20251021240000/{url}", "timestamp": "20251021240000"},
|
||||
{"url": f"https://web.archive.org/web/19951021093000/{url}", "timestamp": "19951021093000"},
|
||||
{"url": snapshot, "timestamp": "20251021093001"},
|
||||
{"url": snapshot, "timestamp": int(stamp)},
|
||||
{"url": snapshot},
|
||||
{"url": f"https://web.archive.org/web/{stamp}/javascript:alert(1)", "timestamp": stamp},
|
||||
{"url": f"https://web.archive.org/web/{stamp}/https://example.com/poc", "timestamp": stamp},
|
||||
{"url": f'{snapshot}"><script>alert(1)</script>', "timestamp": stamp},
|
||||
{"url": f"{snapshot} onmouseover=alert(1)", "timestamp": stamp},
|
||||
snapshot, None, [snapshot, stamp],
|
||||
):
|
||||
with self.subTest(record=record):
|
||||
link = build_pages.archive_link(url, "owner/poc", {url: record})
|
||||
self.assertIn(f'<a class="wayback is-lookup" href="https://web.archive.org/web/*/{url}"', link)
|
||||
self.assertNotIn("is-snapshot", link)
|
||||
self.assertNotIn("<script", link)
|
||||
self.assertNotIn("onmouseover", link)
|
||||
# The Wayback Machine may record the scheme, www. or default port differently.
|
||||
canonical = {"url": f"https://web.archive.org/web/{stamp}/http://www.github.com:80/owner/poc", "timestamp": stamp}
|
||||
self.assertIn('class="wayback is-snapshot"', build_pages.archive_link(url, "owner/poc", {url: canonical}))
|
||||
|
||||
def test_archive_links_escape_urls_and_labels(self) -> None:
|
||||
url = "https://example.com/exploit?id=1&name='x'"
|
||||
snapshot = f"https://web.archive.org/web/{self.STAMP}/{url}"
|
||||
for archives, href, kind in (({}, f"https://web.archive.org/web/*/{url}", "is-lookup"),
|
||||
({url: {"url": snapshot, "timestamp": self.STAMP}}, snapshot, "is-snapshot")):
|
||||
with self.subTest(kind=kind):
|
||||
markup = build_pages.archive_link(url, '<b>"label"</b>', archives)
|
||||
(tag, attrs), = ParsedPage(markup).tags
|
||||
self.assertEqual((tag, attrs["class"], attrs["href"]), ("a", f"wayback {kind}", href))
|
||||
self.assertIn('<b>"label"</b>', attrs["aria-label"])
|
||||
self.assertNotIn("<b>", markup)
|
||||
|
||||
def test_archived_fallbacks_link_their_snapshot_under_the_original_name(self) -> None:
|
||||
original = "https://github.com/owner/gone-poc"
|
||||
fallback = f"https://web.archive.org/web/20240101120000/{original}"
|
||||
live = "https://github.com/owner/live"
|
||||
collection = "https://github.com/google/security-research/tree/master/pocs/linux/kernelctf/CVE-2024-26642_lts"
|
||||
entry = {"cve": "CVE-2026-1234", "desc": "Widget flaw", "poc": [live, fallback],
|
||||
"collections": [f"https://web.archive.org/web/20240101120000/{collection}"]}
|
||||
markup = self.render(entry, {original: {"url": fallback, "timestamp": "20240101120000"}},
|
||||
{"owner/gone-poc": [50, "2024-01-01"], "owner/live": [10, "2026-10-01"]})
|
||||
stamp = ('<span class="wayback is-archived" title="Wayback Machine page snapshot captured 2024-01-01 12:00 UTC, '
|
||||
'linked in place of the original" aria-hidden="true">ARCHIVED</span>')
|
||||
rows = re.findall(r'<li><a href="([^"]*)"([^>]*)>([^<]*)</a>(?:<span>(.*?)</span>)?</li>', markup)
|
||||
# Ranked, named and measured as the original, linked to the snapshot.
|
||||
self.assertEqual(rows[:3], [
|
||||
(fallback, ' rel="nofollow noopener" target="_blank" '
|
||||
'aria-label="owner/gone-poc, archived page snapshot captured 2024-01-01 12:00 UTC"',
|
||||
"owner/gone-poc", f"Repo stars 50★ · pushed 2024-01-01 · {stamp}"),
|
||||
(live, ' rel="nofollow noopener" target="_blank"', "owner/live",
|
||||
"Repo stars 10★ · pushed 2026-10-01 · "
|
||||
f'<a class="wayback is-lookup" href="https://web.archive.org/web/*/{live}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Search the Wayback Machine for captures; none confirmed" '
|
||||
'aria-label="find archive of owner/live on the Wayback Machine">find archive</a>'),
|
||||
(f"https://web.archive.org/web/20240101120000/{collection}",
|
||||
' rel="noopener" target="_blank" aria-label="linux/kernelctf/CVE-2024-26642_lts, archived page snapshot '
|
||||
'captured 2024-01-01 12:00 UTC"',
|
||||
"linux/kernelctf/CVE-2024-26642_lts", f"google/security-research · Repo metadata unavailable · {stamp}"),
|
||||
])
|
||||
self.assertNotIn(f'href="{original}"', markup)
|
||||
self.assertNotIn(f'href="{collection}"', markup)
|
||||
self.assertNotIn("/web/*/https://web.archive.org", markup)
|
||||
self.assertIn("3 public PoC exploits", markup)
|
||||
|
||||
def test_cve_pages_disclose_saved_files_beside_snapshots_and_lookups(self) -> None:
|
||||
pending = "https://github.com/Owner/Other-PoC"
|
||||
report = "https://hackerone.com/reports/123456"
|
||||
newest, spaced, report_json = (saved_file(raw_file("poc.py")), saved_file(raw_file("src/exploit%20one.py")),
|
||||
saved_file(f"{report}.json"))
|
||||
entry = {"cve": "CVE-2026-1234", "desc": "Widget flaw", "poc": [self.URL, pending, report]}
|
||||
markup = self.render(entry, {
|
||||
# One capture per file, the latest, however often the record lists it.
|
||||
self.URL: {"url": self.SNAPSHOT, "timestamp": self.STAMP,
|
||||
"files": [spaced, saved_file(raw_file("poc.py"), "20261021120000"), newest, newest]},
|
||||
# A page whose own snapshot is still pending.
|
||||
pending: {"files": [saved_file(raw_file("run.sh", "owner/other-poc"))]},
|
||||
report: {"files": [report_json]},
|
||||
# Each file is also listed under its own original, which is no row.
|
||||
newest["original"]: {"url": newest["url"], "timestamp": newest["timestamp"]},
|
||||
})
|
||||
self.assertIn(
|
||||
'aria-label="snapshot of owner/poc on the Wayback Machine, captured 2025-10-21 09:30 UTC">snapshot</a></span>'
|
||||
f'<details class="sources" style="--files:2" data-sources="{self.URL}"><summary title="{self.FILES_TITLE}" '
|
||||
'aria-label="2 files saved from owner/poc">2 files</summary><div class="sources-list">'
|
||||
f'<a href="{newest["url"]}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Wayback Machine capture of poc.py at commit 0123456, 2026-10-22 16:10 UTC" '
|
||||
'aria-label="poc.py at commit 0123456 of owner/poc, Wayback Machine capture 2026-10-22 16:10 UTC">poc.py</a>'
|
||||
f'<a href="{spaced["url"]}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Wayback Machine capture of src/exploit one.py at commit 0123456, 2026-10-22 16:10 UTC" '
|
||||
'aria-label="src/exploit one.py at commit 0123456 of owner/poc, Wayback Machine capture 2026-10-22 16:10 UTC">'
|
||||
"src/exploit one.py</a></div></details></li>", markup)
|
||||
self.assertNotIn("20261021120000", markup)
|
||||
self.assertIn(
|
||||
f'find archive</a></span><details class="sources" style="--files:1" data-sources="{pending}">'
|
||||
f'<summary title="{self.FILES_TITLE}" aria-label="1 file saved from Owner/Other-PoC">1 file</summary>', markup)
|
||||
self.assertIn(
|
||||
f'find archive</a></span><details class="sources" style="--files:1" data-sources="{report}">'
|
||||
f'<summary title="Wayback Machine capture of the report JSON" aria-label="1 file saved from {report}">1 file</summary>'
|
||||
f'<div class="sources-list"><a href="{report_json["url"]}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Wayback Machine capture of 123456.json, the report JSON, 2026-10-22 16:10 UTC" '
|
||||
f'aria-label="123456.json, the report JSON of {report}, Wayback Machine capture 2026-10-22 16:10 UTC">'
|
||||
"123456.json</a></div></details></li>", markup)
|
||||
# Saved files are no PoCs of their own, and nothing saved is looked up again.
|
||||
self.assertIn("<title>CVE-2026-1234: 3 public PoC exploits | PoC Index</title>", markup)
|
||||
self.assertEqual(markup.count('class="wayback '), 3)
|
||||
self.assertEqual(markup.count("<details "), 3)
|
||||
self.assertNotRegex(markup, r"/web/\*/https://(?:web\.archive\.org|raw\.githubusercontent\.com|hackerone\.com/reports/123456\.json)")
|
||||
|
||||
def test_saved_files_must_be_captures_of_the_linked_pages_own_files(self) -> None:
|
||||
good = saved_file(raw_file("poc.py"))
|
||||
stamp = good["timestamp"]
|
||||
|
||||
def disclosed(parent: str, files: object) -> str:
|
||||
return build_pages.source_files(parent, "label", {parent: {"files": files}})
|
||||
|
||||
# GitHub names repositories without regard to case or a .git suffix.
|
||||
for parent in (self.URL, "https://github.com/Owner/POC.git", "https://www.github.com/owner/poc/blob/main/README.md"):
|
||||
with self.subTest(parent=parent):
|
||||
self.assertIn("1 file</summary>", disclosed(parent, [good]))
|
||||
for file in (
|
||||
saved_file(raw_file("poc.py", "owner/other")),
|
||||
saved_file(raw_file("poc.py", "other/poc")),
|
||||
saved_file(raw_file("poc.py", commit="main")),
|
||||
saved_file(raw_file("poc.py", commit=COMMIT[1:])),
|
||||
saved_file(raw_file("poc.py", commit=COMMIT.upper())),
|
||||
# A dot segment would open another repository's file.
|
||||
saved_file(raw_file(f"../../other/poc/{COMMIT}/poc.py")),
|
||||
saved_file(raw_file("%2e%2E/poc.py")),
|
||||
saved_file(raw_file("./poc.py")),
|
||||
saved_file(raw_file("src//poc.py")),
|
||||
saved_file(raw_file("poc.py?token=1")),
|
||||
saved_file(raw_file("poc.py#L1")),
|
||||
saved_file(f"http://raw.githubusercontent.com/owner/poc/{COMMIT}/poc.py"),
|
||||
saved_file(f"https://raw.githubusercontent.com.example/owner/poc/{COMMIT}/poc.py"),
|
||||
saved_file(f"https://gist.githubusercontent.com/owner/poc/raw/{COMMIT}/poc.py"),
|
||||
saved_file(f"https://github.com/owner/poc/blob/{COMMIT}/poc.py"),
|
||||
saved_file(f"https://example.com/owner/poc/{COMMIT}/poc.py"),
|
||||
saved_file("https://hackerone.com/reports/123456.json"),
|
||||
# An archive link is never saved again.
|
||||
saved_file(good["url"]),
|
||||
{**good, "url": saved_file(raw_file("other.py"))["url"]},
|
||||
{**good, "url": f"https://web.archive.org/web/{stamp}/https://example.com/owner/poc/{COMMIT}/poc.py"},
|
||||
{**good, "timestamp": "20261022161014"},
|
||||
{**good, "timestamp": int(stamp)},
|
||||
{**good, "url": f"https://web.archive.org/web/{stamp}id_/{good['original']}"},
|
||||
{**good, "url": f"https://web.archive.org/web/*/{good['original']}"},
|
||||
{**good, "url": f"http://web.archive.org/web/{stamp}/{good['original']}"},
|
||||
{**good, "url": f"https://web.archive.org/web/20250230093000/{good['original']}", "timestamp": "20250230093000"},
|
||||
{"url": good["url"], "timestamp": stamp},
|
||||
good["url"], None, [good],
|
||||
):
|
||||
with self.subTest(file=file):
|
||||
self.assertEqual(disclosed(self.URL, [file]), "")
|
||||
for files in (good, "files", None, {}):
|
||||
with self.subTest(files=files):
|
||||
self.assertEqual(disclosed(self.URL, files), "")
|
||||
# Advisories, gists, other hosts and files themselves hold no repository files.
|
||||
for parent in ("https://github.com/owner/poc/security/advisories/GHSA-1234-5678-9abc",
|
||||
"https://gist.github.com/owner/0123", "https://example.com/owner/poc", good["original"]):
|
||||
with self.subTest(parent=parent):
|
||||
self.assertEqual(disclosed(parent, [good]), "")
|
||||
# A HackerOne report holds its own report JSON and nothing else.
|
||||
report_json = saved_file("https://hackerone.com/reports/123456.json")
|
||||
for parent in ("https://hackerone.com/reports/123456", "https://www.hackerone.com/reports/123456/",
|
||||
"https://hackerone.com/reports/123456?subject=x"):
|
||||
with self.subTest(parent=parent):
|
||||
self.assertIn("1 file</summary>", disclosed(parent, [report_json]))
|
||||
for parent, file in (
|
||||
("https://hackerone.com/reports/123456", saved_file("https://hackerone.com/reports/654321.json")),
|
||||
("https://hackerone.com/reports/123456", saved_file("https://hackerone.example/reports/123456.json")),
|
||||
("https://hackerone.com/reports/123456", saved_file("https://hackerone.com/reports/123456")),
|
||||
("https://hackerone.com/reports/123456", saved_file("https://hackerone.com/reports/123456.json?x=1")),
|
||||
("https://hackerone.com/reports/123456", good),
|
||||
("https://hackerone.com/reports/123456.json", report_json),
|
||||
("https://hackerone.com/bugs?report_id=123456", report_json),
|
||||
("https://example.com/reports/123456", report_json),
|
||||
):
|
||||
with self.subTest(parent=parent, file=file):
|
||||
self.assertEqual(disclosed(parent, [file]), "")
|
||||
|
||||
def test_saved_file_names_links_and_labels_are_escaped(self) -> None:
|
||||
odd = saved_file(raw_file("x%22%3E%3Cimg%20src%3Dx%20onerror%3Dalert(1)%3E/it's.py"))
|
||||
malformed = saved_file(raw_file("100%.py"))
|
||||
markup =build_pages.source_files(self.URL, '<b>"label"</b>', {self.URL: {"files": [odd, malformed]}})
|
||||
page = ParsedPage(markup)
|
||||
self.assertEqual([tag for tag, _ in page.tags], ["details", "summary", "div", "a", "a"])
|
||||
self.assertEqual(page.tags[1][1]["aria-label"], '2 files saved from <b>"label"</b>')
|
||||
links = [attrs for tag, attrs in page.tags if tag == "a"]
|
||||
self.assertEqual([attrs["href"] for attrs in links], [malformed["url"], odd["url"]])
|
||||
self.assertEqual(links[1]["aria-label"], 'x"><img src=x onerror=alert(1)>/it\'s.py at commit 0123456 of '
|
||||
'<b>"label"</b>, Wayback Machine capture 2026-10-22 16:10 UTC')
|
||||
# A malformed escape is shown as written.
|
||||
self.assertEqual(page.text[1:], ["100%.py", 'x"><img src=x onerror=alert(1)>/it\'s.py'])
|
||||
self.assertNotIn("<img", markup)
|
||||
self.assertNotIn("<b>", markup)
|
||||
|
||||
def test_archived_fallbacks_disclose_their_original_saved_files_once(self) -> None:
|
||||
original = "https://github.com/owner/gone-poc"
|
||||
fallback = f"https://web.archive.org/web/20240101120000/{original}"
|
||||
exploit = saved_file(raw_file("exploit.c", "owner/gone-poc"))
|
||||
listed, helper = saved_file(raw_file("poc.py")), saved_file(raw_file("helper.py"))
|
||||
entry = {"cve": "CVE-2026-1234", "desc": "Widget flaw", "poc": [fallback, self.URL, listed["original"]]}
|
||||
archives = {original: {"url": fallback, "timestamp": "20240101120000", "files": [exploit]},
|
||||
self.URL: {"files": [listed, helper]},
|
||||
listed["original"]: {"url": listed["url"], "timestamp": listed["timestamp"]}}
|
||||
markup = self.render(entry, archives)
|
||||
self.assertIn('linked in place of the original" aria-hidden="true">ARCHIVED</span></span>'
|
||||
f'<details class="sources" style="--files:1" data-sources="{fallback}"><summary', markup)
|
||||
self.assertIn(f'<a href="{exploit["url"]}" target="_blank" rel="nofollow noopener" '
|
||||
'title="Wayback Machine capture of exploit.c at commit 0123456, 2026-10-22 16:10 UTC" '
|
||||
'aria-label="exploit.c at commit 0123456 of owner/gone-poc, Wayback Machine capture '
|
||||
'2026-10-22 16:10 UTC">exploit.c</a>', markup)
|
||||
# The lost original is never linked, and no capture is looked up again.
|
||||
self.assertNotIn(f'href="{original}"', markup)
|
||||
self.assertNotRegex(markup, r"/web/\*/https://(?:web\.archive\.org|raw\.githubusercontent\.com)")
|
||||
# poc.py has a row and a snapshot of its own, so its page discloses only helper.py.
|
||||
self.assertIn('aria-label="1 file saved from owner/poc">1 file</summary>', markup)
|
||||
self.assertIn(f'<a href="{helper["url"]}"', markup)
|
||||
self.assertEqual(markup.count(f'href="{listed["url"]}"'), 1)
|
||||
self.assertIn("3 public PoC exploits", markup)
|
||||
# The page's fingerprint follows the files it shows.
|
||||
data = {"today": "2026-10-21", "meta": {}, "nuclei": {}, "kev": {}, "repo_meta": {}}
|
||||
without = {**archives, original: {"url": fallback, "timestamp": "20240101120000"}}
|
||||
self.assertNotEqual(build_pages.content_state(entry, {**data, "archives": archives}, {})[0],
|
||||
build_pages.content_state(entry, {**data, "archives": without}, {})[0])
|
||||
|
||||
def test_a_missing_archive_index_is_empty_and_a_malformed_one_stops_the_build(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory, patch.object(build_pages, "DOCS", directory):
|
||||
self.assertEqual(build_pages.load_optional("archives.json"), {})
|
||||
index = Path(directory, "archives.json")
|
||||
records = {self.URL: {"url": self.SNAPSHOT, "timestamp": self.STAMP}}
|
||||
index.write_text(json.dumps(records), encoding="utf-8")
|
||||
self.assertEqual(build_pages.load_optional("archives.json"), records)
|
||||
index.write_text("[]", encoding="utf-8")
|
||||
with self.assertRaisesRegex(ValueError, "archives.json"):
|
||||
build_pages.load_optional("archives.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -77,8 +77,10 @@ const context = vm.createContext({
|
||||
location: { pathname: '/', search: '' },
|
||||
window: { matchMedia: () => ({ matches: false }) },
|
||||
setTimeout() {}, clearTimeout() {},
|
||||
fetch: url => url === '/trending_poc.json'
|
||||
? Promise.resolve({ ok: true, json: async () => input.feed })
|
||||
// With archives, an empty index lets the archive download that follows it run.
|
||||
fetch: url => url === '/trending_poc.json' ? Promise.resolve({ ok: true, json: async () => input.feed })
|
||||
: input.archives && url === '/CVE_list.json' ? Promise.resolve({ ok: true, json: async () => [] })
|
||||
: input.archives && url === '/archives.json' ? Promise.resolve({ ok: true, json: async () => input.archives })
|
||||
: new Promise(() => {})
|
||||
});
|
||||
vm.runInContext(fs.readFileSync(input.script, 'utf8'), context);
|
||||
@@ -100,12 +102,13 @@ setImmediate(() => {
|
||||
"""
|
||||
|
||||
|
||||
def browser(feed: dict) -> dict:
|
||||
"""What logic.js shows once the feed arrives, then for each switch value."""
|
||||
def browser(feed: dict, archives: dict | None = None) -> dict:
|
||||
"""What logic.js shows once the feed, and any archive index, arrives, then
|
||||
for each switch value."""
|
||||
node = shutil.which("node")
|
||||
if node is None:
|
||||
raise unittest.SkipTest("node renders the browser lists")
|
||||
payload = {"script": str(ROOT / "docs" / "logic.js"), "feed": feed, "now": NOW.isoformat()}
|
||||
payload = {"script": str(ROOT / "docs" / "logic.js"), "feed": feed, "now": NOW.isoformat(), "archives": archives}
|
||||
done = subprocess.run([node, "-e", HARNESS], input=json.dumps(payload), capture_output=True,
|
||||
text=True, timeout=60)
|
||||
if done.returncode:
|
||||
@@ -191,6 +194,64 @@ class HomepageListTests(unittest.TestCase):
|
||||
self.assertEqual([self.views[mode]["pressed"] for mode in ("initial", "TRENDING", "RECENT")],
|
||||
[["true", "false"], ["false", "true"], ["true", "false"]])
|
||||
|
||||
def test_wayback_snapshots_and_lookups_match_in_browser_and_prerender(self) -> None:
|
||||
stamp = "20261006160000"
|
||||
archives = {
|
||||
SHARED: {"url": f"https://web.archive.org/web/{stamp}/{SHARED}", "timestamp": stamp},
|
||||
# Recorded with another scheme, www. and port: still this page.
|
||||
LANDED[2]["url"]: {"url": f"https://web.archive.org/web/{stamp}/http://exploit-db.com:80/exploits/50592",
|
||||
"timestamp": stamp},
|
||||
# Another site's capture is not this link's snapshot.
|
||||
KERNELCTF + "mitigation": {"url": f"https://web.archive.org/web/{stamp}/https://example.com/", "timestamp": stamp},
|
||||
}
|
||||
views = browser(FEED, archives)
|
||||
landed = build_pages.trend_rows(LANDED, True, NOW, archives)
|
||||
self.assertEqual(views["initial"]["rows"], as_browser(landed))
|
||||
self.assertEqual(views["TRENDING"]["rows"], as_browser(build_pages.trend_rows(TRENDING, False, NOW, archives)))
|
||||
self.assertIn(landed, build_pages.homepage(INDEXED, {}, FEED, NOW, archives))
|
||||
self.assertEqual(landed.count('class="wayback is-snapshot"'), 2)
|
||||
self.assertIn(f'<a class="wayback is-lookup" href="https://web.archive.org/web/*/{KERNELCTF}mitigation"', landed)
|
||||
self.assertIn('aria-label="snapshot of CVE-2026-21589 on the Wayback Machine, captured 2026-10-06 16:00 UTC"', landed)
|
||||
# Everything else in the rows reads as it does without archives.
|
||||
self.assertEqual(cells(views["initial"]["rows"]), cells(self.views["LANDED"]["rows"]))
|
||||
|
||||
def test_saved_files_match_in_browser_and_prerender(self) -> None:
|
||||
stamp, commit = "20261006160000", "0123456789abcdef0123456789abcdef01234567"
|
||||
exploit = (f"https://raw.githubusercontent.com/google/security-research/{commit}/"
|
||||
"pocs/linux/kernelctf/CVE-2024-26642_mitigation/exploit.c")
|
||||
poc = f"https://raw.githubusercontent.com/owner/cve-2026-21589/{commit}/poc.py"
|
||||
|
||||
def saved(original: str) -> dict:
|
||||
return {"original": original, "url": f"https://web.archive.org/web/{stamp}/{original}", "timestamp": stamp}
|
||||
|
||||
archives = {
|
||||
SHARED: {"url": f"https://web.archive.org/web/{stamp}/{SHARED}", "timestamp": stamp, "files": [saved(poc)]},
|
||||
# Saved files while the page snapshot is pending.
|
||||
KERNELCTF + "mitigation": {"files": [saved(exploit)]},
|
||||
# Not this page's file.
|
||||
LANDED[2]["url"]: {"files": [saved(poc)]},
|
||||
}
|
||||
views = browser(FEED, archives)
|
||||
landed = build_pages.trend_rows(LANDED, True, NOW, archives)
|
||||
trending = build_pages.trend_rows(TRENDING, False, NOW, archives)
|
||||
self.assertEqual(views["initial"]["rows"], as_browser(landed))
|
||||
self.assertEqual(views["TRENDING"]["rows"], as_browser(trending))
|
||||
self.assertIn(landed, build_pages.homepage(INDEXED, {}, FEED, NOW, archives))
|
||||
self.assertEqual((landed.count("<details "), trending.count("<details ")), (2, 1))
|
||||
self.assertIn(
|
||||
f'find archive</a><details class="sources" style="--files:1" data-sources="{KERNELCTF}mitigation">'
|
||||
'<summary title="Wayback Machine captures of individual files at pinned commits, not a copy of the whole '
|
||||
'repository" aria-label="1 file saved from CVE-2024-26642_mitigation">1 file</summary>'
|
||||
f'<div class="sources-list"><a href="https://web.archive.org/web/{stamp}/{exploit}" target="_blank" '
|
||||
'rel="nofollow noopener" title="Wayback Machine capture of pocs/linux/kernelctf/CVE-2024-26642_mitigation/'
|
||||
'exploit.c at commit 0123456, 2026-10-06 16:00 UTC" aria-label="pocs/linux/kernelctf/CVE-2024-26642_mitigation/'
|
||||
'exploit.c at commit 0123456 of CVE-2024-26642_mitigation, Wayback Machine capture 2026-10-06 16:00 UTC">'
|
||||
'pocs/linux/kernelctf/CVE-2024-26642_mitigation/exploit.c</a></div></details></div></div>', landed)
|
||||
self.assertIn(f'snapshot</a><details class="sources" style="--files:1" data-sources="{SHARED}">', trending)
|
||||
# Everything else in the rows reads as it does without archives.
|
||||
self.assertEqual(cells(views["initial"]["rows"]), cells(self.views["LANDED"]["rows"]))
|
||||
self.assertEqual(cells(views["TRENDING"]["rows"]), cells(self.views["TRENDING"]["rows"]))
|
||||
|
||||
def test_an_empty_list_shows_the_empty_state_in_both(self) -> None:
|
||||
empty = build_pages.trend_rows([], True)
|
||||
self.assertIn("No recent PoCs.", empty)
|
||||
|
||||
+385
-2
@@ -9,7 +9,8 @@ const { MessageChannel } = require('node:worker_threads');
|
||||
|
||||
const script = fs.readFileSync(path.join(__dirname, '../docs/logic.js'), 'utf8');
|
||||
|
||||
function startPage(t, { search = '', trending, kev = {}, repositories = {}, indexError = false, cvssError = false, entries }) {
|
||||
function startPage(t, { search = '', trending, kev = {}, repositories = {}, archives = {}, published = [],
|
||||
indexError = false, cvssError = false, archivesError = false, entries }) {
|
||||
const elements = new Map();
|
||||
const calls = [];
|
||||
const document = {
|
||||
@@ -19,7 +20,9 @@ function startPage(t, { search = '', trending, kev = {}, repositories = {}, inde
|
||||
if (!elements.has(selector)) elements.set(selector, {
|
||||
value: '',
|
||||
innerHTML: selector === '[data-trend-rows]' ? '<a href="/CVE-2026-1234">Prerendered PoC</a>' : '',
|
||||
addEventListener() {}, setAttribute() {}, querySelectorAll: () => []
|
||||
addEventListener() {}, setAttribute() {},
|
||||
// The snapshot links a prerendered homepage carries.
|
||||
querySelectorAll: () => selector === '[data-trend-rows]' ? published : []
|
||||
});
|
||||
return elements.get(selector);
|
||||
}
|
||||
@@ -33,9 +36,11 @@ function startPage(t, { search = '', trending, kev = {}, repositories = {}, inde
|
||||
calls.push(url);
|
||||
if (url === '/CVE_list.json' && indexError) throw new Error('Index unavailable');
|
||||
if (url === '/cvss.json' && cvssError) throw new Error('CVSS unavailable');
|
||||
if (url === '/archives.json' && archivesError) throw new Error('Archive index unavailable');
|
||||
const data = url === '/trending_poc.json' ? await trending
|
||||
: url === '/kev.json' ? await kev
|
||||
: url === '/repo_meta.json' ? await repositories
|
||||
: url === '/archives.json' ? await archives
|
||||
: url === '/CVE_list.json' ? entries || [{ cve: 'CVE-2026-1234', desc: 'widget', poc: ['https://github.com/owner/poc'] }]
|
||||
: {};
|
||||
return { ok: true, json: async () => data };
|
||||
@@ -55,6 +60,13 @@ async function searchFixture(t, entries, options = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
// A source file saved at a pinned commit, as archives.json lists it under its page.
|
||||
const COMMIT = '0123456789abcdef0123456789abcdef01234567';
|
||||
const rawFile = (path, repository = 'owner/poc', commit = COMMIT) =>
|
||||
`https://raw.githubusercontent.com/${repository}/${commit}/${path}`;
|
||||
const savedFile = (original, stamp = '20261022161013') =>
|
||||
({ original, url: `https://web.archive.org/web/${stamp}/${original}`, timestamp: stamp });
|
||||
|
||||
test('repository name prefixes beat incidental prose without joining numbered filenames', async t => {
|
||||
const search = await searchFixture(t, [
|
||||
{ cve: 'CVE-2004-0660', desc: 'XSS in (1) show_archives.php, (2) show_news.php', edb: ['https://www.exploit-db.com/exploits/24238'] },
|
||||
@@ -366,6 +378,377 @@ test('curated artifact paths survive search rendering without duplicate reposito
|
||||
assert.equal(vm.runInContext(`normalizedLink(${JSON.stringify(variants[0])}) === normalizedLink(${JSON.stringify(base)})`, context), true);
|
||||
});
|
||||
|
||||
test('PoC rows link a confirmed Wayback snapshot and otherwise offer a labelled lookup', async t => {
|
||||
const archived = 'https://github.com/owner/archived-poc';
|
||||
const other = 'https://www.exploit-db.com/exploits/50592';
|
||||
const snapshot = `https://web.archive.org/web/20251021093000/${archived}`;
|
||||
const entries = [{ cve: 'CVE-2026-1234', desc: 'widget', poc: [archived], edb: [other] }];
|
||||
const { elements } = startPage(t, { search: '?q=widget', entries, trending: new Promise(() => {}),
|
||||
archives: { [archived]: { url: snapshot, timestamp: '20251021093000' } } });
|
||||
await new Promise(setImmediate);
|
||||
const html = elements.get('[data-results]').innerHTML;
|
||||
assert.ok(html.includes(`<a class="wayback is-snapshot" href="${snapshot}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Wayback Machine page snapshot, captured 2025-10-21 09:30 UTC" ' +
|
||||
'aria-label="snapshot of owner/archived-poc on the Wayback Machine, captured 2025-10-21 09:30 UTC">snapshot</a>'));
|
||||
assert.ok(html.includes(`<a class="wayback is-lookup" href="https://web.archive.org/web/*/${other}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Search the Wayback Machine for captures; none confirmed" ' +
|
||||
'aria-label="find archive of exploit-db.com/50592 on the Wayback Machine">find archive</a>'));
|
||||
assert.equal(html.match(/class="wayback /g).length, 2);
|
||||
// The original stays the PoC link and the count is unchanged.
|
||||
assert.ok(html.includes(`<a href="${archived}" target="_blank" rel="nofollow noopener">archived-poc</a>`));
|
||||
assert.match(html, /2 linked PoCs/);
|
||||
assert.doesNotMatch(html, /web\.archive\.org\/save/);
|
||||
});
|
||||
|
||||
test('malformed or untrusted snapshot records fall back to a lookup and links are escaped', async t => {
|
||||
const { context } = startPage(t, { trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
const url = 'https://github.com/owner/poc';
|
||||
const stamp = '20251021093000';
|
||||
const snapshot = `https://web.archive.org/web/${stamp}/${url}`;
|
||||
const untrusted = [
|
||||
{ url: `http://web.archive.org/web/${stamp}/${url}`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org.example/web/${stamp}/${url}`, timestamp: stamp },
|
||||
{ url: `https://example.com/web.archive.org/web/${stamp}/${url}`, timestamp: stamp },
|
||||
{ url: `https://[email protected]/web/${stamp}/${url}`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org/save/${url}`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org/web/*/${url}`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org/web/${stamp}id_/${url}`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org/web/2025102109300/${url}`, timestamp: '2025102109300' },
|
||||
{ url: `https://web.archive.org/web/20250230093000/${url}`, timestamp: '20250230093000' },
|
||||
{ url: `https://web.archive.org/web/20251021240000/${url}`, timestamp: '20251021240000' },
|
||||
{ url: `https://web.archive.org/web/19951021093000/${url}`, timestamp: '19951021093000' },
|
||||
{ url: snapshot, timestamp: '20251021093001' },
|
||||
{ url: snapshot, timestamp: Number(stamp) },
|
||||
{ url: snapshot },
|
||||
{ url: `https://web.archive.org/web/${stamp}/javascript:alert(1)`, timestamp: stamp },
|
||||
{ url: `https://web.archive.org/web/${stamp}/https://example.com/poc`, timestamp: stamp },
|
||||
{ url: `${snapshot}"><script>alert(1)</script>`, timestamp: stamp },
|
||||
{ url: `${snapshot} onmouseover=alert(1)`, timestamp: stamp },
|
||||
snapshot, null, [snapshot, stamp]
|
||||
];
|
||||
for (const record of untrusted) {
|
||||
context.record = record;
|
||||
const row = vm.runInContext(`archives = { ${JSON.stringify(url)}: record }; pocRow(${JSON.stringify(url)})`, context);
|
||||
assert.ok(row.includes(`<a class="wayback is-lookup" href="https://web.archive.org/web/*/${url}"`), JSON.stringify(record));
|
||||
assert.doesNotMatch(row, /is-snapshot|<script|onmouseover/);
|
||||
}
|
||||
// The Wayback Machine may record the scheme, www. or default port differently.
|
||||
context.record = { url: `https://web.archive.org/web/${stamp}/http://www.github.com:80/owner/poc`, timestamp: stamp };
|
||||
assert.match(vm.runInContext(`archives = { ${JSON.stringify(url)}: record }; pocRow(${JSON.stringify(url)})`, context),
|
||||
/class="wayback is-snapshot"/);
|
||||
const query = "https://example.com/exploit?id=1&name='x'";
|
||||
context.record = { url: `https://web.archive.org/web/${stamp}/${query}`, timestamp: stamp };
|
||||
const linked = vm.runInContext(`archives = { ${JSON.stringify(query)}: record }; archiveLink(${JSON.stringify(query)}, '<b>"label"</b>')`, context);
|
||||
assert.ok(linked.includes(`href="https://web.archive.org/web/${stamp}/https://example.com/exploit?id=1&name='x'"`));
|
||||
assert.ok(linked.includes('aria-label="snapshot of <b>"label"</b> on the Wayback Machine, captured 2025-10-21 09:30 UTC"'));
|
||||
const lookup = vm.runInContext(`archives = {}; archiveLink(${JSON.stringify(query)}, 'label')`, context);
|
||||
assert.ok(lookup.includes('href="https://web.archive.org/web/*/https://example.com/exploit?id=1&name='x'"'));
|
||||
});
|
||||
|
||||
test('an archived fallback stands in for its original without a second archive link', async t => {
|
||||
const original = 'https://github.com/owner/php2shell-poc';
|
||||
const fallback = `https://web.archive.org/web/20240101120000/${original}`;
|
||||
const live = 'https://github.com/owner/live';
|
||||
const entries = [
|
||||
{ cve: 'CVE-2026-1000', desc: 'Pending details', poc: [live, fallback] },
|
||||
{ cve: 'CVE-2026-2000', desc: 'A report mentions php2shell in passing.', poc: ['https://github.com/owner/unrelated'] }
|
||||
];
|
||||
const { context, elements } = startPage(t, { search: '?q=php2shell', entries, trending: new Promise(() => {}),
|
||||
repositories: { 'owner/php2shell-poc': [50, '2024-01-01'], 'owner/live': [10, '2026-10-01'] },
|
||||
archives: { [original]: { url: fallback, timestamp: '20240101120000' } } });
|
||||
while (!vm.runInContext('wordIndex.ready', context)) await new Promise(setImmediate);
|
||||
await new Promise(setImmediate);
|
||||
const search = (query, scan = false) => {
|
||||
context.query = query;
|
||||
return Array.from(vm.runInContext(`runSearch(query, ${scan})`, context), row => row.cve);
|
||||
};
|
||||
// Matched, filtered and ranked as the original would be, not as web.archive.org.
|
||||
assert.deepEqual(search('php2shell'), ['CVE-2026-1000', 'CVE-2026-2000']);
|
||||
assert.deepEqual(search('php2shell'), search('php2shell', true));
|
||||
assert.deepEqual(search('archive'), []);
|
||||
assert.deepEqual([...vm.runInContext('entrySources(dataset[0])', context)], ['GITHUB']);
|
||||
assert.deepEqual(Array.from(vm.runInContext('rankedLinks(dataset[0])', context)), [fallback, live]);
|
||||
const html = elements.get('[data-results]').innerHTML;
|
||||
assert.match(html, /2 linked PoCs/);
|
||||
assert.ok(html.includes(`owner<span class="poc-sep"> / </span><a href="${fallback}" ` +
|
||||
'aria-label="php2shell-poc, archived page snapshot captured 2024-01-01 12:00 UTC" target="_blank" rel="nofollow noopener">php2shell-poc</a>'));
|
||||
assert.ok(html.includes('<span class="wayback is-archived" title="Wayback Machine page snapshot captured 2024-01-01 12:00 UTC, ' +
|
||||
'linked in place of the original" aria-hidden="true">ARCHIVED</span>'));
|
||||
// The lost original is never linked, and the snapshot gets no archive link of its own.
|
||||
assert.ok(!html.includes(`href="${original}"`));
|
||||
assert.ok(!html.includes(`/web/*/${fallback}`));
|
||||
assert.equal(html.match(/class="wayback is-lookup"/g).length, 2);
|
||||
const template = 'https://github.com/projectdiscovery/nuclei-templates/blob/main/http/cves/2021/CVE-2021-44228.yaml';
|
||||
assert.match(vm.runInContext(`pocRow('https://web.archive.org/web/20240101120000/${template}', true)`, context),
|
||||
/<span class="poc-tag is-nuclei"[^>]*>NUCLEI<\/span><a href="https:\/\/web\.archive\.org\/web\/20240101120000\/https:[^"]+" aria-label="CVE-2021-44228\.yaml, archived page snapshot captured 2024-01-01 12:00 UTC"[^>]*>CVE-2021-44228\.yaml<\/a>/);
|
||||
// Other archive links, readable or not, get no Wayback control at all.
|
||||
for (const url of ['https://web.archive.org/web/2011/https://example.com/old', 'https://archive.org/details/exploit-kit']) {
|
||||
assert.doesNotMatch(vm.runInContext(`pocRow(${JSON.stringify(url)})`, context), /class="wayback/);
|
||||
}
|
||||
});
|
||||
|
||||
test('saved files are disclosed beside a snapshot or a lookup without counting as PoCs', async t => {
|
||||
const repo = 'https://github.com/owner/poc';
|
||||
const pending = 'https://github.com/Owner/Other-PoC';
|
||||
const report = 'https://hackerone.com/reports/123456';
|
||||
const live = 'https://github.com/owner/live';
|
||||
const page = `https://web.archive.org/web/20261022160913/${repo}`;
|
||||
const newest = savedFile(rawFile('poc.py'));
|
||||
const spaced = savedFile(rawFile('src/exploit%20one.py'));
|
||||
const json = savedFile(`${report}.json`);
|
||||
const entries = [{ cve: 'CVE-2026-1234', desc: 'widget', poc: [repo, pending, report, live] }];
|
||||
const archives = {
|
||||
// One capture per file, the latest, however often the record lists it.
|
||||
[repo]: { url: page, timestamp: '20261022160913',
|
||||
files: [spaced, savedFile(rawFile('poc.py'), '20261021120000'), newest, newest] },
|
||||
// A page whose own snapshot is still pending.
|
||||
[pending]: { files: [savedFile(rawFile('run.sh', 'owner/other-poc'))] },
|
||||
[report]: { files: [json] },
|
||||
// Each file is also listed under its own original, which is no row.
|
||||
[newest.original]: { url: newest.url, timestamp: newest.timestamp }
|
||||
};
|
||||
const { elements } = startPage(t, { search: '?q=widget', entries, archives, trending: new Promise(() => {}),
|
||||
repositories: { 'owner/live': [90, '2026-10-01'], 'owner/poc': [50, '2026-10-01'], 'owner/other-poc': [10, '2026-10-01'] } });
|
||||
await new Promise(setImmediate);
|
||||
const html = elements.get('[data-results]').innerHTML;
|
||||
// Rows keep their star order and count, and each keeps one Wayback control.
|
||||
const order = [live, repo, pending, report].map(url => html.indexOf(`href="${url}"`));
|
||||
assert.ok(order[0] >= 0);
|
||||
assert.deepEqual([...order].sort((a, b) => a - b), order);
|
||||
assert.match(html, /4 linked PoCs/);
|
||||
assert.equal(html.match(/class="wayback /g).length, 4);
|
||||
assert.equal(html.match(/<details /g).length, 3);
|
||||
assert.ok(html.includes(`<div class="poc-archive"><a class="wayback is-snapshot" href="${page}"`));
|
||||
assert.ok(html.includes('snapshot</a><details class="sources" style="--files:2" data-sources="https://github.com/owner/poc">' +
|
||||
'<summary title="Wayback Machine captures of individual files at pinned commits, not a copy of the whole repository" ' +
|
||||
'aria-label="2 files saved from owner/poc">2 files</summary><div class="sources-list">' +
|
||||
`<a href="${newest.url}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Wayback Machine capture of poc.py at commit 0123456, 2026-10-22 16:10 UTC" ' +
|
||||
'aria-label="poc.py at commit 0123456 of owner/poc, Wayback Machine capture 2026-10-22 16:10 UTC">poc.py</a>' +
|
||||
`<a href="${spaced.url}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Wayback Machine capture of src/exploit one.py at commit 0123456, 2026-10-22 16:10 UTC" ' +
|
||||
'aria-label="src/exploit one.py at commit 0123456 of owner/poc, Wayback Machine capture 2026-10-22 16:10 UTC">' +
|
||||
'src/exploit one.py</a></div></details></div>'));
|
||||
assert.ok(!html.includes('20261021120000'));
|
||||
assert.ok(html.includes(`find archive</a><details class="sources" style="--files:1" data-sources="${pending}">` +
|
||||
'<summary title="Wayback Machine captures of individual files at pinned commits, not a copy of the whole repository" ' +
|
||||
'aria-label="1 file saved from Owner/Other-PoC">1 file</summary>'));
|
||||
assert.ok(html.includes(`find archive</a><details class="sources" style="--files:1" data-sources="${report}">` +
|
||||
'<summary title="Wayback Machine capture of the report JSON" aria-label="1 file saved from hackerone.com/reports/123456">' +
|
||||
`1 file</summary><div class="sources-list"><a href="${json.url}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Wayback Machine capture of 123456.json, the report JSON, 2026-10-22 16:10 UTC" ' +
|
||||
'aria-label="123456.json, the report JSON of hackerone.com/reports/123456, Wayback Machine capture 2026-10-22 16:10 UTC">' +
|
||||
'123456.json</a></div></details>'));
|
||||
// No saved file or snapshot is looked up again.
|
||||
assert.doesNotMatch(html, /\/web\/\*\/https:\/\/(?:web\.archive\.org|raw\.githubusercontent\.com|hackerone\.com\/reports\/123456\.json)/);
|
||||
});
|
||||
|
||||
test('saved files must be captures of the linked page\'s own files', async t => {
|
||||
const { context } = startPage(t, { trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
const disclosed = (parent, files) => {
|
||||
context.record = { files };
|
||||
return vm.runInContext(`archives = { ${JSON.stringify(parent)}: record }; sourceFiles(${JSON.stringify(parent)}, 'label')`, context);
|
||||
};
|
||||
const repo = 'https://github.com/owner/poc';
|
||||
const good = savedFile(rawFile('poc.py'));
|
||||
const stamp = good.timestamp;
|
||||
// GitHub names repositories without regard to case or a .git suffix.
|
||||
for (const parent of [repo, 'https://github.com/Owner/POC.git', 'https://www.github.com/owner/poc/blob/main/README.md']) {
|
||||
assert.match(disclosed(parent, [good]), /1 file<\/summary>/, parent);
|
||||
}
|
||||
for (const file of [
|
||||
savedFile(rawFile('poc.py', 'owner/other')),
|
||||
savedFile(rawFile('poc.py', 'other/poc')),
|
||||
savedFile(rawFile('poc.py', 'owner/poc', 'main')),
|
||||
savedFile(rawFile('poc.py', 'owner/poc', COMMIT.slice(1))),
|
||||
savedFile(rawFile('poc.py', 'owner/poc', COMMIT.toUpperCase())),
|
||||
// A dot segment would open another repository's file.
|
||||
savedFile(rawFile(`../../other/poc/${COMMIT}/poc.py`)),
|
||||
savedFile(rawFile('%2e%2E/poc.py')),
|
||||
savedFile(rawFile('./poc.py')),
|
||||
savedFile(rawFile('src//poc.py')),
|
||||
savedFile(rawFile('poc.py?token=1')),
|
||||
savedFile(rawFile('poc.py#L1')),
|
||||
savedFile(`http://raw.githubusercontent.com/owner/poc/${COMMIT}/poc.py`),
|
||||
savedFile(`https://raw.githubusercontent.com.example/owner/poc/${COMMIT}/poc.py`),
|
||||
savedFile(`https://gist.githubusercontent.com/owner/poc/raw/${COMMIT}/poc.py`),
|
||||
savedFile(`https://github.com/owner/poc/blob/${COMMIT}/poc.py`),
|
||||
savedFile(`https://example.com/owner/poc/${COMMIT}/poc.py`),
|
||||
savedFile('https://hackerone.com/reports/123456.json'),
|
||||
// An archive link is never saved again.
|
||||
savedFile(good.url),
|
||||
{ ...good, url: savedFile(rawFile('other.py')).url },
|
||||
{ ...good, url: `https://web.archive.org/web/${stamp}/https://example.com/owner/poc/${COMMIT}/poc.py` },
|
||||
{ ...good, timestamp: '20261022161014' },
|
||||
{ ...good, timestamp: Number(stamp) },
|
||||
{ ...good, url: `https://web.archive.org/web/${stamp}id_/${good.original}` },
|
||||
{ ...good, url: `https://web.archive.org/web/*/${good.original}` },
|
||||
{ ...good, url: `http://web.archive.org/web/${stamp}/${good.original}` },
|
||||
{ ...good, url: `https://web.archive.org/web/20250230093000/${good.original}`, timestamp: '20250230093000' },
|
||||
{ url: good.url, timestamp: stamp },
|
||||
good.url, null, [good]
|
||||
]) {
|
||||
assert.equal(disclosed(repo, [file]), '', JSON.stringify(file));
|
||||
}
|
||||
for (const files of [good, 'files', null, {}]) assert.equal(disclosed(repo, files), '', JSON.stringify(files));
|
||||
// Advisories, gists, other hosts and files themselves hold no repository files.
|
||||
for (const parent of ['https://github.com/owner/poc/security/advisories/GHSA-1234-5678-9abc', 'https://gist.github.com/owner/0123',
|
||||
'https://example.com/owner/poc', good.original]) {
|
||||
assert.equal(disclosed(parent, [good]), '', parent);
|
||||
}
|
||||
// A HackerOne report holds its own report JSON and nothing else.
|
||||
const json = savedFile('https://hackerone.com/reports/123456.json');
|
||||
for (const parent of ['https://hackerone.com/reports/123456', 'https://www.hackerone.com/reports/123456/',
|
||||
'https://hackerone.com/reports/123456?subject=x']) {
|
||||
assert.match(disclosed(parent, [json]), /1 file<\/summary>/, parent);
|
||||
}
|
||||
for (const [parent, file] of [
|
||||
['https://hackerone.com/reports/123456', savedFile('https://hackerone.com/reports/654321.json')],
|
||||
['https://hackerone.com/reports/123456', savedFile('https://hackerone.example/reports/123456.json')],
|
||||
['https://hackerone.com/reports/123456', savedFile('https://hackerone.com/reports/123456')],
|
||||
['https://hackerone.com/reports/123456', savedFile('https://hackerone.com/reports/123456.json?x=1')],
|
||||
['https://hackerone.com/reports/123456', good],
|
||||
['https://hackerone.com/reports/123456.json', json],
|
||||
['https://hackerone.com/bugs?report_id=123456', json],
|
||||
['https://example.com/reports/123456', json]
|
||||
]) {
|
||||
assert.equal(disclosed(parent, [file]), '', `${parent} ${JSON.stringify(file)}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('saved file names, links and labels are escaped', async t => {
|
||||
const { context } = startPage(t, { trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
const repo = 'https://github.com/owner/poc';
|
||||
const odd = savedFile(rawFile("x%22%3E%3Cimg%20src%3Dx%20onerror%3Dalert(1)%3E/it's.py"));
|
||||
context.record = { files: [odd, savedFile(rawFile('100%.py'))] };
|
||||
const html = vm.runInContext(`archives = { ${JSON.stringify(repo)}: record }; sourceFiles(${JSON.stringify(repo)}, '<b>"label"</b>')`, context);
|
||||
assert.ok(html.includes(`<a href="${odd.url.replace("'", ''')}" target="_blank"`));
|
||||
assert.ok(html.includes('aria-label="x"><img src=x onerror=alert(1)>/it's.py at commit 0123456 of ' +
|
||||
'<b>"label"</b>, Wayback Machine capture 2026-10-22 16:10 UTC">' +
|
||||
'x"><img src=x onerror=alert(1)>/it's.py</a>'));
|
||||
assert.ok(html.includes('aria-label="2 files saved from <b>"label"</b>"'));
|
||||
// A malformed escape is shown as written.
|
||||
assert.ok(html.includes('>100%.py</a>'));
|
||||
assert.doesNotMatch(html, /<img|<b>|"label"/);
|
||||
});
|
||||
|
||||
test('an archived fallback discloses its original\'s saved files and no file repeats a row', async t => {
|
||||
const original = 'https://github.com/owner/gone-poc';
|
||||
const fallback = `https://web.archive.org/web/20240101120000/${original}`;
|
||||
const exploit = savedFile(rawFile('exploit.c', 'owner/gone-poc'));
|
||||
const repo = 'https://github.com/owner/poc';
|
||||
const listed = savedFile(rawFile('poc.py'));
|
||||
const helper = savedFile(rawFile('helper.py'));
|
||||
const entries = [{ cve: 'CVE-2026-1000', desc: 'widget', poc: [fallback, repo, listed.original] }];
|
||||
const archives = {
|
||||
[original]: { url: fallback, timestamp: '20240101120000', files: [exploit] },
|
||||
[repo]: { files: [listed, helper] },
|
||||
[listed.original]: { url: listed.url, timestamp: listed.timestamp }
|
||||
};
|
||||
const { elements } = startPage(t, { search: '?q=widget', entries, archives, trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
const html = elements.get('[data-results]').innerHTML;
|
||||
assert.match(html, /3 linked PoCs/);
|
||||
assert.ok(html.includes('<div class="poc-archive"><span class="wayback is-archived" title="Wayback Machine page snapshot captured ' +
|
||||
'2024-01-01 12:00 UTC, linked in place of the original" aria-hidden="true">ARCHIVED</span>' +
|
||||
`<details class="sources" style="--files:1" data-sources="${fallback}"><summary`));
|
||||
assert.ok(html.includes(`<a href="${exploit.url}" target="_blank" rel="nofollow noopener" ` +
|
||||
'title="Wayback Machine capture of exploit.c at commit 0123456, 2026-10-22 16:10 UTC" ' +
|
||||
'aria-label="exploit.c at commit 0123456 of owner/gone-poc, Wayback Machine capture 2026-10-22 16:10 UTC">exploit.c</a>'));
|
||||
// The lost original is never linked, and no capture is looked up again.
|
||||
assert.ok(!html.includes(`href="${original}"`));
|
||||
assert.doesNotMatch(html, /\/web\/\*\/https:\/\/(?:web\.archive\.org|raw\.githubusercontent\.com)/);
|
||||
// poc.py has a row and a snapshot of its own, so its page discloses only helper.py.
|
||||
assert.ok(html.includes('aria-label="1 file saved from owner/poc">1 file</summary>'));
|
||||
assert.ok(html.includes(`<a href="${helper.url}"`));
|
||||
assert.ok(html.includes(`<a class="wayback is-snapshot" href="${listed.url}"`));
|
||||
assert.equal(html.split(`href="${listed.url}"`).length, 2);
|
||||
});
|
||||
|
||||
test('an open saved-files list stays open while results re-render', async t => {
|
||||
const repo = 'https://github.com/owner/poc';
|
||||
const entries = [{ cve: 'CVE-2026-1234', desc: 'widget', poc: [repo] }];
|
||||
const { context, elements } = startPage(t, { search: '?q=widget', entries, trending: new Promise(() => {}),
|
||||
archives: { [repo]: { files: [savedFile(rawFile('poc.py'))] } } });
|
||||
await new Promise(setImmediate);
|
||||
const results = () => elements.get('[data-results]').innerHTML;
|
||||
const toggle = (open, matches = true) => vm.runInContext('rememberSources({ target: { matches: () => ' + matches +
|
||||
`, open: ${open}, dataset: { sources: ${JSON.stringify(repo)} } } }); renderResults(null);`, context);
|
||||
assert.ok(results().includes(`data-sources="${repo}"><summary`));
|
||||
toggle(true);
|
||||
assert.ok(results().includes(`data-sources="${repo}" open><summary`));
|
||||
// A toggle elsewhere changes nothing.
|
||||
toggle(false, false);
|
||||
assert.ok(results().includes(`data-sources="${repo}" open><summary`));
|
||||
toggle(false);
|
||||
assert.ok(results().includes(`data-sources="${repo}"><summary`));
|
||||
});
|
||||
|
||||
test('late or failed archive data never blocks search, reorders results or drops snapshots', async t => {
|
||||
let finishArchives;
|
||||
const archives = new Promise(resolve => { finishArchives = resolve; });
|
||||
const entries = ['CVE-2025-1234', 'CVE-2024-1234'].map(cve => ({ cve, desc: 'widget', poc: [`https://github.com/owner/${cve}`] }));
|
||||
const { context, elements } = startPage(t, { search: '?q=widget', entries, archives, trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
const results = () => elements.get('[data-results]').innerHTML;
|
||||
const ids = () => [...results().matchAll(/class="result-id" href="\/([^"]+)"/g)].map(match => match[1]);
|
||||
assert.deepEqual(ids(), ['CVE-2025-1234', 'CVE-2024-1234']);
|
||||
assert.equal(results().match(/class="wayback is-lookup"/g).length, 2);
|
||||
const status = elements.get('[data-status]').textContent;
|
||||
const original = 'https://github.com/owner/CVE-2024-1234';
|
||||
const snapshot = `https://web.archive.org/web/20251021093000/${original}`;
|
||||
const saved = savedFile(rawFile('poc.py', 'owner/CVE-2024-1234'));
|
||||
finishArchives({ [original]: { url: snapshot, timestamp: '20251021093000', files: [saved] } });
|
||||
await new Promise(setImmediate);
|
||||
assert.deepEqual(ids(), ['CVE-2025-1234', 'CVE-2024-1234']);
|
||||
assert.ok(results().includes(`class="wayback is-snapshot" href="${snapshot}"`));
|
||||
assert.ok(results().includes(`<a href="${saved.url}"`));
|
||||
assert.equal(elements.get('[data-status]').textContent, status);
|
||||
// A refresh whose archive download fails or is unusable keeps the snapshot and saved file shown.
|
||||
for (const [at, failure] of ['error', 'malformed'].entries()) {
|
||||
const feed = { generated: `2026-10-21T1${at}:00:00Z`, with_pocs: 2, total_cves: 2, items: [], landed: [] };
|
||||
context.fetch = async url => {
|
||||
const file = url.split('?')[0];
|
||||
if (file === '/archives.json' && failure === 'error') throw new Error('Unavailable');
|
||||
return { ok: true, json: async () => file === '/trending_poc.json' ? feed : file === '/CVE_list.json' ? entries
|
||||
: file === '/archives.json' ? [] : {} };
|
||||
};
|
||||
assert.equal(await vm.runInContext('refreshData(true)', context), true);
|
||||
await new Promise(setImmediate);
|
||||
assert.deepEqual(ids(), ['CVE-2025-1234', 'CVE-2024-1234']);
|
||||
assert.ok(results().includes(`class="wayback is-snapshot" href="${snapshot}"`), failure);
|
||||
assert.ok(results().includes(`<a href="${saved.url}"`), failure);
|
||||
assert.equal(elements.get('[data-notices]').hidden, true);
|
||||
}
|
||||
});
|
||||
|
||||
test('snapshots and saved files published with the homepage survive a failed archive download', async t => {
|
||||
const original = 'https://github.com/owner/poc';
|
||||
const snapshot = `https://web.archive.org/web/20251021093000/${original}`;
|
||||
const file = savedFile(rawFile('poc.py'));
|
||||
const name = { getAttribute: () => original };
|
||||
const row = { querySelector: () => name };
|
||||
const sources = { getAttribute: attribute => attribute === 'data-sources' ? original : null };
|
||||
const published = [
|
||||
{ getAttribute: () => snapshot, closest: selector => selector === '.trend-row' ? row : null },
|
||||
{ getAttribute: () => file.url, closest: selector => selector === 'details.sources' ? sources : row }
|
||||
];
|
||||
const feed = { generated: '2026-10-21T10:00:00Z', with_pocs: 1, total_cves: 1, items: [], landed: [
|
||||
{ cve: 'CVE-2026-1234', url: original, name: 'poc', stars: 1, released: '2026-10-21T09:00:00Z', page: '/CVE-2026-1234' }] };
|
||||
const { elements, calls } = startPage(t, { trending: feed, published, archivesError: true });
|
||||
await new Promise(setImmediate);
|
||||
assert.ok(calls.includes('/archives.json'));
|
||||
const rows = elements.get('[data-trend-rows]').innerHTML;
|
||||
assert.ok(rows.includes('<div class="trend-sub"><a class="trend-detail" href="/CVE-2026-1234">CVE-2026-1234 details</a>' +
|
||||
`<a class="wayback is-snapshot" href="${snapshot}"`));
|
||||
assert.ok(rows.includes(`<details class="sources" style="--files:1" data-sources="${original}"><summary`));
|
||||
assert.ok(rows.includes(`<a href="${file.url}"`));
|
||||
assert.doesNotMatch(rows, /is-lookup/);
|
||||
});
|
||||
|
||||
test('a stalled trending feed does not block a bookmarked search', async t => {
|
||||
const { elements, calls } = startPage(t, { search: '?q=widget', trending: new Promise(() => {}) });
|
||||
await new Promise(setImmediate);
|
||||
|
||||
Reference in new issue
Block a user