From dcf4068b7a050986a21034b02f5743e518094974 Mon Sep 17 00:00:00 2001 From: 0xMarcio Date: Tue, 1 Sep 2026 22:08:20 +0200 Subject: [PATCH] Deduplicate published sources and improve filters --- docs/index.html | 43 ++++----- docs/logic.js | 203 ++++++++++++++++++++++++++++++++++-------- docs/style.css | 98 +++++++++++++++++--- scripts/build_site.py | 138 ++++++++++++++++++++++++---- 4 files changed, 386 insertions(+), 96 deletions(-) diff --git a/docs/index.html b/docs/index.html index 591055eb80f..2d53582c985 100644 --- a/docs/index.html +++ b/docs/index.html @@ -59,32 +59,23 @@
FILTER - +
+ + +
- +
+ + +
@@ -114,7 +105,7 @@ diff --git a/docs/logic.js b/docs/logic.js index b6975441006..3ae0e177eed 100644 --- a/docs/logic.js +++ b/docs/logic.js @@ -7,6 +7,17 @@ const ADVISORY_PREVIEW = 3; const TREND_ROWS = 20; const MIN_QUERY = 2; const POC_FIELDS = ['poc', 'nuclei', 'msf', 'edb', 'vulhub', 'collections']; +const FILTER_OPTIONS = { + severity: [ + ['CRITICAL', 'CRITICAL'], ['HIGH', 'HIGH'], ['MEDIUM', 'MEDIUM'], + ['LOW', 'LOW'], ['NONE', 'NONE'], ['UNSCORED', 'UNSCORED'] + ], + source: [ + ['GITHUB', 'GITHUB'], ['GHSA', 'GHSA'], ['REFERENCE', 'REFERENCE'], + ['NUCLEI', 'NUCLEI'], ['MSF', 'METASPLOIT'], ['EDB', 'EXPLOITDB'], + ['VULHUB', 'VULHUB'], ['COLLECTIONS', 'COLLECTIONS'] + ] +}; const state = { query: '', @@ -15,9 +26,11 @@ const state = { descOpen: new Set(), pocOpen: new Set(), advisoryOpen: new Set(), - severity: 'ALL', kevOnly: false, - source: 'ALL', + filters: { + severity: { include: new Set(), exclude: new Set() }, + source: { include: new Set(), exclude: new Set() } + }, ready: false, results: [] }; @@ -160,7 +173,7 @@ function pocText(entry) { function entryLinks(entry) { if (entry._allLinks === undefined) { - entry._allLinks = [...new Set(POC_FIELDS.flatMap(field => entry[field] || []))]; + entry._allLinks = uniqueSourceLinks(POC_FIELDS.flatMap(field => entry[field] || [])); } return entry._allLinks; } @@ -236,32 +249,46 @@ function entrySeverity(entry) { } function hasActiveFilters() { - return state.severity !== 'ALL' || state.kevOnly || state.source !== 'ALL'; + return state.kevOnly || Object.values(state.filters).some(filter => + filter.include.size || filter.exclude.size); } -function hasSource(entry, source) { +function entrySources(entry) { + const sources = new Set(); const fields = { NUCLEI: 'nuclei', MSF: 'msf', EDB: 'edb', VULHUB: 'vulhub', COLLECTIONS: 'collections' }; - if (fields[source]) { - const curated = CURATED.find(item => item.tag === source); - return (entry[fields[source]] || []).length > 0 - || (entry.poc || []).some(url => curated && url.includes(curated.match)); + for (const [source, field] of Object.entries(fields)) { + if ((entry[field] || []).length) sources.add(source); } - if (source === 'GITHUB') { - return (entry.poc || []).some(url => /^https?:\/\/(?:www\.)?github\.com\//i.test(url)); + for (const url of entry.poc || []) { + if (/^https?:\/\/(?:www\.)?github\.com\/advisories\/GHSA-/i.test(url)) { + sources.add('GHSA'); + } else if (repoFromUrl(url)) { + sources.add('GITHUB'); + } else { + sources.add('REFERENCE'); + } } - if (source === 'REFERENCE') { - return (entry.poc || []).some(url => !/^https?:\/\/(?:www\.)?github\.com\//i.test(url)); + return sources; +} + +function matchesMultiFilter(values, filter) { + for (const value of filter.exclude) { + if (values.has(value)) return false; } - return true; + if (!filter.include.size) return true; + for (const value of filter.include) { + if (values.has(value)) return true; + } + return false; } function matchesFilters(entry) { if (state.kevOnly && !kev[entry.cve]) return false; - if (state.source !== 'ALL' && !hasSource(entry, state.source)) return false; - if (state.severity === 'UNSCORED') return !entrySeverity(entry); - return state.severity === 'ALL' || entrySeverity(entry) === state.severity; + if (!matchesMultiFilter(entrySources(entry), state.filters.source)) return false; + const severity = entrySeverity(entry) || 'UNSCORED'; + return matchesMultiFilter(new Set([severity]), state.filters.severity); } function runSearch(query) { @@ -354,13 +381,25 @@ const el = { statPocs: document.querySelector('[data-stat-pocs]'), statKev: document.querySelector('[data-stat-kev]'), refreshed: document.querySelector('[data-refreshed]'), - severity: document.querySelector('[data-filter-severity]'), + filterTriggers: { + severity: document.querySelector('[data-filter-trigger="severity"]'), + source: document.querySelector('[data-filter-trigger="source"]') + }, + filterSummaries: { + severity: document.querySelector('[data-filter-summary="severity"]'), + source: document.querySelector('[data-filter-summary="source"]') + }, + filterOptions: { + severity: document.querySelector('[data-filter-options="severity"]'), + source: document.querySelector('[data-filter-options="source"]') + }, kevOnly: document.querySelector('[data-filter-kev]'), - source: document.querySelector('[data-filter-source]'), clearFilters: document.querySelector('[data-clear-filters]') }; const CURATED = [ + { match: 'github.com/advisories/GHSA-', tag: 'GHSA', hint: 'GitHub reviewed advisory with reproducible exploit material', + label: url => url.split('/').pop() }, { match: '/zan8in/afrog/', tag: 'AFROG', hint: 'Runnable afrog CVE template', label: url => decodeURIComponent(url.split('/').pop()) }, { match: '/chaitin/xray/', tag: 'XRAY', hint: 'Runnable xray CVE template', @@ -436,11 +475,11 @@ function rankedLinks(entry) { // runs as it stands, against a target, without reading somebody's code first. // Curated entries lead: templates, exploit archives, modules and runnable // environments can be used as they stand without first reading a repository. - entry._ranked = [...new Set([ + entry._ranked = uniqueSourceLinks([ ...(entry.nuclei || []), ...(entry.msf || []), ...(entry.edb || []), ...(entry.vulhub || []), ...(entry.collections || []), ...scored.map(item => item.url) - ])]; + ]); return entry._ranked; } @@ -466,9 +505,21 @@ function advisoryRow(item) { } function normalizedLink(url) { + const parsed = repoFromUrl(String(url || '')); + if (parsed) return `github:${parsed.owner.toLowerCase()}/${parsed.repo.toLowerCase()}`; return String(url || '').replace(/\/$/, ''); } +function uniqueSourceLinks(urls) { + const seen = new Set(); + return urls.filter(url => { + const key = normalizedLink(url); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + function resultRow(entry) { const id = entry.cve; const open = state.descOpen.has(id); @@ -693,25 +744,88 @@ el.input.addEventListener('input', () => { document.querySelector('.search').addEventListener('submit', event => event.preventDefault()); +function renderFilterOptions() { + for (const [kind, options] of Object.entries(FILTER_OPTIONS)) { + el.filterOptions[kind].innerHTML = + '
VALUEINOUT
' + + options.map(([value, label]) => + `
` + + `${escapeHTML(label)}` + + `` + + `
` + ).join('') + + '

Included values match any. Excluded values always win.

'; + } +} + +function closeFilterMenus(except) { + for (const kind of Object.keys(FILTER_OPTIONS)) { + if (kind === except) continue; + el.filterOptions[kind].hidden = true; + el.filterTriggers[kind].setAttribute('aria-expanded', 'false'); + } +} + +function filterTitle(kind) { + const filter = state.filters[kind]; + const included = [...filter.include]; + const excluded = [...filter.exclude]; + if (!included.length && !excluded.length) return 'All values'; + return [ + included.length ? `Include: ${included.join(', ')}` : '', + excluded.length ? `Exclude: ${excluded.join(', ')}` : '' + ].filter(Boolean).join(' | '); +} + function syncFilterControls() { - el.severity.value = state.severity; - el.source.value = state.source; + for (const kind of Object.keys(FILTER_OPTIONS)) { + const filter = state.filters[kind]; + const summary = [ + filter.include.size ? `+${filter.include.size}` : '', + filter.exclude.size ? `-${filter.exclude.size}` : '' + ].filter(Boolean).join(' ') || 'ALL'; + el.filterSummaries[kind].textContent = summary; + el.filterTriggers[kind].title = filterTitle(kind); + for (const button of el.filterOptions[kind].querySelectorAll('[data-filter-mode]')) { + const selected = filter[button.dataset.filterMode].has(button.dataset.filterValue); + button.setAttribute('aria-pressed', String(selected)); + } + } el.kevOnly.setAttribute('aria-pressed', String(state.kevOnly)); el.clearFilters.hidden = !hasActiveFilters(); } -el.severity.addEventListener('change', () => { - state.severity = el.severity.value; - state.shown = PAGE_SIZE; - syncFilterControls(); - render(); -}); +for (const kind of Object.keys(FILTER_OPTIONS)) { + el.filterTriggers[kind].addEventListener('click', () => { + const opening = el.filterOptions[kind].hidden; + closeFilterMenus(kind); + el.filterOptions[kind].hidden = !opening; + el.filterTriggers[kind].setAttribute('aria-expanded', String(opening)); + }); + el.filterOptions[kind].addEventListener('click', event => { + const button = event.target.closest('[data-filter-mode]'); + if (!button) return; + const filter = state.filters[kind]; + const mode = button.dataset.filterMode; + const other = mode === 'include' ? 'exclude' : 'include'; + const value = button.dataset.filterValue; + if (filter[mode].has(value)) filter[mode].delete(value); + else { + filter[mode].add(value); + filter[other].delete(value); + } + state.shown = PAGE_SIZE; + syncFilterControls(); + render(); + }); +} -el.source.addEventListener('change', () => { - state.source = el.source.value; - state.shown = PAGE_SIZE; - syncFilterControls(); - render(); +document.addEventListener('click', event => { + if (!event.target.closest('.filter-menu')) closeFilterMenus(); }); el.kevOnly.addEventListener('click', () => { @@ -722,14 +836,17 @@ el.kevOnly.addEventListener('click', () => { }); el.clearFilters.addEventListener('click', () => { - state.severity = 'ALL'; state.kevOnly = false; - state.source = 'ALL'; + for (const filter of Object.values(state.filters)) { + filter.include.clear(); + filter.exclude.clear(); + } state.shown = PAGE_SIZE; syncFilterControls(); render(); }); +renderFilterOptions(); syncFilterControls(); // Typing anywhere on the page means typing into the search field. The keystroke @@ -737,6 +854,13 @@ syncFilterControls(); // field it just moved to, so the first letter is never lost. document.addEventListener('keydown', event => { if (event.ctrlKey || event.metaKey || event.altKey) return; + const openFilter = Object.keys(FILTER_OPTIONS).find(kind => + el.filterTriggers[kind].getAttribute('aria-expanded') === 'true'); + if (event.key === 'Escape' && openFilter) { + closeFilterMenus(); + el.filterTriggers[openFilter].focus(); + return; + } if (document.activeElement === el.input) { if (event.key === 'Escape') { el.input.value = ''; @@ -747,7 +871,7 @@ document.addEventListener('keydown', event => { } const active = document.activeElement; const editing = active && (active.isContentEditable - || ['INPUT', 'TEXTAREA', 'SELECT'].includes(active.tagName)); + || ['INPUT', 'TEXTAREA', 'SELECT', 'BUTTON'].includes(active.tagName)); if (editing) return; // A lone slash is the shortcut, so it focuses without typing itself. if (event.key === '/') { @@ -828,12 +952,13 @@ async function loadJSON(url, options) { loadJSON('/cve_metadata.json', { cache: 'no-cache' }).then(data => { metadata = data || {}; - el.severity.disabled = false; + el.filterTriggers.severity.disabled = false; if (state.ready && (state.query || hasActiveFilters())) render(); }).catch(err => console.warn(err.message)); loadJSON('/nuclei.json', { cache: 'no-cache' }).then(data => { ratings = data || {}; + el.filterTriggers.severity.disabled = false; if (state.ready && (state.query || hasActiveFilters())) render(); }).catch(err => console.warn(err.message)); @@ -851,7 +976,7 @@ async function loadJSON(url, options) { try { dataset = prepareDataset(await loadJSON('/CVE_list.json', { cache: 'no-cache' })); state.ready = true; - el.source.disabled = false; + el.filterTriggers.source.disabled = false; render(); } catch (err) { console.warn(err.message); diff --git a/docs/style.css b/docs/style.css index 8f8e87308a9..81a3564b19d 100644 --- a/docs/style.css +++ b/docs/style.css @@ -147,29 +147,101 @@ a:hover { color: var(--accent-hover); } letter-spacing: .07em; } .filter-caption { margin-right: 2px; color: var(--neutral); } -.filter-select { +.filter-menu { position: relative; } +.filter-trigger { display: flex; align-items: center; - gap: 7px; + gap: 9px; height: 31px; - padding-left: 9px; + min-width: 154px; + padding: 0 10px; color: var(--neutral); border: 1px solid var(--border-muted); border-radius: 4px; background: var(--canvas-subtle); -} -.filter-select:focus-within { border-color: var(--accent-emphasis); } -.filter-select select { - height: 29px; - padding: 0 25px 0 0; - color: var(--fg-subtle); - border: 0; - outline: 0; - background: transparent; font: 500 10px/1 var(--mono); letter-spacing: .07em; cursor: pointer; } +.filter-menu[data-filter-menu="source"] .filter-trigger { min-width: 174px; } +.filter-trigger strong { + margin-left: auto; + color: var(--fg-subtle); + font: inherit; +} +.filter-trigger::after { + width: 6px; + height: 6px; + margin: -3px 1px 0 4px; + border-right: 1px solid var(--fg-muted); + border-bottom: 1px solid var(--fg-muted); + transform: rotate(45deg); + content: ''; +} +.filter-trigger:hover, .filter-trigger[aria-expanded="true"] { + color: var(--fg-muted); + border-color: var(--fg-muted); +} +.filter-trigger[aria-expanded="true"]::after { margin-top: 3px; transform: rotate(225deg); } +.filter-popover { + position: absolute; + z-index: 20; + top: calc(100% + 6px); + left: 0; + width: 304px; + padding: 8px; + color: var(--fg-muted); + border: 1px solid var(--border); + border-radius: 5px; + background: var(--canvas-subtle); + box-shadow: 0 12px 28px rgba(1, 4, 9, .55); +} +.filter-popover-head, .filter-option { + display: grid; + grid-template-columns: minmax(0, 1fr) 58px 58px; + gap: 6px; + align-items: center; +} +.filter-popover-head { + padding: 3px 5px 7px; + color: var(--neutral); + font-size: 9px; + letter-spacing: .1em; +} +.filter-popover-head span:not(:first-child) { text-align: center; } +.filter-option { + padding: 5px; + border-top: 1px solid var(--border-muted); +} +.filter-option-label { color: var(--fg-subtle); } +.filter-choice { + height: 24px; + padding: 0; + color: var(--fg-muted); + border: 1px solid var(--border); + border-radius: 3px; + background: transparent; + font: 500 9px/1 var(--mono); + letter-spacing: .06em; + cursor: pointer; +} +.filter-choice:hover { color: var(--fg); border-color: var(--fg-muted); } +.filter-choice[data-filter-mode="include"][aria-pressed="true"] { + color: var(--accent-hover); + border-color: rgba(56, 139, 253, .5); + background: rgba(56, 139, 253, .12); +} +.filter-choice[data-filter-mode="exclude"][aria-pressed="true"] { + color: var(--danger); + border-color: rgba(248, 81, 73, .45); + background: rgba(248, 81, 73, .1); +} +.filter-popover-note { + margin: 8px 5px 2px; + color: var(--neutral); + font: 400 9px/1.4 var(--mono); + letter-spacing: 0; +} .filter-toggle, .filter-clear { height: 31px; padding: 0 10px; @@ -474,4 +546,6 @@ a.chip:hover { color: var(--fg); border-color: var(--fg-muted); } .filters { align-items: stretch; flex-wrap: wrap; } .filter-caption { display: none; } .filter-clear { margin-left: 0; } + .filter-popover { max-width: calc(100vw - 32px); } + .filter-menu[data-filter-menu="source"] .filter-popover { right: 0; left: auto; } } diff --git a/scripts/build_site.py b/scripts/build_site.py index ff824efa8cd..93061c160c0 100644 --- a/scripts/build_site.py +++ b/scripts/build_site.py @@ -33,6 +33,21 @@ METADATA_OUTPUT = DOCS_DIR / "cve_metadata.json" VERIFIED_REFERENCES = INDEX / "reference_pocs.txt" EPSS_OUTPUT = DOCS_DIR / "epss.json" EPSS_FEED = "https://epss.empiricalsecurity.com/epss_scores-current.csv.gz" +GITHUB_NON_REPOS = { + "advisories", "apps", "collections", "marketplace", "notifications", + "orgs", "security", "settings", "sponsors", "topics", "user-attachments", +} +CODE_SUFFIXES = { + ".c", ".cc", ".cpp", ".cs", ".go", ".java", ".js", ".md", ".nse", + ".php", ".pl", ".ps1", ".py", ".rb", ".rs", ".sh", ".ts", ".txt", + ".yaml", ".yml", ".zip", +} +MEDIA_SUFFIXES = { + ".bmp", ".gif", ".jpeg", ".jpg", ".mov", ".mp4", ".pdf", ".png", + ".svg", ".webm", ".webp", +} + + def load_blacklist() -> set[str]: if not BLACKLIST.exists(): return set() @@ -135,9 +150,9 @@ def repo_from_url(url: str) -> str: except Exception: path = url parts = path.strip("/").split("/") - if len(parts) >= 2: - return "/".join(parts[:2]).lower() - return (parts[-1] if parts else "").lower() + if len(parts) < 2 or parts[0].lower() in GITHUB_NON_REPOS: + return "" + return "/".join(parts[:2]).lower() def is_blacklisted(url: str, blacklist: Collection[str]) -> bool: @@ -159,6 +174,84 @@ def link_key(url: str) -> str: return f"github:{repo}{suffix}?{parsed.query}#{parsed.fragment}" +def source_key(url: str) -> str: + """Identity of one source, independent of a path inside its repository.""" + repo = repo_from_url(url) + if repo and "/" in repo: + return f"github:{repo}" + return url.rstrip("/") + + +def poc_link_rank(url: str, cve_id: str) -> tuple[int, int]: + """Choose the most useful path when one repository supplied several.""" + if not repo_from_url(url): + return 0, len(url) + parsed = urlparse(url) + parts = [part for part in parsed.path.split("/") if part] + suffix = "/".join(parts[2:]).lower() + if not suffix: + return 0, len(url) + compact_cve = re.sub(r"[^a-z0-9]", "", cve_id.lower()) + compact_path = re.sub(r"[^a-z0-9]", "", suffix) + if compact_cve and compact_cve in compact_path: + return 1, len(url) + extension = Path(parts[-1]).suffix.lower() + if extension in CODE_SUFFIXES or "/security/advisories/" in parsed.path.lower(): + return 2, len(url) + if re.search(r"(?:^|/)(?:poc|exploit|reproducer)(?:/|$)", suffix): + return 3, len(url) + if extension in MEDIA_SUFFIXES: + return 6, len(url) + return 4, len(url) + + +def dedupe_source_links(urls: Iterable[str], cve_id: str) -> List[str]: + selected: Dict[str, tuple[int, str]] = {} + order: Dict[str, int] = {} + for index, url in enumerate(urls): + key = source_key(url) + order.setdefault(key, index) + current = selected.get(key) + if current is None or poc_link_rank(url, cve_id) < poc_link_rank(current[1], cve_id): + selected[key] = (index, url) + return [selected[key][1] for key in sorted(selected, key=order.get)] + + +def advisory_rank(row: list, cve_id: str) -> tuple[int, int, int]: + url = str(row[0]) + tags = set(row[1] if len(row) > 1 and isinstance(row[1], list) else []) + if "GitHub Advisory" in tags or "/security/advisories/" in url.lower(): + tag_rank = 0 + elif "Exploit" in tags: + tag_rank = 1 + elif "Vendor Advisory" in tags: + tag_rank = 2 + elif "Third Party Advisory" in tags or "Advisory" in tags: + tag_rank = 3 + elif "Issue Tracking" in tags: + tag_rank = 4 + elif "Patch" in tags: + tag_rank = 5 + else: + tag_rank = 6 + path_rank, length = poc_link_rank(url, cve_id) + return tag_rank, path_rank, length + + +def dedupe_advisories(rows: Iterable[list], cve_id: str) -> List[list]: + selected: Dict[str, list] = {} + order: Dict[str, int] = {} + for index, row in enumerate(rows): + if not isinstance(row, list) or not row: + continue + key = source_key(str(row[0])) + order.setdefault(key, index) + current = selected.get(key) + if current is None or advisory_rank(row, cve_id) < advisory_rank(current, cve_id): + selected[key] = row + return [selected[key] for key in sorted(selected, key=order.get)] + + def collect_links(block: str, *, blacklist: Optional[Collection[str]] = None) -> List[str]: links: List[str] = [] blacklist = blacklist or set() @@ -198,7 +291,10 @@ def build_cve_list(blacklist: Collection[str]) -> tuple[List[Dict[str, object]], # Metasploit module are none of them somebody's repository, and should # not be ranked, starred or credited as one. curated = { - field: collect_links(sections.get(header, ""), blacklist=blacklist) + field: dedupe_source_links( + collect_links(sections.get(header, ""), blacklist=blacklist), + cve_id, + ) for header, field in CURATED } curated_keys = { @@ -215,18 +311,15 @@ def build_cve_list(blacklist: Collection[str]) -> tuple[List[Dict[str, object]], and (cve_id, link_key(str(row[0]))) not in verified_references } - poc_entries: List[str] = [] - seen = set() + poc_candidates: List[str] = [] for link in references: key = link_key(link) - if key not in seen and key not in curated_keys and key not in advisory_only: - poc_entries.append(link) - seen.add(key) + if key not in curated_keys and key not in advisory_only: + poc_candidates.append(link) for link in github_links: - key = link_key(link) - if key not in seen and key not in curated_keys: - poc_entries.append(link) - seen.add(key) + if link_key(link) not in curated_keys: + poc_candidates.append(link) + poc_entries = dedupe_source_links(poc_candidates, cve_id) if not poc_entries and not any(curated.values()): continue @@ -298,12 +391,19 @@ def build_nuclei(cve_entries: List[Dict[str, object]]) -> Dict[str, object]: def build_metadata(cve_entries: List[Dict[str, object]]) -> Dict[str, object]: """CVSS generations and vetted advisories for CVEs published on the site.""" have = {str(entry["cve"]) for entry in cve_entries} - keep = ("cvss", "advisories") - return { - cve: {field: value[field] for field in keep if field in value} - for cve, value in load_metadata().items() - if cve in have and isinstance(value, dict) - } + published: Dict[str, object] = {} + for cve, value in load_metadata().items(): + if cve not in have or not isinstance(value, dict): + continue + entry: Dict[str, object] = {} + if value.get("cvss"): + entry["cvss"] = value["cvss"] + advisories = dedupe_advisories(value.get("advisories", []), cve) + if advisories: + entry["advisories"] = advisories + if entry: + published[cve] = entry + return published def build_epss(cve_entries: List[Dict[str, object]]) -> Dict[str, object]: