mirror of
https://github.com/0xMarcio/cve.git
synced 2026-10-27 16:51:03 +00:00
Retire the abandoned second pipeline
Ten modules totalling 2,206 lines, reachable from no workflow and no live script, alongside 18 MB of github-all.txt and references-all.txt that nothing reads and 400 KB of stale api/v1 output that shipped in every Pages artifact. scripts/README.md documented this as the build process. utils.py existed for one caller and duplicated functions build_site.py already had, so it folds into it. Nothing imports a third-party package any more, and requirements.txt goes with it.
This commit is contained in:
0xMarcio committed
1 parent
6d65ae36bb
commit
85fe2b3ad4
22 files changed
+38
-281496
No files matched your search
@@ -1,13 +0,0 @@
|
||||
# Build pipeline
|
||||
|
||||
```
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
python scripts/fetch_kev.py
|
||||
python scripts/fetch_epss.py
|
||||
python scripts/build_site.py
|
||||
python scripts/build_all.py
|
||||
```
|
||||
|
||||
Outputs land in `docs/` and JSON under `docs/api/v1/`. Snapshots live in `docs/api/v1/snapshots/` (last 14 days) and diffs under `docs/api/v1/diffs/`.
|
||||
@@ -1,114 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
import requests
|
||||
|
||||
from pipeline_outputs import (
|
||||
build_diff,
|
||||
prune_old_diffs,
|
||||
prune_old_snapshots,
|
||||
summarise_for_snapshot,
|
||||
write_cve_outputs,
|
||||
write_diff,
|
||||
write_index,
|
||||
write_snapshot,
|
||||
write_top,
|
||||
)
|
||||
from poc_pipeline import PoCPipeline, build_scope, persist_evidence
|
||||
from utils import API_DIR, DOCS_DIR, load_json
|
||||
|
||||
|
||||
def load_existing_results(api_dir: Path) -> List[Dict]:
|
||||
results: List[Dict] = []
|
||||
if not api_dir.exists():
|
||||
return results
|
||||
for path in api_dir.glob("CVE-*.json"):
|
||||
data = load_json(path, default={}) or {}
|
||||
if "pocs" in data:
|
||||
results.append({"cve_id": data.get("cve_id") or path.stem, "pocs": data.get("pocs", []), "last_updated": data.get("last_updated")})
|
||||
return results
|
||||
|
||||
|
||||
def main(argv: List[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Build CVE PoC pipeline outputs, snapshots, and static site")
|
||||
parser.add_argument("--days", type=int, default=7, help="Days window for GitHub discovery windows")
|
||||
parser.add_argument("--mode", choices=["daily", "weekly"], default="daily", help="Run mode to tune scope")
|
||||
parser.add_argument("--limit", type=int, default=50, help="Maximum CVEs to scan per run")
|
||||
parser.add_argument("--cve", action="append", help="Explicit CVE IDs to scan (can be passed multiple times)")
|
||||
parser.add_argument("--skip-discovery", action="store_true", help="Skip GitHub discovery and reuse existing API outputs")
|
||||
parser.add_argument("--check-links", action="store_true", help="Optionally HEAD check repo URLs for dead links")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
pipeline = PoCPipeline()
|
||||
scope: List[str] = []
|
||||
discovery_days = args.days
|
||||
if args.cve:
|
||||
scope = [cve.upper() for cve in args.cve]
|
||||
elif not args.skip_discovery:
|
||||
prefer_recent = True
|
||||
scan_days = args.days
|
||||
limit = args.limit
|
||||
if args.mode == "weekly":
|
||||
scan_days = max(scan_days, 30)
|
||||
discovery_days = scan_days
|
||||
prefer_recent = False
|
||||
limit = None
|
||||
scope = build_scope(scan_days, github_list=Path("github.txt"), existing_api=API_DIR / "cve", prefer_recent_years=prefer_recent, max_cves=limit)
|
||||
|
||||
results: List[Dict] = []
|
||||
if args.skip_discovery:
|
||||
results = load_existing_results(API_DIR / "cve")
|
||||
else:
|
||||
for idx, cve_id in enumerate(scope):
|
||||
try:
|
||||
results.append(pipeline.discover_for_cve(cve_id, days=discovery_days))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"[warn] Failed to process {cve_id}: {exc}", file=sys.stderr)
|
||||
persist_evidence(results)
|
||||
|
||||
if not results:
|
||||
print("No results to write; aborting.")
|
||||
return 1
|
||||
|
||||
write_cve_outputs(results)
|
||||
index_payload = write_index(results)
|
||||
top_payload = write_top(results)
|
||||
|
||||
def maybe_check_links() -> List[Dict]:
|
||||
if not args.check_links:
|
||||
return []
|
||||
urls = []
|
||||
for result in results:
|
||||
for poc in result.get("pocs", []):
|
||||
if poc.get("confidence_tier") in {"high", "medium"} and poc.get("repo_url"):
|
||||
urls.append(poc["repo_url"])
|
||||
urls = urls[:25]
|
||||
dead: List[Dict] = []
|
||||
for url in urls:
|
||||
try:
|
||||
resp = requests.head(url, timeout=5, allow_redirects=True)
|
||||
if resp.status_code >= 400:
|
||||
dead.append({"url": url, "status": resp.status_code})
|
||||
except requests.RequestException as exc: # noqa: BLE001
|
||||
dead.append({"url": url, "error": str(exc)})
|
||||
return dead
|
||||
|
||||
snapshot_payload = summarise_for_snapshot(results, top=top_payload)
|
||||
prev_snapshot = load_json(API_DIR / "snapshots" / "latest.json", default={}) or {}
|
||||
snapshot_path = write_snapshot(snapshot_payload)
|
||||
diff_payload = build_diff(prev_snapshot, snapshot_payload, dead_links=maybe_check_links())
|
||||
write_diff(diff_payload)
|
||||
prune_old_snapshots()
|
||||
prune_old_diffs()
|
||||
|
||||
print(f"Wrote pipeline outputs under {DOCS_DIR}")
|
||||
print(f"Wrote latest snapshot to {snapshot_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,139 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
from utils import API_DIR, SNAPSHOT_DIR, ensure_dirs, load_json, save_json
|
||||
|
||||
DEFAULT_LOOKBACK_DAYS = 14
|
||||
DEFAULT_HIGH_EPSS_THRESHOLD = 0.05
|
||||
DEFAULT_MAX_MOVERS = 50
|
||||
DEFAULT_RECENT_KEV_DAYS = 30
|
||||
|
||||
|
||||
def parse_date(date_str: str) -> date:
|
||||
return datetime.strptime(date_str, "%Y-%m-%d").date()
|
||||
|
||||
|
||||
def load_snapshot(path: Path) -> Dict:
|
||||
return load_json(path, default={}) or {}
|
||||
|
||||
|
||||
def diff_lists(prev: List[Dict], curr: List[Dict], key: str = "cve") -> Dict[str, List[Dict]]:
|
||||
prev_ids = {item[key]: item for item in prev}
|
||||
curr_ids = {item[key]: item for item in curr}
|
||||
new_items = [curr_ids[cve] for cve in sorted(curr_ids.keys() - prev_ids.keys())]
|
||||
removed_items = [prev_ids[cve] for cve in sorted(prev_ids.keys() - curr_ids.keys())]
|
||||
return {"new": new_items, "removed": removed_items}
|
||||
|
||||
|
||||
def compute_epss_movers(prev_epss: Dict[str, Dict], curr_epss: Dict[str, Dict], max_items: int) -> List[Dict]:
|
||||
deltas = []
|
||||
for cve, curr in curr_epss.items():
|
||||
prev = prev_epss.get(cve)
|
||||
if not prev:
|
||||
continue
|
||||
delta = (curr.get("epss") or 0) - (prev.get("epss") or 0)
|
||||
if abs(delta) < 0.0001:
|
||||
continue
|
||||
deltas.append({"cve": cve, "delta": round(delta, 5), "epss": curr.get("epss"), "prev_epss": prev.get("epss")})
|
||||
deltas.sort(key=lambda row: (-row["delta"], row["cve"]))
|
||||
return deltas[:max_items]
|
||||
|
||||
|
||||
def filter_recent_kev(kev_items: List[Dict], *, recent_days: int) -> List[Dict]:
|
||||
cutoff = datetime.utcnow().date() - timedelta(days=recent_days)
|
||||
fresh: List[Tuple[date, Dict]] = []
|
||||
for row in kev_items:
|
||||
date_str = row.get("date_added") or row.get("dateAdded")
|
||||
if not date_str:
|
||||
continue
|
||||
try:
|
||||
added = parse_date(date_str)
|
||||
except ValueError:
|
||||
continue
|
||||
if added >= cutoff:
|
||||
fresh.append((added, row))
|
||||
fresh.sort(key=lambda item: (item[0], item[1].get("percentile") or 0), reverse=True)
|
||||
return [row for _, row in fresh]
|
||||
|
||||
|
||||
def build_diff(
|
||||
snapshots: List[Path],
|
||||
kev_full: List[Dict] | None = None,
|
||||
*,
|
||||
threshold: float,
|
||||
max_movers: int,
|
||||
recent_days: int,
|
||||
) -> Tuple[Dict, Path | None]:
|
||||
if not snapshots:
|
||||
return {}, None
|
||||
latest_path = snapshots[-1]
|
||||
latest = load_snapshot(latest_path)
|
||||
latest_date = latest.get("generated") or latest_path.stem
|
||||
|
||||
if len(snapshots) >= 2:
|
||||
prev = load_snapshot(snapshots[-2])
|
||||
kev_diff = diff_lists(prev.get("kev_top", []), latest.get("kev_top", []))
|
||||
high_epss_diff = diff_lists(prev.get("high_epss", []), latest.get("high_epss", []))
|
||||
else:
|
||||
prev = {}
|
||||
kev_diff = {"new": latest.get("kev_top", []), "removed": []}
|
||||
high_epss_diff = {"new": latest.get("high_epss", []), "removed": []}
|
||||
|
||||
prev_epss_lookup = {row["cve"]: row for row in (prev.get("high_epss", []) if prev else [])}
|
||||
curr_epss_lookup = {row["cve"]: row for row in latest.get("high_epss", [])}
|
||||
epss_movers = compute_epss_movers(prev_epss_lookup, curr_epss_lookup, max_movers)
|
||||
|
||||
kev_recent = filter_recent_kev(kev_full or latest.get("kev_top", []), recent_days=recent_days)
|
||||
|
||||
diff_outputs = {
|
||||
"generated": latest_date,
|
||||
"new_kev_entries": kev_recent,
|
||||
"removed_kev_entries": kev_diff["removed"],
|
||||
"new_high_epss": [row for row in high_epss_diff["new"] if (row.get("epss") or 0) >= threshold],
|
||||
"removed_high_epss": high_epss_diff["removed"],
|
||||
"epss_movers": epss_movers,
|
||||
}
|
||||
|
||||
target = API_DIR / "diff" / f"{latest_date}.json"
|
||||
ensure_dirs(target.parent)
|
||||
save_json(target, diff_outputs)
|
||||
# also write a stable latest pointer
|
||||
save_json(target.parent / "latest.json", diff_outputs)
|
||||
|
||||
return diff_outputs, target
|
||||
|
||||
|
||||
def prune_snapshots(snapshots: List[Path], *, lookback_days: int) -> None:
|
||||
cutoff = datetime.utcnow().date() - timedelta(days=lookback_days)
|
||||
for snap in snapshots:
|
||||
snap_date = parse_date(snap.stem)
|
||||
if snap_date < cutoff:
|
||||
snap.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build daily diff JSON from snapshots")
|
||||
parser.add_argument("--threshold", type=float, default=DEFAULT_HIGH_EPSS_THRESHOLD, help="High EPSs minimum threshold")
|
||||
parser.add_argument("--lookback", type=int, default=DEFAULT_LOOKBACK_DAYS, help="How many days of snapshots to keep")
|
||||
parser.add_argument("--max-movers", type=int, default=DEFAULT_MAX_MOVERS, help="Max EPSs movers to keep")
|
||||
parser.add_argument("--recent-days", type=int, default=DEFAULT_RECENT_KEV_DAYS, help="Days of KEV entries to surface as new")
|
||||
args = parser.parse_args()
|
||||
|
||||
ensure_dirs(SNAPSHOT_DIR)
|
||||
snapshots = sorted(SNAPSHOT_DIR.glob("*.json"))
|
||||
diff, target = build_diff(snapshots, kev_full=None, threshold=args.threshold, max_movers=args.max_movers, recent_days=args.recent_days)
|
||||
if target:
|
||||
print(f"Wrote diff to {target}")
|
||||
else:
|
||||
print("No snapshots available to diff")
|
||||
|
||||
prune_snapshots(snapshots, lookback_days=args.lookback)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,345 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Set, Tuple
|
||||
|
||||
from utils import (
|
||||
API_DIR,
|
||||
DATA_DIR,
|
||||
DOCS_DIR,
|
||||
SNAPSHOT_DIR,
|
||||
load_json,
|
||||
save_json,
|
||||
today_str,
|
||||
ensure_dirs,
|
||||
load_poc_index,
|
||||
slugify,
|
||||
stable_unique,
|
||||
)
|
||||
|
||||
KEV_PATH = DATA_DIR / "kev.json"
|
||||
EPSS_PATH = DATA_DIR / "epss.json"
|
||||
|
||||
DEFAULT_TOP_KEV = 75
|
||||
DEFAULT_HIGH_EPSS_LIMIT = 50
|
||||
DEFAULT_HIGH_EPSS_THRESHOLD = 0.05
|
||||
RECENT_YEAR_WINDOW = 1
|
||||
|
||||
|
||||
def load_inputs(kev_path: Path, epss_path: Path) -> Tuple[Dict, Dict]:
|
||||
kev_data = load_json(kev_path, default={}) or {}
|
||||
epss_data = load_json(epss_path, default={}) or {}
|
||||
return kev_data, epss_data
|
||||
|
||||
|
||||
def enrich_kev(kev_items: List[Dict], epss_lookup: Dict[str, Dict], poc_index: Dict[str, Dict]) -> List[Dict]:
|
||||
enriched = []
|
||||
current_year = today_str()
|
||||
current_year_int = int(current_year.split("-")[0])
|
||||
def is_recent(cve_id: str) -> bool:
|
||||
try:
|
||||
year = int(cve_id.split("-")[1])
|
||||
except Exception:
|
||||
return False
|
||||
return year >= current_year_int - RECENT_YEAR_WINDOW
|
||||
|
||||
for entry in kev_items:
|
||||
cve = entry.get("cve") or entry.get("cveID") or ""
|
||||
if not cve:
|
||||
continue
|
||||
cve = cve.upper()
|
||||
if not is_recent(cve):
|
||||
continue
|
||||
epss_info = epss_lookup.get(cve, {})
|
||||
poc_info = poc_index.get(cve)
|
||||
if not poc_info or not poc_info.get("poc"):
|
||||
continue
|
||||
poc_count = len(poc_info["poc"])
|
||||
enriched.append(
|
||||
{
|
||||
"cve": cve,
|
||||
"vendor": entry.get("vendor") or entry.get("vendorProject", ""),
|
||||
"product": entry.get("product", ""),
|
||||
"date_added": entry.get("date_added") or entry.get("dateAdded"),
|
||||
"due_date": entry.get("due_date") or entry.get("dueDate"),
|
||||
"short_description": entry.get("short_description") or entry.get("shortDescription", ""),
|
||||
"required_action": entry.get("required_action") or entry.get("requiredAction", ""),
|
||||
"notes": entry.get("notes", ""),
|
||||
"epss": epss_info.get("epss"),
|
||||
"percentile": epss_info.get("percentile"),
|
||||
"poc_count": poc_count,
|
||||
}
|
||||
)
|
||||
enriched.sort(key=lambda row: (-float(row.get("percentile") or 0), row["cve"]))
|
||||
return enriched
|
||||
|
||||
|
||||
def build_epss_lookup(epss_items: List[Dict]) -> Dict[str, Dict]:
|
||||
return {row.get("cve", "").upper(): row for row in epss_items if row.get("cve")}
|
||||
|
||||
|
||||
def build_high_epss_not_in_kev(
|
||||
epss_items: List[Dict],
|
||||
kev_set: Set[str],
|
||||
poc_index: Dict[str, Dict],
|
||||
*,
|
||||
threshold: float,
|
||||
limit: int,
|
||||
) -> List[Dict]:
|
||||
current_year_int = int(today_str().split("-")[0])
|
||||
def is_recent(cve_id: str) -> bool:
|
||||
try:
|
||||
year = int(cve_id.split("-")[1])
|
||||
except Exception:
|
||||
return False
|
||||
return year >= current_year_int - RECENT_YEAR_WINDOW
|
||||
|
||||
ranked = sorted(
|
||||
(
|
||||
row
|
||||
for row in epss_items
|
||||
if row.get("cve")
|
||||
and row.get("cve", "").upper() not in kev_set
|
||||
and (row.get("epss") is not None)
|
||||
and is_recent(row.get("cve", ""))
|
||||
),
|
||||
key=lambda row: (-float(row.get("epss") or 0), row.get("cve", "")),
|
||||
)
|
||||
|
||||
def build_rows(source: List[Dict]) -> List[Dict]:
|
||||
output: List[Dict] = []
|
||||
for row in source:
|
||||
cve = row.get("cve", "").upper()
|
||||
if not cve:
|
||||
continue
|
||||
epss_score = row.get("epss") or 0.0
|
||||
if epss_score < threshold:
|
||||
continue
|
||||
poc_info = poc_index.get(cve)
|
||||
if not poc_info or not poc_info.get("poc"):
|
||||
continue
|
||||
poc_count = len(poc_info["poc"])
|
||||
output.append(
|
||||
{
|
||||
"cve": cve,
|
||||
"epss": row.get("epss"),
|
||||
"percentile": row.get("percentile"),
|
||||
"summary": truncate_description(poc_info.get("desc", "")),
|
||||
"poc_count": poc_count,
|
||||
}
|
||||
)
|
||||
if len(output) >= limit:
|
||||
break
|
||||
return output
|
||||
|
||||
rows = build_rows(ranked)
|
||||
if not rows and threshold > 0:
|
||||
# If the threshold is too strict for a given day, fall back to the top ranked set.
|
||||
rows = build_rows([dict(row, epss=row.get("epss", 0) or 0) for row in ranked[:limit]])
|
||||
return rows
|
||||
|
||||
|
||||
def build_cve_details(
|
||||
kev_enriched: Iterable[Dict],
|
||||
high_epss: Iterable[Dict],
|
||||
poc_index: Dict[str, Dict],
|
||||
) -> Dict[str, Dict]:
|
||||
details: Dict[str, Dict] = {}
|
||||
|
||||
def ensure_detail(cve: str) -> Dict:
|
||||
if cve not in details:
|
||||
data = poc_index.get(cve, {})
|
||||
details[cve] = {
|
||||
"cve": cve,
|
||||
"description": data.get("desc", ""),
|
||||
"poc_links": data.get("poc", []),
|
||||
"poc_count": len(data.get("poc", [])),
|
||||
"kev": None,
|
||||
"epss": None,
|
||||
"percentile": None,
|
||||
"vendor": None,
|
||||
"product": None,
|
||||
}
|
||||
return details[cve]
|
||||
|
||||
for entry in kev_enriched:
|
||||
cve = entry["cve"]
|
||||
detail = ensure_detail(cve)
|
||||
detail.update(
|
||||
{
|
||||
"kev": {
|
||||
"date_added": entry.get("date_added"),
|
||||
"due_date": entry.get("due_date"),
|
||||
"short_description": entry.get("short_description"),
|
||||
"required_action": entry.get("required_action"),
|
||||
"notes": entry.get("notes"),
|
||||
},
|
||||
"epss": entry.get("epss"),
|
||||
"percentile": entry.get("percentile"),
|
||||
"vendor": entry.get("vendor"),
|
||||
"product": entry.get("product"),
|
||||
}
|
||||
)
|
||||
|
||||
for entry in high_epss:
|
||||
cve = entry["cve"]
|
||||
detail = ensure_detail(cve)
|
||||
if detail.get("epss") is None:
|
||||
detail["epss"] = entry.get("epss")
|
||||
detail["percentile"] = entry.get("percentile")
|
||||
|
||||
return details
|
||||
|
||||
|
||||
def build_vendor_map(details: Dict[str, Dict]) -> Dict[str, Dict]:
|
||||
vendors: Dict[str, Dict] = {}
|
||||
for detail in details.values():
|
||||
vendor_name = detail.get("vendor")
|
||||
if not vendor_name:
|
||||
continue
|
||||
slug = slugify(vendor_name)
|
||||
entry = vendors.setdefault(slug, {"vendor": vendor_name, "cves": []})
|
||||
entry["cves"].append(detail["cve"])
|
||||
|
||||
for value in vendors.values():
|
||||
value["cves"].sort()
|
||||
return dict(sorted(vendors.items(), key=lambda kv: kv[0]))
|
||||
|
||||
|
||||
def truncate_description(text: str, limit: int = 220) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
text = " ".join(text.split())
|
||||
return text if len(text) <= limit else text[: limit - 3].rstrip() + "..."
|
||||
|
||||
|
||||
def build_joined(
|
||||
kev_data: Dict,
|
||||
epss_data: Dict,
|
||||
poc_index: Dict[str, Dict],
|
||||
*,
|
||||
top_kev: int = DEFAULT_TOP_KEV,
|
||||
high_epss_threshold: float = DEFAULT_HIGH_EPSS_THRESHOLD,
|
||||
high_epss_limit: int = DEFAULT_HIGH_EPSS_LIMIT,
|
||||
extra_cves: Iterable[str] | None = None,
|
||||
) -> Dict:
|
||||
kev_items = kev_data.get("items") or []
|
||||
epss_items = epss_data.get("items") or []
|
||||
|
||||
epss_lookup = build_epss_lookup(epss_items)
|
||||
kev_enriched = enrich_kev(kev_items, epss_lookup, poc_index)
|
||||
kev_top = kev_enriched[:top_kev]
|
||||
|
||||
kev_set = {row["cve"] for row in kev_enriched}
|
||||
high_epss = build_high_epss_not_in_kev(epss_items, kev_set, poc_index, threshold=high_epss_threshold, limit=high_epss_limit)
|
||||
|
||||
details = build_cve_details(kev_top, high_epss, poc_index)
|
||||
|
||||
if extra_cves:
|
||||
extra_set = {cve.upper() for cve in extra_cves}
|
||||
epss_lookup = build_epss_lookup(epss_items)
|
||||
kev_lookup = {row["cve"]: row for row in kev_enriched}
|
||||
for cve in sorted(extra_set):
|
||||
if cve in details:
|
||||
continue
|
||||
epss_row = epss_lookup.get(cve, {})
|
||||
kev_row = kev_lookup.get(cve)
|
||||
details[cve] = {
|
||||
"cve": cve,
|
||||
"description": poc_index.get(cve, {}).get("desc", ""),
|
||||
"poc_links": poc_index.get(cve, {}).get("poc", []),
|
||||
"poc_count": len(poc_index.get(cve, {}).get("poc", [])),
|
||||
"kev": None,
|
||||
"epss": epss_row.get("epss"),
|
||||
"percentile": epss_row.get("percentile"),
|
||||
"vendor": None,
|
||||
"product": None,
|
||||
}
|
||||
if kev_row:
|
||||
details[cve]["kev"] = {
|
||||
"date_added": kev_row.get("date_added"),
|
||||
"due_date": kev_row.get("due_date"),
|
||||
"short_description": kev_row.get("short_description"),
|
||||
"required_action": kev_row.get("required_action"),
|
||||
"notes": kev_row.get("notes"),
|
||||
}
|
||||
details[cve]["vendor"] = kev_row.get("vendor")
|
||||
details[cve]["product"] = kev_row.get("product")
|
||||
|
||||
vendors = build_vendor_map(details)
|
||||
|
||||
# add display summary
|
||||
for collection in (kev_top, high_epss):
|
||||
for row in collection:
|
||||
desc = poc_index.get(row["cve"], {}).get("desc") or ""
|
||||
row["summary"] = truncate_description(desc)
|
||||
|
||||
joined = {
|
||||
"generated": today_str(),
|
||||
"kev_top": kev_top,
|
||||
"high_epss": high_epss,
|
||||
}
|
||||
|
||||
return {
|
||||
"joined": joined,
|
||||
"kev_enriched": kev_enriched,
|
||||
"epss_items": epss_items,
|
||||
"details": details,
|
||||
"vendors": vendors,
|
||||
}
|
||||
|
||||
|
||||
def write_api_outputs(payload: Dict, *, api_dir: Path = API_DIR) -> None:
|
||||
ensure_dirs(api_dir, api_dir / "cve", SNAPSHOT_DIR)
|
||||
joined = payload["joined"]
|
||||
save_json(api_dir / "kev.json", {"generated": joined["generated"], "items": payload["kev_enriched"]})
|
||||
save_json(
|
||||
api_dir / "epss_top.json",
|
||||
{
|
||||
"generated": joined["generated"],
|
||||
"items": payload["joined"]["high_epss"],
|
||||
},
|
||||
)
|
||||
save_json(api_dir / "joined_top.json", joined)
|
||||
|
||||
for cve, detail in payload["details"].items():
|
||||
save_json(api_dir / "cve" / f"{cve}.json", detail)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Join KEV and EPSS with PoC data")
|
||||
parser.add_argument("--kev", type=Path, default=KEV_PATH, help="Path to KEV JSON")
|
||||
parser.add_argument("--epss", type=Path, default=EPSS_PATH, help="Path to EPSS JSON")
|
||||
parser.add_argument("--top-kev", type=int, default=DEFAULT_TOP_KEV, help="How many KEV rows to surface on top list")
|
||||
parser.add_argument(
|
||||
"--high-epss-threshold",
|
||||
type=float,
|
||||
default=DEFAULT_HIGH_EPSS_THRESHOLD,
|
||||
help="Minimum EPSS to include when selecting high EPSs CVEs",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--high-epss-limit",
|
||||
type=int,
|
||||
default=DEFAULT_HIGH_EPSS_LIMIT,
|
||||
help="Maximum number of high EPSs CVEs to keep",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
poc_index = load_poc_index()
|
||||
kev_data, epss_data = load_inputs(args.kev, args.epss)
|
||||
payload = build_joined(
|
||||
kev_data,
|
||||
epss_data,
|
||||
poc_index,
|
||||
top_kev=args.top_kev,
|
||||
high_epss_threshold=args.high_epss_threshold,
|
||||
high_epss_limit=args.high_epss_limit,
|
||||
)
|
||||
write_api_outputs(payload)
|
||||
print("Generated joined JSON endpoints under docs/api/v1/")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+38
-5
@@ -8,12 +8,45 @@ from pathlib import Path
|
||||
from typing import Collection, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from utils import DOCS_DIR, ensure_dirs, load_blacklist, parse_trending_from_readme, is_blacklisted_repo
|
||||
|
||||
ROOT = DOCS_DIR.parent
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DOCS_DIR = ROOT / "docs"
|
||||
BLACKLIST = ROOT / "blacklist.txt"
|
||||
README_PATH = ROOT / "README.md"
|
||||
CVE_OUTPUT = DOCS_DIR / "CVE_list.json"
|
||||
TRENDING_OUTPUT = DOCS_DIR / "trending_poc.json"
|
||||
TREND_ROW_RE = re.compile(
|
||||
r"^\|\s*(?P<stars>\d+)\s*⭐\s*\|\s*(?P<updated>[^|]+)\|\s*"
|
||||
r"\[(?P<name>[^\]]+)\]\((?P<url>[^)]+)\)\s*\|\s*(?P<desc>.*)\|$"
|
||||
)
|
||||
|
||||
|
||||
def load_blacklist() -> set[str]:
|
||||
if not BLACKLIST.exists():
|
||||
return set()
|
||||
return {
|
||||
line.strip().lower()
|
||||
for line in BLACKLIST.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip() and not line.lstrip().startswith("#")
|
||||
}
|
||||
|
||||
|
||||
def parse_trending_from_readme(readme_path: Path) -> List[Dict[str, str]]:
|
||||
"""Read the trending table the hot-CVEs job writes into README.md."""
|
||||
if not readme_path.exists():
|
||||
return []
|
||||
rows: List[Dict[str, str]] = []
|
||||
year: Optional[str] = None
|
||||
for raw in readme_path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if line.startswith("## ") and line[3:].strip().isdigit():
|
||||
year = line[3:].strip()
|
||||
continue
|
||||
match = TREND_ROW_RE.match(line)
|
||||
if match and year:
|
||||
row = match.groupdict()
|
||||
row["year"] = year
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def normalise_block(text: str) -> str:
|
||||
@@ -139,7 +172,7 @@ def build_trending(blacklist: Collection[str]) -> List[Dict[str, object]]:
|
||||
continue
|
||||
year = int(year_text)
|
||||
url = (row.get("url") or "").strip()
|
||||
if url and is_blacklisted_repo(url, blacklist):
|
||||
if url and is_blacklisted(url, blacklist):
|
||||
continue
|
||||
stars_text = str(row.get("stars") or "").strip()
|
||||
stars = int(re.sub(r"\D", "", stars_text) or 0)
|
||||
@@ -170,7 +203,7 @@ def write_json(path: Path, data, *, indent: Optional[int] = None) -> None:
|
||||
def main() -> int:
|
||||
argparse.ArgumentParser(description="Build the CVE PoC site data files").parse_args()
|
||||
|
||||
ensure_dirs(DOCS_DIR)
|
||||
DOCS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
blacklist = load_blacklist()
|
||||
|
||||
cve_payload = build_cve_list(blacklist)
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
import requests
|
||||
|
||||
from utils import DATA_DIR, maybe_float, save_json, today_str
|
||||
|
||||
API_URL = "https://api.first.org/data/v1/epss"
|
||||
DEFAULT_LIMIT = 2000
|
||||
DEFAULT_BATCH = 1000
|
||||
|
||||
|
||||
def fetch_batch(offset: int, limit: int) -> Dict:
|
||||
params = {
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"sort": "epss",
|
||||
"order": "desc",
|
||||
}
|
||||
response = requests.get(API_URL, params=params, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def normalise_rows(raw_rows: List[Dict]) -> List[Dict]:
|
||||
normalised = []
|
||||
for row in raw_rows:
|
||||
cve = str(row.get("cve", "")).upper()
|
||||
if not cve:
|
||||
continue
|
||||
epss = maybe_float(row.get("epss"))
|
||||
pct = maybe_float(row.get("percentile"))
|
||||
normalised.append(
|
||||
{
|
||||
"cve": cve,
|
||||
"epss": epss,
|
||||
"percentile": pct,
|
||||
"date": row.get("date"),
|
||||
}
|
||||
)
|
||||
return normalised
|
||||
|
||||
|
||||
def fetch_epss(limit: int = DEFAULT_LIMIT, batch_size: int = DEFAULT_BATCH) -> Dict:
|
||||
rows: List[Dict] = []
|
||||
offset = 0
|
||||
while offset < limit:
|
||||
size = min(batch_size, limit - offset)
|
||||
payload = fetch_batch(offset, size)
|
||||
data_rows = payload.get("data") or []
|
||||
rows.extend(normalise_rows(data_rows))
|
||||
if len(data_rows) < size:
|
||||
break
|
||||
offset += size
|
||||
|
||||
rows.sort(key=lambda row: (-row.get("epss", 0.0), row["cve"]))
|
||||
return {
|
||||
"source": API_URL,
|
||||
"fetched": today_str(),
|
||||
"count": len(rows),
|
||||
"limit": limit,
|
||||
"items": rows,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Fetch EPSS top list")
|
||||
parser.add_argument("--limit", type=int, default=DEFAULT_LIMIT, help="Number of EPSS rows to fetch")
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=DEFAULT_BATCH,
|
||||
help="Batch size for paginated EPSS API calls",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=DATA_DIR / "epss.json",
|
||||
help="Where to store the downloaded EPSS JSON",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
payload = fetch_epss(args.limit, args.batch_size)
|
||||
save_json(args.output, payload)
|
||||
print(f"Saved {payload['count']} EPSS rows to {args.output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,60 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from utils import DATA_DIR, fetch_json, save_json, today_str
|
||||
|
||||
DEFAULT_SOURCE = "https://raw.githubusercontent.com/cisagov/kev-data/main/known_exploited_vulnerabilities.json"
|
||||
|
||||
|
||||
def fetch_kev(source: str = DEFAULT_SOURCE) -> dict:
|
||||
data = fetch_json(source)
|
||||
items = data.get("vulnerabilities") or data.get("data") or data
|
||||
|
||||
normalised = []
|
||||
for entry in items:
|
||||
cve_id = (entry.get("cveID") or "").upper()
|
||||
if not cve_id:
|
||||
continue
|
||||
normalised.append(
|
||||
{
|
||||
"cve": cve_id,
|
||||
"vendor": entry.get("vendorProject", "").strip(),
|
||||
"product": entry.get("product", "").strip(),
|
||||
"date_added": entry.get("dateAdded"),
|
||||
"due_date": entry.get("dueDate"),
|
||||
"short_description": entry.get("shortDescription", "").strip(),
|
||||
"required_action": entry.get("requiredAction", "").strip(),
|
||||
"notes": entry.get("notes", "").strip(),
|
||||
}
|
||||
)
|
||||
|
||||
normalised.sort(key=lambda row: row["cve"])
|
||||
return {
|
||||
"source": source,
|
||||
"fetched": today_str(),
|
||||
"count": len(normalised),
|
||||
"items": normalised,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Fetch CISA KEV catalogue")
|
||||
parser.add_argument("--source", default=DEFAULT_SOURCE, help="KEV JSON source URL")
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=DATA_DIR / "kev.json",
|
||||
help="Where to store the downloaded KEV JSON",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
payload = fetch_kev(args.source)
|
||||
save_json(args.output, payload)
|
||||
print(f"Saved {payload['count']} KEV entries to {args.output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,210 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
from utils import CACHE_DIR, chunked, hash_key, isoformat
|
||||
|
||||
|
||||
TEXT_MATCH_HEADER = "application/vnd.github.text-match+json"
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
def __init__(self, calls_per_minute: int) -> None:
|
||||
self.min_interval = 60.0 / max(calls_per_minute, 1)
|
||||
self.last_call: Dict[str, float] = {}
|
||||
|
||||
def wait(self, bucket: str) -> None:
|
||||
last = self.last_call.get(bucket, 0.0)
|
||||
elapsed = time.time() - last
|
||||
if elapsed < self.min_interval:
|
||||
time.sleep(self.min_interval - elapsed)
|
||||
self.last_call[bucket] = time.time()
|
||||
|
||||
|
||||
class FileCache:
|
||||
def __init__(self, base: Path) -> None:
|
||||
self.base = base
|
||||
self.base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _path_for(self, key: str) -> Path:
|
||||
digest = hash_key(key)
|
||||
return self.base / digest[:2] / f"{digest}.json"
|
||||
|
||||
def load(self, key: str, *, ttl: int) -> Optional[Dict]:
|
||||
path = self._path_for(key)
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
expires_at = data.get("expires_at")
|
||||
if expires_at:
|
||||
try:
|
||||
expires_ts = time.mktime(time.strptime(expires_at, "%Y-%m-%dT%H:%M:%S"))
|
||||
if time.time() > expires_ts:
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
return data.get("payload")
|
||||
|
||||
def save(self, key: str, payload: Dict, *, ttl: int) -> None:
|
||||
path = self._path_for(key)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = {
|
||||
"fetched_at": isoformat(),
|
||||
"expires_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.time() + ttl)),
|
||||
"payload": payload,
|
||||
}
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
kind: str
|
||||
query: str
|
||||
page: int
|
||||
payload: Dict
|
||||
|
||||
|
||||
class GitHubClient:
|
||||
def __init__(
|
||||
self,
|
||||
token: Optional[str],
|
||||
*,
|
||||
cache_dir: Path | None = None,
|
||||
code_search_rpm: int = 10,
|
||||
general_rpm: int = 30,
|
||||
) -> None:
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({"Accept": TEXT_MATCH_HEADER})
|
||||
if token:
|
||||
self.session.headers["Authorization"] = f"Bearer {token}"
|
||||
self.base_url = "https://api.github.com"
|
||||
self.graphql_url = f"{self.base_url}/graphql"
|
||||
cache_root = cache_dir or CACHE_DIR / "github"
|
||||
self.cache = FileCache(cache_root)
|
||||
self.rate_limiters = {
|
||||
"code": RateLimiter(code_search_rpm),
|
||||
"search": RateLimiter(general_rpm),
|
||||
"graphql": RateLimiter(general_rpm),
|
||||
}
|
||||
|
||||
def _request(self, method: str, url: str, *, bucket: str, **kwargs) -> requests.Response:
|
||||
self.rate_limiters[bucket].wait(bucket)
|
||||
attempts = 0
|
||||
while True:
|
||||
attempts += 1
|
||||
try:
|
||||
response = self.session.request(method, url, timeout=30, **kwargs)
|
||||
except requests.RequestException:
|
||||
if attempts >= 3:
|
||||
raise
|
||||
time.sleep(2 * attempts)
|
||||
continue
|
||||
|
||||
if response.status_code == 403 and "X-RateLimit-Remaining" in response.headers:
|
||||
remaining = int(response.headers.get("X-RateLimit-Remaining") or "0")
|
||||
reset = response.headers.get("X-RateLimit-Reset")
|
||||
if remaining <= 0 and reset:
|
||||
try:
|
||||
reset_ts = int(reset)
|
||||
wait_for = max(0, reset_ts - int(time.time()) + 1)
|
||||
time.sleep(wait_for)
|
||||
continue
|
||||
except ValueError:
|
||||
pass
|
||||
if response.status_code >= 500 and attempts < 3:
|
||||
time.sleep(1 + attempts)
|
||||
continue
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
def _cached_search(self, kind: str, query: str, page: int, per_page: int, ttl: int) -> Dict:
|
||||
cache_key = f"{kind}:{query}:p{page}:n{per_page}"
|
||||
cached = self.cache.load(cache_key, ttl=ttl)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
url = f"{self.base_url}/search/{kind}"
|
||||
params = {"q": query, "page": page, "per_page": per_page}
|
||||
resp = self._request("GET", url, params=params, bucket="code" if kind == "code" else "search")
|
||||
payload = resp.json()
|
||||
self.cache.save(cache_key, payload, ttl=ttl)
|
||||
return payload
|
||||
|
||||
def search_repositories(self, query: str, *, page: int = 1, per_page: int = 100, ttl: int = 3600) -> SearchResult:
|
||||
return SearchResult("repositories", query, page, self._cached_search("repositories", query, page, per_page, ttl))
|
||||
|
||||
def search_code(self, query: str, *, page: int = 1, per_page: int = 100, ttl: int = 3600) -> SearchResult:
|
||||
return SearchResult("code", query, page, self._cached_search("code", query, page, per_page, ttl))
|
||||
|
||||
def search_topics(self, query: str, *, page: int = 1, per_page: int = 100, ttl: int = 3600) -> SearchResult:
|
||||
return SearchResult("repositories", query, page, self._cached_search("repositories", query, page, per_page, ttl))
|
||||
|
||||
def fetch_repo_metadata(self, full_names: Iterable[str], *, ttl: int = 6 * 3600) -> Dict[str, Dict]:
|
||||
results: Dict[str, Dict] = {}
|
||||
to_fetch: List[str] = []
|
||||
for name in full_names:
|
||||
cache_key = f"repo-meta:{name}"
|
||||
cached = self.cache.load(cache_key, ttl=ttl)
|
||||
if cached is not None:
|
||||
results[name] = cached
|
||||
else:
|
||||
to_fetch.append(name)
|
||||
|
||||
if not to_fetch:
|
||||
return results
|
||||
|
||||
fields = """
|
||||
nameWithOwner
|
||||
url
|
||||
stargazerCount
|
||||
description
|
||||
forkCount
|
||||
isFork
|
||||
isArchived
|
||||
pushedAt
|
||||
updatedAt
|
||||
primaryLanguage { name }
|
||||
parent { nameWithOwner url }
|
||||
repositoryTopics(first: 20) { nodes { topic { name } } }
|
||||
"""
|
||||
|
||||
for batch in chunked(to_fetch, 12):
|
||||
parts = []
|
||||
for idx, full_name in enumerate(batch):
|
||||
if "/" not in full_name:
|
||||
continue
|
||||
owner, name = full_name.split("/", 1)
|
||||
owner = owner.replace('"', "")
|
||||
name = name.replace('"', "")
|
||||
parts.append(f'repo_{idx}: repository(owner: "{owner}", name: "{name}") {{ {fields} }}')
|
||||
if not parts:
|
||||
continue
|
||||
query = "query { " + " ".join(parts) + " }"
|
||||
resp = self._request("POST", self.graphql_url, json={"query": query}, bucket="graphql")
|
||||
data = resp.json()
|
||||
repos = data.get("data", {})
|
||||
for idx, full_name in enumerate(batch):
|
||||
key = f"repo_{idx}"
|
||||
meta = repos.get(key) or {}
|
||||
cache_key = f"repo-meta:{full_name}"
|
||||
self.cache.save(cache_key, meta, ttl=ttl)
|
||||
results[full_name] = meta
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def build_client(token_env: str = "GITHUB_TOKEN") -> GitHubClient:
|
||||
token = os.environ.get(token_env)
|
||||
return GitHubClient(token, cache_dir=CACHE_DIR / "github")
|
||||
@@ -1,220 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Tuple
|
||||
|
||||
from utils import API_DIR, DIFFS_DIR, SNAPSHOT_DIR, TOP_DIR, ensure_dirs, load_json, save_json, today_str
|
||||
|
||||
|
||||
def write_cve_outputs(results: List[Dict], *, base_dir: Path | None = None) -> None:
|
||||
target_dir = base_dir or API_DIR / "cve"
|
||||
ensure_dirs(target_dir)
|
||||
for result in results:
|
||||
last_updated = result.get("last_updated") or today_str()
|
||||
output = {
|
||||
"cve_id": result["cve_id"],
|
||||
"last_updated": last_updated,
|
||||
"pocs": [
|
||||
{
|
||||
"repo_full_name": poc.get("repo_full_name"),
|
||||
"repo_url": poc.get("repo_url"),
|
||||
"is_fork": poc.get("is_fork"),
|
||||
"parent_repo_url": poc.get("parent_repo_url"),
|
||||
"stars": poc.get("stars"),
|
||||
"forks": poc.get("forks"),
|
||||
"archived": poc.get("archived"),
|
||||
"pushed_at": poc.get("pushed_at") or poc.get("updated_at"),
|
||||
"topics": poc.get("topics", []),
|
||||
"primary_language": poc.get("primary_language"),
|
||||
"matches": poc.get("matches", []),
|
||||
"confidence_score": poc.get("confidence_score"),
|
||||
"confidence_tier": poc.get("confidence_tier"),
|
||||
}
|
||||
for poc in result.get("pocs", [])
|
||||
],
|
||||
}
|
||||
save_json(target_dir / f"{result['cve_id']}.json", output)
|
||||
|
||||
|
||||
def build_index(results: List[Dict]) -> Dict:
|
||||
items: List[Dict] = []
|
||||
for result in results:
|
||||
poc_entries = result.get("pocs", [])
|
||||
high = [p for p in poc_entries if p.get("confidence_tier") == "high"]
|
||||
medium = [p for p in poc_entries if p.get("confidence_tier") == "medium"]
|
||||
langs = Counter()
|
||||
max_score = 0.0
|
||||
for poc in poc_entries:
|
||||
lang = poc.get("primary_language")
|
||||
if lang:
|
||||
langs[lang] += 1
|
||||
max_score = max(max_score, poc.get("confidence_score") or 0)
|
||||
items.append(
|
||||
{
|
||||
"cve_id": result["cve_id"],
|
||||
"poc_count": len(poc_entries),
|
||||
"high_confidence": len(high),
|
||||
"medium_confidence": len(medium),
|
||||
"top_languages": [lang for lang, _ in langs.most_common(3)],
|
||||
"max_score": max_score,
|
||||
"last_updated": result.get("last_updated"),
|
||||
}
|
||||
)
|
||||
return {"generated": today_str(), "items": sorted(items, key=lambda r: r["cve_id"], reverse=True)}
|
||||
|
||||
|
||||
def write_index(results: List[Dict]) -> Dict:
|
||||
ensure_dirs(API_DIR)
|
||||
payload = build_index(results)
|
||||
save_json(API_DIR / "index.json", payload)
|
||||
return payload
|
||||
|
||||
|
||||
def write_top(results: List[Dict], *, limit: int = 100) -> Dict:
|
||||
ensure_dirs(TOP_DIR)
|
||||
entries: List[Dict] = []
|
||||
for result in results:
|
||||
for poc in result.get("pocs", []):
|
||||
if poc.get("confidence_tier") not in {"high", "medium"}:
|
||||
continue
|
||||
entries.append(
|
||||
{
|
||||
"cve_id": result["cve_id"],
|
||||
"repo_full_name": poc.get("repo_full_name"),
|
||||
"repo_url": poc.get("repo_url"),
|
||||
"score": poc.get("confidence_score"),
|
||||
"tier": poc.get("confidence_tier"),
|
||||
"stars": poc.get("stars"),
|
||||
"primary_language": poc.get("primary_language"),
|
||||
}
|
||||
)
|
||||
entries.sort(key=lambda e: (-(e.get("score") or 0), -(e.get("stars") or 0)))
|
||||
payload = {"generated": today_str(), "items": entries[:limit]}
|
||||
save_json(TOP_DIR / "today.json", payload)
|
||||
return payload
|
||||
|
||||
|
||||
def summarise_for_snapshot(results: List[Dict], *, top: Dict | None = None) -> Dict:
|
||||
summary: Dict[str, Dict[str, Dict]] = {}
|
||||
for result in results:
|
||||
repo_map: Dict[str, Dict] = {}
|
||||
for poc in result.get("pocs", []):
|
||||
repo_map[poc.get("repo_full_name")] = {
|
||||
"score": poc.get("confidence_score"),
|
||||
"tier": poc.get("confidence_tier"),
|
||||
}
|
||||
summary[result["cve_id"]] = repo_map
|
||||
payload = {"generated": today_str(), "entries": summary}
|
||||
if top:
|
||||
payload["top"] = top
|
||||
return payload
|
||||
|
||||
|
||||
def write_snapshot(summary: Dict) -> Path:
|
||||
ensure_dirs(SNAPSHOT_DIR)
|
||||
target = SNAPSHOT_DIR / f"{summary['generated']}.json"
|
||||
save_json(target, summary)
|
||||
save_json(SNAPSHOT_DIR / "latest.json", summary)
|
||||
return target
|
||||
|
||||
|
||||
def prune_old_snapshots(days: int = 14) -> None:
|
||||
if not SNAPSHOT_DIR.exists():
|
||||
return
|
||||
cutoff = datetime.utcnow().date() - timedelta(days=days)
|
||||
for snap in SNAPSHOT_DIR.glob("*.json"):
|
||||
try:
|
||||
snap_date = datetime.strptime(snap.stem, "%Y-%m-%d").date()
|
||||
except ValueError:
|
||||
continue
|
||||
if snap_date < cutoff:
|
||||
snap.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def prune_old_diffs(days: int = 14) -> None:
|
||||
if not DIFFS_DIR.exists():
|
||||
return
|
||||
cutoff = datetime.now().date() - timedelta(days=days)
|
||||
for diff in DIFFS_DIR.glob("*.json"):
|
||||
try:
|
||||
diff_date = datetime.strptime(diff.stem, "%Y-%m-%d").date()
|
||||
except ValueError:
|
||||
continue
|
||||
if diff_date < cutoff:
|
||||
diff.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _load_snapshot(path: Path) -> Dict:
|
||||
return load_json(path, default={}) or {}
|
||||
|
||||
|
||||
def build_diff(prev: Dict, curr: Dict, *, dead_links: List[Dict] | None = None) -> Dict:
|
||||
prev_entries = prev.get("entries", {})
|
||||
curr_entries = curr.get("entries", {})
|
||||
|
||||
new_high: List[Dict] = []
|
||||
promoted: List[Dict] = []
|
||||
demoted: List[Dict] = []
|
||||
|
||||
for cve_id, repos in curr_entries.items():
|
||||
for repo_name, info in repos.items():
|
||||
tier = info.get("tier")
|
||||
if tier != "high":
|
||||
continue
|
||||
prev_info = (prev_entries.get(cve_id) or {}).get(repo_name)
|
||||
if not prev_info:
|
||||
new_high.append({"cve_id": cve_id, "repo_full_name": repo_name, "score": info.get("score")})
|
||||
elif prev_info.get("tier") != "high":
|
||||
promoted.append(
|
||||
{
|
||||
"cve_id": cve_id,
|
||||
"repo_full_name": repo_name,
|
||||
"score": info.get("score"),
|
||||
"previous_tier": prev_info.get("tier"),
|
||||
}
|
||||
)
|
||||
|
||||
for cve_id, repos in prev_entries.items():
|
||||
for repo_name, info in repos.items():
|
||||
if info.get("tier") != "high":
|
||||
continue
|
||||
curr_info = (curr_entries.get(cve_id) or {}).get(repo_name)
|
||||
if not curr_info or curr_info.get("tier") != "high":
|
||||
demoted.append(
|
||||
{
|
||||
"cve_id": cve_id,
|
||||
"repo_full_name": repo_name,
|
||||
"previous_score": info.get("score"),
|
||||
"previous_tier": info.get("tier"),
|
||||
"current_tier": curr_info.get("tier") if curr_info else None,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"generated": curr.get("generated"),
|
||||
"new_high_conf_pocs": new_high,
|
||||
"promoted_to_high": promoted,
|
||||
"demoted_or_removed": demoted,
|
||||
"dead_links": dead_links or [],
|
||||
}
|
||||
|
||||
|
||||
def write_diff(diff: Dict) -> Path:
|
||||
ensure_dirs(DIFFS_DIR)
|
||||
target = DIFFS_DIR / f"{diff['generated']}.json"
|
||||
save_json(target, diff)
|
||||
save_json(DIFFS_DIR / "latest.json", diff)
|
||||
return target
|
||||
|
||||
|
||||
def latest_snapshots() -> Tuple[Dict, Dict]:
|
||||
if not SNAPSHOT_DIR.exists():
|
||||
return {}, {}
|
||||
snaps = sorted(SNAPSHOT_DIR.glob("*.json"))
|
||||
if not snaps:
|
||||
return {}, {}
|
||||
curr = _load_snapshot(snaps[-1])
|
||||
prev = _load_snapshot(snaps[-2]) if len(snaps) > 1 else {}
|
||||
return prev, curr
|
||||
@@ -1,274 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Optional, Set, Tuple
|
||||
|
||||
from github_client import GitHubClient, SearchResult, build_client
|
||||
from poc_scoring import match_score, score_repo
|
||||
from utils import API_DIR, EVIDENCE_DIR, chunked, cve_year, ensure_dirs, isoformat, load_blacklist, load_json, save_json, today_str
|
||||
|
||||
|
||||
LANG_PARTITIONS = ("python", "go", "c", "shell", "powershell", "java", "ruby", "js")
|
||||
CVE_RE = re.compile(r"CVE-\d{4}-\d{4,}", re.IGNORECASE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MatchEvidence:
|
||||
path: str
|
||||
match_type: str
|
||||
query: str
|
||||
score: float | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RepoCandidate:
|
||||
cve_id: str
|
||||
repo_full_name: str
|
||||
repo_url: str
|
||||
matches: List[MatchEvidence] = field(default_factory=list)
|
||||
metadata: Dict[str, object] = field(default_factory=dict)
|
||||
|
||||
def add_match(self, path: str, match_type: str, query: str) -> None:
|
||||
key = (path, match_type)
|
||||
existing = {(m.path, m.match_type) for m in self.matches}
|
||||
if key in existing:
|
||||
return
|
||||
self.matches.append(MatchEvidence(path=path, match_type=match_type, query=query))
|
||||
|
||||
|
||||
def build_created_ranges(days: int, *, window: int = 7) -> List[Tuple[str, str]]:
|
||||
end = date.today()
|
||||
start = end - timedelta(days=max(days, 1))
|
||||
ranges: List[Tuple[str, str]] = []
|
||||
cursor = start
|
||||
while cursor <= end:
|
||||
window_end = min(cursor + timedelta(days=window - 1), end)
|
||||
ranges.append((cursor.isoformat(), window_end.isoformat()))
|
||||
cursor = window_end + timedelta(days=1)
|
||||
return ranges or [(start.isoformat(), end.isoformat())]
|
||||
|
||||
|
||||
def build_query_pack(cve_id: str, created_range: Tuple[str, str] | None = None) -> List[Dict[str, str]]:
|
||||
base_repo = f'{cve_id} in:name,description,readme fork:false'
|
||||
enriched_repo = f'{cve_id} (poc OR exploit) in:name,description,readme fork:false'
|
||||
topic_query = f"topic:{cve_id.lower()} fork:false"
|
||||
created_suffix = ""
|
||||
if created_range:
|
||||
created_suffix = f" created:{created_range[0]}..{created_range[1]}"
|
||||
|
||||
queries = [
|
||||
{"kind": "repositories", "query": base_repo + created_suffix, "match_type": "name"},
|
||||
{"kind": "repositories", "query": enriched_repo + created_suffix, "match_type": "description"},
|
||||
{"kind": "repositories", "query": topic_query + created_suffix, "match_type": "topic"},
|
||||
]
|
||||
|
||||
for lang in LANG_PARTITIONS:
|
||||
base_code = f'{cve_id} in:file language:{lang}{created_suffix}'
|
||||
queries.append({"kind": "code", "query": base_code, "match_type": "code"})
|
||||
|
||||
# generic code search without language partition for the most recent window
|
||||
queries.append({"kind": "code", "query": f"{cve_id} in:file{created_suffix}", "match_type": "code"})
|
||||
return queries
|
||||
|
||||
|
||||
def parse_repo_from_item(item: Dict) -> Tuple[str | None, str | None]:
|
||||
repo_full_name = item.get("full_name") or item.get("repository", {}).get("full_name")
|
||||
repo_url = item.get("html_url") or item.get("repository", {}).get("html_url")
|
||||
if not repo_full_name and "repository" in item:
|
||||
repo_full_name = item["repository"].get("owner", {}).get("login", "")
|
||||
if repo_full_name:
|
||||
repo_full_name = f"{repo_full_name}/{item['repository'].get('name', '')}"
|
||||
return repo_full_name, repo_url
|
||||
|
||||
|
||||
def extract_matches(item: Dict, default_type: str, query: str) -> List[MatchEvidence]:
|
||||
matches: List[MatchEvidence] = []
|
||||
for text_match in item.get("text_matches", []) or []:
|
||||
prop = text_match.get("property") or text_match.get("object_type") or ""
|
||||
fragment = text_match.get("fragment") or text_match.get("path") or prop or ""
|
||||
match_type = prop if prop else default_type
|
||||
matches.append(MatchEvidence(path=str(fragment), match_type=str(match_type), query=query))
|
||||
if not matches:
|
||||
path = item.get("path") or default_type
|
||||
matches.append(MatchEvidence(path=str(path), match_type=default_type, query=query))
|
||||
return matches
|
||||
|
||||
|
||||
def normalise_metadata(meta: Dict, fallback_full_name: str, fallback_url: str) -> Dict:
|
||||
topics = []
|
||||
if meta.get("repositoryTopics"):
|
||||
for node in meta["repositoryTopics"].get("nodes", []):
|
||||
topic = (node.get("topic") or {}).get("name")
|
||||
if topic:
|
||||
topics.append(topic)
|
||||
primary_language = None
|
||||
if meta.get("primaryLanguage"):
|
||||
primary_language = meta["primaryLanguage"].get("name")
|
||||
parent = meta.get("parent") or {}
|
||||
return {
|
||||
"repo_full_name": meta.get("nameWithOwner") or fallback_full_name,
|
||||
"repo_url": meta.get("url") or fallback_url,
|
||||
"description": meta.get("description") or "",
|
||||
"is_fork": bool(meta.get("isFork")),
|
||||
"parent_repo_url": parent.get("url"),
|
||||
"stars": meta.get("stargazerCount") or 0,
|
||||
"forks": meta.get("forkCount") or 0,
|
||||
"archived": bool(meta.get("isArchived")),
|
||||
"pushed_at": meta.get("pushedAt"),
|
||||
"updated_at": meta.get("updatedAt"),
|
||||
"topics": topics,
|
||||
"primary_language": primary_language,
|
||||
}
|
||||
|
||||
|
||||
class PoCPipeline:
|
||||
def __init__(
|
||||
self,
|
||||
client: GitHubClient | None = None,
|
||||
*,
|
||||
blacklist_path: Path | None = None,
|
||||
search_ttl: int = 3 * 3600,
|
||||
) -> None:
|
||||
self.client = client or build_client()
|
||||
self.blacklist = load_blacklist(blacklist_path)
|
||||
self.search_ttl = search_ttl
|
||||
|
||||
def _run_query(self, query: Dict, page: int) -> SearchResult:
|
||||
if query["kind"] == "repositories":
|
||||
return self.client.search_repositories(query["query"], page=page, per_page=50, ttl=self.search_ttl)
|
||||
if query["kind"] == "code":
|
||||
return self.client.search_code(query["query"], page=page, per_page=50, ttl=self.search_ttl)
|
||||
return self.client.search_topics(query["query"], page=page, per_page=50, ttl=self.search_ttl)
|
||||
|
||||
def discover_for_cve(self, cve_id: str, *, days: int, max_pages_repo: int = 2, max_pages_code: int = 2) -> Dict:
|
||||
ranges = build_created_ranges(days)
|
||||
candidates: Dict[str, RepoCandidate] = {}
|
||||
query_log: List[Dict] = []
|
||||
|
||||
for created_range in ranges:
|
||||
query_pack = build_query_pack(cve_id, created_range)
|
||||
for query in query_pack:
|
||||
query_log.append({"query": query["query"], "kind": query["kind"], "window": created_range})
|
||||
page_limit = max_pages_code if query["kind"] == "code" else max_pages_repo
|
||||
for page in range(1, page_limit + 1):
|
||||
result = self._run_query(query, page)
|
||||
items = result.payload.get("items", [])
|
||||
for item in items:
|
||||
repo_full_name, repo_url = parse_repo_from_item(item)
|
||||
if not repo_full_name or not repo_url:
|
||||
continue
|
||||
candidate = candidates.setdefault(
|
||||
repo_full_name,
|
||||
RepoCandidate(cve_id=cve_id, repo_full_name=repo_full_name, repo_url=repo_url),
|
||||
)
|
||||
for match in extract_matches(item, query["match_type"], query["query"]):
|
||||
candidate.add_match(match.path, match.match_type, match.query)
|
||||
if len(items) < 50:
|
||||
break
|
||||
|
||||
metadata = self.client.fetch_repo_metadata(candidates.keys())
|
||||
for repo_full_name, candidate in candidates.items():
|
||||
meta = metadata.get(repo_full_name, {})
|
||||
candidate.metadata = normalise_metadata(meta, repo_full_name, candidate.repo_url)
|
||||
|
||||
repos: List[Dict] = []
|
||||
for candidate in candidates.values():
|
||||
matches_dicts = []
|
||||
for m in candidate.matches:
|
||||
m.score = match_score({"path": m.path, "match_type": m.match_type})
|
||||
matches_dicts.append({"path": m.path, "match_type": m.match_type, "query": m.query, "score": m.score})
|
||||
score, tier = score_repo(candidate.metadata, matches_dicts, self.blacklist)
|
||||
repo_entry = {
|
||||
**candidate.metadata,
|
||||
"matches": matches_dicts,
|
||||
"confidence_score": score,
|
||||
"confidence_tier": tier,
|
||||
"cve_id": cve_id,
|
||||
}
|
||||
repos.append(repo_entry)
|
||||
|
||||
repos.sort(key=lambda r: (-r["confidence_score"], -r.get("stars", 0)))
|
||||
|
||||
evidence = {
|
||||
"queries": query_log,
|
||||
"candidates": [
|
||||
{
|
||||
"repo_full_name": r["repo_full_name"],
|
||||
"matches": r["matches"],
|
||||
"match_count": len(r["matches"]),
|
||||
"score": r["confidence_score"],
|
||||
"tier": r["confidence_tier"],
|
||||
}
|
||||
for r in repos
|
||||
],
|
||||
}
|
||||
return {"cve_id": cve_id, "last_updated": isoformat(), "pocs": repos, "evidence": evidence}
|
||||
|
||||
def discover_many(self, cve_ids: Iterable[str], *, days: int, limit: Optional[int] = None) -> List[Dict]:
|
||||
results: List[Dict] = []
|
||||
for idx, cve_id in enumerate(cve_ids):
|
||||
if limit and idx >= limit:
|
||||
break
|
||||
results.append(self.discover_for_cve(cve_id, days=days))
|
||||
return results
|
||||
|
||||
|
||||
def persist_evidence(results: List[Dict]) -> None:
|
||||
ensure_dirs(EVIDENCE_DIR)
|
||||
for result in results:
|
||||
cve_id = result["cve_id"]
|
||||
evidence_path = EVIDENCE_DIR / f"{cve_id}.json"
|
||||
save_json(evidence_path, result.get("evidence", {}))
|
||||
|
||||
|
||||
def discover_from_github_list(path: Path) -> List[str]:
|
||||
if not path.exists():
|
||||
return []
|
||||
ids: List[str] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
matches = CVE_RE.findall(line)
|
||||
for match in matches:
|
||||
if match.upper() not in ids:
|
||||
ids.append(match.upper())
|
||||
return ids
|
||||
|
||||
|
||||
def load_existing_cves(api_dir: Path = API_DIR / "cve") -> List[str]:
|
||||
if not api_dir.exists():
|
||||
return []
|
||||
return sorted({p.stem.upper() for p in api_dir.glob("CVE-*.json") if CVE_RE.match(p.stem)})
|
||||
|
||||
|
||||
def build_scope(
|
||||
days: int,
|
||||
*,
|
||||
github_list: Path,
|
||||
existing_api: Path,
|
||||
prefer_recent_years: bool = True,
|
||||
max_cves: int | None = None,
|
||||
low_conf_threshold: int = 1,
|
||||
) -> List[str]:
|
||||
seeds = discover_from_github_list(github_list)
|
||||
existing = load_existing_cves(existing_api)
|
||||
candidates = seeds or existing
|
||||
|
||||
if prefer_recent_years:
|
||||
current_year = date.today().year
|
||||
candidates = [cve for cve in candidates if cve_year(cve) and cve_year(cve) >= current_year - 2] or candidates
|
||||
|
||||
index_path = API_DIR / "index.json"
|
||||
low_conf: List[str] = []
|
||||
if index_path.exists():
|
||||
index_payload = load_json(index_path, default={}) or {}
|
||||
for item in index_payload.get("items", []):
|
||||
score = (item.get("high_confidence", 0) or 0) + (item.get("medium_confidence", 0) or 0)
|
||||
if score <= low_conf_threshold:
|
||||
low_conf.append(item.get("cve_id"))
|
||||
|
||||
scoped = candidates + [cve for cve in low_conf if cve and cve not in candidates]
|
||||
if max_cves:
|
||||
scoped = scoped[:max_cves]
|
||||
return scoped
|
||||
@@ -1,121 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, Iterable, List, Tuple
|
||||
|
||||
from utils import clamp, parse_date
|
||||
|
||||
DOC_EXTS = {"md", "txt", "rst", "adoc", "markdown", "mkd", "mdown"}
|
||||
POSITIVE_KEYWORDS = ("poc", "exploit", "rce", "lpe", "auth bypass", "bypass")
|
||||
NEGATIVE_KEYWORDS = ("report", "writeup", "advisory", "changelog")
|
||||
|
||||
|
||||
def is_doc_path(path: str) -> bool:
|
||||
lower = path.lower()
|
||||
if lower.endswith("/"):
|
||||
return True
|
||||
if "." not in lower:
|
||||
return False
|
||||
ext = lower.rsplit(".", 1)[-1]
|
||||
return ext in DOC_EXTS
|
||||
|
||||
|
||||
def match_score(match: Dict) -> float:
|
||||
path = str(match.get("path", ""))
|
||||
match_type = str(match.get("match_type", "")).lower()
|
||||
base = 50 if not is_doc_path(path) else 30
|
||||
if match_type in ("code",):
|
||||
base += 10
|
||||
if "readme" in match_type:
|
||||
base += 5
|
||||
if "topic" in match_type:
|
||||
base -= 5
|
||||
return clamp(base, 0, 100)
|
||||
|
||||
|
||||
def tier_for_score(score: float) -> str:
|
||||
if score >= 75:
|
||||
return "high"
|
||||
if score >= 45:
|
||||
return "medium"
|
||||
return "low"
|
||||
|
||||
|
||||
def keyword_hits(text: str, keywords: Iterable[str]) -> int:
|
||||
if not text:
|
||||
return 0
|
||||
lower = text.lower()
|
||||
return sum(1 for kw in keywords if kw in lower)
|
||||
|
||||
|
||||
def recency_bonus(pushed_at: str | None) -> float:
|
||||
if not pushed_at:
|
||||
return 0.0
|
||||
dt = parse_date(pushed_at)
|
||||
if not dt:
|
||||
return 0.0
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
delta = datetime.now(timezone.utc) - dt
|
||||
if delta <= timedelta(days=30):
|
||||
return 18.0
|
||||
if delta <= timedelta(days=90):
|
||||
return 10.0
|
||||
if delta <= timedelta(days=180):
|
||||
return 5.0
|
||||
return 0.0
|
||||
|
||||
|
||||
def score_repo(repo: Dict, matches: List[Dict], blacklist: List[str]) -> Tuple[float, str]:
|
||||
stars = repo.get("stargazerCount") or repo.get("stars") or 0
|
||||
forks = repo.get("forkCount") or repo.get("forks") or 0
|
||||
is_fork = bool(repo.get("isFork"))
|
||||
archived = bool(repo.get("isArchived"))
|
||||
topics = [t.lower() for t in repo.get("topics", []) if t]
|
||||
name = str(repo.get("nameWithOwner") or repo.get("repo_full_name") or "").lower()
|
||||
description = str(repo.get("description") or "").lower()
|
||||
|
||||
non_doc_matches = [m for m in matches if not is_doc_path(str(m.get("path", "")))]
|
||||
doc_matches = [m for m in matches if is_doc_path(str(m.get("path", "")))]
|
||||
|
||||
score = 12.0
|
||||
if non_doc_matches:
|
||||
score += 25 + min(len(non_doc_matches) * 2, 10)
|
||||
if doc_matches and not non_doc_matches:
|
||||
score -= 20
|
||||
|
||||
score += recency_bonus(repo.get("pushed_at") or repo.get("pushedAt") or repo.get("updated_at"))
|
||||
|
||||
score += min(stars / 50.0, 25.0)
|
||||
score += min(forks / 200.0, 5.0)
|
||||
|
||||
score += keyword_hits(description, POSITIVE_KEYWORDS) * 4.0
|
||||
score += keyword_hits(" ".join(topics), POSITIVE_KEYWORDS) * 4.0
|
||||
|
||||
negative_bias = keyword_hits(description, NEGATIVE_KEYWORDS)
|
||||
if negative_bias and not non_doc_matches:
|
||||
score -= 15
|
||||
|
||||
if is_fork:
|
||||
score -= 12
|
||||
if archived:
|
||||
score -= 30
|
||||
|
||||
lowered_blacklist = [entry.lower() for entry in blacklist]
|
||||
for forbidden in lowered_blacklist:
|
||||
if not forbidden:
|
||||
continue
|
||||
if forbidden.endswith("*"):
|
||||
prefix = forbidden[:-1]
|
||||
if prefix and name.startswith(prefix):
|
||||
score -= 40
|
||||
break
|
||||
elif forbidden in name:
|
||||
score -= 40
|
||||
break
|
||||
|
||||
for match in matches:
|
||||
score += match_score(match) / 25.0
|
||||
|
||||
return clamp(score, 0, 100), tier_for_score(score)
|
||||
@@ -1,310 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Collection, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DATA_DIR = ROOT / "data"
|
||||
DOCS_DIR = ROOT / "docs"
|
||||
API_DIR = DOCS_DIR / "api" / "v1"
|
||||
SNAPSHOT_DIR = API_DIR / "snapshots"
|
||||
DIFFS_DIR = API_DIR / "diffs"
|
||||
TOP_DIR = API_DIR / "top"
|
||||
ASSETS_DIR = DOCS_DIR / "assets"
|
||||
CACHE_DIR = DATA_DIR / "cache"
|
||||
STATE_DIR = DATA_DIR / "state"
|
||||
EVIDENCE_DIR = DATA_DIR / "evidence"
|
||||
|
||||
|
||||
def ensure_dirs(*paths: Path) -> None:
|
||||
for path in paths:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def load_json(path: Path, default=None):
|
||||
if not path.exists():
|
||||
return default
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def save_json(path: Path, data, *, sort_keys: bool = True) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, ensure_ascii=False, indent=2, sort_keys=sort_keys)
|
||||
|
||||
|
||||
def fetch_json(url: str, *, timeout: int = 30, headers: Optional[Dict[str, str]] = None):
|
||||
import requests
|
||||
|
||||
response = requests.get(url, timeout=timeout, headers=headers or {})
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def today_str() -> str:
|
||||
return datetime.now(timezone.utc).date().isoformat()
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def isoformat(dt: datetime | None = None) -> str:
|
||||
return (dt or now_utc()).isoformat()
|
||||
|
||||
|
||||
def parse_date(value: str) -> datetime | None:
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
cleaned = re.sub(r"[^A-Za-z0-9]+", "-", text.strip().lower())
|
||||
cleaned = cleaned.strip("-")
|
||||
return cleaned or "unknown"
|
||||
|
||||
|
||||
def stable_unique(items: Iterable[str]) -> List[str]:
|
||||
seen = set()
|
||||
output = []
|
||||
for item in items:
|
||||
if item and item not in seen:
|
||||
seen.add(item)
|
||||
output.append(item)
|
||||
return output
|
||||
|
||||
|
||||
def maybe_float(value: str | float | int | None) -> Optional[float]:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
# --- PoC data helpers ----------------------------------------------------
|
||||
|
||||
|
||||
CVE_SECTION_RE = re.compile(r"^CVE-\d{4}-\d{4,}$", re.IGNORECASE)
|
||||
|
||||
|
||||
def load_poc_index() -> Dict[str, Dict[str, object]]:
|
||||
"""Load CVE → {desc, poc} mapping from docs/CVE_list.json or markdown files."""
|
||||
cve_json = DOCS_DIR / "CVE_list.json"
|
||||
blacklist = load_blacklist()
|
||||
if cve_json.exists():
|
||||
data = load_json(cve_json, default=[]) or []
|
||||
mapping = {}
|
||||
for entry in data:
|
||||
cve = str(entry.get("cve", "")).upper()
|
||||
if not is_valid_cve(cve):
|
||||
continue
|
||||
desc = (entry.get("desc") or "").strip()
|
||||
poc_links = stable_unique(entry.get("poc", []) or [])
|
||||
poc_links = filter_links_by_blacklist(poc_links, blacklist)
|
||||
if not desc or not poc_links:
|
||||
continue
|
||||
mapping[cve] = {
|
||||
"desc": desc,
|
||||
"poc": poc_links,
|
||||
}
|
||||
return mapping
|
||||
|
||||
return build_poc_index_from_markdown(blacklist=blacklist)
|
||||
|
||||
|
||||
def build_poc_index_from_markdown(*, blacklist: Optional[List[str]] = None) -> Dict[str, Dict[str, object]]:
|
||||
mapping: Dict[str, Dict[str, object]] = {}
|
||||
for md_path in sorted(ROOT.glob("[12][0-9][0-9][0-9]/CVE-*.md")):
|
||||
cve = md_path.stem.upper()
|
||||
if not is_valid_cve(cve):
|
||||
continue
|
||||
desc, poc_links = parse_cve_markdown(md_path, blacklist=blacklist)
|
||||
mapping[cve] = {"desc": desc, "poc": poc_links}
|
||||
return mapping
|
||||
|
||||
|
||||
def parse_cve_markdown(path: Path, *, blacklist: Optional[List[str]] = None) -> Tuple[str, List[str]]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
sections = parse_sections(text)
|
||||
description = normalise_block(sections.get("### Description", ""))
|
||||
blacklist = blacklist or []
|
||||
references = collect_links(sections.get("#### Reference", ""), blacklist=blacklist)
|
||||
github_links = collect_links(sections.get("#### Github", ""), blacklist=blacklist)
|
||||
poc_links = stable_unique([*references, *github_links])
|
||||
return description, poc_links
|
||||
|
||||
|
||||
def normalise_block(text: str) -> str:
|
||||
text = text.replace("\r\n", "\n")
|
||||
text = re.sub(r"\n{2,}", "\n", text.strip())
|
||||
lines = [line.lstrip("- ").rstrip() for line in text.split("\n")]
|
||||
return "\n".join(line for line in lines if line)
|
||||
|
||||
|
||||
def parse_sections(content: str) -> Dict[str, str]:
|
||||
sections: Dict[str, str] = {}
|
||||
current: Optional[str] = None
|
||||
buffer: List[str] = []
|
||||
|
||||
for raw_line in content.splitlines():
|
||||
line = raw_line.strip()
|
||||
if line.startswith("### ") or line.startswith("#### "):
|
||||
if current is not None:
|
||||
sections[current] = "\n".join(buffer).strip()
|
||||
current = line
|
||||
buffer = []
|
||||
else:
|
||||
buffer.append(raw_line)
|
||||
|
||||
if current is not None:
|
||||
sections[current] = "\n".join(buffer).strip()
|
||||
|
||||
return sections
|
||||
|
||||
|
||||
def collect_links(block: str, *, blacklist: Optional[List[str]] = None) -> List[str]:
|
||||
links: List[str] = []
|
||||
for raw in block.splitlines():
|
||||
entry = raw.strip()
|
||||
if not entry or "No PoCs" in entry:
|
||||
continue
|
||||
if entry.startswith("- "):
|
||||
entry = entry[2:].strip()
|
||||
if entry and entry not in links:
|
||||
links.append(entry)
|
||||
return filter_links_by_blacklist(links, blacklist or [])
|
||||
|
||||
|
||||
def is_valid_cve(cve_id: str) -> bool:
|
||||
parts = cve_id.split("-")
|
||||
if len(parts) != 3:
|
||||
return False
|
||||
year = parts[1]
|
||||
return year.isdigit() and parts[2].isdigit()
|
||||
|
||||
|
||||
def cve_year(cve_id: str) -> int | None:
|
||||
if not is_valid_cve(cve_id):
|
||||
return None
|
||||
try:
|
||||
return int(cve_id.split("-")[1])
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
# --- Trending PoCs -------------------------------------------------------
|
||||
|
||||
TREND_ROW_RE = re.compile(r"^\|\s*(?P<stars>\d+)\s*⭐\s*\|\s*(?P<updated>[^|]+)\|\s*\[(?P<name>[^\]]+)\]\((?P<url>[^)]+)\)\s*\|\s*(?P<desc>.*)\|$")
|
||||
|
||||
|
||||
def parse_trending_from_readme(readme_path: Path) -> List[Dict[str, str]]:
|
||||
if not readme_path.exists():
|
||||
return []
|
||||
results: List[Dict[str, str]] = []
|
||||
current_year: Optional[str] = None
|
||||
for line in readme_path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("## ") and line[3:].strip().isdigit():
|
||||
current_year = line[3:].strip()
|
||||
continue
|
||||
match = TREND_ROW_RE.match(line)
|
||||
if match and current_year:
|
||||
entry = match.groupdict()
|
||||
entry["year"] = current_year
|
||||
results.append(entry)
|
||||
# Keep deterministic order (README already ordered newest first)
|
||||
return results
|
||||
|
||||
|
||||
# --- Misc helpers --------------------------------------------------------
|
||||
|
||||
|
||||
def read_text(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8") if path.exists() else ""
|
||||
|
||||
|
||||
def write_text(path: Path, content: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
# --- New helpers for PoC discovery -------------------------------------------------
|
||||
|
||||
|
||||
def clamp(value: float, minimum: float = 0, maximum: float = 100) -> float:
|
||||
return max(minimum, min(maximum, value))
|
||||
|
||||
|
||||
def chunked(iterable: Iterable, size: int) -> Iterable[List]:
|
||||
chunk: List = []
|
||||
for item in iterable:
|
||||
chunk.append(item)
|
||||
if len(chunk) >= size:
|
||||
yield chunk
|
||||
chunk = []
|
||||
if chunk:
|
||||
yield chunk
|
||||
|
||||
|
||||
def hash_key(text: str) -> str:
|
||||
import hashlib
|
||||
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def load_blacklist(path: Path | None = None) -> set[str]:
|
||||
target = path or ROOT / "blacklist.txt"
|
||||
if not target.exists():
|
||||
return set()
|
||||
entries: set[str] = set()
|
||||
for raw in target.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if line and not line.startswith("#"):
|
||||
entries.add(line.lower())
|
||||
return entries
|
||||
|
||||
|
||||
def extract_repo_from_url(url: str) -> str:
|
||||
"""Return the owner and repository segments from a URL."""
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(url)
|
||||
host = (parsed.netloc or "").lower()
|
||||
if host and "github" not in host:
|
||||
return ""
|
||||
path = parsed.path or url
|
||||
except Exception:
|
||||
path = url
|
||||
parts = path.strip("/").split("/")
|
||||
if len(parts) >= 2:
|
||||
return "/".join(parts[:2]).lower()
|
||||
if parts:
|
||||
return parts[-1].lower()
|
||||
return ""
|
||||
|
||||
|
||||
def is_blacklisted_repo(url: str, blacklist: Collection[str]) -> bool:
|
||||
full_name = extract_repo_from_url(url)
|
||||
if not full_name:
|
||||
return False
|
||||
return full_name in blacklist
|
||||
|
||||
|
||||
def filter_links_by_blacklist(links: List[str], blacklist: Collection[str]) -> List[str]:
|
||||
if not blacklist:
|
||||
return links
|
||||
filtered: List[str] = []
|
||||
for link in links:
|
||||
if is_blacklisted_repo(link, blacklist):
|
||||
continue
|
||||
filtered.append(link)
|
||||
return filtered
|
||||
Reference in new issue
Block a user