Remove confirmed dead PoC links during metadata refresh

.github/getTrending.py

@@ -312,6 +312,7 @@ def just_landed(token: str, ledger: dict | None = None) -> list[dict]:
 
 
 def ledger_landed(ledger: dict, candidates: list[dict]) -> list[dict]:
+    import link_health
     from build_site import build_cve_list, build_landed, dedupe_source_links, source_key, CURATED, REPO_META
     entries, _ = build_cve_list(load_blacklist())
     published = {entry["cve"]: entry for entry in entries}
@@ -326,6 +327,8 @@ def ledger_landed(ledger: dict, candidates: list[dict]) -> list[dict]:
     for repo in candidates:
         cve = cve_of(repo)
         url = repo.get("_artifact_url") or repo["html_url"]
+        if link_health.unavailable(url):
+            continue
         entry = published.get(cve, {})
         curated = [link for _, field in CURATED for link in entry.get(field, [])
                    if source_key(link) == source_key(url)]
@@ -865,6 +868,7 @@ repository URL.
 
 
 def main(*, history_limit: int = HISTORY_LIMIT, retry_history: bool = False) -> int:
+    import link_health
     global HISTORY_REMAINING, HISTORY_RETRY_HOURS
     HISTORY_REMAINING = history_limit
     HISTORY_RETRY_HOURS = 0 if retry_history else 4
@@ -884,6 +888,7 @@ def main(*, history_limit: int = HISTORY_LIMIT, retry_history: bool = False) ->
         searched = len(repositories)
         repositories, paths = qualifying_repositories(repositories, token)
         repositories = published_repositories(repositories)
+        repositories = [repo for repo in repositories if not link_health.unavailable(artifact_link(repo)[0])]
         observations.extend(repositories)
         release_dates(repositories, token, paths, ledger)
         shipped = code_pushed(

.github/workflows/audit_poc_links.yml

@@ -29,13 +29,15 @@ jobs:
         with:
           python-version: '3.13'
 
-      # 4,000 repositories and 3,000 reference links a day sweeps the whole
-      # index about weekly, in around half an hour. A sample found one link in
-      # eight pointing at a deleted repository under the old fortnightly sweep.
+      # The hourly sweep checks front-page repositories plus 2,000 older repos
+      # and 500 artifact URLs. This deeper scan also includes reference links,
+      # curated file paths and recovery checks for previously unavailable URLs.
       - name: Drop dead links and refresh repository metadata
+        id: link_audit
+        continue-on-error: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: python3 scripts/audit_poc_links.py --limit 4000 --references 3000
+        run: python3 scripts/audit_poc_links.py --limit 4000 --references 10000 --workers 32 --timeout 5
 
       - name: Verify the generated index
         run: python3 scripts/build_site.py --offline
@@ -56,10 +58,13 @@ jobs:
 
           git add --all -- \
             .github/link_audit_state.json \
+            index/link_health.json \
             index/github.txt \
             index/repo_meta.json \
             index/kev.json \
             index/references.txt \
+            index/reference_pocs.txt \
+            index/trending.json \
             README.md \
             docs/hero.svg \
             docs/social-card.svg \
@@ -93,3 +98,7 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
         run: gh workflow run site.yml --ref main
+
+      - name: Report incomplete repository checks
+        if: always() && steps.link_audit.outcome == 'failure'
+        run: exit 1

.github/workflows/hot_cves.yml

@@ -45,7 +45,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: python3 scripts/update_cves.py --lookback-days 1 --years 2 --skip-cvelist
 
-      - name: Refresh repository stars and push dates
+      - name: Refresh repository metadata and remove confirmed dead links
         id: repository_metadata
         continue-on-error: true
         env:
@@ -88,6 +88,7 @@ jobs:
           git add --all -- \
             README.md \
             .github/repo_metadata_state.json \
+            index/link_health.json \
             index/repo_meta.json \
             index/metadata/ \
             index/trending.json \