代码改进 (不含 workflow)

run.py:
- 移除硬编码 doonsec session cookie (公开仓库凭据泄露, RSS 无需 cookie)
- 新增 canon_url() 规范化去重: 长链按 __biz/mid/idx/sn 判重,
  避免同一文章因 chksm/scene 跟踪参数不同被重复入库
- 新增 chainreactors 源: 从 archive 分支 dump frontmatter 提取短链
  (长链对机房IP有 wappoc 验证码, CI 只收短链)
- requests 全部加 30s 超时; 裸 except 改打印错误
- data.json 解析失败保留原文件并退出, 不再覆盖清空
- 修复 URL 提取正则字符类; 零抓取 exit 1 让 workflow 失败通知
- 删除 get_issue_url / issue 模式; 临时目录清理; 子进程 120s 超时
- rep_filename 同名冲突加序号防覆盖

README 更新数据源说明
新增: backfill.py (断档补录), crawl_todo.py (断点续爬)

README.md

@@ -3,7 +3,7 @@
 [![GitHub Actions](https://github.com/gelusus/wxvl/actions/workflows/update_today.yml/badge.svg)](https://github.com/gelusus/wxvl/actions)
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 
-本项目基于 [原版wxvl](https://github.com/20142995/wxvl) 进行扩展，新增了2022年4月至2025年4月期间的安全漏洞文章。
+本项目基于 [原wxvl](https://github.com/20142995/wxvl) 进行扩展
 
 ## ✨ 项目功能
 
@@ -12,10 +12,9 @@
 ## 📰 数据来源
 
 数据来自以下渠道的公众号文章：
-- [chainreactors/picker](https://github.com/chainreactors/picker) 每日归档
 - [BruceFeIix/picker](https://github.com/BruceFeIix/picker) 每日归档
 - [Doonsec](https://doonsec.com) RSS订阅源
-- 从GitHub Issues中提取的公众号链接
+
 
 ## 🔍 内容筛选规则
 

backfill.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""一次性补档脚本 v2: 从 chainreactors picker 的 daily dump 直接复制文章。
+
+数据源:
+  2025-08, 2025-09          -> 本地 /mnt/d/Learn/picker-chainreactors (master)
+  2025-10-26~31, 11, 12 月  -> /root/picker-dump (上游 archive 分支导出)
+  2026-06                   -> /root/picker-dump
+
+URL 映射: 优先 daily.json ({feed: {title: url}}), 其次 daily.md 的 [title](url) 行;
+映射不上的用 dump:// 合成键。标题按项目关键词规则过滤, 与 run.py 语义一致。
+"""
+import os
+import re
+import json
+import shutil
+
+WXVL = '/mnt/d/Tools/Vuln-db/wxvl'
+DATA_FILE = os.path.join(WXVL, 'data.json')
+DOC = os.path.join(WXVL, 'doc')
+
+SOURCES = [
+    ('2025-08', '/mnt/d/Learn/picker-chainreactors/archive/2025/08'),
+    ('2025-09', '/mnt/d/Learn/picker-chainreactors/archive/2025/09'),
+    ('2025-10', '/root/picker-dump/archive/2025/10'),
+    ('2025-11', '/root/picker-dump/archive/2025/11'),
+    ('2025-12', '/root/picker-dump/archive/2025/12'),
+    ('2026-06', '/root/picker-dump/archive/2026/06'),
+]
+
+KEYWORDS = re.compile(
+    r'(?:复现|漏洞|CVE-\d+|CNVD-\d+|CNNVD-\d+|XVE-\d+|QVD-\d+|POC|EXP|0day|1day|nday|RCE|代码执行|命令执行)',
+    re.I)
+MD_LINK = re.compile(r'- \[ \] \[(?P<title>[^\]]+)\]\((?P<url>https?://[^)]+)\)')
+ILLEGAL = re.compile(r'[\/\\\:\*\?\"\<\>\|]')
+
+
+def load_day_map(day_dir):
+    """返回 (title->url, exact_filename->url) 两个映射"""
+    title2url, file2url = {}, {}
+
+    jf = os.path.join(day_dir, 'daily.json')
+    if os.path.exists(jf):
+        try:
+            feeds = json.load(open(jf, encoding='utf-8'))
+            for feed, arts in feeds.items():
+                if isinstance(arts, dict):
+                    for title, url in arts.items():
+                        if isinstance(url, str) and url.startswith('http'):
+                            title2url[title] = url
+                            file2url['%s_%s.md' % (feed, title)] = url
+                elif isinstance(arts, list):
+                    for a in arts:
+                        t, u = a.get('title'), a.get('url')
+                        if t and u:
+                            title2url[t] = u
+                            file2url['%s_%s.md' % (feed, t)] = u
+        except Exception as e:
+            print('  daily.json 解析失败:', e)
+
+    mf = os.path.join(day_dir, 'daily.md')
+    if os.path.exists(mf):
+        for m in MD_LINK.finditer(open(mf, encoding='utf-8').read()):
+            title2url.setdefault(m.group('title'), m.group('url'))
+
+    return title2url, file2url
+
+
+def safe_dest(month_dir, name):
+    name = ILLEGAL.sub('', name).strip() or 'untitled'
+    dest = os.path.join(month_dir, name + '.md')
+    i = 2
+    while os.path.exists(dest):
+        dest = os.path.join(month_dir, '%s (%d).md' % (name, i))
+        i += 1
+    return dest
+
+
+def main():
+    data = json.load(open(DATA_FILE, encoding='utf-8'))
+    base = len(data)
+    stats = {}
+
+    for month, src in SOURCES:
+        if not os.path.isdir(src):
+            print('!! 缺少源目录:', src)
+            continue
+        month_dir = os.path.join(DOC, month)
+        os.makedirs(month_dir, exist_ok=True)
+        s = stats.setdefault(month, {'copied': 0, 'skip_dup': 0, 'no_keyword': 0, 'unmapped': 0})
+
+        for day in sorted(os.listdir(src)):
+            day_dir = os.path.join(src, day)
+            daily = os.path.join(day_dir, 'daily')
+            if not os.path.isdir(daily):
+                continue
+            date = '%s-%s' % (month, day)
+            title2url, file2url = load_day_map(day_dir)
+
+            for fn in sorted(os.listdir(daily)):
+                if not fn.endswith('.md'):
+                    continue
+                raw_title = fn[:-3]
+                url = file2url.get(fn)
+                if url is None and '_' in raw_title:
+                    url = title2url.get(raw_title.split('_', 1)[1])
+                if url is None:
+                    url = title2url.get(raw_title)
+                # 展示标题: 去掉 feed 前缀
+                title = raw_title.split('_', 1)[1] if (url and '_' in raw_title) else raw_title
+
+                if not KEYWORDS.search(title) and not KEYWORDS.search(raw_title):
+                    s['no_keyword'] += 1
+                    continue
+                key = url if url else 'dump://%s/%s' % (date, fn)
+                if key in data:
+                    s['skip_dup'] += 1
+                    continue
+                try:
+                    shutil.copy2(os.path.join(daily, fn), safe_dest(month_dir, title))
+                    data[key] = title
+                    s['copied'] += 1
+                    if url is None:
+                        s['unmapped'] += 1
+                except Exception as e:
+                    print('  复制失败 %s: %s' % (fn[:50], e))
+
+        print('%s: 新增 %d (无URL映射 %d), 已存在跳过 %d, 非漏洞类跳过 %d'
+              % (month, s['copied'], s['unmapped'], s['skip_dup'], s['no_keyword']))
+
+    tmp = DATA_FILE + '.tmp'
+    with open(tmp, 'w', encoding='utf-8') as f:
+        json.dump(data, f, ensure_ascii=False, indent=4)
+    os.replace(tmp, DATA_FILE)
+    print('data.json: %d -> %d (+%d)' % (base, len(data), len(data) - base))
+
+
+if __name__ == '__main__':
+    main()

crawl_todo.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""按 /tmp/backfill_todo.json 抓取: python3 crawl_todo.py short|long|sample"""
+import os, re, json, time, shutil, tempfile, subprocess, sys
+
+WXVL = '/mnt/d/Tools/Vuln-db/wxvl'
+EXE = os.path.join(WXVL, 'bin/wechatmp2markdown-v1.1.11_linux_amd64')
+ILLEGAL = re.compile(r'[\/\\\:\*\?\"\<\>\|]')
+MODE = sys.argv[1] if len(sys.argv) > 1 else 'short'
+
+todo = json.load(open('/tmp/backfill_todo.json'))
+if MODE == 'sample':
+    items = sorted(todo['long'].items())
+    step = max(1, len(items) // 20)
+    items = items[::step][:20]
+    items = [(u, d) for u, d in items if u not in json.load(open(os.path.join(WXVL, 'data.json')))]
+else:
+    data = json.load(open(os.path.join(WXVL, 'data.json')))
+    items = [(u, d) for u, d in sorted(todo[MODE].items()) if u not in data]
+
+print(f'[{MODE}] 本次处理 {len(items)} 条')
+ok = dead = 0
+t0 = time.time()
+for i, (url, date) in enumerate(items, 1):
+    with tempfile.TemporaryDirectory() as tmp:
+        try:
+            subprocess.run([EXE, url, tmp, '--image=url'], capture_output=True, timeout=90)
+        except subprocess.TimeoutExpired:
+            pass
+        got = None
+        for root, _, files in os.walk(tmp):
+            for fn in files:
+                if fn.endswith('.md') and fn != '.md':
+                    got = (fn[:-3], os.path.join(root, fn))
+        if got is None:
+            dead += 1
+        else:
+            data = json.load(open(os.path.join(WXVL, 'data.json')))
+            if url in data:
+                continue
+            month_dir = os.path.join(WXVL, 'doc', date[:7])
+            os.makedirs(month_dir, exist_ok=True)
+            title = ILLEGAL.sub('', got[0]).strip() or 'untitled'
+            dest = os.path.join(month_dir, title + '.md')
+            n = 2
+            while os.path.exists(dest):
+                dest = os.path.join(month_dir, '%s (%d).md' % (title, n)); n += 1
+            shutil.copy2(got[1], dest)
+            data[url] = got[0]
+            with open(os.path.join(WXVL, 'data.json') + '.tmp', 'w', encoding='utf8') as f:
+                json.dump(data, f, ensure_ascii=False, indent=4)
+            os.replace(os.path.join(WXVL, 'data.json') + '.tmp', os.path.join(WXVL, 'data.json'))
+            ok += 1
+            print('OK', date, got[0][:40])
+    if MODE != 'sample':
+        # 触发微信风控后需低速; SLOW=1 时 4s/条
+        time.sleep(4.0 if os.environ.get('SLOW') else 0.8)
+print(f'[{MODE}] 完成: 成功 {ok} / 失败(死链或拦截) {dead}, 耗时 {(time.time()-t0)/60:.1f} 分钟')

run.py

@@ -9,6 +9,23 @@ import requests
 import shutil
 import subprocess
 import datetime
+from urllib.parse import urlparse, parse_qsl, urlencode
+
+
+def canon_url(u):
+    '''规范化微信文章URL: 短链保留token; 长链只留 __biz/mid/idx/sn,
+    去掉 chksm/scene 等跟踪参数和 #fragment —— 同一篇文章的多种写法归一,
+    否则不同源(裸参数版/带跟踪参数版)会被当成两篇导致重复入库'''
+    try:
+        p = urlparse(u)
+        if p.path.startswith('/s/') and len(p.path) > 3:
+            return 'https://mp.weixin.qq.com' + p.path
+        q = [(k, v) for k, v in parse_qsl(p.query) if k in ('__biz', 'mid', 'idx', 'sn')]
+        if not q:
+            return u
+        return 'https://mp.weixin.qq.com/s?' + urlencode(q)
+    except Exception:
+        return u
 
 
 def write_json(path, data, encoding="utf8"):
@@ -18,19 +35,14 @@ def write_json(path, data, encoding="utf8"):
 
 
 def read_json(path, default_data={}, encoding="utf8"):
-    """读取json"""
+    """读取json, 解析失败时保留原文件并报错退出, 避免误清空历史记录"""
-    data = {}
+    if not os.path.exists(path):
-    if os.path.exists(path):
+        return default_data
-        try:
+    try:
-            data = json.loads(open(path, "r", encoding=encoding).read())
+        return json.loads(open(path, "r", encoding=encoding).read())
-        except:
+    except Exception as e:
-            data = default_data
+        print("data.json 解析失败, 已停止以免覆盖历史记录: {}".format(e))
-            write_json(path, data, encoding=encoding)
+        sys.exit(1)
-
-    else:
-        data = default_data
-        write_json(path, data, encoding=encoding)
-    return data
 
 def get_executable_path():
     '''获取可执行文件路径'''
@@ -46,46 +58,55 @@ def get_executable_path():
 
 def get_md_path(executable_path,url):
     '''获取md文件路径'''
+    # finally 保证临时目录被清理, 子进程 120s 超时防止卡死整个 workflow
     temp_directory = tempfile.mkdtemp()
     command = [executable_path, url, temp_directory, '--image=url']
-    subprocess.check_output(command)
+    try:
-    for root, _, files in os.walk(temp_directory):
+        subprocess.check_output(command, timeout=120)
-        for file in files:
+        for root, _, files in os.walk(temp_directory):
-            if file.endswith(".md"):
+            for file in files:
-                file_path = os.path.join(root, file)
+                if file.endswith(".md"):
-                yield file_path
+                    file_path = os.path.join(root, file)
+                    yield file_path
+    finally:
+        shutil.rmtree(temp_directory, ignore_errors=True)
 
 def get_chainreactors_url():
-    '''获取今日url'''
+    '''从 chainreactors picker (archive 分支) 当日 dump 的 frontmatter 提取微信文章链接。
-    current_date = datetime.datetime.now().strftime("%Y-%m-%d")
+
-    base_url = 'https://raw.githubusercontent.com/chainreactors/picker/refs/heads/master/archive/daily/{}/{}.md'.format(current_date[:4], current_date)
+    dump 结构: archive/{年}/{月}/{日}/daily/{公众号_标题}.md, frontmatter 带 url 字段。
-    headers = {
+    目录滞后一天生成(次日约 12:02 CST 提交), 故取昨天+今天两个目录。
-        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+    只收 /s/ 短链: s?__biz= 长链自 2025-07 起对机房IP(含 GitHub runners)启用
-        'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
+    wappoc 验证码, 家用IP实测约85%可爬但 CI 必失败, 为防烧坏 runner IP 直接过滤。
-        'cache-control': 'no-cache',
+    先按文件名(含标题)做关键词过滤, 只下载命中文件的 frontmatter, 减少请求量。
-        'pragma': 'no-cache',
+    '''
-        'priority': 'u=0, i',
+    keyword = re.compile(r'(?:复现|漏洞|CVE-\d+|CNVD-\d+|CNNVD-\d+|XVE-\d+|QVD-\d+|POC|EXP|0day|1day|nday|RCE|代码执行|命令执行)', re.I)
-        'referer': 'https://github.com/chainreactors/picker',
+    api_headers = {'Accept': 'application/vnd.github+json'}
-        'sec-ch-ua': '"Chromium";v="130", "Microsoft Edge";v="130", "Not?A_Brand";v="99"',
+    token = os.environ.get('GH_TOKEN')
-        'sec-ch-ua-mobile': '?0',
+    if token:
-        'sec-ch-ua-platform': '"Windows"',
+        api_headers['Authorization'] = 'token ' + token
-        'sec-fetch-dest': 'document',
+    urls = []
-        'sec-fetch-mode': 'navigate',
+    today = datetime.date.today()
-        'sec-fetch-site': 'cross-site',
+    for delta in (1, 0):
-        'sec-fetch-user': '?1',
+        d = today - datetime.timedelta(days=delta)
-        'upgrade-insecure-requests': '1',
+        api = 'https://api.github.com/repos/chainreactors/picker/contents/archive/{}/{:02d}/{:02d}/daily?ref=archive'.format(d.year, d.month, d.day)
-        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0',
+        try:
-    }
+            resp = requests.get(api, headers=api_headers, timeout=30)
-    try:
+            if resp.status_code != 200:
-        response = requests.get(
+                continue
-            base_url,
+            for item in resp.json():
-            headers=headers,
+                name = item.get('name', '')
-        )
+                if not name.endswith('.md') or not keyword.search(name):
-        urls = re.findall('(?:复现|漏洞|CVE-\d+|CNVD-\d+|CNNVD-\d+|XVE-\d+|QVD-\d+|POC|EXP|0day|1day|nday|RCE|代码执行|命令执行).*?(https://mp.weixin.qq.com/(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)',response.text,re.I)
+                    continue
-        urls = [url.rstrip(')') for url in urls]
+                # Range 只取文件头, frontmatter (含 url 行) 在前 2KB 内
-        return urls
+                raw = requests.get(item['download_url'], timeout=30,
-    except:
+                                   headers={'Range': 'bytes=0-2047'})
-        return []
+                m = re.search(r'^url:\s*(https://mp\.weixin\.qq\.com/s/\S+)', raw.text, re.M)
+                if m:
+                    urls.append(m.group(1).rstrip(').,，'))
+        except Exception as e:
+            print("chainreactors 源({})获取失败: {}".format(d, e))
+    return urls
 
 def get_BruceFeIix_url():
     '''获取今日url'''
@@ -112,23 +133,19 @@ def get_BruceFeIix_url():
         response = requests.get(
             base_url,
             headers=headers,
+            timeout=30,
         )
-        urls = re.findall('(?:复现|漏洞|CVE-\d+|CNVD-\d+|CNNVD-\d+|XVE-\d+|QVD-\d+|POC|EXP|0day|1day|nday|RCE|代码执行|命令执行).*?(https://mp.weixin.qq.com/(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)',response.text,re.I)
+        # 注意: 字符类必须覆盖 URL 全部合法字符 (/ ? = & 等), 否则链接被截断
+        urls = re.findall('(?:复现|漏洞|CVE-\d+|CNVD-\d+|CNNVD-\d+|XVE-\d+|QVD-\d+|POC|EXP|0day|1day|nday|RCE|代码执行|命令执行).*?(https://mp.weixin.qq.com/[A-Za-z0-9\-._~:/?#@!$&*+,;=%]+)',response.text,re.I)
         urls = [url.rstrip(')') for url in urls]
         return urls
-    except:
+    except Exception as e:
+        print("BruceFeIix 源获取失败: {}".format(e))
         return []
 
-import xml.etree.ElementTree as ET
 
 def get_doonsec_url():
     '''从 Doonsec RSS 获取今日URL，使用 XML 解析'''
-    cookies = {
-        'UM_follow': 'True',
-        'UM_distinctids': 'fgmr',
-        'session': 'eyJfcGVybWFuZW50Ijp0cnVlLCJjc3JmX3Rva2VuIjoiMzU2ZDE4OTcwZjliZDljY2NjN2M3YzlkMzRhOGVlZWQyZDk1NmI1ZSIsInZpc3RvciI6ImZHTXJGQXBlVndRUnZrWjJHdWplV2gifQ.ZzidRw.GyjS15N12JYU0TByO31rrwBIiPY',
-    }
-
     headers = {
         'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
         'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
@@ -147,7 +164,7 @@ def get_doonsec_url():
     }
 
     try:
-        response = requests.get('https://wechat.doonsec.com/rss.xml', cookies=cookies, headers=headers)
+        response = requests.get('https://wechat.doonsec.com/rss.xml', headers=headers, timeout=30)
         response.encoding = response.apparent_encoding
 
         # XML 解析
@@ -165,25 +182,25 @@ def get_doonsec_url():
         return []
 
 
-def get_issue_url():
-    file = '/tmp/issue_content.txt'
-    if os.path.exists(file):
-        content = open(file,'r',encoding='utf8').read()
-        urls = re.findall('(https://mp.weixin.qq.com/(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)',content,re.I)
-        urls = [url.rstrip(')') for url in urls]
-        return urls
-    return []
     
 def rep_filename(result_path):
-    ''' 
+    '''
-    替换不能用于文件名的字符
+    替换不能用于文件名的字符; 同名冲突时追加序号避免覆盖已有文章
     '''
     for root, _, files in os.walk(result_path):
         for file in files:
             if file.endswith(".md"):
                 file_path = os.path.join(root, file)
                 new_file = re.sub(r'[\/\\\:\*\?\"\<\>\|]', '', file)
-                shutil.move(os.path.join(root, file), os.path.join(root, new_file))
+                if new_file == file:
+                    continue
+                dest = os.path.join(root, new_file)
+                i = 2
+                while os.path.exists(dest):
+                    stem, ext = os.path.splitext(new_file)
+                    dest = os.path.join(root, '{} ({}).{}'.format(stem, i, ext))
+                    i += 1
+                shutil.move(file_path, dest)
                 
 def main():
     '''主函数'''
@@ -197,22 +214,43 @@ def main():
     os.makedirs(result_path, exist_ok=True)
     # 读取历史记录
     data = read_json(data_file, default_data=data)
-    if len(sys.argv) == 2:
+    urls = []
-        if sys.argv[1] == 'today':
+    if len(sys.argv) == 2 and sys.argv[1] == 'today':
-            urls = list(set(get_chainreactors_url() + get_BruceFeIix_url() + get_doonsec_url()))
+        print("正在获取URL...")
-        else:
+        chainreactors_urls = get_chainreactors_url()
-            urls = get_issue_url()
+        print(f"chainreactors: {len(chainreactors_urls)} 个URL")
-        for url in urls:
+
-            if url in data:
+        brucefelix_urls = get_BruceFeIix_url()
-                continue
+        print(f"BruceFeIix: {len(brucefelix_urls)} 个URL")
+
+        doonsec_urls = get_doonsec_url()
+        print(f"Doonsec: {len(doonsec_urls)} 个URL")
+
+        urls = list(set(chainreactors_urls + brucefelix_urls + doonsec_urls))
+        print(f"总共去重后: {len(urls)} 个URL")
+        # 三个源同时归零说明数据源已失效, 退出非零让 workflow 失败并通知,
+        # 防止再次出现断档空转无人察觉 (2025-08~12 的教训)
+        if not urls:
+            sys.exit(1)
+    # 规范化去重: 同一篇文章在不同源里可能带不同跟踪参数, 按规范化键判重
+    known = {canon_url(u) for u in data}
+    for url in urls:
+        if url in data or canon_url(url) in known:
+            continue
+        try:
             for file_path in get_md_path(executable_path, url):
                 name = os.path.splitext(os.path.basename(file_path))[0]
                 if name == '.md':
                     continue
                 shutil.copy2(file_path,result_path)
                 data[url] = name
+                known.add(canon_url(url))
                 write_json(data_file,data)
                 print(name,end='、')
+        except Exception as e:
+            print(f"\n处理URL失败 {url}: {e}")
+            continue
     rep_filename(result_path)
+    
 if __name__ == '__main__':
     main()

run_history.py

@@ -56,8 +56,8 @@ def get_md_path(executable_path,url):
 def get_chainreactors_url():
     '''获取多日url，返回URL和对应日期的元组列表'''
     # 定义日期范围
-    start_date = "2022-04-07"
+    start_date = "2025-07-09"
-    end_date = "2025-04-16"
+    end_date = "2025-07-28"
     
     # 转换为datetime对象
     start = datetime.datetime.strptime(start_date, "%Y-%m-%d")