mirror of
https://github.com/gelusus/wxvl.git
synced 2026-10-24 08:54:27 +00:00
代码改进 (不含 workflow)
run.py: - 移除硬编码 doonsec session cookie (公开仓库凭据泄露, RSS 无需 cookie) - 新增 canon_url() 规范化去重: 长链按 __biz/mid/idx/sn 判重, 避免同一文章因 chksm/scene 跟踪参数不同被重复入库 - 新增 chainreactors 源: 从 archive 分支 dump frontmatter 提取短链 (长链对机房IP有 wappoc 验证码, CI 只收短链) - requests 全部加 30s 超时; 裸 except 改打印错误 - data.json 解析失败保留原文件并退出, 不再覆盖清空 - 修复 URL 提取正则字符类; 零抓取 exit 1 让 workflow 失败通知 - 删除 get_issue_url / issue 模式; 临时目录清理; 子进程 120s 超时 - rep_filename 同名冲突加序号防覆盖 README 更新数据源说明 新增: backfill.py (断档补录), crawl_todo.py (断点续爬)
This commit is contained in:
Your Name committed
1 parent
cb40caa95f
commit
ae2f047a3f
5 files changed
+315
-83
No files matched your search
@@ -3,7 +3,7 @@
|
||||
[](https://github.com/gelusus/wxvl/actions)
|
||||
[](LICENSE)
|
||||
|
||||
本项目基于 [原版wxvl](https://github.com/20142995/wxvl) 进行扩展，新增了2022年4月至2025年4月期间的安全漏洞文章。
|
||||
本项目基于 [原wxvl](https://github.com/20142995/wxvl) 进行扩展
|
||||
|
||||
## ✨ 项目功能
|
||||
|
||||
@@ -12,10 +12,9 @@
|
||||
## 📰 数据来源
|
||||
|
||||
数据来自以下渠道的公众号文章：
|
||||
- [chainreactors/picker](https://github.com/chainreactors/picker) 每日归档
|
||||
- [BruceFeIix/picker](https://github.com/BruceFeIix/picker) 每日归档
|
||||
- [Doonsec](https://doonsec.com) RSS订阅源
|
||||
- 从GitHub Issues中提取的公众号链接
|
||||
|
||||
|
||||
## 🔍 内容筛选规则
|
||||
|
||||
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
"""一次性补档脚本 v2: 从 chainreactors picker 的 daily dump 直接复制文章。
|
||||
|
||||
数据源:
|
||||
2025-08, 2025-09 -> 本地 /mnt/d/Learn/picker-chainreactors (master)
|
||||
2025-10-26~31, 11, 12 月 -> /root/picker-dump (上游 archive 分支导出)
|
||||
2026-06 -> /root/picker-dump
|
||||
|
||||
URL 映射: 优先 daily.json ({feed: {title: url}}), 其次 daily.md 的 [title](url) 行;
|
||||
映射不上的用 dump:// 合成键。标题按项目关键词规则过滤, 与 run.py 语义一致。
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import shutil
|
||||
|
||||
WXVL = '/mnt/d/Tools/Vuln-db/wxvl'
|
||||
DATA_FILE = os.path.join(WXVL, 'data.json')
|
||||
DOC = os.path.join(WXVL, 'doc')
|
||||
|
||||
SOURCES = [
|
||||
('2025-08', '/mnt/d/Learn/picker-chainreactors/archive/2025/08'),
|
||||
('2025-09', '/mnt/d/Learn/picker-chainreactors/archive/2025/09'),
|
||||
('2025-10', '/root/picker-dump/archive/2025/10'),
|
||||
('2025-11', '/root/picker-dump/archive/2025/11'),
|
||||
('2025-12', '/root/picker-dump/archive/2025/12'),
|
||||
('2026-06', '/root/picker-dump/archive/2026/06'),
|
||||
]
|
||||
|
||||
KEYWORDS = re.compile(
|
||||
r'(?:复现|漏洞|CVE-\d+|CNVD-\d+|CNNVD-\d+|XVE-\d+|QVD-\d+|POC|EXP|0day|1day|nday|RCE|代码执行|命令执行)',
|
||||
re.I)
|
||||
MD_LINK = re.compile(r'- \[ \] \[(?P<title>[^\]]+)\]\((?P<url>https?://[^)]+)\)')
|
||||
ILLEGAL = re.compile(r'[\/\\\:\*\?\"\<\>\|]')
|
||||
|
||||
|
||||
def load_day_map(day_dir):
|
||||
"""返回 (title->url, exact_filename->url) 两个映射"""
|
||||
title2url, file2url = {}, {}
|
||||
|
||||
jf = os.path.join(day_dir, 'daily.json')
|
||||
if os.path.exists(jf):
|
||||
try:
|
||||
feeds = json.load(open(jf, encoding='utf-8'))
|
||||
for feed, arts in feeds.items():
|
||||
if isinstance(arts, dict):
|
||||
for title, url in arts.items():
|
||||
if isinstance(url, str) and url.startswith('http'):
|
||||
title2url[title] = url
|
||||
file2url['%s_%s.md' % (feed, title)] = url
|
||||
elif isinstance(arts, list):
|
||||
for a in arts:
|
||||
t, u = a.get('title'), a.get('url')
|
||||
if t and u:
|
||||
title2url[t] = u
|
||||
file2url['%s_%s.md' % (feed, t)] = u
|
||||
except Exception as e:
|
||||
print(' daily.json 解析失败:', e)
|
||||
|
||||
mf = os.path.join(day_dir, 'daily.md')
|
||||
if os.path.exists(mf):
|
||||
for m in MD_LINK.finditer(open(mf, encoding='utf-8').read()):
|
||||
title2url.setdefault(m.group('title'), m.group('url'))
|
||||
|
||||
return title2url, file2url
|
||||
|
||||
|
||||
def safe_dest(month_dir, name):
|
||||
name = ILLEGAL.sub('', name).strip() or 'untitled'
|
||||
dest = os.path.join(month_dir, name + '.md')
|
||||
i = 2
|
||||
while os.path.exists(dest):
|
||||
dest = os.path.join(month_dir, '%s (%d).md' % (name, i))
|
||||
i += 1
|
||||
return dest
|
||||
|
||||
|
||||
def main():
|
||||
data = json.load(open(DATA_FILE, encoding='utf-8'))
|
||||
base = len(data)
|
||||
stats = {}
|
||||
|
||||
for month, src in SOURCES:
|
||||
if not os.path.isdir(src):
|
||||
print('!! 缺少源目录:', src)
|
||||
continue
|
||||
month_dir = os.path.join(DOC, month)
|
||||
os.makedirs(month_dir, exist_ok=True)
|
||||
s = stats.setdefault(month, {'copied': 0, 'skip_dup': 0, 'no_keyword': 0, 'unmapped': 0})
|
||||
|
||||
for day in sorted(os.listdir(src)):
|
||||
day_dir = os.path.join(src, day)
|
||||
daily = os.path.join(day_dir, 'daily')
|
||||
if not os.path.isdir(daily):
|
||||
continue
|
||||
date = '%s-%s' % (month, day)
|
||||
title2url, file2url = load_day_map(day_dir)
|
||||
|
||||
for fn in sorted(os.listdir(daily)):
|
||||
if not fn.endswith('.md'):
|
||||
continue
|
||||
raw_title = fn[:-3]
|
||||
url = file2url.get(fn)
|
||||
if url is None and '_' in raw_title:
|
||||
url = title2url.get(raw_title.split('_', 1)[1])
|
||||
if url is None:
|
||||
url = title2url.get(raw_title)
|
||||
# 展示标题: 去掉 feed 前缀
|
||||
title = raw_title.split('_', 1)[1] if (url and '_' in raw_title) else raw_title
|
||||
|
||||
if not KEYWORDS.search(title) and not KEYWORDS.search(raw_title):
|
||||
s['no_keyword'] += 1
|
||||
continue
|
||||
key = url if url else 'dump://%s/%s' % (date, fn)
|
||||
if key in data:
|
||||
s['skip_dup'] += 1
|
||||
continue
|
||||
try:
|
||||
shutil.copy2(os.path.join(daily, fn), safe_dest(month_dir, title))
|
||||
data[key] = title
|
||||
s['copied'] += 1
|
||||
if url is None:
|
||||
s['unmapped'] += 1
|
||||
except Exception as e:
|
||||
print(' 复制失败 %s: %s' % (fn[:50], e))
|
||||
|
||||
print('%s: 新增 %d (无URL映射 %d), 已存在跳过 %d, 非漏洞类跳过 %d'
|
||||
% (month, s['copied'], s['unmapped'], s['skip_dup'], s['no_keyword']))
|
||||
|
||||
tmp = DATA_FILE + '.tmp'
|
||||
with open(tmp, 'w', encoding='utf-8') as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=4)
|
||||
os.replace(tmp, DATA_FILE)
|
||||
print('data.json: %d -> %d (+%d)' % (base, len(data), len(data) - base))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""按 /tmp/backfill_todo.json 抓取: python3 crawl_todo.py short|long|sample"""
|
||||
import os, re, json, time, shutil, tempfile, subprocess, sys
|
||||
|
||||
WXVL = '/mnt/d/Tools/Vuln-db/wxvl'
|
||||
EXE = os.path.join(WXVL, 'bin/wechatmp2markdown-v1.1.11_linux_amd64')
|
||||
ILLEGAL = re.compile(r'[\/\\\:\*\?\"\<\>\|]')
|
||||
MODE = sys.argv[1] if len(sys.argv) > 1 else 'short'
|
||||
|
||||
todo = json.load(open('/tmp/backfill_todo.json'))
|
||||
if MODE == 'sample':
|
||||
items = sorted(todo['long'].items())
|
||||
step = max(1, len(items) // 20)
|
||||
items = items[::step][:20]
|
||||
items = [(u, d) for u, d in items if u not in json.load(open(os.path.join(WXVL, 'data.json')))]
|
||||
else:
|
||||
data = json.load(open(os.path.join(WXVL, 'data.json')))
|
||||
items = [(u, d) for u, d in sorted(todo[MODE].items()) if u not in data]
|
||||
|
||||
print(f'[{MODE}] 本次处理 {len(items)} 条')
|
||||
ok = dead = 0
|
||||
t0 = time.time()
|
||||
for i, (url, date) in enumerate(items, 1):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
try:
|
||||
subprocess.run([EXE, url, tmp, '--image=url'], capture_output=True, timeout=90)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
got = None
|
||||
for root, _, files in os.walk(tmp):
|
||||
for fn in files:
|
||||
if fn.endswith('.md') and fn != '.md':
|
||||
got = (fn[:-3], os.path.join(root, fn))
|
||||
if got is None:
|
||||
dead += 1
|
||||
else:
|
||||
data = json.load(open(os.path.join(WXVL, 'data.json')))
|
||||
if url in data:
|
||||
continue
|
||||
month_dir = os.path.join(WXVL, 'doc', date[:7])
|
||||
os.makedirs(month_dir, exist_ok=True)
|
||||
title = ILLEGAL.sub('', got[0]).strip() or 'untitled'
|
||||
dest = os.path.join(month_dir, title + '.md')
|
||||
n = 2
|
||||
while os.path.exists(dest):
|
||||
dest = os.path.join(month_dir, '%s (%d).md' % (title, n)); n += 1
|
||||
shutil.copy2(got[1], dest)
|
||||
data[url] = got[0]
|
||||
with open(os.path.join(WXVL, 'data.json') + '.tmp', 'w', encoding='utf8') as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=4)
|
||||
os.replace(os.path.join(WXVL, 'data.json') + '.tmp', os.path.join(WXVL, 'data.json'))
|
||||
ok += 1
|
||||
print('OK', date, got[0][:40])
|
||||
if MODE != 'sample':
|
||||
# 触发微信风控后需低速; SLOW=1 时 4s/条
|
||||
time.sleep(4.0 if os.environ.get('SLOW') else 0.8)
|
||||
print(f'[{MODE}] 完成: 成功 {ok} / 失败(死链或拦截) {dead}, 耗时 {(time.time()-t0)/60:.1f} 分钟')
|
||||
@@ -9,6 +9,23 @@ import requests
|
||||
import shutil
|
||||
import subprocess
|
||||
import datetime
|
||||
from urllib.parse import urlparse, parse_qsl, urlencode
|
||||
|
||||
|
||||
def canon_url(u):
|
||||
'''规范化微信文章URL: 短链保留token; 长链只留 __biz/mid/idx/sn,
|
||||
去掉 chksm/scene 等跟踪参数和 #fragment —— 同一篇文章的多种写法归一,
|
||||
否则不同源(裸参数版/带跟踪参数版)会被当成两篇导致重复入库'''
|
||||
try:
|
||||
p = urlparse(u)
|
||||
if p.path.startswith('/s/') and len(p.path) > 3:
|
||||
return 'https://mp.weixin.qq.com' + p.path
|
||||
q = [(k, v) for k, v in parse_qsl(p.query) if k in ('__biz', 'mid', 'idx', 'sn')]
|
||||
if not q:
|
||||
return u
|
||||
return 'https://mp.weixin.qq.com/s?' + urlencode(q)
|
||||
except Exception:
|
||||
return u
|
||||
|
||||
|
||||
def write_json(path, data, encoding="utf8"):
|
||||
@@ -18,19 +35,14 @@ def write_json(path, data, encoding="utf8"):
|
||||
|
||||
|
||||
def read_json(path, default_data={}, encoding="utf8"):
|
||||
"""读取json"""
|
||||
data = {}
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
data = json.loads(open(path, "r", encoding=encoding).read())
|
||||
except:
|
||||
data = default_data
|
||||
write_json(path, data, encoding=encoding)
|
||||
|
||||
else:
|
||||
data = default_data
|
||||
write_json(path, data, encoding=encoding)
|
||||
return data
|
||||
"""读取json, 解析失败时保留原文件并报错退出, 避免误清空历史记录"""
|
||||
if not os.path.exists(path):
|
||||
return default_data
|
||||
try:
|
||||
return json.loads(open(path, "r", encoding=encoding).read())
|
||||
except Exception as e:
|
||||
print("data.json 解析失败, 已停止以免覆盖历史记录: {}".format(e))
|
||||
sys.exit(1)
|
||||
|
||||
def get_executable_path():
|
||||
'''获取可执行文件路径'''
|
||||
@@ -46,46 +58,55 @@ def get_executable_path():
|
||||
|
||||
def get_md_path(executable_path,url):
|
||||
'''获取md文件路径'''
|
||||
# finally 保证临时目录被清理, 子进程 120s 超时防止卡死整个 workflow
|
||||
temp_directory = tempfile.mkdtemp()
|
||||
command = [executable_path, url, temp_directory, '--image=url']
|
||||
subprocess.check_output(command)
|
||||
for root, _, files in os.walk(temp_directory):
|
||||
for file in files:
|
||||
if file.endswith(".md"):
|
||||
file_path = os.path.join(root, file)
|
||||
yield file_path
|
||||
try:
|
||||
subprocess.check_output(command, timeout=120)
|
||||
for root, _, files in os.walk(temp_directory):
|
||||
for file in files:
|
||||
if file.endswith(".md"):
|
||||
file_path = os.path.join(root, file)
|
||||
yield file_path
|
||||
finally:
|
||||
shutil.rmtree(temp_directory, ignore_errors=True)
|
||||
|
||||
def get_chainreactors_url():
|
||||
'''获取今日url'''
|
||||
current_date = datetime.datetime.now().strftime("%Y-%m-%d")
|
||||
base_url = 'https://raw.githubusercontent.com/chainreactors/picker/refs/heads/master/archive/daily/{}/{}.md'.format(current_date[:4], current_date)
|
||||
headers = {
|
||||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'cache-control': 'no-cache',
|
||||
'pragma': 'no-cache',
|
||||
'priority': 'u=0, i',
|
||||
'referer': 'https://github.com/chainreactors/picker',
|
||||
'sec-ch-ua': '"Chromium";v="130", "Microsoft Edge";v="130", "Not?A_Brand";v="99"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"Windows"',
|
||||
'sec-fetch-dest': 'document',
|
||||
'sec-fetch-mode': 'navigate',
|
||||
'sec-fetch-site': 'cross-site',
|
||||
'sec-fetch-user': '?1',
|
||||
'upgrade-insecure-requests': '1',
|
||||
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0',
|
||||
}
|
||||
try:
|
||||
response = requests.get(
|
||||
base_url,
|
||||
headers=headers,
|
||||
)
|
||||
urls = re.findall('(?:复现|漏洞|CVE-\d+|CNVD-\d+|CNNVD-\d+|XVE-\d+|QVD-\d+|POC|EXP|0day|1day|nday|RCE|代码执行|命令执行).*?(https://mp.weixin.qq.com/(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)',response.text,re.I)
|
||||
urls = [url.rstrip(')') for url in urls]
|
||||
return urls
|
||||
except:
|
||||
return []
|
||||
'''从 chainreactors picker (archive 分支) 当日 dump 的 frontmatter 提取微信文章链接。
|
||||
|
||||
dump 结构: archive/{年}/{月}/{日}/daily/{公众号_标题}.md, frontmatter 带 url 字段。
|
||||
目录滞后一天生成(次日约 12:02 CST 提交), 故取昨天+今天两个目录。
|
||||
只收 /s/ 短链: s?__biz= 长链自 2025-07 起对机房IP(含 GitHub runners)启用
|
||||
wappoc 验证码, 家用IP实测约85%可爬但 CI 必失败, 为防烧坏 runner IP 直接过滤。
|
||||
先按文件名(含标题)做关键词过滤, 只下载命中文件的 frontmatter, 减少请求量。
|
||||
'''
|
||||
keyword = re.compile(r'(?:复现|漏洞|CVE-\d+|CNVD-\d+|CNNVD-\d+|XVE-\d+|QVD-\d+|POC|EXP|0day|1day|nday|RCE|代码执行|命令执行)', re.I)
|
||||
api_headers = {'Accept': 'application/vnd.github+json'}
|
||||
token = os.environ.get('GH_TOKEN')
|
||||
if token:
|
||||
api_headers['Authorization'] = 'token ' + token
|
||||
urls = []
|
||||
today = datetime.date.today()
|
||||
for delta in (1, 0):
|
||||
d = today - datetime.timedelta(days=delta)
|
||||
api = 'https://api.github.com/repos/chainreactors/picker/contents/archive/{}/{:02d}/{:02d}/daily?ref=archive'.format(d.year, d.month, d.day)
|
||||
try:
|
||||
resp = requests.get(api, headers=api_headers, timeout=30)
|
||||
if resp.status_code != 200:
|
||||
continue
|
||||
for item in resp.json():
|
||||
name = item.get('name', '')
|
||||
if not name.endswith('.md') or not keyword.search(name):
|
||||
continue
|
||||
# Range 只取文件头, frontmatter (含 url 行) 在前 2KB 内
|
||||
raw = requests.get(item['download_url'], timeout=30,
|
||||
headers={'Range': 'bytes=0-2047'})
|
||||
m = re.search(r'^url:\s*(https://mp\.weixin\.qq\.com/s/\S+)', raw.text, re.M)
|
||||
if m:
|
||||
urls.append(m.group(1).rstrip(').,，'))
|
||||
except Exception as e:
|
||||
print("chainreactors 源({})获取失败: {}".format(d, e))
|
||||
return urls
|
||||
|
||||
def get_BruceFeIix_url():
|
||||
'''获取今日url'''
|
||||
@@ -112,23 +133,19 @@ def get_BruceFeIix_url():
|
||||
response = requests.get(
|
||||
base_url,
|
||||
headers=headers,
|
||||
timeout=30,
|
||||
)
|
||||
urls = re.findall('(?:复现|漏洞|CVE-\d+|CNVD-\d+|CNNVD-\d+|XVE-\d+|QVD-\d+|POC|EXP|0day|1day|nday|RCE|代码执行|命令执行).*?(https://mp.weixin.qq.com/(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)',response.text,re.I)
|
||||
# 注意: 字符类必须覆盖 URL 全部合法字符 (/ ? = & 等), 否则链接被截断
|
||||
urls = re.findall('(?:复现|漏洞|CVE-\d+|CNVD-\d+|CNNVD-\d+|XVE-\d+|QVD-\d+|POC|EXP|0day|1day|nday|RCE|代码执行|命令执行).*?(https://mp.weixin.qq.com/[A-Za-z0-9\-._~:/?#@!$&*+,;=%]+)',response.text,re.I)
|
||||
urls = [url.rstrip(')') for url in urls]
|
||||
return urls
|
||||
except:
|
||||
except Exception as e:
|
||||
print("BruceFeIix 源获取失败: {}".format(e))
|
||||
return []
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
def get_doonsec_url():
|
||||
'''从 Doonsec RSS 获取今日URL，使用 XML 解析'''
|
||||
cookies = {
|
||||
'UM_follow': 'True',
|
||||
'UM_distinctids': 'fgmr',
|
||||
'session': 'eyJfcGVybWFuZW50Ijp0cnVlLCJjc3JmX3Rva2VuIjoiMzU2ZDE4OTcwZjliZDljY2NjN2M3YzlkMzRhOGVlZWQyZDk1NmI1ZSIsInZpc3RvciI6ImZHTXJGQXBlVndRUnZrWjJHdWplV2gifQ.ZzidRw.GyjS15N12JYU0TByO31rrwBIiPY',
|
||||
}
|
||||
|
||||
headers = {
|
||||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
@@ -147,7 +164,7 @@ def get_doonsec_url():
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get('https://wechat.doonsec.com/rss.xml', cookies=cookies, headers=headers)
|
||||
response = requests.get('https://wechat.doonsec.com/rss.xml', headers=headers, timeout=30)
|
||||
response.encoding = response.apparent_encoding
|
||||
|
||||
# XML 解析
|
||||
@@ -165,25 +182,25 @@ def get_doonsec_url():
|
||||
return []
|
||||
|
||||
|
||||
def get_issue_url():
|
||||
file = '/tmp/issue_content.txt'
|
||||
if os.path.exists(file):
|
||||
content = open(file,'r',encoding='utf8').read()
|
||||
urls = re.findall('(https://mp.weixin.qq.com/(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)',content,re.I)
|
||||
urls = [url.rstrip(')') for url in urls]
|
||||
return urls
|
||||
return []
|
||||
|
||||
def rep_filename(result_path):
|
||||
'''
|
||||
替换不能用于文件名的字符
|
||||
'''
|
||||
替换不能用于文件名的字符; 同名冲突时追加序号避免覆盖已有文章
|
||||
'''
|
||||
for root, _, files in os.walk(result_path):
|
||||
for file in files:
|
||||
if file.endswith(".md"):
|
||||
file_path = os.path.join(root, file)
|
||||
new_file = re.sub(r'[\/\\\:\*\?\"\<\>\|]', '', file)
|
||||
shutil.move(os.path.join(root, file), os.path.join(root, new_file))
|
||||
if new_file == file:
|
||||
continue
|
||||
dest = os.path.join(root, new_file)
|
||||
i = 2
|
||||
while os.path.exists(dest):
|
||||
stem, ext = os.path.splitext(new_file)
|
||||
dest = os.path.join(root, '{} ({}).{}'.format(stem, i, ext))
|
||||
i += 1
|
||||
shutil.move(file_path, dest)
|
||||
|
||||
def main():
|
||||
'''主函数'''
|
||||
@@ -197,22 +214,43 @@ def main():
|
||||
os.makedirs(result_path, exist_ok=True)
|
||||
# 读取历史记录
|
||||
data = read_json(data_file, default_data=data)
|
||||
if len(sys.argv) == 2:
|
||||
if sys.argv[1] == 'today':
|
||||
urls = list(set(get_chainreactors_url() + get_BruceFeIix_url() + get_doonsec_url()))
|
||||
else:
|
||||
urls = get_issue_url()
|
||||
for url in urls:
|
||||
if url in data:
|
||||
continue
|
||||
urls = []
|
||||
if len(sys.argv) == 2 and sys.argv[1] == 'today':
|
||||
print("正在获取URL...")
|
||||
chainreactors_urls = get_chainreactors_url()
|
||||
print(f"chainreactors: {len(chainreactors_urls)} 个URL")
|
||||
|
||||
brucefelix_urls = get_BruceFeIix_url()
|
||||
print(f"BruceFeIix: {len(brucefelix_urls)} 个URL")
|
||||
|
||||
doonsec_urls = get_doonsec_url()
|
||||
print(f"Doonsec: {len(doonsec_urls)} 个URL")
|
||||
|
||||
urls = list(set(chainreactors_urls + brucefelix_urls + doonsec_urls))
|
||||
print(f"总共去重后: {len(urls)} 个URL")
|
||||
# 三个源同时归零说明数据源已失效, 退出非零让 workflow 失败并通知,
|
||||
# 防止再次出现断档空转无人察觉 (2025-08~12 的教训)
|
||||
if not urls:
|
||||
sys.exit(1)
|
||||
# 规范化去重: 同一篇文章在不同源里可能带不同跟踪参数, 按规范化键判重
|
||||
known = {canon_url(u) for u in data}
|
||||
for url in urls:
|
||||
if url in data or canon_url(url) in known:
|
||||
continue
|
||||
try:
|
||||
for file_path in get_md_path(executable_path, url):
|
||||
name = os.path.splitext(os.path.basename(file_path))[0]
|
||||
if name == '.md':
|
||||
continue
|
||||
shutil.copy2(file_path,result_path)
|
||||
data[url] = name
|
||||
known.add(canon_url(url))
|
||||
write_json(data_file,data)
|
||||
print(name,end='、')
|
||||
except Exception as e:
|
||||
print(f"\n处理URL失败 {url}: {e}")
|
||||
continue
|
||||
rep_filename(result_path)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+2
-2
@@ -56,8 +56,8 @@ def get_md_path(executable_path,url):
|
||||
def get_chainreactors_url():
|
||||
'''获取多日url，返回URL和对应日期的元组列表'''
|
||||
# 定义日期范围
|
||||
start_date = "2022-04-07"
|
||||
end_date = "2025-04-16"
|
||||
start_date = "2025-07-09"
|
||||
end_date = "2025-07-28"
|
||||
|
||||
# 转换为datetime对象
|
||||
start = datetime.datetime.strptime(start_date, "%Y-%m-%d")
|
||||
|
||||
Reference in new issue
Block a user