mirror of
https://github.com/Hxnxe/CyberSentinel-AI.git
synced 2026-10-23 16:35:20 +00:00
chore: publish clean CyberSentinel AI release
This commit is contained in:
ubuntu-master committed
1 parent
9f8a3486d1
commit
526b634da2
393 files changed
+9226
-1389179
No files matched your search
@@ -0,0 +1,38 @@
|
||||
# CyberSentinel AI public configuration template.
|
||||
# Copy this file to `.env`, then replace placeholder values with your own.
|
||||
# Keep `.env` private. It is intentionally ignored by git.
|
||||
|
||||
# Runtime
|
||||
# Monitoring loop interval in seconds. Default: 7200.
|
||||
MONITOR_INTERVAL=7200
|
||||
|
||||
# SQLite database path. Relative paths are resolved from the project root.
|
||||
DATABASE_PATH=cve_database.db
|
||||
|
||||
# Primary OpenAI-compatible provider.
|
||||
# Leave PRIMARY_AI_KEY empty if you only want to run tests or inspect the project.
|
||||
PRIMARY_AI_KEY=
|
||||
PRIMARY_AI_BASE=https://api.openai.com/v1
|
||||
PRIMARY_AI_MODEL=gpt-4.1-nano
|
||||
|
||||
# Backup OpenAI-compatible providers.
|
||||
# JSON array format:
|
||||
# [{"api_key":"replace-with-backup-key","api_base":"https://api.example.com/v1","model":"model-name"}]
|
||||
BACKUP_AI_CONFIGS=[]
|
||||
|
||||
# Gemini provider.
|
||||
# Leave GEMINI_AI_KEY empty if Gemini fallback is not used.
|
||||
GEMINI_AI_KEY=
|
||||
GEMINI_AI_MODEL=gemini-2.5-flash
|
||||
GEMINI_AI_BASE_URL=https://generativelanguage.googleapis.com
|
||||
|
||||
# GitHub API credentials.
|
||||
# Recommended for real monitoring because unauthenticated GitHub requests are heavily rate limited.
|
||||
# Use either GITHUB_TOKENS as a JSON array or GITHUB_TOKEN as a single fallback token.
|
||||
GITHUB_TOKENS=[]
|
||||
GITHUB_TOKEN=
|
||||
|
||||
# Optional blog publishing integration.
|
||||
# Leave BLOG_TOKEN empty to keep report generation local only.
|
||||
BLOG_BASE_URL=http://127.0.0.1
|
||||
BLOG_TOKEN=
|
||||
@@ -0,0 +1,31 @@
|
||||
name: Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
pytest:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12"]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: pip
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Run tests
|
||||
run: python -m pytest -q
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# Local secrets
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Python cache/runtime
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
*$py.class
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# Virtual environments
|
||||
venv/
|
||||
.venv/
|
||||
env/
|
||||
.envrc
|
||||
|
||||
# Local databases and generated state
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
:memory:
|
||||
logs/
|
||||
results/
|
||||
data/
|
||||
reports/daily/
|
||||
|
||||
# Build/package artifacts
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
*.egg
|
||||
|
||||
# Editor and OS files
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# Local backups and temporary files
|
||||
*.bak
|
||||
*.tmp
|
||||
*.log
|
||||
@@ -0,0 +1,26 @@
|
||||
# Contributing
|
||||
|
||||
## Local setup
|
||||
|
||||
```bash
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
cp .env.example .env
|
||||
python -m pytest -q
|
||||
```
|
||||
|
||||
## Development rules
|
||||
|
||||
- Do not commit `.env`, API keys, tokens, logs, generated databases, or runtime reports.
|
||||
- Keep changes focused and covered by tests when behavior changes.
|
||||
- Run `python -m pytest -q` before opening a pull request.
|
||||
- Use `.env.example` for documentation of configuration keys, not real secrets.
|
||||
|
||||
## Useful commands
|
||||
|
||||
```bash
|
||||
python main.py --check-config
|
||||
python scripts/doctor.py
|
||||
python -m pytest -q
|
||||
```
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 CyberSentinel AI contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,14 @@
|
||||
.PHONY: install test run clean
|
||||
|
||||
install:
|
||||
python3 -m venv .venv
|
||||
. .venv/bin/activate && pip install -r requirements.txt
|
||||
|
||||
test:
|
||||
python -m pytest -q
|
||||
|
||||
run:
|
||||
python main.py
|
||||
|
||||
clean:
|
||||
rm -rf .pytest_cache __pycache__ */__pycache__ .coverage htmlcov build dist *.egg-info
|
||||
@@ -1,54 +1,115 @@
|
||||
# CyberSentinel AI - 自动化安全监控与AI分析系统
|
||||
# CyberSentinel AI
|
||||
|
||||
[](LICENSE)
|
||||
CyberSentinel AI is an automated security monitoring system for tracking CVE-related repositories, security-focused GitHub projects, and security articles. It can analyze collected data with AI providers, generate Markdown/HTML reports, and optionally publish daily reports to a blog service.
|
||||
|
||||
[English README](README_EN.md) | [中文 README](README_ZH.md)
|
||||
- 中文说明: [README_ZH.md](README_ZH.md)
|
||||
- English guide: [README_EN.md](README_EN.md)
|
||||
|
||||
---
|
||||
## What it does
|
||||
|
||||
**CyberSentinel AI** 是一个强大的自动化安全监控与AI分析系统，旨在帮助安全研究人员和爱好者 **实时追踪最新的安全漏洞 (CVE)** 和 **GitHub 上的安全相关仓库**，并利用 **人工智能技术**进行深度分析，最终将有价值的安全情报 **自动发布到博客平台**。
|
||||
- Monitors GitHub for CVE proof-of-concept repositories and security tools.
|
||||
- Tracks configured high-priority repositories.
|
||||
- Uses OpenAI-compatible providers and Gemini for AI-assisted triage.
|
||||
- Stores monitor state in SQLite.
|
||||
- Generates local Markdown and HTML reports.
|
||||
- Optionally publishes reports to a blog API when `BLOG_TOKEN` is configured.
|
||||
|
||||
**🚀 功能亮点**
|
||||
## Quick start
|
||||
|
||||
* **多源数据监控** (CVE & GitHub 仓库)
|
||||
* **智能 AI 分析** (OpenAI & Gemini 双引擎)
|
||||
* **全自动化工作流** (7x24 监控、每日简报、动态黑名单)
|
||||
* **灵活的配置与管理** (多 Token 支持、可配置参数、详细日志)
|
||||
* **博客自动发布** (集成博客平台、Markdown 报告)
|
||||
```bash
|
||||
git clone https://github.com/Hxnxe/CyberSentinel-AI.git
|
||||
cd CyberSentinel-AI
|
||||
|
||||
**🛠️ 技术实现**
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
||||
* **监控模块 (Monitors)**: `cve_monitor.py`, `github_monitor.py`
|
||||
* **AI 分析模块 (AI)**: `analyzer.py`
|
||||
* **数据处理与管理 (Utils)**: `logger.py`, `csv_writer.py`, `article_fetcher.py`, `article_manager.py`, `blog_manager.py`
|
||||
* **数据库 (Database)**: `database/models.py`
|
||||
* **配置文件 (Config)**: `config.py`
|
||||
* **主程序 (Main)**: `main.py`
|
||||
cp .env.example .env
|
||||
python main.py --check-config
|
||||
python -m pytest -q
|
||||
python main.py
|
||||
```
|
||||
|
||||
**⚙️ 运行环境**
|
||||
The project imports and tests without private credentials. Real monitoring needs your own `.env` values.
|
||||
|
||||
* Python 3.8+
|
||||
* 依赖库 (见 `requirements.txt`)
|
||||
You can also let the bootstrap script perform the setup:
|
||||
|
||||
**📦 安装步骤**
|
||||
```bash
|
||||
bash scripts/bootstrap.sh
|
||||
```
|
||||
|
||||
请参考 [English README](README_EN.md) 或 [中文 README](README_ZH.md) 中的详细安装步骤。
|
||||
## Required configuration for real monitoring
|
||||
|
||||
**📝 后续计划**
|
||||
Edit `.env` after copying `.env.example`:
|
||||
|
||||
* 更多数据源支持
|
||||
* 更精细化的 AI 分析
|
||||
* 更丰富的功能
|
||||
* Web UI 管理界面
|
||||
```dotenv
|
||||
GITHUB_TOKENS=["replace-with-your-github-token"]
|
||||
PRIMARY_AI_KEY=replace-with-your-openai-compatible-key
|
||||
PRIMARY_AI_BASE=https://api.openai.com/v1
|
||||
PRIMARY_AI_MODEL=gpt-4.1-nano
|
||||
```
|
||||
|
||||
**🤝 贡献**
|
||||
Gemini and blog publishing are optional:
|
||||
|
||||
欢迎参与项目贡献！如果您有任何建议或 Bug 报告，请提交 Issue 或 Pull Request。
|
||||
```dotenv
|
||||
GEMINI_AI_KEY=
|
||||
BLOG_BASE_URL=http://127.0.0.1
|
||||
BLOG_TOKEN=
|
||||
```
|
||||
|
||||
**📜 许可证**
|
||||
If `BLOG_TOKEN` is empty, the system keeps reports local and skips blog API calls.
|
||||
|
||||
本项目采用 MIT 许可证，详情请见 [LICENSE](LICENSE) 文件。
|
||||
## Generated files
|
||||
|
||||
---
|
||||
Runtime files are intentionally ignored by git:
|
||||
|
||||
**感谢您的关注！** ⭐ **Star** 本项目以支持我们的工作！
|
||||
```text
|
||||
.env
|
||||
cve_database.db
|
||||
logs/
|
||||
results/
|
||||
data/
|
||||
venv/
|
||||
.venv/
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
```
|
||||
|
||||
Do not commit generated databases, logs, reports, or private credentials.
|
||||
|
||||
## Development checks
|
||||
|
||||
```bash
|
||||
python main.py --check-config
|
||||
python -m pytest -q
|
||||
```
|
||||
|
||||
The public release includes tests for:
|
||||
|
||||
- configuration loading without private credentials;
|
||||
- `.env.example` completeness and sanitization;
|
||||
- `.gitignore` protection for private/generated files;
|
||||
- GitHub token rotation behavior;
|
||||
- monitor quality filtering;
|
||||
- database persistence;
|
||||
- optional blog publishing behavior.
|
||||
|
||||
## Project layout
|
||||
|
||||
```text
|
||||
ai/ AI client, prompts, schemas, result validation, analysis logic
|
||||
clients/ GitHub API client
|
||||
database/ SQLAlchemy models, sessions, repositories, initialization
|
||||
monitors/ CVE and GitHub repository monitors
|
||||
reports/ Markdown/HTML report generation
|
||||
utils/ logging, article, blacklist, blog, CSV, and Markdown helpers
|
||||
scripts/ bootstrap and configuration doctor helpers
|
||||
docs/ configuration, quick-start, and deployment notes
|
||||
tests/ pytest suite
|
||||
main.py runtime entrypoint
|
||||
config.py environment-backed configuration
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
MIT. See [LICENSE](LICENSE).
|
||||
+107
-143
@@ -1,177 +1,141 @@
|
||||
# 🔥 CyberSentinel AI - Automated Security Monitoring and AI Analysis System
|
||||
# CyberSentinel AI
|
||||
|
||||
[](LICENSE)
|
||||
CyberSentinel AI is an automated security monitoring and AI analysis system. It watches CVE-related repositories, security-focused GitHub repositories, and security articles, then produces local Markdown/HTML reports. Blog publishing is optional.
|
||||
|
||||
CyberSentinel AI is an **automated security monitoring and AI analysis system** designed to **track the latest security vulnerabilities (CVEs)** and **security-related repositories on GitHub** in real-time. It leverages **Artificial Intelligence (AI) technology** for in-depth analysis and **automatically publishes valuable security intelligence to a blog platform**.
|
||||
## Features
|
||||
|
||||
## 🚀 Key Features
|
||||
- Monitor GitHub for CVE/POC-related repositories.
|
||||
- Search GitHub by security keywords.
|
||||
- Track configured high-priority repositories.
|
||||
- Analyze findings with OpenAI-compatible providers and Gemini.
|
||||
- Store monitor state in SQLite.
|
||||
- Generate Markdown and HTML reports.
|
||||
- Publish reports to a blog API only when `BLOG_TOKEN` is configured.
|
||||
|
||||
* **Multi-Source Data Monitoring**:
|
||||
* **CVE Monitoring**: Real-time scraping of the latest CVE-related information from GitHub, enabling rapid discovery and tracking of the latest vulnerability trends.
|
||||
* **GitHub Repository Monitoring**: Comprehensive monitoring of security-related open-source projects on GitHub through keyword searches and predefined watch lists.
|
||||
* **Intelligent AI Analysis**:
|
||||
* **OpenAI & Gemini Dual Engine**: Integrated with OpenAI and Gemini AI models, providing powerful natural language processing capabilities for in-depth security data analysis.
|
||||
* **Multi-Dimensional Security Assessment**: Evaluation of CVEs and repositories from multiple dimensions, including **vulnerability principles**, **exploitation methods**, **risk levels**, and **impact scope**, ensuring depth and breadth of analysis.
|
||||
* **Value Judgment and Filtering**: Intelligent AI-driven judgment of security information value, automatically filtering out low-value information and focusing on truly noteworthy security threats and technologies.
|
||||
* **Automated Workflow**:
|
||||
* **Fully Automated Monitoring**: The system runs **24/7** unattended, automating security information collection, analysis, and report generation.
|
||||
* **Daily Security Briefing**: Generates daily security briefing reports on a schedule, summarizing the latest CVE vulnerabilities and GitHub security repository dynamics, and pushing them to a blog platform.
|
||||
* **Dynamic Blacklisting**: Automatically updates blacklists based on AI analysis results, reducing interference from invalid information and improving monitoring efficiency.
|
||||
* **Flexible Configuration and Management**:
|
||||
* **Multi-GitHub Token Support**: Supports configuration of multiple GitHub Tokens, intelligently rotating usage to effectively avoid API rate limits.
|
||||
* **Configurable Monitoring Parameters**: Keywords, watch repository lists, blacklists, etc., can be flexibly adjusted through configuration files to meet different monitoring needs.
|
||||
* **Detailed Logging**: Detailed logs are recorded for all critical steps of system operation, facilitating troubleshooting and system monitoring.
|
||||
* **Automated Blog Publishing**:
|
||||
* **Integrated Blog Platform**: Integrated with a blog platform API to automatically publish daily security briefing reports, quickly sharing security intelligence.
|
||||
* **Markdown Reports**: Analysis results and security briefings are generated in **Markdown format**, making them easy to read and edit.
|
||||
## Quick start
|
||||
|
||||
## 🛠️ Technical Implementation
|
||||
```bash
|
||||
git clone https://github.com/Hxnxe/CyberSentinel-AI.git
|
||||
cd CyberSentinel-AI
|
||||
|
||||
### 1. Monitoring Modules (Monitors)
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
||||
* **`cve_monitor.py`**: **CVE Monitor**
|
||||
* **GitHub API Interaction**: Uses the GitHub API to search for CVE-related repositories, keyword `CVE-202+`, and sorts by `updated` time.
|
||||
* **CVE Information Extraction**: Extracts CVE numbers from repository names and descriptions using regular expressions.
|
||||
* **Repository Information Crawling**: Retrieves repository descriptions, star counts, update times, recent commits, and other information.
|
||||
* **Blacklist Filtering**: Supports **user blacklists** and **repository blacklists** to filter out invalid information sources.
|
||||
* **File Content Analysis**: Clones repositories locally and **intelligently analyzes** **README.md** and other **high-priority files**, calculates file **relevance scores**, and initially filters high-value repositories.
|
||||
* **Intelligent Token Management**: Implements automatic rotation and status checking of GitHub Tokens, dynamically switching available tokens to ensure the continuity of monitoring tasks.
|
||||
* **Database Storage**: Uses the **SQLite** database **`database/cve_record.db`** to store CVE records, including CVE numbers, descriptions, publication dates, last modified dates, repository URLs, and other information.
|
||||
cp .env.example .env
|
||||
python main.py --check-config
|
||||
python -m pytest -q
|
||||
python main.py
|
||||
```
|
||||
|
||||
* **`github_monitor.py`**: **GitHub Repository Monitor**
|
||||
* **Keyword Search**: Periodically searches GitHub repositories based on the `GITHUB_KEYWORDS` list defined in the configuration file **`config.py`**.
|
||||
* **Watch List**: Supports the `WATCHED_REPOSITORIES` list in the configuration file **`config.py`** to **focus monitoring** on predefined security repositories.
|
||||
* **Repository Information Crawling**: Retrieves detailed repository information, including descriptions, star counts, last update times, recent commit records, and more.
|
||||
* **Commit Record Analysis**: Crawls the **recent commit records** of repositories, **intelligently analyzes** commit information and file changes, and initially judges the **security relevance** of repositories.
|
||||
* **Blacklist Filtering**: Supports **user blacklists** and **repository blacklists** to filter out invalid information sources.
|
||||
* **Intelligent Token Management**: Shares the Token management mechanism with the CVE monitor.
|
||||
* **Database Storage**: Uses the **SQLite** database **`database/github_repo.db`** to store GitHub repository records, including repository names, URLs, descriptions, last update times, star counts, whether they are high-value repositories, and other information.
|
||||
The project can be imported and tested without private credentials. Add your own `.env` values before real monitoring.
|
||||
|
||||
### 2. AI Analysis Module (AI)
|
||||
You can also use the bootstrap script:
|
||||
|
||||
* **`analyzer.py`**: **AI Analyzer**
|
||||
* **OpenAI & Gemini API**: Integrates **OpenAI API** (primary) and **Gemini API** (backup), supports **multi-model** switching, such as `gpt-4o-mini-2024-07-18` (fallback model).
|
||||
* **Prompt Engineering**: Designed **different Prompt templates** for **different analysis scenarios** (CVE analysis, new repository analysis, repository update analysis, specific watch repository analysis) to optimize AI analysis results.
|
||||
* **JSON Format Output**: Requires AI to **strictly output analysis results in JSON format** for easy program parsing and data processing.
|
||||
* **Multi-Dimensional Security Analysis**: AI analysis results include rich information such as **brief descriptions of vulnerabilities/repositories**, **detailed summaries**, **risk levels**, **key points**, **technical details**, **affected components**, **value assessments**, **security types**, **update types**, and **vulnerability exploitation status**.
|
||||
* **Result Validation and Standardization**: Performs **strict format validation** and **content standardization** on the JSON results returned by AI to ensure the accuracy and usability of the data.
|
||||
* **Dynamic Blacklist Update**: Based on AI analysis results, **automatically judges** whether repositories or users should be added to the blacklist and **dynamically updates the blacklist file**.
|
||||
* **Analysis Result Persistence**: **Saves** AI analysis results as **JSON files** and **updates** corresponding records in the **database**.
|
||||
* **Article Title Classification**: Supports **AI classification** of security article titles for generating security briefing reports.
|
||||
* **API Failover**: When OpenAI API calls fail, **automatically switches** to **backup OpenAI API** or **Gemini API** to improve system **stability and availability**.
|
||||
```bash
|
||||
bash scripts/bootstrap.sh
|
||||
```
|
||||
|
||||
### 3. Data Processing and Management (Utils)
|
||||
## Minimal production configuration
|
||||
|
||||
* **`logger.py`**: **Logger**
|
||||
* Uses the `logging` module to provide **complete logging** functionality, covering **DEBUG**, **INFO**, **WARNING**, **ERROR**, and other levels.
|
||||
* Log information is **detailed** and **structured**, making it easy to troubleshoot and monitor the system.
|
||||
* Logs are output to the file **`logs/security_monitor.log`** and rolled over daily.
|
||||
```dotenv
|
||||
GITHUB_TOKENS=["replace-with-your-github-token"]
|
||||
PRIMARY_AI_KEY=replace-with-your-openai-compatible-key
|
||||
PRIMARY_AI_BASE=https://api.openai.com/v1
|
||||
PRIMARY_AI_MODEL=gpt-4.1-nano
|
||||
```
|
||||
|
||||
* **`csv_writer.py`**: **CSV Result Writer** (currently not used, can be extended)
|
||||
* Provides the function of **exporting analysis results to CSV files** for easy data analysis and sharing.
|
||||
Optional providers and integrations:
|
||||
|
||||
* **`article_fetcher.py`**: **Article Fetcher**
|
||||
* **Multi-Source Fetching**: Currently supports fetching security articles from **BruceFeIix** and **D洞见 (doonsec)** WeChat official accounts.
|
||||
* **Article Title and URL Extraction**: Uses regular expressions to extract article titles and URLs from web page content.
|
||||
* **Retry Mechanism**: Uses a **backoff strategy** retry mechanism to improve the **stability and success rate** of article fetching.
|
||||
* **Article Title Cleaning**: **Standardizes** and **cleans** article titles, removing redundant markers and formats.
|
||||
```dotenv
|
||||
BACKUP_AI_CONFIGS=[]
|
||||
GEMINI_AI_KEY=
|
||||
BLOG_BASE_URL=http://127.0.0.1
|
||||
BLOG_TOKEN=
|
||||
```
|
||||
|
||||
* **`article_manager.py`**: **Article Manager**
|
||||
* **Article De-duplication**: **Automatically filters** processed article URLs to avoid duplicate analysis and pushing.
|
||||
* **AI Classification Result Processing**: Processes AI article title classification results and **organizes article lists by category**.
|
||||
* **Daily Security Briefing Report Generation**: **Regularly** generates **Markdown format** daily security briefing reports, summarizing the latest security articles and AI analysis results.
|
||||
* **Automated Blog Publishing**: Calls the `blog_manager.py` module to **automatically publish daily security briefing reports to a blog platform**.
|
||||
* **Article Data Persistence**: **Saves** processed URLs and classified articles as **JSON files** for easy subsequent use and management.
|
||||
When `BLOG_TOKEN` is empty, the application skips blog API calls and keeps reports local.
|
||||
|
||||
* **`blog_manager.py`**: **Blog Manager**
|
||||
* **Blog Platform API Interaction**: Encapsulates **common functions** for interacting with blog platform APIs, such as **creating articles** and **updating articles**.
|
||||
* **Article ID Mapping Management**: **Records** the **article IDs** of daily security briefing reports on the blog platform for easy subsequent updates and management.
|
||||
* **Automated Blog Publishing**: Implements the function of **automatically publishing daily security briefing reports to a blog platform**.
|
||||
## Commands
|
||||
|
||||
### 4. Database (Database)
|
||||
Run tests:
|
||||
|
||||
* **`database/models.py`**: **Database Model Definition**
|
||||
* Uses **SQLAlchemy** to define two data models, **`CVERecord`** (CVE record) and **`Repository`** (GitHub repository record), to facilitate data storage and querying.
|
||||
* The database uses **SQLite**, and the file paths are **`database/cve_record.db`** and **`database/github_repo.db`**.
|
||||
```bash
|
||||
python main.py --check-config
|
||||
python -m pytest -q
|
||||
```
|
||||
|
||||
### 5. Configuration File (Config)
|
||||
Start the monitor:
|
||||
|
||||
* **`config.py`**: **System Configuration File**
|
||||
* Centrally manages the system's **configuration parameters**, such as database paths, API keys, monitoring intervals, keyword lists, blacklists, etc.
|
||||
* Facilitates users to **customize** and **adjust** system behavior.
|
||||
* Includes the following main configuration items:
|
||||
* `DATABASE_PATH`: Database file path
|
||||
* `MONITOR_INTERVAL`: Monitoring cycle interval (seconds)
|
||||
* `GITHUB_TOKEN`: GitHub API Token (supports list `GITHUB_TOKENS`)
|
||||
* `GITHUB_KEYWORDS`: List of GitHub repository search keywords
|
||||
* `WATCHED_REPOSITORIES`: List of GitHub repositories to focus on monitoring
|
||||
* `BLACKLIST_USERS`: User blacklist
|
||||
* `BLACKLIST_REPOSITORIES`: Repository blacklist
|
||||
* `PRIMARY_AI_CONFIG`: Primary AI service (OpenAI) configuration
|
||||
* `BACKUP_AI_CONFIGS`: List of backup AI service (OpenAI) configurations
|
||||
* `GEMINI_AI_CONFIG`: Gemini AI service configuration
|
||||
* `BLOG_TOKEN`: Blog platform API Token
|
||||
```bash
|
||||
python main.py
|
||||
```
|
||||
|
||||
### 6. Main Program (Main)
|
||||
Install as an editable package:
|
||||
|
||||
* **`main.py`**: **System Main Program**
|
||||
* **Initializes** each module (monitors, AI analyzer, article manager, etc.).
|
||||
* **Starts** the monitoring cycle, **regularly** executing CVE monitoring, GitHub repository monitoring, AI analysis, article crawling, and blog publishing tasks.
|
||||
* Uses **multi-threading** to achieve **concurrent monitoring** and **AI analysis**, improving system efficiency.
|
||||
* **Exception handling** and **retry mechanisms** ensure stable system operation.
|
||||
* **Status Monitoring Thread**: **Regularly checks** system operating status and records logs.
|
||||
* **Daily Blog Publishing**: **Regularly** automatically publishes daily security briefing reports to a blog platform.
|
||||
* **Command-Line Startup**: Users can start and stop the monitoring system via the command line.
|
||||
```bash
|
||||
pip install -e .
|
||||
cybersentinel-ai
|
||||
```
|
||||
|
||||
## ⚙️ Running Environment
|
||||
## Project layout
|
||||
|
||||
* Python 3.8+
|
||||
* Dependencies (see `requirements.txt`)
|
||||
```text
|
||||
ai/ AI clients, prompts, schemas, validation, and analysis
|
||||
clients/ GitHub API client
|
||||
database/ SQLAlchemy models, sessions, repositories, initialization
|
||||
monitors/ CVE and GitHub repository monitors
|
||||
reports/ Markdown/HTML report generation
|
||||
utils/ logging, article, blacklist, blog, CSV, and Markdown helpers
|
||||
scripts/ bootstrap and configuration doctor helpers
|
||||
docs/ configuration, quick-start, and deployment notes
|
||||
tests/ pytest suite
|
||||
main.py runtime entrypoint
|
||||
config.py environment-backed configuration
|
||||
```
|
||||
|
||||
## 📦 Installation Steps
|
||||
More documentation:
|
||||
|
||||
1. **Clone the code repository**
|
||||
- [Configuration guide](docs/CONFIGURATION.md)
|
||||
- [Deployment notes](docs/DEPLOYMENT.md)
|
||||
|
||||
```bash
|
||||
git clone [Project Repository Address]
|
||||
cd [Project Directory]
|
||||
```
|
||||
## Private and generated files
|
||||
|
||||
2. **Install dependencies**
|
||||
The release excludes local secrets and runtime artifacts:
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
```text
|
||||
.env
|
||||
.env.*
|
||||
venv/
|
||||
.venv/
|
||||
logs/
|
||||
results/
|
||||
data/
|
||||
cve_database.db
|
||||
*.db
|
||||
*.sqlite
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
```
|
||||
|
||||
3. **Configure the `config.py` file**
|
||||
Never commit real tokens, generated databases, logs, reports, runtime data, or virtual environments.
|
||||
|
||||
* Configure GitHub API Token (`GITHUB_TOKEN` or `GITHUB_TOKENS`)
|
||||
* Configure OpenAI API key and Base URL (`PRIMARY_AI_CONFIG`, `BACKUP_AI_CONFIGS`)
|
||||
* Configure Gemini API key and Base URL (`GEMINI_AI_CONFIG`)
|
||||
* Configure Blog platform API Token (`BLOG_TOKEN`) (if you need to automatically publish to a blog)
|
||||
* Modify other configuration items as needed, such as monitoring interval, keyword list, blacklist, etc.
|
||||
## Configuration reference
|
||||
|
||||
4. **Run the system**
|
||||
| Variable | Purpose |
|
||||
| --- | --- |
|
||||
| `MONITOR_INTERVAL` | Monitoring interval in seconds. Default: 7200 |
|
||||
| `DATABASE_PATH` | SQLite database path. Default: `cve_database.db` |
|
||||
| `GITHUB_TOKENS` | JSON array of GitHub tokens |
|
||||
| `GITHUB_TOKEN` | Single fallback GitHub token |
|
||||
| `PRIMARY_AI_KEY` | Primary AI provider API key |
|
||||
| `PRIMARY_AI_BASE` | Primary AI provider base URL |
|
||||
| `PRIMARY_AI_MODEL` | Primary AI model |
|
||||
| `BACKUP_AI_CONFIGS` | JSON array of backup OpenAI-compatible providers |
|
||||
| `GEMINI_AI_KEY` | Gemini API key |
|
||||
| `BLOG_BASE_URL` | Blog API base URL |
|
||||
| `BLOG_TOKEN` | Blog API token. Leave empty to disable blog publishing |
|
||||
|
||||
```bash
|
||||
python main.py
|
||||
```
|
||||
## License
|
||||
|
||||
## 📝 Future Plans
|
||||
|
||||
* **More Data Source Support**: Expand support for more security information sources, such as security communities, vulnerability platforms, etc.
|
||||
* **More Refined AI Analysis**: Continuously optimize Prompt engineering to improve the accuracy and depth of AI analysis.
|
||||
* **Richer Features**: Such as vulnerability early warning, threat intelligence visualization, custom reports, etc.
|
||||
* **Web UI Management Interface**: Develop a Web UI management interface to facilitate users to configure and manage the monitoring system.
|
||||
|
||||
## 🤝 Contribution
|
||||
|
||||
Contributions are welcome! If you have any suggestions or bug reports, please submit an Issue or Pull Request.
|
||||
|
||||
## 📜 License
|
||||
|
||||
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
||||
|
||||
---
|
||||
|
||||
**Thank you for your attention!** ⭐ **Star** this project to support our work!
|
||||
MIT. See [LICENSE](LICENSE).
|
||||
+112
-140
@@ -1,177 +1,149 @@
|
||||
# CyberSentinel AI 🔥 自动化安全监控与AI分析系统
|
||||
# CyberSentinel AI
|
||||
|
||||
[](LICENSE)
|
||||
CyberSentinel AI 是一个自动化安全监控与 AI 分析系统，用于跟踪 CVE 相关仓库、安全工具仓库和安全文章，生成本地 Markdown/HTML 报告，并可选发布到博客服务。
|
||||
|
||||
本项目是一个**自动化安全监控与AI分析系统**，旨在**实时追踪最新的安全漏洞（CVE）**和 **GitHub 上的安全相关仓库**，并利用 **人工智能技术**进行深度分析，最终将有价值的安全情报 **自动发布到博客平台**。
|
||||
## 功能概览
|
||||
|
||||
## 🚀 功能亮点
|
||||
- 监控 GitHub 上的 CVE/POC 相关仓库。
|
||||
- 按关键词搜索安全工具、漏洞利用、攻防研究等仓库。
|
||||
- 重点跟踪配置中的固定仓库列表。
|
||||
- 使用 OpenAI 兼容接口和 Gemini 做 AI 辅助分析。
|
||||
- 使用 SQLite 保存监控状态和分析记录。
|
||||
- 生成 Markdown/HTML 报告。
|
||||
- 配置 `BLOG_TOKEN` 后可自动同步到博客；未配置时自动跳过博客发布。
|
||||
|
||||
* **多源数据监控**:
|
||||
* **CVE 监控**: 实时抓取 GitHub 上最新的 CVE 相关信息，快速发现和跟踪最新的漏洞动态。
|
||||
* **GitHub 仓库监控**: 通过关键词搜索和预定义的监控列表，全面监控 GitHub 上与网络安全相关的开源项目。
|
||||
* **智能AI分析**:
|
||||
* **OpenAI & Gemini 双引擎**: 集成 OpenAI 和 Gemini AI 模型，提供强大的自然语言处理能力，用于深入分析安全数据。
|
||||
* **多维度安全评估**: 从**漏洞原理**、**利用方式**、**风险等级**、**影响范围**等多个维度对 CVE 和仓库进行评估，确保分析结果的深度和广度。
|
||||
* **价值判断与筛选**: 通过 AI 智能判断安全信息的价值，**自动过滤**低价值信息，**聚焦**真正值得关注的安全威胁和技术。
|
||||
* **自动化工作流**:
|
||||
* **全自动化监控**: 系统 **7x24 小时**不间断运行，无需人工值守，自动化完成安全信息收集、分析和报告生成。
|
||||
* **每日安全简报**: **每日定时**生成安全资讯报告，汇总最新的 CVE 漏洞和 GitHub 安全仓库动态，并推送到博客平台。
|
||||
* **动态黑名单**: 根据 AI 分析结果，**自动更新**黑名单，**降低**无效信息干扰，提升监控效率。
|
||||
* **灵活的配置与管理**:
|
||||
* **多 GitHub Token 支持**: 支持配置 **多个 GitHub Token**，**智能轮换**使用，有效**避免 API 调用频率限制**。
|
||||
* **可配置监控参数**: 关键词、监控仓库列表、黑名单等均可通过配置文件灵活调整，满足不同的监控需求。
|
||||
* **详细日志记录**: 系统运行的**所有关键步骤**均有详细日志记录，方便问题排查和系统监控。
|
||||
* **博客自动发布**:
|
||||
* **集成博客平台**: 集成博客平台 API，**自动发布**每日安全资讯报告，快速分享安全情报。
|
||||
* **Markdown 报告**: 分析结果和安全简报均以 **Markdown 格式**生成，方便阅读和编辑。
|
||||
## 快速开始
|
||||
|
||||
## 🛠️ 技术实现
|
||||
```bash
|
||||
git clone https://github.com/Hxnxe/CyberSentinel-AI.git
|
||||
cd CyberSentinel-AI
|
||||
|
||||
### 1. 监控模块 (Monitors)
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
||||
* **`cve_monitor.py`**: **CVE 监控器**
|
||||
* **GitHub API 交互**: 使用 GitHub API 搜索 CVE 相关的仓库，关键词 `CVE-202+`，并按 `updated` 时间排序。
|
||||
* **CVE 信息提取**: 通过正则表达式从仓库名称和描述中提取 CVE 编号。
|
||||
* **仓库信息抓取**: 获取仓库的描述、星数、更新时间、最近提交等信息。
|
||||
* **黑名单过滤**: 支持**用户黑名单**和**仓库黑名单**，过滤无效信息源。
|
||||
* **文件内容分析**: 克隆仓库到本地，**智能分析**仓库中的 **README.md** 和其他**高优先级文件**，计算文件**相关性得分**，初步筛选高价值仓库。
|
||||
* **Token 智能管理**: 实现 GitHub Token 的**自动轮换**和**状态检查**，**动态切换**可用 Token，确保监控任务的连续性。
|
||||
* **数据库存储**: 使用 **SQLite** 数据库 **`database/cve_record.db`** 存储 CVE 记录，包括 CVE 编号、描述、发布时间、最后修改时间、仓库 URL 等信息。
|
||||
cp .env.example .env
|
||||
python main.py --check-config
|
||||
python -m pytest -q
|
||||
python main.py
|
||||
```
|
||||
|
||||
* **`github_monitor.py`**: **GitHub 仓库监控器**
|
||||
* **关键词搜索**: 根据配置文件 **`config.py`** 中定义的 `GITHUB_KEYWORDS` 列表，定期搜索 GitHub 仓库。
|
||||
* **监控列表**: 支持配置文件 **`config.py`** 中 `WATCHED_REPOSITORIES` 列表，**重点监控**预定义的安全仓库。
|
||||
* **仓库信息抓取**: 获取仓库的详细信息，包括描述、星数、最后更新时间、最近提交记录等。
|
||||
* **提交记录分析**: 抓取仓库的**最近提交记录**，**智能分析**提交信息和文件变更，初步判断仓库的**安全相关性**。
|
||||
* **黑名单过滤**: 支持**用户黑名单**和**仓库黑名单**，过滤无效信息源。
|
||||
* **Token 智能管理**: 与 CVE 监控器共享 Token 管理机制。
|
||||
* **数据库存储**: 使用 **SQLite** 数据库 **`database/github_repo.db`** 存储 GitHub 仓库记录，包括仓库名称、URL、描述、最后更新时间、星数、是否为高价值仓库等信息。
|
||||
项目在没有真实密钥时也可以导入和运行测试。正式监控前请编辑 `.env`。
|
||||
|
||||
### 2. AI 分析模块 (AI)
|
||||
也可以直接使用引导脚本：
|
||||
|
||||
* **`analyzer.py`**: **AI 分析器**
|
||||
* **OpenAI & Gemini API**: 集成 **OpenAI API** (主要) 和 **Gemini API** (备用)，支持 **多模型** 切换，例如 `gpt-4o-mini-2024-07-18` (保底模型)。
|
||||
* **Prompt 工程**: 针对 **不同分析场景** (CVE 分析、新仓库分析、仓库更新分析、特定监控仓库分析)，设计了**不同的 Prompt 模板**，优化 AI 分析效果。
|
||||
* **JSON 格式输出**: 要求 AI **严格按照 JSON 格式** 输出分析结果，方便程序解析和数据处理。
|
||||
* **多维度安全分析**: AI 分析结果包含 **漏洞/仓库的简要描述**、**详细总结**、**风险等级**、**关键要点**、**技术细节**、**影响组件**、**价值评估**、**安全类型**、**更新类型**、**漏洞利用状态**等丰富信息。
|
||||
* **结果验证与规范化**: 对 AI 返回的 JSON 结果进行**严格的格式校验**和**内容规范化**，确保数据的准确性和可用性。
|
||||
* **动态黑名单更新**: 根据 AI 分析结果，**自动判断**仓库或用户是否应该加入黑名单，并**动态更新黑名单文件**。
|
||||
* **分析结果持久化**: 将 AI 分析结果 **保存为 JSON 文件**，并 **更新数据库** 中的相应记录。
|
||||
* **文章标题分类**: 支持对安全文章标题进行 **AI 分类**，用于生成安全资讯报告。
|
||||
* **API 故障切换**: 当 OpenAI API 调用失败时，**自动切换**到 **备用 OpenAI API** 或 **Gemini API**，提高系统的**稳定性和可用性**。
|
||||
```bash
|
||||
bash scripts/bootstrap.sh
|
||||
```
|
||||
|
||||
### 3. 数据处理与管理 (Utils)
|
||||
## 最小可用配置
|
||||
|
||||
* **`logger.py`**: **日志记录器**
|
||||
* 使用 `logging` 模块，提供**完善的日志记录**功能，覆盖**DEBUG**、**INFO**、**WARNING**、**ERROR** 等多个级别。
|
||||
* 日志信息**详细**且**结构化**，方便问题排查和系统监控。
|
||||
* 日志输出到文件 **`logs/security_monitor.log`**，并按日期滚动。
|
||||
复制模板：
|
||||
|
||||
* **`csv_writer.py`**: **CSV 结果写入器** (目前未使用，可扩展)
|
||||
* 提供将分析结果 **导出为 CSV 文件** 的功能，方便数据分析和共享。
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
* **`article_fetcher.py`**: **文章抓取器**
|
||||
* **多来源抓取**: 目前支持抓取 **BruceFeIix** 和 **洞见 (doonsec)** 微信公众号的安全文章。
|
||||
* **文章标题和 URL 提取**: 使用正则表达式从网页内容中提取文章标题和 URL。
|
||||
* **重试机制**: 使用 **退避策略** 的重试机制，提高文章抓取的**稳定性和成功率**。
|
||||
* **文章标题清理**: **标准化**和**清理**文章标题，移除多余的标记和格式。
|
||||
编辑 `.env`：
|
||||
|
||||
* **`article_manager.py`**: **文章管理器**
|
||||
* **文章去重**: **自动过滤**已处理过的文章 URL，避免重复分析和推送。
|
||||
* **AI 分类结果处理**: 处理 AI 文章标题分类结果，**按类别整理**文章列表。
|
||||
* **每日安全资讯报告生成**: **定时**生成 **Markdown 格式** 的每日安全资讯报告，汇总最新的安全文章和 AI 分析结果。
|
||||
* **博客自动发布**: 调用 `blog_manager.py` 模块，将每日安全资讯报告 **自动发布到博客平台**。
|
||||
* **文章数据持久化**: 将已处理的 URL 和分类文章 **保存为 JSON 文件**，方便后续使用和管理。
|
||||
```dotenv
|
||||
GITHUB_TOKENS=["替换为你自己的 GitHub Token"]
|
||||
PRIMARY_AI_KEY=替换为你自己的 AI API Key
|
||||
PRIMARY_AI_BASE=https://api.openai.com/v1
|
||||
PRIMARY_AI_MODEL=gpt-4.1-nano
|
||||
```
|
||||
|
||||
* **`blog_manager.py`**: **博客管理器**
|
||||
* **博客平台 API 交互**: 封装了与博客平台 API 交互的 **常用功能**，例如 **创建文章**、**更新文章**。
|
||||
* **文章 ID 映射管理**: **记录**每日安全资讯报告在博客平台上的 **文章 ID**，方便后续更新和管理。
|
||||
* **博客自动发布**: 实现将每日安全资讯报告 **自动发布到博客平台** 的功能。
|
||||
说明：
|
||||
|
||||
### 4. 数据库 (Database)
|
||||
- `GITHUB_TOKENS` 推荐配置，否则 GitHub API 很容易触发频率限制。
|
||||
- `PRIMARY_AI_KEY` 用于 OpenAI 兼容接口。
|
||||
- `BACKUP_AI_CONFIGS` 可以配置多个备用 OpenAI 兼容服务。
|
||||
- `GEMINI_AI_KEY` 是可选 Gemini 兜底。
|
||||
- `BLOG_TOKEN` 为空时不会请求博客接口，只在本地生成报告。
|
||||
|
||||
* **`database/models.py`**: **数据库模型定义**
|
||||
* 使用 **SQLAlchemy** 定义了 **`CVERecord`** (CVE 记录) 和 **`Repository`** (GitHub 仓库记录) 两个数据模型，方便数据存储和查询。
|
||||
* 数据库使用 **SQLite**，文件路径为 **`database/cve_record.db`** 和 **`database/github_repo.db`**。
|
||||
## 常用命令
|
||||
|
||||
### 5. 配置文件 (Config)
|
||||
运行测试：
|
||||
|
||||
* **`config.py`**: **系统配置文件**
|
||||
* 集中管理系统的**各项配置参数**，例如数据库路径、API 密钥、监控间隔、关键词列表、黑名单等。
|
||||
* 方便用户**自定义**和**调整**系统行为。
|
||||
* 包括以下主要配置项：
|
||||
* `DATABASE_PATH`: 数据库文件路径
|
||||
* `MONITOR_INTERVAL`: 监控循环间隔 (秒)
|
||||
* `GITHUB_TOKEN`: GitHub API Token (支持列表 `GITHUB_TOKENS`)
|
||||
* `GITHUB_KEYWORDS`: GitHub 仓库搜索关键词列表
|
||||
* `WATCHED_REPOSITORIES`: 重点监控的 GitHub 仓库列表
|
||||
* `BLACKLIST_USERS`: 用户黑名单
|
||||
* `BLACKLIST_REPOSITORIES`: 仓库黑名单
|
||||
* `PRIMARY_AI_CONFIG`: 主要 AI 服务 (OpenAI) 配置
|
||||
* `BACKUP_AI_CONFIGS`: 备用 AI 服务 (OpenAI) 配置列表
|
||||
* `GEMINI_AI_CONFIG`: Gemini AI 服务配置
|
||||
* `BLOG_TOKEN`: 博客平台 API Token
|
||||
```bash
|
||||
python main.py --check-config
|
||||
python -m pytest -q
|
||||
```
|
||||
|
||||
### 6. 主程序 (Main)
|
||||
启动监控：
|
||||
|
||||
* **`main.py`**: **系统主程序**
|
||||
* **初始化**各个模块 (监控器、AI 分析器、文章管理器等)。
|
||||
* **启动**监控循环，**定时**执行 CVE 监控、GitHub 仓库监控、AI 分析、文章抓取和博客发布等任务。
|
||||
* 使用 **多线程** 实现 **并发监控** 和 **AI 分析**，提高系统效率。
|
||||
* **异常处理**和**重试机制**，确保系统稳定运行。
|
||||
* **状态监控线程**: **定时检查**系统运行状态，记录日志。
|
||||
* **每日博客发布**: **定时**将每日安全资讯报告 **自动发布到博客平台**。
|
||||
* **命令行启动**: 用户可通过命令行启动和停止监控系统。
|
||||
```bash
|
||||
python main.py
|
||||
```
|
||||
|
||||
## ⚙️ 运行环境
|
||||
如果安装为可编辑包，也可以运行：
|
||||
|
||||
* Python 3.8+
|
||||
* 依赖库 (见 `requirements.txt`)
|
||||
```bash
|
||||
pip install -e .
|
||||
cybersentinel-ai
|
||||
```
|
||||
|
||||
## 📦 安装步骤
|
||||
## 目录结构
|
||||
|
||||
1. **克隆代码库**
|
||||
```text
|
||||
ai/ AI 客户端、Prompt、Schema、结果验证和分析逻辑
|
||||
clients/ GitHub API 客户端
|
||||
database/ SQLAlchemy 模型、会话、仓储和初始化逻辑
|
||||
monitors/ CVE 监控器和 GitHub 仓库监控器
|
||||
reports/ Markdown/HTML 报告生成
|
||||
utils/ 日志、文章、黑名单、博客、CSV、Markdown 工具
|
||||
scripts/ 初始化脚本和配置自检脚本
|
||||
docs/ 配置、快速开始和部署说明
|
||||
tests/ pytest 测试套件
|
||||
main.py 主程序入口
|
||||
config.py 基于环境变量的配置入口
|
||||
```
|
||||
|
||||
```bash
|
||||
git clone [项目仓库地址]
|
||||
cd [项目目录]
|
||||
```
|
||||
更多说明：
|
||||
|
||||
2. **安装依赖库**
|
||||
- [中文快速开始](docs/QUICKSTART_ZH.md)
|
||||
- [配置说明](docs/CONFIGURATION.md)
|
||||
- [部署说明](docs/DEPLOYMENT.md)
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
## 不会发布的文件
|
||||
|
||||
3. **配置 `config.py` 文件**
|
||||
以下内容属于本地运行产物或敏感信息，已在 `.gitignore` 中排除：
|
||||
|
||||
* 配置 GitHub API Token (`GITHUB_TOKEN` 或 `GITHUB_TOKENS`)
|
||||
* 配置 OpenAI API 密钥和 Base URL (`PRIMARY_AI_CONFIG`, `BACKUP_AI_CONFIGS`)
|
||||
* 配置 Gemini API 密钥和 Base URL (`GEMINI_AI_CONFIG`)
|
||||
* 配置博客平台 API Token (`BLOG_TOKEN`) (如果需要自动发布博客)
|
||||
* 根据需要修改其他配置项，例如监控间隔、关键词列表、黑名单等。
|
||||
```text
|
||||
.env
|
||||
.env.*
|
||||
venv/
|
||||
.venv/
|
||||
logs/
|
||||
results/
|
||||
data/
|
||||
cve_database.db
|
||||
*.db
|
||||
*.sqlite
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
```
|
||||
|
||||
4. **运行系统**
|
||||
请不要提交真实 Token、数据库、日志、运行结果或虚拟环境。
|
||||
|
||||
```bash
|
||||
python main.py
|
||||
```
|
||||
## 配置说明
|
||||
|
||||
## 📝 后续计划
|
||||
主要配置来自 `.env`：
|
||||
|
||||
* **更多数据源支持**: 扩展支持更多安全信息来源，例如安全社区、漏洞平台等。
|
||||
* **更精细化的 AI 分析**: 持续优化 Prompt 工程，提升 AI 分析的准确性和深度。
|
||||
* **更丰富的功能**: 例如漏洞预警、威胁情报可视化、自定义报告等。
|
||||
* **Web UI 管理界面**: 开发 Web UI 管理界面，方便用户配置和管理监控系统。
|
||||
| 变量 | 说明 |
|
||||
| --- | --- |
|
||||
| `MONITOR_INTERVAL` | 监控循环间隔，默认 7200 秒 |
|
||||
| `DATABASE_PATH` | SQLite 数据库路径，默认 `cve_database.db` |
|
||||
| `GITHUB_TOKENS` | GitHub Token JSON 数组 |
|
||||
| `GITHUB_TOKEN` | 单 GitHub Token 兜底 |
|
||||
| `PRIMARY_AI_KEY` | 主 AI 服务 API Key |
|
||||
| `PRIMARY_AI_BASE` | 主 AI 服务 Base URL |
|
||||
| `PRIMARY_AI_MODEL` | 主 AI 服务模型名 |
|
||||
| `BACKUP_AI_CONFIGS` | 备用 AI 服务 JSON 数组 |
|
||||
| `GEMINI_AI_KEY` | Gemini API Key |
|
||||
| `BLOG_BASE_URL` | 博客服务地址 |
|
||||
| `BLOG_TOKEN` | 博客发布 Token，留空则跳过博客发布 |
|
||||
|
||||
## 🤝 贡献
|
||||
## 许可证
|
||||
|
||||
欢迎参与项目贡献！如果您有任何建议或 Bug 报告，请提交 Issue 或 Pull Request。
|
||||
|
||||
## 📜 许可证
|
||||
|
||||
本项目采用 MIT 许可证，详情请见 [LICENSE](LICENSE) 文件。
|
||||
|
||||
---
|
||||
|
||||
**感谢您的关注！** ⭐ **Star** 本项目以支持我们的工作！
|
||||
本项目使用 MIT 许可证。详见 [LICENSE](LICENSE)。
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
# Security Policy
|
||||
|
||||
## Secret handling
|
||||
|
||||
CyberSentinel AI reads credentials from `.env` or environment variables. The public repository must never contain:
|
||||
|
||||
- GitHub tokens;
|
||||
- OpenAI-compatible API keys;
|
||||
- Gemini API keys;
|
||||
- blog publishing tokens;
|
||||
- private keys;
|
||||
- generated SQLite databases;
|
||||
- logs or runtime reports that may include operational data.
|
||||
|
||||
Before publishing changes, run:
|
||||
|
||||
```bash
|
||||
python -m pytest -q
|
||||
python main.py --check-config
|
||||
```
|
||||
|
||||
Also inspect `git status --short` and confirm that `.env`, `logs/`, `results/`, `data/`, `*.db`, and virtual environments are not staged.
|
||||
|
||||
## Reporting issues
|
||||
|
||||
If you find a security issue, do not include real secrets in public issue text. Describe the affected component, expected behavior, and reproduction steps using placeholder credentials.
|
||||
@@ -0,0 +1,6 @@
|
||||
"""AI 模块导出。"""
|
||||
|
||||
from ai.analyzer import SecurityAnalyzer
|
||||
from ai.llm_client import LLMClient
|
||||
|
||||
__all__ = ["SecurityAnalyzer", "LLMClient"]
|
||||
+509
@@ -0,0 +1,509 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from ai.llm_client import LLMClient
|
||||
from ai.prompts import (
|
||||
build_repo_user_prompt,
|
||||
get_article_system_prompt,
|
||||
get_cve_system_prompt,
|
||||
get_cve_user_prompt,
|
||||
get_repo_system_prompt,
|
||||
)
|
||||
from ai.result_validator import validate_article_result, validate_cve_result, validate_repo_result
|
||||
from ai.schemas import ArticleClassificationResult, SecurityAnalysisResult
|
||||
from database.repositories import CVERepo, RepositoryRepo
|
||||
from database.session import SessionFactory, get_engine, get_session
|
||||
from utils.blacklist_manager import BlacklistManager
|
||||
from utils.logger import get_logger
|
||||
|
||||
logger = get_logger("analyzer")
|
||||
logger.propagate = False
|
||||
|
||||
|
||||
class SecurityAnalyzer:
|
||||
"""安全分析业务编排层。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
llm_client: LLMClient | None = None,
|
||||
blacklist_manager: BlacklistManager | None = None,
|
||||
engine: Any | None = None,
|
||||
session_factory: Any | None = None,
|
||||
) -> None:
|
||||
self.engine = engine or get_engine()
|
||||
self.session_factory = session_factory or SessionFactory(self.engine)
|
||||
self.llm_client = llm_client or LLMClient()
|
||||
self.blacklist_manager = blacklist_manager or BlacklistManager()
|
||||
|
||||
self.report_dir = os.path.join("reports", "daily")
|
||||
self.analysis_history_dir = os.path.join("data", "analysis_history")
|
||||
self.ai_analysis_dir = os.path.join("results", "ai_analysis")
|
||||
|
||||
os.makedirs(self.report_dir, exist_ok=True)
|
||||
os.makedirs(self.analysis_history_dir, exist_ok=True)
|
||||
os.makedirs(self.ai_analysis_dir, exist_ok=True)
|
||||
|
||||
def _safe_json_dumps(self, data: Any, ensure_ascii: bool = False) -> str:
|
||||
def _default(obj: Any) -> str:
|
||||
if isinstance(obj, datetime):
|
||||
return obj.isoformat()
|
||||
return str(obj)
|
||||
|
||||
try:
|
||||
return json.dumps(data, ensure_ascii=ensure_ascii, default=_default, indent=2)
|
||||
except Exception as exc:
|
||||
logger.warning("JSON序列化失败: %s", str(exc))
|
||||
return str(data)
|
||||
|
||||
def _clean_title_for_analysis(self, title: str) -> str:
|
||||
if not title:
|
||||
return ""
|
||||
text = title.replace("\n", " ").replace("\r", " ").replace("\t", " ")
|
||||
text = re.sub(r"[\x00-\x1F\x7F-\x9F]", "", text)
|
||||
return " ".join(text.split())
|
||||
|
||||
def _process_commits(
|
||||
self,
|
||||
commits: Any,
|
||||
max_commits: int = 5,
|
||||
max_file_changes: int = 1000,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""统一清洗提交数据。"""
|
||||
if not commits:
|
||||
return []
|
||||
|
||||
if isinstance(commits, str):
|
||||
try:
|
||||
commits = json.loads(commits)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
|
||||
if not isinstance(commits, list):
|
||||
return []
|
||||
|
||||
processed_commits: list[dict[str, Any]] = []
|
||||
for commit in commits[:max_commits]:
|
||||
if not isinstance(commit, dict):
|
||||
continue
|
||||
|
||||
author = commit.get("author", "")
|
||||
if isinstance(author, dict):
|
||||
author_name = author.get("name", "")
|
||||
else:
|
||||
author_name = str(author or "")
|
||||
|
||||
files: list[dict[str, Any]] = []
|
||||
for file_info in commit.get("files", []):
|
||||
if not isinstance(file_info, dict):
|
||||
continue
|
||||
|
||||
patch = str(file_info.get("patch", ""))
|
||||
if patch and len(patch) > max_file_changes:
|
||||
patch = patch[:max_file_changes] + "..."
|
||||
|
||||
files.append(
|
||||
{
|
||||
"path": str(file_info.get("path", "")),
|
||||
"status": str(file_info.get("status", "")),
|
||||
"changes": int(file_info.get("changes", 0) or 0),
|
||||
"patch": patch,
|
||||
"additions": int(file_info.get("additions", 0) or 0),
|
||||
"deletions": int(file_info.get("deletions", 0) or 0),
|
||||
}
|
||||
)
|
||||
|
||||
processed_commits.append(
|
||||
{
|
||||
"message": str(commit.get("message", "")),
|
||||
"date": str(commit.get("date", "")),
|
||||
"author": author_name,
|
||||
"files": files,
|
||||
"sha": str(commit.get("sha", "")),
|
||||
}
|
||||
)
|
||||
|
||||
return processed_commits
|
||||
|
||||
def _validate_repo_data(self, repo_data: Any) -> bool:
|
||||
"""验证仓库输入数据格式。"""
|
||||
if not isinstance(repo_data, dict):
|
||||
return False
|
||||
if not repo_data.get("name"):
|
||||
return False
|
||||
|
||||
commits = repo_data.get("recent_commits")
|
||||
if commits is None:
|
||||
repo_data["recent_commits"] = []
|
||||
return True
|
||||
|
||||
if isinstance(commits, str):
|
||||
try:
|
||||
commits = json.loads(commits)
|
||||
except json.JSONDecodeError:
|
||||
repo_data["recent_commits"] = []
|
||||
return False
|
||||
|
||||
if not isinstance(commits, list):
|
||||
repo_data["recent_commits"] = []
|
||||
return False
|
||||
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for commit in commits:
|
||||
if isinstance(commit, str):
|
||||
try:
|
||||
commit = json.loads(commit)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(commit, dict):
|
||||
normalized.append(commit)
|
||||
|
||||
repo_data["recent_commits"] = normalized
|
||||
return True
|
||||
|
||||
def _get_default_repo_result(self, repo_data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"repository_name": repo_data.get("name", "Unknown"),
|
||||
"is_valuable": False,
|
||||
"value_reason": "AI分析服务暂时不可用",
|
||||
"summary": "由于AI服务不可用，无法进行详细分析",
|
||||
"risk_level": "LOW",
|
||||
"key_points": ["AI服务暂时不可用"],
|
||||
"technical_details": ["无法获取技术细节"],
|
||||
"affected_components": ["未知"],
|
||||
"security_type": "未知",
|
||||
"update_type": "GENERAL_UPDATE",
|
||||
"search_keyword": repo_data.get("search_keyword", "N/A"),
|
||||
"relevance_score": "LOW",
|
||||
}
|
||||
|
||||
def _get_default_cve_result(self, cve_data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"cve_id": cve_data.get("cve_id"),
|
||||
"is_valuable": False,
|
||||
"value_reason": "AI分析服务暂时不可用",
|
||||
"short_description": "AI服务不可用，无法分析",
|
||||
"summary": "由于AI服务不可用，无法进行详细的CVE分析",
|
||||
"risk_level": "LOW",
|
||||
"exploit_status": "未知",
|
||||
"key_points": ["AI服务暂时不可用"],
|
||||
"technical_details": ["无法获取技术细节"],
|
||||
"affected_components": ["未知"],
|
||||
"repository_url": cve_data.get("repository_url"),
|
||||
"published_date": cve_data.get("published_date"),
|
||||
"last_modified": cve_data.get("last_modified"),
|
||||
"stars": cve_data.get("stars", 0),
|
||||
}
|
||||
|
||||
def analyze_repository(self, repo_data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""分析仓库数据。"""
|
||||
try:
|
||||
repo_name = str(repo_data.get("name", ""))
|
||||
owner = str(repo_data.get("owner", ""))
|
||||
|
||||
self.blacklist_manager.reload_if_needed()
|
||||
if repo_name and self.blacklist_manager.is_blacklisted(repo_name, "repository"):
|
||||
logger.info("仓库 %s 在黑名单中，跳过分析", repo_name)
|
||||
return None
|
||||
if owner and self.blacklist_manager.is_blacklisted(owner, "user"):
|
||||
logger.info("作者 %s 在黑名单中，跳过分析", owner)
|
||||
return None
|
||||
|
||||
if not self._validate_repo_data(repo_data):
|
||||
logger.error("仓库数据验证失败")
|
||||
return None
|
||||
|
||||
processed_commits = self._process_commits(repo_data.get("recent_commits", []))
|
||||
repo_data["recent_commits"] = processed_commits
|
||||
|
||||
if repo_data.get("is_watched"):
|
||||
prompt_type = "watched"
|
||||
elif repo_data.get("is_update"):
|
||||
prompt_type = "update"
|
||||
else:
|
||||
prompt_type = "normal"
|
||||
|
||||
system_prompt = get_repo_system_prompt(prompt_type)
|
||||
user_prompt = build_repo_user_prompt(repo_data, processed_commits, prompt_type)
|
||||
raw_result = self.llm_client.generate_json(
|
||||
system_prompt=system_prompt,
|
||||
user_prompt=user_prompt,
|
||||
response_schema=SecurityAnalysisResult,
|
||||
)
|
||||
|
||||
if raw_result is None:
|
||||
logger.error("仓库 %s 分析失败，返回默认结果", repo_name)
|
||||
result = self._get_default_repo_result(repo_data)
|
||||
else:
|
||||
result = validate_repo_result(raw_result, repo_data, processed_commits)
|
||||
if not result:
|
||||
logger.error("仓库 %s 结果校验失败，返回默认结果", repo_name)
|
||||
result = self._get_default_repo_result(repo_data)
|
||||
|
||||
self._save_ai_response(result, "repository", repo_name or "unknown")
|
||||
|
||||
with get_session(self.session_factory) as session:
|
||||
repo_repo = RepositoryRepo(session)
|
||||
existing = repo_repo.get_by_name(repo_name)
|
||||
if existing:
|
||||
repo_repo.update_analysis(
|
||||
repo_name,
|
||||
{
|
||||
"ai_summary": self._safe_json_dumps(result),
|
||||
"is_valuable": bool(result.get("is_valuable", False)),
|
||||
"last_analyzed": datetime.now(),
|
||||
},
|
||||
)
|
||||
|
||||
if not result.get("is_valuable", False):
|
||||
self._record_worthless_repository(repo_name)
|
||||
|
||||
return result
|
||||
except Exception as exc:
|
||||
logger.error("分析仓库 %s 失败: %s", repo_data.get("name", "Unknown"), str(exc), exc_info=True)
|
||||
return None
|
||||
|
||||
def analyze_cve(self, cve_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""分析 CVE 数据。"""
|
||||
try:
|
||||
system_prompt = get_cve_system_prompt()
|
||||
user_prompt = get_cve_user_prompt(cve_data)
|
||||
|
||||
raw_result = self.llm_client.generate_json(
|
||||
system_prompt=system_prompt,
|
||||
user_prompt=user_prompt,
|
||||
response_schema=SecurityAnalysisResult,
|
||||
)
|
||||
if raw_result is None:
|
||||
logger.error("CVE %s 分析失败，返回默认结果", cve_data.get("cve_id"))
|
||||
result = self._get_default_cve_result(cve_data)
|
||||
else:
|
||||
result = validate_cve_result(raw_result, cve_data)
|
||||
if not result:
|
||||
logger.error("CVE %s 结果校验失败，返回默认结果", cve_data.get("cve_id"))
|
||||
result = self._get_default_cve_result(cve_data)
|
||||
|
||||
self._save_ai_response(result, "cve", str(cve_data.get("cve_id", "unknown")))
|
||||
|
||||
with get_session(self.session_factory) as session:
|
||||
cve_repo = CVERepo(session)
|
||||
cve_id = str(cve_data.get("cve_id", ""))
|
||||
if cve_id:
|
||||
existing = cve_repo.get_by_id(cve_id)
|
||||
if existing:
|
||||
cve_repo.update(
|
||||
cve_id,
|
||||
{
|
||||
"ai_summary": self._safe_json_dumps(result),
|
||||
"severity": result.get("risk_level", "MEDIUM"),
|
||||
"last_analyzed": datetime.now(),
|
||||
},
|
||||
)
|
||||
|
||||
return result
|
||||
except Exception as exc:
|
||||
logger.error("分析CVE失败: %s", str(exc), exc_info=True)
|
||||
return {
|
||||
"cve_id": cve_data.get("cve_id"),
|
||||
"is_valuable": False,
|
||||
"value_reason": f"分析过程出错: {str(exc)}",
|
||||
}
|
||||
|
||||
def _build_article_user_prompt(self, articles_batch: list[tuple[str, str]]) -> str:
|
||||
lines = ["请分析以下文章标题：", ""]
|
||||
for title, _ in articles_batch:
|
||||
lines.append(f"- {title}")
|
||||
return "\n".join(lines)
|
||||
|
||||
def _default_classify_articles(self, articles_batch: list[tuple[str, str]]) -> list[dict[str, str]]:
|
||||
rules = {
|
||||
"漏洞分析": ["漏洞", "cve", "poc", "exploit", "复现", "注入", "rce", "xss", "提权"],
|
||||
"威胁情报": ["apt", "恶意软件", "勒索", "攻击", "入侵", "僵尸网络", "黑客组织"],
|
||||
"安全工具": ["工具", "扫描", "检测", "审计", "nmap", "burp", "metasploit"],
|
||||
"安全研究": ["研究", "分析", "技术", "原理", "方法", "框架"],
|
||||
"最佳实践": ["防护", "配置", "指南", "教程", "实践", "建设"],
|
||||
"吃瓜新闻": ["事件", "新闻", "企业", "泄露", "被黑", "入侵"],
|
||||
}
|
||||
|
||||
classified = []
|
||||
for title, url in articles_batch:
|
||||
clean_title = self._clean_title_for_analysis(title).lower()
|
||||
category = "其他"
|
||||
max_matches = 0
|
||||
for current, keywords in rules.items():
|
||||
matches = sum(1 for keyword in keywords if keyword in clean_title)
|
||||
if matches > max_matches:
|
||||
max_matches = matches
|
||||
category = current
|
||||
classified.append({"title": title, "url": url, "category": category})
|
||||
return classified
|
||||
|
||||
def analyze_article_titles(self, articles: list[tuple[str, str]]) -> dict[str, Any]:
|
||||
"""分析文章标题并分类。"""
|
||||
if not articles:
|
||||
return {
|
||||
"filtered_articles": [],
|
||||
"categories": {},
|
||||
"stats": {"total": 0, "filtered": 0, "ad_count": 0},
|
||||
}
|
||||
|
||||
total_articles = len(articles)
|
||||
batch_size = 100
|
||||
all_results: list[dict[str, str]] = []
|
||||
|
||||
for start in range(0, total_articles, batch_size):
|
||||
batch = articles[start : start + batch_size]
|
||||
user_prompt = self._build_article_user_prompt(batch)
|
||||
title_to_url = {self._clean_title_for_analysis(title): url for title, url in batch}
|
||||
|
||||
raw_result = self.llm_client.generate_json(
|
||||
system_prompt=get_article_system_prompt(),
|
||||
user_prompt=user_prompt,
|
||||
response_schema=ArticleClassificationResult,
|
||||
)
|
||||
validated = validate_article_result(raw_result if isinstance(raw_result, dict) else {})
|
||||
|
||||
if not validated:
|
||||
all_results.extend(self._default_classify_articles(batch))
|
||||
continue
|
||||
|
||||
for item in validated.get("articles", []):
|
||||
title = self._clean_title_for_analysis(str(item.get("title", "")))
|
||||
category = str(item.get("category", "其他")).strip() or "其他"
|
||||
if not title:
|
||||
continue
|
||||
|
||||
url = title_to_url.get(title)
|
||||
if not url:
|
||||
for raw_title, raw_url in batch:
|
||||
if self._clean_title_for_analysis(raw_title) == title:
|
||||
url = raw_url
|
||||
break
|
||||
if not url:
|
||||
continue
|
||||
|
||||
all_results.append({"title": title, "url": url, "category": category})
|
||||
|
||||
ad_keywords = ["培训", "课程", "报名", "直播课", "优惠", "加群", "微信咨询", "认证班"]
|
||||
filtered_articles: list[dict[str, str]] = []
|
||||
categories_count: dict[str, int] = {}
|
||||
ad_count = 0
|
||||
|
||||
for article in all_results:
|
||||
title = article.get("title", "")
|
||||
category = article.get("category", "其他")
|
||||
if any(keyword in title for keyword in ad_keywords):
|
||||
ad_count += 1
|
||||
continue
|
||||
|
||||
filtered_articles.append(article)
|
||||
categories_count[category] = categories_count.get(category, 0) + 1
|
||||
|
||||
return {
|
||||
"filtered_articles": filtered_articles,
|
||||
"categories": categories_count,
|
||||
"stats": {
|
||||
"total": total_articles,
|
||||
"filtered": len(filtered_articles),
|
||||
"ad_count": ad_count,
|
||||
},
|
||||
}
|
||||
|
||||
def _save_ai_response(self, result: dict[str, Any], analysis_type: str, identifier: str) -> None:
|
||||
"""保存 AI 分析结果到按月分组的 JSON 文件。"""
|
||||
try:
|
||||
current_month = datetime.now().strftime("%Y_%m")
|
||||
base_dir = os.path.join(self.ai_analysis_dir, analysis_type, current_month)
|
||||
os.makedirs(base_dir, exist_ok=True)
|
||||
|
||||
valuable_file = os.path.join(base_dir, "valuable_analysis.json")
|
||||
worthless_file = os.path.join(base_dir, "worthless_analysis.json")
|
||||
|
||||
model_info = dict(self.llm_client.last_used_info)
|
||||
analysis_record = {
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"identifier": identifier,
|
||||
"model_info": model_info,
|
||||
"analysis_result": result,
|
||||
}
|
||||
|
||||
if analysis_type == "repository":
|
||||
search_keyword = result.get("search_keyword", "N/A")
|
||||
analysis_record["search_info"] = {
|
||||
"keyword": search_keyword,
|
||||
"discovery_method": "keyword_search" if search_keyword != "N/A" else "watched_repository",
|
||||
}
|
||||
|
||||
target_file = valuable_file if result.get("is_valuable", False) else worthless_file
|
||||
|
||||
records: list[dict[str, Any]] = []
|
||||
if os.path.exists(target_file):
|
||||
try:
|
||||
loaded = json.loads(open(target_file, "r", encoding="utf-8").read())
|
||||
if isinstance(loaded, list):
|
||||
records = loaded
|
||||
except Exception:
|
||||
records = []
|
||||
|
||||
records.append(analysis_record)
|
||||
with open(target_file, "w", encoding="utf-8") as f:
|
||||
json.dump(records, f, ensure_ascii=False, indent=2, default=str)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("保存AI分析结果失败: %s", str(exc), exc_info=True)
|
||||
|
||||
def _record_worthless_repository(self, repo_name: str) -> None:
|
||||
"""记录无价值仓库并按阈值更新动态黑名单。"""
|
||||
if not repo_name:
|
||||
return
|
||||
|
||||
stats_file = os.path.join(self.analysis_history_dir, "worthless_repository_stats.json")
|
||||
stats: dict[str, Any] = {"repos": {}, "owners": {}}
|
||||
|
||||
if os.path.exists(stats_file):
|
||||
try:
|
||||
loaded = json.loads(open(stats_file, "r", encoding="utf-8").read())
|
||||
if isinstance(loaded, dict):
|
||||
stats = loaded
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
repo_stats = stats.setdefault("repos", {})
|
||||
owner_stats = stats.setdefault("owners", {})
|
||||
|
||||
current = repo_stats.get(repo_name, {"count": 0, "last_updated": None})
|
||||
current["count"] = int(current.get("count", 0)) + 1
|
||||
current["last_updated"] = datetime.now().isoformat()
|
||||
repo_stats[repo_name] = current
|
||||
|
||||
owner = repo_name.split("/", 1)[0] if "/" in repo_name else ""
|
||||
if owner:
|
||||
owner_info = owner_stats.get(owner, {"repos": []})
|
||||
repos = set(owner_info.get("repos", []))
|
||||
repos.add(repo_name)
|
||||
owner_info["repos"] = sorted(repos)
|
||||
owner_stats[owner] = owner_info
|
||||
|
||||
try:
|
||||
with open(stats_file, "w", encoding="utf-8") as f:
|
||||
json.dump(stats, f, ensure_ascii=False, indent=2)
|
||||
except Exception:
|
||||
logger.warning("写入无价值仓库统计失败")
|
||||
|
||||
if int(current.get("count", 0)) >= 3:
|
||||
self.blacklist_manager.add_to_blacklist(repo_name, "repository", "Consecutive worthless analysis")
|
||||
|
||||
if owner:
|
||||
owner_repo_count = len(owner_stats.get(owner, {}).get("repos", []))
|
||||
if owner_repo_count >= 5:
|
||||
self.blacklist_manager.add_to_blacklist(owner, "user", "Multiple worthless repositories")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
analyzer = SecurityAnalyzer()
|
||||
print(analyzer is not None)
|
||||
@@ -0,0 +1,216 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
from config import BACKUP_AI_CONFIGS, GEMINI_AI_CONFIG, PRIMARY_AI_CONFIG
|
||||
from utils.logger import get_logger
|
||||
|
||||
logger = get_logger("ai.llm_client")
|
||||
|
||||
try:
|
||||
from google import genai
|
||||
except Exception: # pragma: no cover - 运行环境可选依赖
|
||||
genai = None
|
||||
|
||||
|
||||
class LLMClient:
|
||||
"""统一封装 Gemini/OpenAI 的调用入口。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
gemini_config: dict[str, Any] | None = None,
|
||||
primary_openai_config: dict[str, Any] | None = None,
|
||||
backup_openai_configs: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
self.gemini_config = gemini_config or GEMINI_AI_CONFIG
|
||||
self.primary_openai_config = primary_openai_config or PRIMARY_AI_CONFIG
|
||||
self.backup_openai_configs = backup_openai_configs or BACKUP_AI_CONFIGS
|
||||
|
||||
self._primary_client = self._build_openai_client(self.primary_openai_config)
|
||||
self._backup_clients: list[tuple[OpenAI, dict[str, Any]]] = []
|
||||
for cfg in self.backup_openai_configs:
|
||||
client = self._build_openai_client(cfg)
|
||||
if client is not None:
|
||||
self._backup_clients.append((client, cfg))
|
||||
|
||||
self.last_used_info: dict[str, Any] = {
|
||||
"provider": None,
|
||||
"model": None,
|
||||
"api_base": None,
|
||||
"is_backup": False,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _build_openai_client(config: dict[str, Any] | None) -> OpenAI | None:
|
||||
if not config:
|
||||
return None
|
||||
api_key = str(config.get("api_key", "")).strip()
|
||||
api_base = str(config.get("api_base", "")).strip()
|
||||
if not api_key or not api_base:
|
||||
return None
|
||||
return OpenAI(api_key=api_key, base_url=api_base)
|
||||
|
||||
@staticmethod
|
||||
def _extract_json(text: str) -> dict[str, Any] | list[Any] | None:
|
||||
if not text:
|
||||
return None
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return None
|
||||
try:
|
||||
return json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
def _set_last_used_info(
|
||||
self,
|
||||
*,
|
||||
provider: str | None,
|
||||
model: str | None,
|
||||
api_base: str | None,
|
||||
is_backup: bool,
|
||||
) -> None:
|
||||
self.last_used_info = {
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"api_base": api_base,
|
||||
"is_backup": is_backup,
|
||||
}
|
||||
|
||||
def _call_gemini(
|
||||
self,
|
||||
*,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
response_schema: type[Any] | None = None,
|
||||
) -> dict[str, Any] | list[Any] | None:
|
||||
api_key = str(self.gemini_config.get("api_key", "")).strip()
|
||||
model = str(self.gemini_config.get("model", "")).strip()
|
||||
base_url = str(self.gemini_config.get("base_url", "")).strip()
|
||||
if not api_key or not model or genai is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
http_options = {"base_url": base_url} if base_url else None
|
||||
client = genai.Client(api_key=api_key, http_options=http_options)
|
||||
content = f"系统提示:\n{system_prompt}\n\n用户提示:\n{user_prompt}"
|
||||
config: dict[str, Any] = {"response_mime_type": "application/json"}
|
||||
if response_schema is not None:
|
||||
config["response_schema"] = response_schema
|
||||
|
||||
response = client.models.generate_content(
|
||||
model=model,
|
||||
contents=content,
|
||||
config=config,
|
||||
)
|
||||
|
||||
parsed = getattr(response, "parsed", None)
|
||||
if parsed is not None:
|
||||
if hasattr(parsed, "model_dump"):
|
||||
result = parsed.model_dump()
|
||||
elif isinstance(parsed, (dict, list)):
|
||||
result = parsed
|
||||
else:
|
||||
result = None
|
||||
else:
|
||||
result = self._extract_json(getattr(response, "text", ""))
|
||||
|
||||
if result is not None:
|
||||
self._set_last_used_info(
|
||||
provider="gemini",
|
||||
model=model,
|
||||
api_base=base_url or None,
|
||||
is_backup=False,
|
||||
)
|
||||
return result
|
||||
except Exception as exc:
|
||||
logger.warning("Gemini 调用失败: %s", str(exc))
|
||||
return None
|
||||
|
||||
def _call_openai_client(
|
||||
self,
|
||||
client: OpenAI,
|
||||
config: dict[str, Any],
|
||||
*,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
is_backup: bool,
|
||||
) -> dict[str, Any] | list[Any] | None:
|
||||
model = str(config.get("model", "")).strip()
|
||||
if not model:
|
||||
return None
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
response_format={"type": "json_object"},
|
||||
)
|
||||
content = ""
|
||||
if response and response.choices:
|
||||
message = response.choices[0].message
|
||||
content = message.content if message else ""
|
||||
result = self._extract_json(content)
|
||||
if result is not None:
|
||||
self._set_last_used_info(
|
||||
provider="openai",
|
||||
model=model,
|
||||
api_base=str(config.get("api_base", "")).strip() or None,
|
||||
is_backup=is_backup,
|
||||
)
|
||||
return result
|
||||
|
||||
def generate_json(
|
||||
self,
|
||||
*,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
response_schema: type[Any] | None = None,
|
||||
) -> dict[str, Any] | list[Any] | None:
|
||||
"""按优先级调用 LLM：Gemini -> 主 OpenAI -> 备用 OpenAI。"""
|
||||
self._set_last_used_info(provider=None, model=None, api_base=None, is_backup=False)
|
||||
|
||||
result = self._call_gemini(
|
||||
system_prompt=system_prompt,
|
||||
user_prompt=user_prompt,
|
||||
response_schema=response_schema,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
if self._primary_client is not None:
|
||||
try:
|
||||
result = self._call_openai_client(
|
||||
self._primary_client,
|
||||
self.primary_openai_config,
|
||||
system_prompt=system_prompt,
|
||||
user_prompt=user_prompt,
|
||||
is_backup=False,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
except Exception as exc:
|
||||
logger.warning("主 OpenAI 调用失败: %s", str(exc))
|
||||
|
||||
for idx, (client, cfg) in enumerate(self._backup_clients, start=1):
|
||||
try:
|
||||
result = self._call_openai_client(
|
||||
client,
|
||||
cfg,
|
||||
system_prompt=system_prompt,
|
||||
user_prompt=user_prompt,
|
||||
is_backup=True,
|
||||
)
|
||||
if result is not None:
|
||||
logger.info("已切换到备用 OpenAI 配置 #%s", idx)
|
||||
return result
|
||||
except Exception as exc:
|
||||
logger.warning("备用 OpenAI #%s 调用失败: %s", idx, str(exc))
|
||||
|
||||
return None
|
||||
+531
@@ -0,0 +1,531 @@
|
||||
import json
|
||||
from textwrap import dedent
|
||||
|
||||
|
||||
def get_repo_system_prompt(prompt_type: str) -> str:
|
||||
"""获取仓库分析系统提示词
|
||||
|
||||
Args:
|
||||
prompt_type: 'watched' | 'update' | 'normal'
|
||||
"""
|
||||
prompts = {
|
||||
"watched": dedent(
|
||||
"""
|
||||
你是资深网络安全专家，拥有10年+漏洞研究和威胁分析经验。专注于监控高价值安全仓库的动态变更。
|
||||
请用专业、精准的视角分析此监控仓库的最新更新。
|
||||
|
||||
**重要要求：所有分析结果必须使用中文输出，包括所有字段内容。**
|
||||
|
||||
## 🎯 严格筛选标准
|
||||
|
||||
### 1. 技术价值评估（必须满足其一，否则 is_valuable = false）
|
||||
**高价值更新（CRITICAL/HIGH）**：
|
||||
- 新增0day/1day漏洞利用代码，包含完整攻击链
|
||||
- 突破性绕过技术（如新型EDR/沙箱逃逸）
|
||||
- 重大安全工具功能升级（影响攻防格局）
|
||||
- 关键安全漏洞修复（影响广泛使用组件）
|
||||
|
||||
**中等价值更新（MEDIUM）**：
|
||||
- 现有POC的改进优化，提升成功率
|
||||
- 增强的检测规避能力
|
||||
- 扩展攻击面的新模块
|
||||
|
||||
**无价值内容（直接过滤）**：
|
||||
- 纯文档/README更新、翻译
|
||||
- 代码格式化、重构、注释修改
|
||||
- 依赖版本升级、CI/CD配置
|
||||
- 测试用例增删、LOGO更换
|
||||
- 自动生成的IP/域名黑名单更新
|
||||
|
||||
### 2. 实战可用性评估
|
||||
- 代码完整性：能否直接编译运行
|
||||
- 利用稳定性：成功率和重现性
|
||||
- 环境要求：部署难度和兼容性
|
||||
- 检测难度：被防护软件发现的概率
|
||||
|
||||
### 3. 影响范围分析
|
||||
- 目标广泛性：影响主流系统/组件数量
|
||||
- 用户基数：受影响的用户规模
|
||||
- 业务关键性：对关键基础设施的影响
|
||||
|
||||
## 📋 精准输出格式
|
||||
|
||||
{
|
||||
"short_description": "核心功能描述(≤15字，突出技术特点)",
|
||||
"summary": "详细分析更新内容的安全价值和技术改进(≤150字)",
|
||||
"risk_level": "CRITICAL/HIGH/MEDIUM/LOW",
|
||||
"key_points": [
|
||||
"技术核心：具体技术突破点",
|
||||
"安全影响：对攻防的实际影响",
|
||||
"应用价值：实战应用场景"
|
||||
],
|
||||
"technical_details": [
|
||||
"实现机制：核心技术原理分析",
|
||||
"改进要点：相比之前版本的提升",
|
||||
"利用条件：成功利用的前置条件"
|
||||
],
|
||||
"affected_components": [
|
||||
"具体受影响的系统/软件/版本"
|
||||
],
|
||||
"is_valuable": true/false,
|
||||
"value_reason": "基于技术价值、实战可用性、影响范围的综合评估(≤80字)",
|
||||
"security_type": "0day利用/1day利用/绕过技术/安全工具/防护增强",
|
||||
"update_type": "SECURITY_CRITICAL/SECURITY_IMPROVEMENT/GENERAL_UPDATE"
|
||||
}
|
||||
|
||||
**关键原则**：宁缺毋滥，只推送真正有价值的安全更新。每个字段必须包含实质性信息，避免空泛描述。
|
||||
"""
|
||||
).strip(),
|
||||
"update": dedent(
|
||||
"""
|
||||
你是经验丰富的安全工程师，专注于评估安全仓库更新的实战价值。
|
||||
请用严谨的技术视角分析此仓库的最新更新。
|
||||
|
||||
**重要要求：所有分析结果必须使用中文输出，包括所有字段内容。**
|
||||
|
||||
## 🔄 仓库更新价值量化
|
||||
|
||||
### 1. 功能价值评估（必须满足其一，否则 is_valuable = false）
|
||||
**突破性更新（CRITICAL/HIGH）**：
|
||||
- 重大功能突破：全新攻击向量或检测能力
|
||||
- 架构升级：性能/稳定性显著提升
|
||||
- 兼容性扩展：支持新系统/版本/环境
|
||||
- 关键Bug修复：影响核心功能的严重问题
|
||||
|
||||
**改进性更新（MEDIUM）**：
|
||||
- 功能增强：现有能力的改进优化
|
||||
- 稳定性提升：减少崩溃或提高成功率
|
||||
- 用户体验：操作流程优化
|
||||
|
||||
**无价值更新（直接过滤）**：
|
||||
- 纯依赖升级：package.json、requirements.txt更新
|
||||
- 代码整理：格式化、lint修复、重构
|
||||
- 文档维护：README、注释、翻译
|
||||
- 配置调整：CI/CD、编译配置
|
||||
- 自动化数据：威胁情报列表、IP黑名单
|
||||
|
||||
### 2. 安全影响力评估
|
||||
**实战导向**：
|
||||
- 直接可用：生产环境可立即部署
|
||||
- 攻击效果：对目标系统的实际影响
|
||||
- 防护绕过：突破现有安全防线的能力
|
||||
- 检测规避：避免被监控系统发现
|
||||
|
||||
### 3. 代码质量分析
|
||||
**必须检查**：
|
||||
- 代码完整性：关键模块是否齐全
|
||||
- 错误处理：异常情况的处理机制
|
||||
- 安全考虑：是否存在安全隐患
|
||||
- 文档说明：使用方法是否明确
|
||||
|
||||
## 📋 结构化输出格式
|
||||
|
||||
{
|
||||
"short_description": "核心功能概述(≤15字，体现技术特色)",
|
||||
"summary": "仓库功能简介+本次更新的具体改进内容分析(≤180字)",
|
||||
"risk_level": "CRITICAL/HIGH/MEDIUM/LOW",
|
||||
"key_points": [
|
||||
"功能定位：仓库的核心能力和应用场景",
|
||||
"更新亮点：本次更新的主要技术改进",
|
||||
"安全价值：对实际安全工作的帮助",
|
||||
"应用建议：推荐的使用方式或注意事项"
|
||||
],
|
||||
"technical_details": [
|
||||
"技术架构：核心实现原理和技术栈",
|
||||
"改进机制：具体的技术改进点分析",
|
||||
"部署要求：运行环境和依赖条件"
|
||||
],
|
||||
"affected_components": [
|
||||
"涉及的具体技术组件、系统或协议"
|
||||
],
|
||||
"is_valuable": true/false,
|
||||
"value_reason": "基于功能创新、安全影响、代码质量的综合评估(≤90字)",
|
||||
"security_type": "攻击工具/防护工具/分析工具/研究框架/漏洞利用",
|
||||
"update_type": "SECURITY_CRITICAL/SECURITY_IMPROVEMENT/GENERAL_UPDATE"
|
||||
}
|
||||
|
||||
**评估重点**：关注增量价值，重点分析相比之前版本的实质性提升。只推送对安全工作有实际帮助的更新。
|
||||
"""
|
||||
).strip(),
|
||||
"normal": dedent(
|
||||
"""
|
||||
你是顶级网络安全研究员，拥有15年+红队实战和漏洞挖掘经验。
|
||||
请用极其严格的标准评估这个新发现的安全仓库，确保只推荐真正有价值的项目。
|
||||
|
||||
**重要要求：所有分析结果必须使用中文输出，包括所有字段内容。**
|
||||
|
||||
## 🎯 严苛评估框架（总分100分，85分以上才有价值）
|
||||
|
||||
### 1. 核心价值评估（50分，必须≥40分）
|
||||
**顶级项目（45-50分）**：
|
||||
- 原创0day/1day利用代码，包含完整攻击链
|
||||
- 突破性攻击技术，改变现有攻防格局
|
||||
- 系统性漏洞研究，具有重大理论突破
|
||||
- 影响广泛的安全工具，解决实际痛点
|
||||
|
||||
**优质项目（35-44分）**：
|
||||
- 高质量POC集合，覆盖多个漏洞类型
|
||||
- 创新的攻击方法或绕过技术
|
||||
- 实用的安全检测/分析工具
|
||||
- 深度的技术研究和方法论
|
||||
|
||||
**低价值项目（<35分，直接过滤）**：
|
||||
- 简单脚本集合，无技术含量
|
||||
- 过时的攻击技术或工具
|
||||
- 纯文档项目，缺乏代码实现
|
||||
- 基础功能工具，无创新点
|
||||
|
||||
### 2. 技术质量评估（25分，必须≥20分）
|
||||
**代码质量**：
|
||||
- 架构设计：模块化、可扩展性
|
||||
- 代码规范：注释、错误处理
|
||||
- 功能完整性：核心功能是否齐全
|
||||
- 实际可用性：能否直接运行部署
|
||||
|
||||
**创新程度**：
|
||||
- 技术新颖度：是否为首创或重大改进
|
||||
- 解决方案：是否解决了现有痛点
|
||||
- 实现难度：技术门槛和复杂度
|
||||
|
||||
### 3. 实战价值评估（15分，必须≥12分）
|
||||
**应用场景**：
|
||||
- 攻击效果：对目标的实际影响
|
||||
- 使用频率：在实战中的应用价值
|
||||
- 环境适配：支持的系统和版本范围
|
||||
|
||||
**影响范围**：
|
||||
- 目标广泛性：影响的系统/组件数量
|
||||
- 用户基数：潜在受影响的用户规模
|
||||
|
||||
### 4. 关键词相关性（10分，必须≥8分）
|
||||
**相关性评估**：
|
||||
- 核心功能与关键词匹配度（5分）
|
||||
- 实际应用场景契合度（3分）
|
||||
- 项目描述和文档相关性（2分）
|
||||
|
||||
## 🚫 严格排除标准（出现即判定为无价值）
|
||||
|
||||
**技术门槛不足**：
|
||||
- 简单的脚本集合，无复杂逻辑
|
||||
- 仅调用现有工具的包装脚本
|
||||
- 缺乏核心算法或创新实现
|
||||
|
||||
**项目质量问题**：
|
||||
- 6个月以上无更新的废弃项目
|
||||
- 代码结构混乱，缺乏基本规范
|
||||
- 功能不完整，仅有框架代码
|
||||
|
||||
**安全性问题**：
|
||||
- 明显的教学练习项目
|
||||
- 重复的已有项目或简单fork
|
||||
- 恶意软件或后门程序
|
||||
|
||||
**内容质量问题**：
|
||||
- 自动生成的数据列表（IP黑名单等）
|
||||
- 纯翻译或转载的文档项目
|
||||
- 无实质内容的空仓库
|
||||
|
||||
## 📋 精炼输出格式
|
||||
|
||||
{
|
||||
"short_description": "精准功能定位(≤12字，突出核心价值)",
|
||||
"summary": "全面评估：仓库核心能力+技术特点+实战价值+创新点(≤200字)",
|
||||
"risk_level": "CRITICAL/HIGH/MEDIUM/LOW",
|
||||
"key_points": [
|
||||
"技术亮点：核心技术突破和创新点",
|
||||
"实战价值：在攻防中的具体应用场景",
|
||||
"质量评估：代码质量和可用性分析",
|
||||
"相关性评估：与搜索关键词的精确匹配度"
|
||||
],
|
||||
"technical_details": [
|
||||
"核心技术：关键算法或实现原理深度分析",
|
||||
"架构设计：系统架构和技术栈评估",
|
||||
"创新评估：相比同类项目的突出优势"
|
||||
],
|
||||
"affected_components": [
|
||||
"精确的目标系统/软件/协议/版本范围"
|
||||
],
|
||||
"is_valuable": true/false,
|
||||
"value_reason": "基于核心价值、技术质量、实战价值、相关性的综合评分和详细理由(≤120字)",
|
||||
"security_type": "漏洞利用/渗透工具/防护工具/研究框架/威胁分析",
|
||||
"relevance_score": "HIGH/MEDIUM/LOW"
|
||||
}
|
||||
|
||||
**铁律**：宁可错过100个普通项目，也不放过1个真正的高价值项目。每个评分维度都必须达标，任一不符合标准即判定为无价值。
|
||||
"""
|
||||
).strip(),
|
||||
}
|
||||
return prompts.get(prompt_type, prompts["normal"])
|
||||
|
||||
|
||||
def get_cve_system_prompt() -> str:
|
||||
"""获取 CVE 分析系统提示词"""
|
||||
return dedent(
|
||||
"""
|
||||
你是首席漏洞研究专家，拥有深厚的0day挖掘和威胁分析经验。
|
||||
请用极其严格的标准评估这个CVE漏洞的实战威胁价值。
|
||||
|
||||
**重要要求：所有分析结果必须使用中文输出，包括所有字段内容。**
|
||||
|
||||
## 🎯 CVE威胁价值评估（总分100分，≥80分才有关注价值）
|
||||
|
||||
### 1. 影响范围权重（35分，必须≥28分）
|
||||
**广泛影响（30-35分）**：
|
||||
- 全球广泛使用的基础组件（操作系统、Web服务器、数据库）
|
||||
- 关键基础设施软件（网络设备、工控系统）
|
||||
- 主流开发框架和中间件
|
||||
|
||||
**特定影响（20-29分）**：
|
||||
- 行业特定的重要软件
|
||||
- 企业级应用和服务
|
||||
- 流行的开源项目
|
||||
|
||||
**有限影响（<20分，直接过滤）**：
|
||||
- 小众软件或过时产品
|
||||
- 测试工具或学习项目
|
||||
- 已停止维护的软件
|
||||
|
||||
### 2. 利用难度权重（30分，必须≥24分）
|
||||
**极易利用（25-30分）**：
|
||||
- 一键利用，无需特殊条件
|
||||
- 现成工具和脚本可直接使用
|
||||
- 网络可达，无需本地访问
|
||||
|
||||
**可利用（18-24分）**：
|
||||
- 需要简单脚本，技术门槛低
|
||||
- 条件相对宽松，容易满足
|
||||
- 有详细的技术文档
|
||||
|
||||
**困难利用（<18分，价值有限）**：
|
||||
- 复杂的利用条件
|
||||
- 需要深度技术知识
|
||||
- 成功率不稳定
|
||||
|
||||
### 3. 危害程度权重（25分，必须≥20分）
|
||||
**严重危害（20-25分）**：
|
||||
- 远程代码执行，完全控制系统
|
||||
- 权限提升至系统管理员
|
||||
- 绕过所有安全防护
|
||||
|
||||
**中等危害（15-19分）**：
|
||||
- 敏感信息泄露，影响业务安全
|
||||
- 部分权限提升或功能绕过
|
||||
- 拒绝服务，影响业务可用性
|
||||
|
||||
### 4. 时效性权重（10分，必须≥8分）
|
||||
**高时效性（8-10分）**：
|
||||
- 0day漏洞，无补丁可用
|
||||
- 1day漏洞，补丁覆盖率低
|
||||
- 新发现的攻击面
|
||||
|
||||
## 🚫 严格排除标准
|
||||
|
||||
**技术门槛过低**：
|
||||
- 仅DoS漏洞，无其他危害
|
||||
- 需要物理接触的漏洞
|
||||
- 过于复杂的利用链
|
||||
|
||||
**影响有限**：
|
||||
- 无具体受影响版本信息
|
||||
- 描述模糊，缺乏技术细节
|
||||
- 已修复且广泛更新的漏洞
|
||||
|
||||
**代码质量问题**：
|
||||
- POC不完整或无法运行
|
||||
- 缺乏验证的概念性漏洞
|
||||
- 明显的误报或重复
|
||||
|
||||
## 📋 威胁报告格式
|
||||
|
||||
{
|
||||
"cve_id": "CVE编号",
|
||||
"short_description": "漏洞产品+类型(≤15字，如：Apache Struts2 RCE)",
|
||||
"summary": "漏洞影响和利用要点的深度分析，包括技术原理和实战威胁(≤200字)",
|
||||
"risk_level": "CRITICAL/HIGH/MEDIUM/LOW",
|
||||
"exploit_status": "EXP可用/POC可用/理论可行/条件苛刻/无法利用",
|
||||
"key_points": [
|
||||
"漏洞机制：核心漏洞原理和触发条件",
|
||||
"攻击条件：成功利用的具体要求",
|
||||
"威胁影响：对目标系统的实际危害",
|
||||
"防护状态：补丁发布和修复情况"
|
||||
],
|
||||
"technical_details": [
|
||||
"漏洞原理：深度技术分析和成因机制",
|
||||
"利用方法：具体的攻击步骤和技术细节",
|
||||
"修复方案：官方补丁和缓解措施评估"
|
||||
],
|
||||
"affected_components": [
|
||||
"具体受影响的产品名称和版本范围"
|
||||
],
|
||||
"is_valuable": true/false,
|
||||
"value_reason": "基于影响范围、利用难度、危害程度、时效性的综合威胁评估(≤100字)",
|
||||
"code_analysis": [
|
||||
"POC/EXP可用性：代码完整性和执行效果",
|
||||
"攻击成功率：在不同环境下的成功概率",
|
||||
"检测规避：绕过安全防护的能力"
|
||||
]
|
||||
}
|
||||
|
||||
**评估铁律**：从攻击者视角评估利用价值，从防护者视角评估威胁影响。只推送真正具有威胁价值的CVE。
|
||||
"""
|
||||
).strip()
|
||||
|
||||
|
||||
def get_cve_user_prompt(cve_data: dict) -> str:
|
||||
"""构建 CVE 分析用户提示词"""
|
||||
user_prompt = dedent(
|
||||
f"""
|
||||
CVE信息:
|
||||
- CVE编号: {cve_data.get('cve_id')}
|
||||
- 描述: {cve_data.get('description')}
|
||||
- 发布时间: {cve_data.get('published_date')}
|
||||
- 最后修改: {cve_data.get('last_modified')}
|
||||
- 相关仓库: {cve_data.get('repository_url')}
|
||||
- Star数: {cve_data.get('stars')}
|
||||
|
||||
最新提交信息:
|
||||
"""
|
||||
).strip()
|
||||
|
||||
recent_commits = cve_data.get("recent_commits", [])
|
||||
if recent_commits:
|
||||
for commit in recent_commits:
|
||||
user_prompt += dedent(
|
||||
f"""
|
||||
|
||||
提交: {commit.get('message')}
|
||||
作者: {commit.get('author')}
|
||||
时间: {commit.get('date')}
|
||||
修改文件:
|
||||
"""
|
||||
)
|
||||
for file in commit.get("files", []):
|
||||
user_prompt += dedent(
|
||||
f"""
|
||||
- {file.get('path')} ({file.get('status')}, +{file.get('additions')}/-{file.get('deletions')})
|
||||
变更内容:
|
||||
{file.get('patch', '无变更内容')}
|
||||
"""
|
||||
)
|
||||
else:
|
||||
user_prompt += "\n无最新提交信息"
|
||||
return user_prompt
|
||||
|
||||
|
||||
def get_article_system_prompt() -> str:
|
||||
"""获取文章分类系统提示词"""
|
||||
return dedent(
|
||||
"""
|
||||
你是一个安全研究专家。请对以下安全文章标题进行分类，如果标题是课程，培训，推广，营销，直接过滤掉。
|
||||
|
||||
**重要要求：所有分类结果必须使用中文输出。**
|
||||
|
||||
分类规则：
|
||||
1. 将文章分为以下类别（必须严格使用这些分类名称）：
|
||||
- 漏洞分析：漏洞分析、POC、漏洞利用、漏洞复现、CVE、补丁分析等
|
||||
- 安全研究：技术研究、漏洞挖掘、攻防对抗、安全原理、代码审计等
|
||||
- 威胁情报：APT活动、攻击分析、恶意软件、威胁动态、黑客组织等
|
||||
- 安全工具：工具发布、使用教程、工具分享、自动化工具等
|
||||
- 最佳实践：防护方案、配置指南、安全建设、技术教程、最佳实践等
|
||||
- 吃瓜新闻：安全事件、行业动态、政策法规、企业新闻等
|
||||
- 其他：不属于上述类别的安全内容
|
||||
|
||||
2. 分类原则：
|
||||
- 根据文章主要内容和核心主题进行分类
|
||||
- 有漏洞相关内容优先归为"漏洞分析"
|
||||
- 有APT或恶意软件相关优先归为"威胁情报"
|
||||
- 有工具相关内容优先归为"安全工具"
|
||||
- 有教程指南相关优先归为"最佳实践"
|
||||
- 新闻事件相关归为"吃瓜新闻"
|
||||
- 研究性或理论性内容归为"安全研究"
|
||||
- 其他内容归为"其他"
|
||||
|
||||
注意：分类名称必须完全匹配，不能有多余的空格或变体。
|
||||
|
||||
请以JSON格式输出，格式如下：
|
||||
{
|
||||
"articles": [
|
||||
{
|
||||
"title": "原文标题",
|
||||
"category": "分类名称"
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
).strip()
|
||||
|
||||
|
||||
def build_repo_user_prompt(repo_data: dict, processed_commits: list, prompt_type: str) -> str:
|
||||
"""构建仓库分析用户提示词"""
|
||||
prompt = dedent(
|
||||
f"""
|
||||
仓库名称: {repo_data.get('name', 'N/A')}
|
||||
描述: {repo_data.get('description', 'N/A')}
|
||||
"""
|
||||
).strip()
|
||||
|
||||
if repo_data.get("search_keyword"):
|
||||
prompt += f"\n搜索关键词: {repo_data['search_keyword']}"
|
||||
|
||||
if repo_data.get("url"):
|
||||
prompt += f"\n仓库URL: {repo_data['url']}"
|
||||
|
||||
prompt += "\n\n提交历史:"
|
||||
total_patch_length = 0
|
||||
max_patch_length = 30000
|
||||
max_single_patch = 5000
|
||||
|
||||
for commit in processed_commits:
|
||||
prompt += f"\n- {commit['message']} (作者: {commit['author']}, 时间: {commit['date']})"
|
||||
if commit["files"]:
|
||||
prompt += "\n 修改的文件:"
|
||||
for file in commit["files"]:
|
||||
prompt += f"\n * {file['path']} ({file['status']}, +{file['additions']}/-{file['deletions']})"
|
||||
if file["patch"]:
|
||||
patch = file["patch"]
|
||||
is_security_file = any(
|
||||
keyword in file["path"].lower()
|
||||
for keyword in ["poc", "exp", "exploit", "payload", "vuln", "cve", "security", "hack"]
|
||||
)
|
||||
max_patch = max_single_patch * 2 if is_security_file else max_single_patch
|
||||
if len(patch) > max_patch:
|
||||
patch = patch[:max_patch] + "\n... (内容已截断)"
|
||||
|
||||
if total_patch_length + len(patch) > max_patch_length:
|
||||
prompt += "\n (后续变更内容已省略以适应长度限制)"
|
||||
break
|
||||
|
||||
prompt += f"\n 变更内容:\n{patch}"
|
||||
total_patch_length += len(patch)
|
||||
|
||||
if total_patch_length >= max_patch_length:
|
||||
prompt += "\n(后续提交的变更内容已省略以适应长度限制)"
|
||||
break
|
||||
|
||||
if prompt_type == "normal" and repo_data.get("readme"):
|
||||
readme_content = repo_data["readme"]
|
||||
if len(readme_content) > 5000:
|
||||
readme_content = readme_content[:5000] + "\n... (README内容已截断)"
|
||||
prompt += f"\n\nREADME内容:\n{readme_content}"
|
||||
|
||||
if prompt_type == "normal":
|
||||
repo_info = {
|
||||
"language": repo_data.get("language"),
|
||||
"topics": repo_data.get("topics", []),
|
||||
"stars": repo_data.get("stars", 0),
|
||||
"created_at": repo_data.get("created_at"),
|
||||
"size": repo_data.get("size"),
|
||||
"forks_count": repo_data.get("forks_count", 0),
|
||||
}
|
||||
prompt += f"\n\n仓库信息:\n{json.dumps(repo_info, indent=2, ensure_ascii=False)}"
|
||||
|
||||
if prompt_type == "normal" and repo_data.get("search_keyword"):
|
||||
prompt = dedent(
|
||||
f"""
|
||||
搜索关键词: {repo_data['search_keyword']}
|
||||
请特别关注仓库与该关键词的相关性。
|
||||
|
||||
{prompt}
|
||||
"""
|
||||
).strip()
|
||||
return prompt
|
||||
@@ -0,0 +1,235 @@
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from utils.logger import get_logger
|
||||
|
||||
|
||||
logger = get_logger("analyzer")
|
||||
|
||||
|
||||
def _normalize_common_result(result: dict, source_type: str) -> dict | None:
|
||||
"""规范化分析结果并校验关键字段"""
|
||||
try:
|
||||
if not result:
|
||||
logger.error("分析结果为空")
|
||||
return None
|
||||
|
||||
required_fields: dict[str, type] = {
|
||||
"summary": str,
|
||||
"risk_level": str,
|
||||
"key_points": list,
|
||||
"technical_details": list,
|
||||
"affected_components": list,
|
||||
"is_valuable": bool,
|
||||
"value_reason": str,
|
||||
}
|
||||
|
||||
if source_type == "cve":
|
||||
required_fields.update({"cve_id": str, "exploit_status": str})
|
||||
|
||||
for field, field_type in required_fields.items():
|
||||
if field not in result:
|
||||
logger.error(f"分析结果缺少必需字段: {field}")
|
||||
return None
|
||||
if not isinstance(result[field], field_type):
|
||||
logger.error(
|
||||
f"字段 {field} 的类型错误，期望 {field_type.__name__}，实际为 {type(result[field]).__name__}"
|
||||
)
|
||||
return None
|
||||
|
||||
valid_risk_levels = {"CRITICAL", "HIGH", "MEDIUM", "LOW"}
|
||||
risk_level = result["risk_level"].upper()
|
||||
if risk_level not in valid_risk_levels:
|
||||
logger.error(f"无效的风险等级: {risk_level}，使用默认值 MEDIUM")
|
||||
risk_level = "MEDIUM"
|
||||
result["risk_level"] = risk_level
|
||||
|
||||
if source_type == "repository" and "security_type" in result:
|
||||
valid_security_types = {
|
||||
"POC收集",
|
||||
"漏洞利用框架",
|
||||
"安全工具",
|
||||
"安全研究",
|
||||
"POC更新",
|
||||
"漏洞利用",
|
||||
"安全修复",
|
||||
"安全功能",
|
||||
}
|
||||
security_type = result["security_type"]
|
||||
if security_type not in valid_security_types:
|
||||
logger.warning(f"无效的安全类型: {security_type}，使用默认值 安全研究")
|
||||
security_type = "安全研究"
|
||||
result["security_type"] = security_type
|
||||
|
||||
if source_type == "cve":
|
||||
valid_exploit_status = {"POC可用", "漏洞利用可用", "理论可行", "未知"}
|
||||
exploit_status = result["exploit_status"]
|
||||
if exploit_status not in valid_exploit_status:
|
||||
logger.warning(f"无效的利用状态: {exploit_status}，使用默认值 未知")
|
||||
exploit_status = "未知"
|
||||
result["exploit_status"] = exploit_status
|
||||
|
||||
for field in ["key_points", "technical_details", "affected_components"]:
|
||||
if not result[field] or not any(item for item in result[field] if item and item != "Unknown"):
|
||||
logger.error(f"字段 {field} 为空或仅包含无效值")
|
||||
return None
|
||||
|
||||
result["is_valuable"] = bool(result.get("is_valuable", False))
|
||||
if not result.get("value_reason"):
|
||||
result["value_reason"] = "未提供价值说明"
|
||||
|
||||
if source_type == "repository":
|
||||
# watched/update 仓库不依赖 relevance_score 判定价值
|
||||
is_watched = result.get("is_watched", False)
|
||||
is_update = result.get("is_update", False)
|
||||
|
||||
relevance_score = result.get("relevance_score", "LOW")
|
||||
if relevance_score not in ["HIGH", "MEDIUM", "LOW"]:
|
||||
relevance_score = "LOW"
|
||||
result["relevance_score"] = relevance_score
|
||||
|
||||
if not is_watched and not is_update and relevance_score == "LOW":
|
||||
result["is_valuable"] = False
|
||||
result["value_reason"] = f"与搜索关键词 '{result.get('search_keyword', 'N/A')}' 相关性不足"
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"验证分析结果时出错: {str(e)}")
|
||||
return None
|
||||
|
||||
|
||||
def _process_repo_analysis_result(result: dict | str, repo_data: dict, processed_commits: list) -> dict | None:
|
||||
"""清理并补齐仓库分析结果"""
|
||||
try:
|
||||
if not result:
|
||||
logger.error("分析结果为空")
|
||||
return None
|
||||
|
||||
if isinstance(result, str):
|
||||
logger.info("解析JSON字符串结果")
|
||||
result = json.loads(result)
|
||||
|
||||
cleaned_result: dict[str, Any] = {}
|
||||
for key, value in result.items():
|
||||
if value is None or value == "Unknown":
|
||||
continue
|
||||
if isinstance(value, list):
|
||||
cleaned_list = [item for item in value if item and item != "Unknown"]
|
||||
if cleaned_list:
|
||||
cleaned_result[key] = cleaned_list
|
||||
else:
|
||||
cleaned_result[key] = value
|
||||
|
||||
repo_name = repo_data.get("name", "unknown")
|
||||
if "/" in repo_name:
|
||||
repo_name = repo_name.split("/")[-1]
|
||||
|
||||
if not cleaned_result.get("short_description"):
|
||||
description = str(repo_data.get("description", "")).strip()
|
||||
if description:
|
||||
short_desc = description[:20] + ("..." if len(description) > 20 else "")
|
||||
else:
|
||||
security_type = cleaned_result.get("security_type", "安全工具")
|
||||
if "POC" in security_type or "poc" in security_type.lower():
|
||||
short_desc = "漏洞利用验证工具"
|
||||
elif "漏洞利用" in security_type:
|
||||
short_desc = "漏洞利用框架"
|
||||
elif "安全工具" in security_type:
|
||||
short_desc = "网络安全工具集"
|
||||
elif "安全研究" in security_type:
|
||||
short_desc = "安全研究项目"
|
||||
else:
|
||||
short_desc = "安全相关项目"
|
||||
cleaned_result["short_description"] = short_desc
|
||||
|
||||
cleaned_result.update(
|
||||
{
|
||||
"repository_name": repo_name,
|
||||
"repository_url": repo_data.get("url"),
|
||||
"commits_analyzed": len(processed_commits),
|
||||
"files_changed": sum(len(commit.get("files", [])) for commit in processed_commits),
|
||||
"is_watched": repo_data.get("is_watched", False),
|
||||
"is_update": repo_data.get("is_update", False),
|
||||
"search_keyword": repo_data.get("search_keyword", "N/A"),
|
||||
}
|
||||
)
|
||||
|
||||
if not cleaned_result.get("summary") and cleaned_result.get("one_line_summary"):
|
||||
cleaned_result["summary"] = cleaned_result["one_line_summary"]
|
||||
|
||||
return cleaned_result
|
||||
except Exception as e:
|
||||
logger.error(f"处理分析结果时出错: {str(e)}")
|
||||
return None
|
||||
|
||||
|
||||
def validate_cve_result(result: dict, cve_data: dict) -> dict | None:
|
||||
"""验证并增强 CVE 分析结果"""
|
||||
if not result:
|
||||
return None
|
||||
|
||||
result.update(
|
||||
{
|
||||
"cve_id": cve_data.get("cve_id"),
|
||||
"repository_url": cve_data.get("repository_url"),
|
||||
"published_date": cve_data.get("published_date"),
|
||||
"last_modified": cve_data.get("last_modified"),
|
||||
"stars": cve_data.get("stars", 0),
|
||||
}
|
||||
)
|
||||
|
||||
normalized_result = _normalize_common_result(result, "cve")
|
||||
if not normalized_result:
|
||||
return None
|
||||
|
||||
if not normalized_result.get("is_valuable", False):
|
||||
normalized_result["value_reason"] = "不满足推送条件"
|
||||
logger.info(f"CVE {normalized_result['cve_id']} 不满足推送条件")
|
||||
elif normalized_result["risk_level"] not in ["CRITICAL", "HIGH"]:
|
||||
normalized_result["is_valuable"] = False
|
||||
normalized_result["value_reason"] = f"风险等级较低 ({normalized_result['risk_level']})"
|
||||
logger.info(f"CVE {normalized_result['cve_id']} 风险等级较低")
|
||||
elif normalized_result["exploit_status"] == "未知":
|
||||
normalized_result["is_valuable"] = False
|
||||
normalized_result["value_reason"] = "利用状态未知"
|
||||
logger.info(f"CVE {normalized_result['cve_id']} 利用状态未知")
|
||||
elif not any(comp and comp != "Unknown" for comp in normalized_result["affected_components"]):
|
||||
normalized_result["is_valuable"] = False
|
||||
normalized_result["value_reason"] = "没有明确的受影响组件"
|
||||
logger.info(f"CVE {normalized_result['cve_id']} 没有明确的受影响组件")
|
||||
|
||||
return normalized_result
|
||||
|
||||
|
||||
def validate_repo_result(result: dict, repo_data: dict, processed_commits: list) -> dict | None:
|
||||
"""验证并规范化仓库分析结果"""
|
||||
processed_result = _process_repo_analysis_result(result, repo_data, processed_commits)
|
||||
if not processed_result:
|
||||
return None
|
||||
return _normalize_common_result(processed_result, "repository")
|
||||
|
||||
|
||||
def validate_article_result(result: dict) -> dict | None:
|
||||
"""验证文章分类结果结构"""
|
||||
try:
|
||||
if not result or not isinstance(result, dict):
|
||||
return None
|
||||
|
||||
articles = result.get("articles")
|
||||
if not isinstance(articles, list):
|
||||
return None
|
||||
|
||||
valid_articles = []
|
||||
for article in articles:
|
||||
if not isinstance(article, dict):
|
||||
continue
|
||||
title = article.get("title")
|
||||
category = article.get("category")
|
||||
if isinstance(title, str) and isinstance(category, str) and title.strip() and category.strip():
|
||||
valid_articles.append({"title": title, "category": category})
|
||||
|
||||
if not valid_articles:
|
||||
return None
|
||||
return {"articles": valid_articles}
|
||||
except Exception as e:
|
||||
logger.error(f"验证文章分类结果失败: {str(e)}")
|
||||
return None
|
||||
@@ -0,0 +1,27 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class SecurityAnalysisResult(BaseModel):
|
||||
short_description: str
|
||||
summary: str
|
||||
risk_level: str
|
||||
key_points: List[str]
|
||||
technical_details: List[str]
|
||||
affected_components: List[str]
|
||||
is_valuable: bool
|
||||
value_reason: str
|
||||
security_type: str
|
||||
update_type: str
|
||||
exploit_status: str
|
||||
relevance_score: Optional[str] = None
|
||||
|
||||
|
||||
class ArticleClassificationItem(BaseModel):
|
||||
title: str
|
||||
category: str
|
||||
|
||||
|
||||
class ArticleClassificationResult(BaseModel):
|
||||
articles: List[ArticleClassificationItem]
|
||||
@@ -0,0 +1,5 @@
|
||||
"""外部服务客户端导出。"""
|
||||
|
||||
from clients.github_client import GithubClient
|
||||
|
||||
__all__ = ["GithubClient"]
|
||||
@@ -0,0 +1,285 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util import Retry
|
||||
|
||||
from config import GITHUB_TOKEN, GITHUB_TOKENS
|
||||
from utils.logger import get_logger
|
||||
|
||||
logger = get_logger("clients.github_client")
|
||||
|
||||
|
||||
@dataclass
|
||||
class _TokenState:
|
||||
remaining: int = 5000
|
||||
available_at: float = 0.0
|
||||
reset_at: float = 0.0
|
||||
|
||||
|
||||
class GithubClient:
|
||||
"""GitHub API 统一客户端。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokens: list[str] | None = None,
|
||||
*,
|
||||
min_interval: float = 0.2,
|
||||
timeout: int = 30,
|
||||
max_retries: int = 3,
|
||||
) -> None:
|
||||
raw_tokens = tokens if tokens is not None else (GITHUB_TOKENS or [GITHUB_TOKEN])
|
||||
self.tokens = [token.strip() for token in raw_tokens if isinstance(token, str) and token.strip()]
|
||||
self.min_interval = max(0.0, min_interval)
|
||||
self.timeout = timeout
|
||||
self.max_retries = max(1, max_retries)
|
||||
|
||||
self._lock = threading.Lock()
|
||||
self._current_index = 0
|
||||
self._next_request_time = 0.0
|
||||
self._token_state = {token: _TokenState() for token in self.tokens}
|
||||
|
||||
self.session = requests.Session()
|
||||
retry = Retry(
|
||||
total=5,
|
||||
backoff_factor=0.5,
|
||||
status_forcelist=[500, 502, 503, 504],
|
||||
allowed_methods=frozenset(["GET"]),
|
||||
)
|
||||
self.session.mount("https://", HTTPAdapter(max_retries=retry))
|
||||
self.session.verify = True
|
||||
|
||||
def _wait_interval(self) -> None:
|
||||
if self.min_interval <= 0:
|
||||
return
|
||||
with self._lock:
|
||||
now = time.time()
|
||||
wait = max(0.0, self._next_request_time - now)
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
self._next_request_time = time.time() + self.min_interval
|
||||
|
||||
def _get_token(self) -> str | None:
|
||||
if not self.tokens:
|
||||
return None
|
||||
|
||||
while True:
|
||||
with self._lock:
|
||||
now = time.time()
|
||||
wait_seconds = None
|
||||
chosen_token: str | None = None
|
||||
|
||||
for offset in range(len(self.tokens)):
|
||||
idx = (self._current_index + offset) % len(self.tokens)
|
||||
token = self.tokens[idx]
|
||||
state = self._token_state[token]
|
||||
if state.available_at <= now:
|
||||
self._current_index = (idx + 1) % len(self.tokens)
|
||||
chosen_token = token
|
||||
break
|
||||
token_wait = max(0.0, state.available_at - now)
|
||||
wait_seconds = token_wait if wait_seconds is None else min(wait_seconds, token_wait)
|
||||
|
||||
if chosen_token:
|
||||
return chosen_token
|
||||
if wait_seconds is None:
|
||||
return None
|
||||
time.sleep(max(wait_seconds, 0.1))
|
||||
|
||||
def _update_token_state_from_headers(self, token: str | None, response: requests.Response) -> None:
|
||||
if not token:
|
||||
return
|
||||
remaining_header = response.headers.get("X-RateLimit-Remaining")
|
||||
reset_header = response.headers.get("X-RateLimit-Reset")
|
||||
with self._lock:
|
||||
state = self._token_state[token]
|
||||
if remaining_header is not None:
|
||||
try:
|
||||
state.remaining = int(remaining_header)
|
||||
except ValueError:
|
||||
pass
|
||||
if reset_header is not None:
|
||||
try:
|
||||
state.reset_at = float(reset_header)
|
||||
except ValueError:
|
||||
pass
|
||||
if state.remaining <= 0 and state.reset_at > time.time():
|
||||
state.available_at = state.reset_at
|
||||
elif state.available_at < time.time():
|
||||
state.available_at = 0.0
|
||||
|
||||
def _mark_token_cooldown(self, token: str | None, cooldown_seconds: float, remaining: int = 0) -> None:
|
||||
if not token:
|
||||
return
|
||||
with self._lock:
|
||||
state = self._token_state[token]
|
||||
state.remaining = remaining
|
||||
state.available_at = max(state.available_at, time.time() + cooldown_seconds)
|
||||
|
||||
def _request(
|
||||
self,
|
||||
endpoint: str,
|
||||
*,
|
||||
params: dict[str, Any] | None = None,
|
||||
absolute: bool = False,
|
||||
) -> requests.Response | None:
|
||||
url = endpoint if absolute else f"https://api.github.com{endpoint}"
|
||||
last_error: Exception | None = None
|
||||
|
||||
for attempt in range(1, self.max_retries + 1):
|
||||
token = self._get_token()
|
||||
headers = {"Accept": "application/vnd.github.v3+json"}
|
||||
if token:
|
||||
headers["Authorization"] = f"token {token}"
|
||||
|
||||
self._wait_interval()
|
||||
|
||||
try:
|
||||
response = self.session.get(url, headers=headers, params=params, timeout=self.timeout)
|
||||
except requests.RequestException as exc:
|
||||
last_error = exc
|
||||
logger.warning("GitHub 请求异常，第 %s 次重试: %s", attempt, str(exc))
|
||||
self._mark_token_cooldown(token, cooldown_seconds=30.0)
|
||||
if attempt < self.max_retries:
|
||||
time.sleep(min(2 ** attempt, 8))
|
||||
continue
|
||||
|
||||
self._update_token_state_from_headers(token, response)
|
||||
|
||||
if response.status_code == 401:
|
||||
logger.warning("GitHub Token 认证失败，切换 Token")
|
||||
self._mark_token_cooldown(token, cooldown_seconds=3600.0)
|
||||
if attempt < self.max_retries:
|
||||
continue
|
||||
return None
|
||||
|
||||
if response.status_code == 403:
|
||||
reset_header = response.headers.get("X-RateLimit-Reset")
|
||||
wait = 60.0
|
||||
if reset_header:
|
||||
try:
|
||||
wait = max(1.0, float(reset_header) - time.time())
|
||||
except ValueError:
|
||||
wait = 60.0
|
||||
logger.warning("GitHub 速率限制触发，切换 Token")
|
||||
self._mark_token_cooldown(token, cooldown_seconds=wait)
|
||||
if attempt < self.max_retries:
|
||||
continue
|
||||
return None
|
||||
|
||||
if 500 <= response.status_code < 600:
|
||||
logger.warning("GitHub 服务端错误 %s，第 %s 次重试", response.status_code, attempt)
|
||||
if attempt < self.max_retries:
|
||||
time.sleep(min(2 ** attempt, 8))
|
||||
continue
|
||||
return None
|
||||
|
||||
return response
|
||||
|
||||
if last_error:
|
||||
logger.error("GitHub 请求失败: %s", str(last_error))
|
||||
return None
|
||||
|
||||
def _request_json(
|
||||
self,
|
||||
endpoint: str,
|
||||
*,
|
||||
params: dict[str, Any] | None = None,
|
||||
absolute: bool = False,
|
||||
) -> dict[str, Any] | list[Any] | None:
|
||||
response = self._request(endpoint, params=params, absolute=absolute)
|
||||
if not response:
|
||||
return None
|
||||
if response.status_code >= 400:
|
||||
logger.warning("GitHub 返回错误状态码: %s", response.status_code)
|
||||
return None
|
||||
try:
|
||||
return response.json()
|
||||
except ValueError:
|
||||
logger.warning("GitHub 返回非 JSON 响应")
|
||||
return None
|
||||
|
||||
def get_rate_limit(self) -> dict[str, Any] | None:
|
||||
data = self._request_json("/rate_limit")
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
def search_repos(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
sort: str = "updated",
|
||||
order: str = "desc",
|
||||
page: int = 1,
|
||||
per_page: int = 30,
|
||||
) -> dict[str, Any] | None:
|
||||
params = {
|
||||
"q": query,
|
||||
"sort": sort,
|
||||
"order": order,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
}
|
||||
data = self._request_json("/search/repositories", params=params)
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
def get_commits(
|
||||
self,
|
||||
repo_full_name: str,
|
||||
*,
|
||||
since: datetime | str | None = None,
|
||||
per_page: int = 10,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
params: dict[str, Any] = {"per_page": per_page}
|
||||
if since:
|
||||
if isinstance(since, datetime):
|
||||
params["since"] = since.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
else:
|
||||
params["since"] = since
|
||||
data = self._request_json(f"/repos/{repo_full_name}/commits", params=params)
|
||||
if isinstance(data, list):
|
||||
return [item for item in data if isinstance(item, dict)]
|
||||
return None
|
||||
|
||||
def get_commit_detail(self, repo_full_name: str, sha_or_url: str) -> dict[str, Any] | None:
|
||||
if sha_or_url.startswith("http://") or sha_or_url.startswith("https://"):
|
||||
data = self._request_json(sha_or_url, absolute=True)
|
||||
else:
|
||||
data = self._request_json(f"/repos/{repo_full_name}/commits/{sha_or_url}")
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
def get_readme(self, repo_full_name: str) -> str | None:
|
||||
data = self._request_json(f"/repos/{repo_full_name}/readme")
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
encoded = data.get("content")
|
||||
if not encoded:
|
||||
return None
|
||||
try:
|
||||
normalized = str(encoded).replace("\n", "")
|
||||
return base64.b64decode(normalized).decode("utf-8", errors="ignore")
|
||||
except Exception:
|
||||
logger.warning("README 解码失败: %s", repo_full_name)
|
||||
return None
|
||||
|
||||
def get_repository(self, repo_full_name: str) -> dict[str, Any] | None:
|
||||
data = self._request_json(f"/repos/{repo_full_name}")
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
def get_contents(self, repo_full_name: str, path: str = "") -> list[dict[str, Any]] | None:
|
||||
endpoint = f"/repos/{repo_full_name}/contents/{path}" if path else f"/repos/{repo_full_name}/contents"
|
||||
data = self._request_json(endpoint)
|
||||
if isinstance(data, list):
|
||||
return [item for item in data if isinstance(item, dict)]
|
||||
if isinstance(data, dict):
|
||||
return [data]
|
||||
return None
|
||||
|
||||
def get_json_by_url(self, url: str) -> dict[str, Any] | list[Any] | None:
|
||||
return self._request_json(url, absolute=True)
|
||||
@@ -0,0 +1,381 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
try:
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
except ModuleNotFoundError:
|
||||
from dotenv import dotenv_values
|
||||
|
||||
class SettingsConfigDict(dict):
|
||||
"""兼容缺少 pydantic-settings 的运行环境。"""
|
||||
|
||||
class BaseSettings(BaseModel):
|
||||
"""简化版 BaseSettings：优先读取环境变量，其次读取 .env。"""
|
||||
|
||||
model_config = {"extra": "ignore", "populate_by_name": True}
|
||||
|
||||
def __init__(self, **values: Any) -> None:
|
||||
env_values: dict[str, Any] = {}
|
||||
|
||||
# 构建 alias -> field_name 映射
|
||||
alias_to_name: dict[str, str] = {}
|
||||
for field_name, field in self.__class__.model_fields.items():
|
||||
if field.alias:
|
||||
alias_to_name[field.alias] = field_name
|
||||
|
||||
# 从 .env 文件加载，将 alias key 转为 field_name key
|
||||
env_file = getattr(self.__class__, "model_config", {}).get("env_file")
|
||||
if env_file and Path(env_file).exists():
|
||||
loaded = dotenv_values(str(env_file))
|
||||
for key, val in loaded.items():
|
||||
if val is None:
|
||||
continue
|
||||
field_name = alias_to_name.get(key, key)
|
||||
env_values[field_name] = val
|
||||
|
||||
# 从环境变量加载（优先级高于 .env）
|
||||
for alias, field_name in alias_to_name.items():
|
||||
if alias in os.environ:
|
||||
env_values[field_name] = os.environ[alias]
|
||||
|
||||
# 显式传入的值优先级最高
|
||||
env_values.update(values)
|
||||
super().__init__(**env_values)
|
||||
|
||||
|
||||
class Paths:
|
||||
"""Centralized path management for the project."""
|
||||
|
||||
ROOT_DIR: ClassVar[Path] = Path(__file__).resolve().parent
|
||||
DATA_DIR: ClassVar[Path] = ROOT_DIR / "data"
|
||||
LOGS_DIR: ClassVar[Path] = ROOT_DIR / "logs"
|
||||
RESULTS_DIR: ClassVar[Path] = ROOT_DIR / "results"
|
||||
REPORTS_DIR: ClassVar[Path] = ROOT_DIR / "reports"
|
||||
BLACKLIST_DIR: ClassVar[Path] = DATA_DIR / "blacklist"
|
||||
ANALYSIS_HISTORY_DIR: ClassVar[Path] = DATA_DIR / "analysis_history"
|
||||
DATABASE_PATH: ClassVar[Path] = ROOT_DIR / "cve_database.db"
|
||||
|
||||
@classmethod
|
||||
def abs_path(cls, value: str | Path) -> str:
|
||||
path = Path(value).expanduser()
|
||||
if not path.is_absolute():
|
||||
path = cls.ROOT_DIR / path
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
class StaticConfig:
|
||||
"""Non-sensitive configuration kept in code."""
|
||||
|
||||
BLACKLIST_USERS: ClassVar[list[str]] = [
|
||||
"Kvi74",
|
||||
"huihuo123",
|
||||
"Live-Hack-CVE",
|
||||
"yrtsec",
|
||||
"l00neyhacker",
|
||||
"Shmily-ing",
|
||||
"romainmarcoux",
|
||||
"criminalip",
|
||||
"20142995",
|
||||
"Siwakorn2545",
|
||||
"moraveczd",
|
||||
"Vitrius",
|
||||
]
|
||||
|
||||
BLACKLIST_REPOSITORIES: ClassVar[list[str]] = [
|
||||
"1992513/www",
|
||||
"Ifem2BXvz4N4gh1gGn0bkR3Lp/cihna-dictatorshrip-8",
|
||||
"cirosantilli/china-dictatorhsip-7",
|
||||
"cirosantilli/china-dictatorship",
|
||||
"huangxy7/jd_scripts",
|
||||
"jerryy2577/jd_scripts",
|
||||
"snalfword/t_v_b_o_x_u_r_l_2",
|
||||
"SingLoveMyself/C2Panel",
|
||||
"lxw15337674/weibo-trending-hot-history",
|
||||
"btou666/I0QdsjFIVY",
|
||||
"JoeZang520/PHA_dm",
|
||||
"psk107520/p",
|
||||
"mhieu102/C2_NNPTUDM_24Mar",
|
||||
"dick318/awesome-stars",
|
||||
"romainmarcoux/malicious-outgoing-ip",
|
||||
"criminalip/C2-Daily-Feed",
|
||||
"huqi-pr/trending-in-one",
|
||||
"drb-ra/C2IntelFeeds",
|
||||
"szzdmj/szzdmj",
|
||||
"cxyfreedom/website-hot-hub",
|
||||
"XiangDongCJC/Conditional_competition-bypass-defender",
|
||||
"drrgtg/weibo_hot",
|
||||
"AZeC4/TelegramGroup",
|
||||
"Ro1eee/AcademyManager",
|
||||
"FlashNightModReborn/CrazyFlashNight",
|
||||
"adminlove520/Poc-Monitor_v1.0.1",
|
||||
"projectdiscovery/nuclei-templates",
|
||||
"romainmarcoux/malicious-outgoing-ip",
|
||||
"criminalip/C2-Daily-Feed",
|
||||
"lintx0/link-tools",
|
||||
"rodanmaharjan/ThreatIntelligence",
|
||||
"TencentBlueKing/blueking-dbm",
|
||||
"dimpal001/c2w_com",
|
||||
"duggytuxy/malicious_ip_addresses",
|
||||
"trickest/cve",
|
||||
"XiaomingX/data-cve-poc",
|
||||
"adysec/nuclei_poc",
|
||||
]
|
||||
|
||||
WATCHED_REPOSITORIES: ClassVar[list[str]] = [
|
||||
"wy876/POC",
|
||||
"knownsec/404StarLink",
|
||||
"shadow1ng/fscan",
|
||||
"SafeGroceryStore/MDUT",
|
||||
"gentilkiwi/mimikatz",
|
||||
"ehang-io/nps",
|
||||
"chaitin/xray",
|
||||
"BeichenDream/Godzilla",
|
||||
"FunnyWolf/pystinger",
|
||||
"Adaptix-Framework/AdaptixC2",
|
||||
]
|
||||
|
||||
GITHUB_KEYWORDS: ClassVar[list[str]] = [
|
||||
"免杀",
|
||||
"Sql注入",
|
||||
"RCE",
|
||||
"反序列化",
|
||||
"任意文件上传",
|
||||
"漏洞复现",
|
||||
"漏洞",
|
||||
"edr evasion",
|
||||
"安全工具",
|
||||
"security tool",
|
||||
"c2",
|
||||
"反向shell",
|
||||
"cnvd",
|
||||
"post-exploitation command",
|
||||
"AI Security",
|
||||
"外网打点",
|
||||
"资产+AND+指纹识别",
|
||||
"shellcode Loader",
|
||||
"后渗透",
|
||||
]
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
extra="ignore",
|
||||
populate_by_name=True,
|
||||
)
|
||||
|
||||
# Database
|
||||
database_path: str = Field(default=str(Paths.DATABASE_PATH), alias="DATABASE_PATH")
|
||||
|
||||
# Runtime
|
||||
monitor_interval: int = Field(default=7200, alias="MONITOR_INTERVAL", gt=0)
|
||||
|
||||
# Primary AI config
|
||||
primary_ai_key: str = Field(default="", alias="PRIMARY_AI_KEY")
|
||||
primary_ai_base: str = Field(default="https://ai.897010.xyz/v1", alias="PRIMARY_AI_BASE")
|
||||
primary_ai_model: str = Field(default="qwen3-next", alias="PRIMARY_AI_MODEL")
|
||||
|
||||
# Backup AI configs (JSON list)
|
||||
backup_ai_configs: list[dict[str, str]] = Field(default_factory=list, alias="BACKUP_AI_CONFIGS")
|
||||
|
||||
# Gemini config
|
||||
gemini_ai_key: str = Field(default="", alias="GEMINI_AI_KEY")
|
||||
gemini_ai_model: str = Field(default="gemini-2.5-flash", alias="GEMINI_AI_MODEL")
|
||||
gemini_ai_base_url: str = Field(
|
||||
default="https://generativelanguage.googleapis.com",
|
||||
alias="GEMINI_AI_BASE_URL",
|
||||
)
|
||||
|
||||
# GitHub tokens
|
||||
github_tokens: list[str] = Field(default_factory=list, alias="GITHUB_TOKENS")
|
||||
github_token: str = Field(default="", alias="GITHUB_TOKEN")
|
||||
|
||||
# Blog
|
||||
blog_base_url: str = Field(default="http://127.0.0.1", alias="BLOG_BASE_URL")
|
||||
blog_token: str = Field(default="", alias="BLOG_TOKEN")
|
||||
|
||||
@field_validator("database_path", mode="before")
|
||||
@classmethod
|
||||
def _normalize_database_path(cls, value: Any) -> str:
|
||||
if value in (None, ""):
|
||||
return str(Paths.DATABASE_PATH)
|
||||
return Paths.abs_path(value)
|
||||
|
||||
@field_validator(
|
||||
"primary_ai_key",
|
||||
"primary_ai_base",
|
||||
"primary_ai_model",
|
||||
"gemini_ai_key",
|
||||
"gemini_ai_model",
|
||||
"gemini_ai_base_url",
|
||||
"github_token",
|
||||
"blog_base_url",
|
||||
"blog_token",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
def _strip_string_fields(cls, value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return str(value).strip()
|
||||
|
||||
@field_validator("github_tokens", mode="before")
|
||||
@classmethod
|
||||
def _parse_github_tokens(cls, value: Any) -> list[str]:
|
||||
if value in (None, ""):
|
||||
return []
|
||||
|
||||
if isinstance(value, list):
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
|
||||
if isinstance(value, str):
|
||||
raw = value.strip()
|
||||
if not raw:
|
||||
return []
|
||||
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return [item.strip() for item in raw.split(",") if item.strip()]
|
||||
|
||||
if isinstance(parsed, list):
|
||||
return [str(item).strip() for item in parsed if str(item).strip()]
|
||||
if isinstance(parsed, str):
|
||||
return [item.strip() for item in parsed.split(",") if item.strip()]
|
||||
|
||||
raise ValueError("GITHUB_TOKENS must be a JSON array or comma-separated string")
|
||||
|
||||
@field_validator("backup_ai_configs", mode="before")
|
||||
@classmethod
|
||||
def _parse_backup_ai_configs(cls, value: Any) -> list[dict[str, str]]:
|
||||
if value in (None, ""):
|
||||
return []
|
||||
|
||||
parsed: Any = value
|
||||
if isinstance(value, str):
|
||||
raw = value.strip()
|
||||
if not raw:
|
||||
return []
|
||||
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError("BACKUP_AI_CONFIGS must be a JSON array") from exc
|
||||
|
||||
if not isinstance(parsed, list):
|
||||
raise ValueError("BACKUP_AI_CONFIGS must be a list of objects")
|
||||
|
||||
normalized: list[dict[str, str]] = []
|
||||
for item in parsed:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("Each BACKUP_AI_CONFIGS item must be an object")
|
||||
|
||||
api_key = str(item.get("api_key", "")).strip()
|
||||
if not api_key:
|
||||
continue
|
||||
|
||||
api_base = str(item.get("api_base", "https://api.openai.com/v1")).strip()
|
||||
model = str(item.get("model", "gpt-4.1-nano-2025-04-14")).strip()
|
||||
normalized.append(
|
||||
{
|
||||
"api_key": api_key,
|
||||
"api_base": api_base,
|
||||
"model": model,
|
||||
}
|
||||
)
|
||||
|
||||
return normalized
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _sync_github_token_fields(self) -> "Settings":
|
||||
if self.github_tokens:
|
||||
self.github_tokens = [token for token in self.github_tokens if token]
|
||||
self.github_token = self.github_tokens[0] if self.github_tokens else ""
|
||||
elif self.github_token:
|
||||
self.github_token = self.github_token.strip()
|
||||
self.github_tokens = [self.github_token] if self.github_token else []
|
||||
else:
|
||||
self.github_tokens = []
|
||||
self.github_token = ""
|
||||
|
||||
return self
|
||||
|
||||
@property
|
||||
def primary_ai_config(self) -> dict[str, str]:
|
||||
return {
|
||||
"api_key": self.primary_ai_key,
|
||||
"api_base": self.primary_ai_base,
|
||||
"model": self.primary_ai_model,
|
||||
}
|
||||
|
||||
@property
|
||||
def gemini_ai_config(self) -> dict[str, str]:
|
||||
return {
|
||||
"api_key": self.gemini_ai_key,
|
||||
"model": self.gemini_ai_model,
|
||||
"base_url": self.gemini_ai_base_url,
|
||||
}
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
# Paths
|
||||
PATHS = Paths
|
||||
ROOT_DIR = str(Paths.ROOT_DIR)
|
||||
DATA_DIR = str(Paths.DATA_DIR)
|
||||
LOGS_DIR = str(Paths.LOGS_DIR)
|
||||
RESULTS_DIR = str(Paths.RESULTS_DIR)
|
||||
REPORTS_DIR = str(Paths.REPORTS_DIR)
|
||||
BLACKLIST_DIR = str(Paths.BLACKLIST_DIR)
|
||||
ANALYSIS_HISTORY_DIR = str(Paths.ANALYSIS_HISTORY_DIR)
|
||||
|
||||
# Backward-compatible exports
|
||||
DATABASE_PATH = settings.database_path
|
||||
MONITOR_INTERVAL = settings.monitor_interval
|
||||
|
||||
PRIMARY_AI_CONFIG = settings.primary_ai_config
|
||||
BACKUP_AI_CONFIGS = settings.backup_ai_configs
|
||||
GEMINI_AI_CONFIG = settings.gemini_ai_config
|
||||
|
||||
GITHUB_TOKENS = settings.github_tokens
|
||||
GITHUB_TOKEN = settings.github_token
|
||||
|
||||
BLACKLIST_USERS = list(StaticConfig.BLACKLIST_USERS)
|
||||
BLACKLIST_REPOSITORIES = list(StaticConfig.BLACKLIST_REPOSITORIES)
|
||||
WATCHED_REPOSITORIES = list(StaticConfig.WATCHED_REPOSITORIES)
|
||||
GITHUB_KEYWORDS = list(StaticConfig.GITHUB_KEYWORDS)
|
||||
|
||||
BLOG_TOKEN = settings.blog_token
|
||||
BLOG_BASE_URL = settings.blog_base_url
|
||||
|
||||
__all__ = [
|
||||
"settings",
|
||||
"PATHS",
|
||||
"ROOT_DIR",
|
||||
"DATA_DIR",
|
||||
"LOGS_DIR",
|
||||
"RESULTS_DIR",
|
||||
"REPORTS_DIR",
|
||||
"BLACKLIST_DIR",
|
||||
"ANALYSIS_HISTORY_DIR",
|
||||
"DATABASE_PATH",
|
||||
"MONITOR_INTERVAL",
|
||||
"PRIMARY_AI_CONFIG",
|
||||
"BACKUP_AI_CONFIGS",
|
||||
"GEMINI_AI_CONFIG",
|
||||
"GITHUB_TOKENS",
|
||||
"GITHUB_TOKEN",
|
||||
"BLACKLIST_USERS",
|
||||
"BLACKLIST_REPOSITORIES",
|
||||
"WATCHED_REPOSITORIES",
|
||||
"GITHUB_KEYWORDS",
|
||||
"BLOG_TOKEN",
|
||||
"BLOG_BASE_URL",
|
||||
]
|
||||
@@ -0,0 +1,20 @@
|
||||
"""数据库模块导出。"""
|
||||
|
||||
from database.init_db import init_database
|
||||
from database.models import Base, CVERecord, MonitorState, Repository
|
||||
from database.repositories import CVERepo, MonitorStateRepo, RepositoryRepo
|
||||
from database.session import SessionFactory, get_engine, get_session
|
||||
|
||||
__all__ = [
|
||||
"Base",
|
||||
"Repository",
|
||||
"CVERecord",
|
||||
"MonitorState",
|
||||
"get_engine",
|
||||
"SessionFactory",
|
||||
"get_session",
|
||||
"init_database",
|
||||
"CVERepo",
|
||||
"RepositoryRepo",
|
||||
"MonitorStateRepo",
|
||||
]
|
||||
@@ -0,0 +1,22 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from config import DATABASE_PATH
|
||||
from database.models import Base
|
||||
from database.session import get_engine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def init_database(db_path: str | Path = DATABASE_PATH) -> None:
|
||||
"""统一数据库初始化入口。"""
|
||||
engine = get_engine(db_path)
|
||||
Base.metadata.create_all(engine)
|
||||
logger.info("数据库初始化完成: %s", db_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
init_database()
|
||||
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Integer, String, Text
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
"""数据库模型基类。"""
|
||||
|
||||
|
||||
class Repository(Base):
|
||||
"""仓库监控记录。"""
|
||||
|
||||
__tablename__ = "repositories"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), unique=True, nullable=False)
|
||||
url: Mapped[str | None] = mapped_column(String(255))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
stars: Mapped[int | None] = mapped_column(Integer, default=0)
|
||||
last_update: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
created_at: Mapped[datetime | None] = mapped_column(DateTime, default=datetime.now)
|
||||
is_valuable: Mapped[bool | None] = mapped_column(Boolean, default=True)
|
||||
ai_summary: Mapped[str | None] = mapped_column(Text)
|
||||
last_analyzed: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
last_fetched: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
push_count: Mapped[int | None] = mapped_column(Integer, default=0)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"Repository(id={self.id!r}, name={self.name!r}, "
|
||||
f"stars={self.stars!r}, last_update={self.last_update!r})"
|
||||
)
|
||||
|
||||
|
||||
class CVERecord(Base):
|
||||
"""CVE 监控记录。"""
|
||||
|
||||
__tablename__ = "cve_records"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
cve_id: Mapped[str] = mapped_column(String(50), unique=True, nullable=False)
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
published_date: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
last_modified: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
repository_url: Mapped[str | None] = mapped_column(String(255))
|
||||
created_at: Mapped[datetime | None] = mapped_column(DateTime, default=datetime.now)
|
||||
severity: Mapped[str | None] = mapped_column(String(50))
|
||||
ai_summary: Mapped[str | None] = mapped_column(Text)
|
||||
last_analyzed: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
stars: Mapped[int | None] = mapped_column(Integer, default=0)
|
||||
last_fetched: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"CVERecord(id={self.id!r}, cve_id={self.cve_id!r}, "
|
||||
f"severity={self.severity!r}, last_modified={self.last_modified!r})"
|
||||
)
|
||||
|
||||
|
||||
class MonitorState(Base):
|
||||
"""监控器运行状态。"""
|
||||
|
||||
__tablename__ = "monitor_state"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
monitor_type: Mapped[str] = mapped_column(String(50), unique=True, nullable=False)
|
||||
last_run: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
last_success: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
status: Mapped[str | None] = mapped_column(String(50))
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"MonitorState(id={self.id!r}, monitor_type={self.monitor_type!r}, "
|
||||
f"status={self.status!r}, last_run={self.last_run!r})"
|
||||
)
|
||||
@@ -0,0 +1,138 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from database.models import CVERecord, MonitorState, Repository
|
||||
|
||||
|
||||
class CVERepo:
|
||||
"""CVE 相关数据访问封装。"""
|
||||
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
|
||||
def get_by_id(self, cve_id: str) -> CVERecord | None:
|
||||
return self.session.query(CVERecord).filter_by(cve_id=cve_id).first()
|
||||
|
||||
def create(self, data: dict[str, Any]) -> CVERecord:
|
||||
record = CVERecord(**data)
|
||||
self.session.add(record)
|
||||
self.session.flush()
|
||||
return record
|
||||
|
||||
def update(self, cve_id: str, data: dict[str, Any]) -> CVERecord | None:
|
||||
record = self.get_by_id(cve_id)
|
||||
if not record:
|
||||
return None
|
||||
for key, value in data.items():
|
||||
if hasattr(record, key):
|
||||
setattr(record, key, value)
|
||||
self.session.flush()
|
||||
return record
|
||||
|
||||
def needs_analysis(
|
||||
self,
|
||||
cve_id: str,
|
||||
latest_modified: datetime | None = None,
|
||||
interval_hours: int = 24,
|
||||
) -> bool:
|
||||
record = self.get_by_id(cve_id)
|
||||
if not record:
|
||||
return True
|
||||
if record.last_analyzed is None:
|
||||
return True
|
||||
if latest_modified and record.last_modified != latest_modified:
|
||||
return True
|
||||
elapsed = datetime.now() - record.last_analyzed
|
||||
return elapsed.total_seconds() >= interval_hours * 3600
|
||||
|
||||
|
||||
class RepositoryRepo:
|
||||
"""仓库相关数据访问封装。"""
|
||||
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
|
||||
def get_by_name(self, name: str) -> Repository | None:
|
||||
return self.session.query(Repository).filter_by(name=name).first()
|
||||
|
||||
def create(self, data: dict[str, Any]) -> Repository:
|
||||
record = Repository(**data)
|
||||
self.session.add(record)
|
||||
self.session.flush()
|
||||
return record
|
||||
|
||||
def update_analysis(self, name: str, data: dict[str, Any]) -> Repository | None:
|
||||
record = self.get_by_name(name)
|
||||
if not record:
|
||||
return None
|
||||
for key, value in data.items():
|
||||
if hasattr(record, key):
|
||||
setattr(record, key, value)
|
||||
if "last_analyzed" not in data:
|
||||
record.last_analyzed = datetime.now()
|
||||
self.session.flush()
|
||||
return record
|
||||
|
||||
def check_daily_limit(self, repo_name: str, now: datetime | None = None) -> bool:
|
||||
if not repo_name:
|
||||
return True
|
||||
|
||||
current_time = now or datetime.now()
|
||||
record = self.get_by_name(repo_name)
|
||||
|
||||
if not record:
|
||||
self.create(
|
||||
{
|
||||
"name": repo_name,
|
||||
"last_fetched": current_time,
|
||||
"push_count": 1,
|
||||
}
|
||||
)
|
||||
return True
|
||||
|
||||
if record.last_fetched and record.last_fetched.date() == current_time.date():
|
||||
return False
|
||||
|
||||
if record.last_fetched and record.last_fetched.date() != current_time.date():
|
||||
record.push_count = 0
|
||||
|
||||
record.last_fetched = current_time
|
||||
record.push_count = (record.push_count or 0) + 1
|
||||
self.session.flush()
|
||||
return True
|
||||
|
||||
|
||||
class MonitorStateRepo:
|
||||
"""监控状态数据访问封装。"""
|
||||
|
||||
def __init__(self, session: Session) -> None:
|
||||
self.session = session
|
||||
|
||||
def get_state(self, monitor_type: str) -> MonitorState | None:
|
||||
return self.session.query(MonitorState).filter_by(monitor_type=monitor_type).first()
|
||||
|
||||
def update_state(
|
||||
self,
|
||||
monitor_type: str,
|
||||
status: str,
|
||||
error_message: str | None = None,
|
||||
*,
|
||||
success: bool = False,
|
||||
last_run: datetime | None = None,
|
||||
) -> MonitorState:
|
||||
record = self.get_state(monitor_type)
|
||||
if not record:
|
||||
record = MonitorState(monitor_type=monitor_type)
|
||||
self.session.add(record)
|
||||
|
||||
record.status = status
|
||||
record.error_message = error_message
|
||||
record.last_run = last_run or datetime.now()
|
||||
if success:
|
||||
record.last_success = record.last_run
|
||||
self.session.flush()
|
||||
return record
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
from sqlalchemy import Engine, create_engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from config import DATABASE_PATH
|
||||
|
||||
|
||||
def get_engine(db_path: str | Path = DATABASE_PATH) -> Engine:
|
||||
"""创建数据库引擎。"""
|
||||
path = Path(db_path).expanduser()
|
||||
if not path.is_absolute():
|
||||
path = Path.cwd() / path
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
return create_engine(f"sqlite:///{path}", future=True)
|
||||
|
||||
|
||||
def SessionFactory(engine: Engine) -> sessionmaker[Session]:
|
||||
"""创建 Session 工厂。"""
|
||||
return sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_session(factory: sessionmaker[Session]) -> Iterator[Session]:
|
||||
"""统一事务管理：自动提交、异常回滚、始终关闭。"""
|
||||
session = factory()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
@@ -0,0 +1,63 @@
|
||||
# Configuration Guide
|
||||
|
||||
CyberSentinel AI is configured through environment variables. The recommended local flow is:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
python main.py --check-config
|
||||
```
|
||||
|
||||
## Runtime
|
||||
|
||||
| Variable | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `MONITOR_INTERVAL` | `7200` | Seconds between monitoring loops. |
|
||||
| `DATABASE_PATH` | `cve_database.db` | SQLite database path. Relative paths are resolved from the project root. |
|
||||
|
||||
## GitHub
|
||||
|
||||
Use a token for real monitoring; unauthenticated requests are rate-limited.
|
||||
|
||||
```dotenv
|
||||
GITHUB_TOKENS=["replace-with-token-1","replace-with-token-2"]
|
||||
GITHUB_TOKEN=
|
||||
```
|
||||
|
||||
`GITHUB_TOKENS` takes priority. `GITHUB_TOKEN` is a single-token fallback.
|
||||
|
||||
## AI providers
|
||||
|
||||
Primary OpenAI-compatible provider:
|
||||
|
||||
```dotenv
|
||||
PRIMARY_AI_KEY=
|
||||
PRIMARY_AI_BASE=https://api.openai.com/v1
|
||||
PRIMARY_AI_MODEL=gpt-4.1-nano
|
||||
```
|
||||
|
||||
Backup OpenAI-compatible providers:
|
||||
|
||||
```dotenv
|
||||
BACKUP_AI_CONFIGS=[{"api_key":"replace-with-key","api_base":"https://api.example.com/v1","model":"model-name"}]
|
||||
```
|
||||
|
||||
Gemini fallback:
|
||||
|
||||
```dotenv
|
||||
GEMINI_AI_KEY=
|
||||
GEMINI_AI_MODEL=gemini-2.5-flash
|
||||
GEMINI_AI_BASE_URL=https://generativelanguage.googleapis.com
|
||||
```
|
||||
|
||||
At least one AI provider key is needed for AI analysis. Tests and configuration checks can run without AI keys.
|
||||
|
||||
## Blog publishing
|
||||
|
||||
Blog publishing is optional:
|
||||
|
||||
```dotenv
|
||||
BLOG_BASE_URL=http://127.0.0.1
|
||||
BLOG_TOKEN=
|
||||
```
|
||||
|
||||
If `BLOG_TOKEN` is empty, blog API calls are skipped and reports stay local.
|
||||
@@ -0,0 +1,51 @@
|
||||
# Deployment Notes
|
||||
|
||||
## Foreground run
|
||||
|
||||
```bash
|
||||
source .venv/bin/activate
|
||||
python main.py
|
||||
```
|
||||
|
||||
## systemd example
|
||||
|
||||
Create `/etc/systemd/system/cybersentinel-ai.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=CyberSentinel AI security monitor
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/CyberSentinel-AI
|
||||
EnvironmentFile=/opt/CyberSentinel-AI/.env
|
||||
ExecStart=/opt/CyberSentinel-AI/.venv/bin/python main.py
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
Enable and start:
|
||||
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now cybersentinel-ai
|
||||
sudo systemctl status cybersentinel-ai
|
||||
```
|
||||
|
||||
## Runtime data
|
||||
|
||||
Runtime data is generated locally and ignored by git:
|
||||
|
||||
```text
|
||||
cve_database.db
|
||||
logs/
|
||||
results/
|
||||
data/
|
||||
```
|
||||
|
||||
Back up those files separately if they are important for your deployment.
|
||||
@@ -0,0 +1,55 @@
|
||||
# 快速开始
|
||||
|
||||
## 1. 安装
|
||||
|
||||
```bash
|
||||
git clone https://github.com/Hxnxe/CyberSentinel-AI.git
|
||||
cd CyberSentinel-AI
|
||||
|
||||
bash scripts/bootstrap.sh
|
||||
```
|
||||
|
||||
脚本会完成：
|
||||
|
||||
- 创建 `.venv`
|
||||
- 安装依赖
|
||||
- 如果 `.env` 不存在，则从 `.env.example` 复制
|
||||
- 运行配置自检
|
||||
|
||||
## 2. 编辑配置
|
||||
|
||||
```bash
|
||||
nano .env
|
||||
```
|
||||
|
||||
至少建议配置：
|
||||
|
||||
```dotenv
|
||||
GITHUB_TOKENS=["你的 GitHub Token"]
|
||||
PRIMARY_AI_KEY=你的 AI API Key
|
||||
PRIMARY_AI_BASE=https://api.openai.com/v1
|
||||
PRIMARY_AI_MODEL=gpt-4.1-nano
|
||||
```
|
||||
|
||||
## 3. 自检
|
||||
|
||||
```bash
|
||||
source .venv/bin/activate
|
||||
python main.py --check-config
|
||||
```
|
||||
|
||||
如果 `github` 或 `ai` 显示 `WARN`，说明对应 Token/API Key 未配置。测试仍然可以运行，但正式监控效果会受影响。
|
||||
|
||||
## 4. 测试
|
||||
|
||||
```bash
|
||||
python -m pytest -q
|
||||
```
|
||||
|
||||
## 5. 启动
|
||||
|
||||
```bash
|
||||
python main.py
|
||||
```
|
||||
|
||||
停止时按 `Ctrl+C`。
|
||||
@@ -0,0 +1,303 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import signal
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from queue import Empty, Queue
|
||||
from threading import Event, Thread
|
||||
from typing import Any
|
||||
|
||||
from ai.analyzer import SecurityAnalyzer
|
||||
from ai.llm_client import LLMClient
|
||||
from clients.github_client import GithubClient
|
||||
from config import BLOG_BASE_URL, BLOG_TOKEN, MONITOR_INTERVAL, settings
|
||||
from database.init_db import init_database
|
||||
from database.session import SessionFactory, get_engine
|
||||
from monitors.cve_monitor import CVEMonitor
|
||||
from monitors.github_monitor import GithubMonitor
|
||||
from reports.generator import ReportGenerator
|
||||
from utils.article_fetcher import ArticleFetcher
|
||||
from utils.article_manager import ArticleManager
|
||||
from utils.blacklist_manager import BlacklistManager
|
||||
from utils.blog_manager import BlogManager
|
||||
from utils.config_diagnostics import build_config_diagnostics, format_config_diagnostics
|
||||
from utils.logger import get_logger
|
||||
|
||||
logger = get_logger("security_monitor.main")
|
||||
ARTICLE_FETCH_INTERVAL = 2 * 60 * 60
|
||||
VERSION = "0.1.0"
|
||||
|
||||
|
||||
class SecurityMonitorSystem:
|
||||
"""安全监控系统编排层。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.engine = get_engine()
|
||||
init_database()
|
||||
self.session_factory = SessionFactory(self.engine)
|
||||
|
||||
self.github_client = GithubClient()
|
||||
self.blacklist_manager = BlacklistManager()
|
||||
self.llm_client = LLMClient()
|
||||
|
||||
self.blog_manager = BlogManager(base_url=BLOG_BASE_URL, token=BLOG_TOKEN)
|
||||
self.report_generator = ReportGenerator(
|
||||
session_factory=self.session_factory,
|
||||
blog_manager=self.blog_manager,
|
||||
)
|
||||
|
||||
self.analyzer = SecurityAnalyzer(
|
||||
llm_client=self.llm_client,
|
||||
blacklist_manager=self.blacklist_manager,
|
||||
engine=self.engine,
|
||||
session_factory=self.session_factory,
|
||||
)
|
||||
|
||||
self.cve_monitor = CVEMonitor(
|
||||
self.engine,
|
||||
github_client=self.github_client,
|
||||
blacklist_manager=self.blacklist_manager,
|
||||
session_factory=self.session_factory,
|
||||
)
|
||||
self.github_monitor = GithubMonitor(
|
||||
self.engine,
|
||||
github_client=self.github_client,
|
||||
blacklist_manager=self.blacklist_manager,
|
||||
session_factory=self.session_factory,
|
||||
)
|
||||
|
||||
self.article_fetcher = ArticleFetcher()
|
||||
self.article_manager = ArticleManager(self.analyzer, blog_manager=self.blog_manager)
|
||||
|
||||
self.data_queue: Queue[tuple[str, dict[str, Any]]] = Queue()
|
||||
self.stop_event = Event()
|
||||
self.last_article_fetch: datetime | None = None
|
||||
|
||||
def _handle_signal(self, signum: int, _frame: Any) -> None:
|
||||
logger.info("接收到退出信号: %s", signum)
|
||||
self.stop_event.set()
|
||||
|
||||
def _setup_signals(self) -> None:
|
||||
signal.signal(signal.SIGINT, self._handle_signal)
|
||||
signal.signal(signal.SIGTERM, self._handle_signal)
|
||||
|
||||
def run_analysis(self) -> None:
|
||||
logger.info("AI分析线程启动")
|
||||
consecutive_errors = 0
|
||||
|
||||
while not self.stop_event.is_set():
|
||||
task_acquired = False
|
||||
source_type = "unknown"
|
||||
|
||||
try:
|
||||
source_type, data = self.data_queue.get(timeout=1)
|
||||
task_acquired = True
|
||||
|
||||
if source_type == "cve":
|
||||
result = self.analyzer.analyze_cve(data)
|
||||
else:
|
||||
result = self.analyzer.analyze_repository(data)
|
||||
|
||||
if result and result.get("is_valuable", False):
|
||||
self.report_generator.save_analysis_result(result, source_type)
|
||||
consecutive_errors = 0
|
||||
except Empty:
|
||||
continue
|
||||
except Exception as exc:
|
||||
consecutive_errors += 1
|
||||
logger.error(
|
||||
"AI分析线程异常(连续%s次): %s",
|
||||
consecutive_errors,
|
||||
str(exc),
|
||||
exc_info=True,
|
||||
)
|
||||
if self.stop_event.wait(min(consecutive_errors, 5)):
|
||||
break
|
||||
finally:
|
||||
if task_acquired:
|
||||
try:
|
||||
self.data_queue.task_done()
|
||||
except ValueError:
|
||||
logger.warning("task_done 调用异常，source_type=%s", source_type)
|
||||
|
||||
def process_articles(self) -> None:
|
||||
logger.info("开始处理安全文章")
|
||||
today_str = datetime.now().strftime("%Y-%m-%d")
|
||||
daily_report_path = os.path.join("results/daily_articles", f"{today_str}.md")
|
||||
ai_report_path = os.path.join(self.report_generator.ai_analysis_dir, f"Ai_{datetime.now().strftime('%Y_%m_%d')}.md")
|
||||
|
||||
try:
|
||||
os.makedirs(os.path.dirname(daily_report_path), exist_ok=True)
|
||||
articles = self.article_fetcher.get_doonsec_articles()
|
||||
new_articles = self.article_manager.filter_new_articles(articles)
|
||||
|
||||
if new_articles:
|
||||
logger.info("发现 %s 篇新文章", len(new_articles))
|
||||
analysis_result = self.article_manager.process_articles(new_articles)
|
||||
self.article_manager.generate_daily_report(analysis_result, daily_report_path)
|
||||
return
|
||||
|
||||
logger.info("没有发现新文章")
|
||||
if os.path.exists(ai_report_path) and os.path.getsize(ai_report_path) > 0:
|
||||
with open(ai_report_path, "r", encoding="utf-8") as ai_file:
|
||||
ai_content = ai_file.read()
|
||||
with open(daily_report_path, "w", encoding="utf-8") as daily_file:
|
||||
daily_file.write(f"# 每日安全资讯 ({today_str})\n\n")
|
||||
daily_file.write("今日未发现新的安全文章，以下是 AI 分析结果：\n\n")
|
||||
daily_file.write(ai_content)
|
||||
elif not os.path.exists(daily_report_path):
|
||||
with open(daily_report_path, "w", encoding="utf-8") as daily_file:
|
||||
daily_file.write(f"# 每日安全资讯 ({today_str})\n\n")
|
||||
daily_file.write("今日未发现新的安全文章，也无 AI 分析结果。\n\n")
|
||||
except Exception as exc:
|
||||
logger.error("处理文章失败: %s", str(exc), exc_info=True)
|
||||
|
||||
def run_monitoring(self) -> None:
|
||||
logger.info("监控线程启动")
|
||||
while not self.stop_event.is_set():
|
||||
try:
|
||||
self.data_queue.join()
|
||||
data_found = False
|
||||
start_time = datetime.now()
|
||||
|
||||
logger.info("开始CVE监控")
|
||||
cve_data = self.cve_monitor.fetch_latest_cves()
|
||||
if cve_data:
|
||||
cves_to_analyze = self.cve_monitor.process_cve_data(cve_data)
|
||||
for cve in cves_to_analyze:
|
||||
self.data_queue.put(("cve", cve))
|
||||
data_found = data_found or bool(cves_to_analyze)
|
||||
logger.info("本轮CVE待分析: %s", len(cves_to_analyze))
|
||||
|
||||
logger.info("开始GitHub仓库监控")
|
||||
repos_to_analyze = self.github_monitor.run()
|
||||
for repo in repos_to_analyze:
|
||||
if isinstance(repo, dict) and repo.get("name"):
|
||||
self.data_queue.put(("repository", repo))
|
||||
data_found = True
|
||||
logger.info("本轮仓库待分析: %s", len(repos_to_analyze))
|
||||
|
||||
if data_found:
|
||||
logger.info("等待本轮分析任务完成")
|
||||
self.data_queue.join()
|
||||
|
||||
if self.last_article_fetch is None or (start_time - self.last_article_fetch).total_seconds() >= ARTICLE_FETCH_INTERVAL:
|
||||
self.process_articles()
|
||||
self.last_article_fetch = start_time
|
||||
|
||||
self.report_generator.convert_and_update_blog()
|
||||
|
||||
end_time = datetime.now()
|
||||
duration = (end_time - start_time).total_seconds()
|
||||
logger.info(
|
||||
"监控循环完成，用时 %.2f 秒，下次运行: %s",
|
||||
duration,
|
||||
(end_time + timedelta(seconds=MONITOR_INTERVAL)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
)
|
||||
|
||||
if self.stop_event.wait(MONITOR_INTERVAL):
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.error("监控线程异常: %s", str(exc), exc_info=True)
|
||||
if self.stop_event.wait(300):
|
||||
break
|
||||
|
||||
def _run_status_check(self) -> None:
|
||||
logger.info("状态检查线程启动")
|
||||
last_status_time: datetime | None = None
|
||||
|
||||
while not self.stop_event.is_set():
|
||||
try:
|
||||
current_time = datetime.now()
|
||||
if current_time.minute == 0 and current_time.second == 0:
|
||||
if last_status_time is None or (current_time - last_status_time).total_seconds() >= 3600:
|
||||
logger.info("监控系统运行正常，待分析队列: %s", self.data_queue.qsize())
|
||||
last_status_time = current_time
|
||||
if self.stop_event.wait(10):
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.error("状态检查异常: %s", str(exc), exc_info=True)
|
||||
if self.stop_event.wait(60):
|
||||
break
|
||||
|
||||
def run(self) -> None:
|
||||
self._setup_signals()
|
||||
logger.info("安全监控系统启动")
|
||||
|
||||
threads = [
|
||||
Thread(target=self.run_monitoring, name="MonitorThread", daemon=True),
|
||||
Thread(target=self.run_analysis, name="AnalysisThread", daemon=True),
|
||||
Thread(target=self._run_status_check, name="StatusThread", daemon=True),
|
||||
]
|
||||
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
|
||||
try:
|
||||
while not self.stop_event.wait(1):
|
||||
continue
|
||||
finally:
|
||||
logger.info("正在停止监控系统")
|
||||
self.stop_event.set()
|
||||
for thread in threads:
|
||||
thread.join(timeout=5)
|
||||
if thread.is_alive():
|
||||
logger.warning("线程 %s 未在超时内退出", thread.name)
|
||||
logger.info("监控系统已停止")
|
||||
|
||||
|
||||
|
||||
def build_arg_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="cybersentinel-ai",
|
||||
description="Automated CVE and security repository monitoring with AI analysis.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--check-config",
|
||||
action="store_true",
|
||||
help="print a non-sensitive configuration diagnostic report and exit",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-db",
|
||||
action="store_true",
|
||||
help="initialize the SQLite database tables and exit",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
action="store_true",
|
||||
help="print version information and exit",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_arg_parser().parse_args(argv)
|
||||
|
||||
if args.version:
|
||||
print(f"CyberSentinel AI {VERSION}")
|
||||
return 0
|
||||
|
||||
if args.check_config:
|
||||
print(format_config_diagnostics(build_config_diagnostics(settings)), end="")
|
||||
return 0
|
||||
|
||||
if args.init_db:
|
||||
init_database()
|
||||
print(f"Database initialized at {settings.database_path}")
|
||||
return 0
|
||||
|
||||
try:
|
||||
system = SecurityMonitorSystem()
|
||||
system.run()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("程序被用户中断")
|
||||
except Exception as exc:
|
||||
logger.error("程序发生致命错误: %s", str(exc), exc_info=True)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,6 @@
|
||||
"""监控模块导出。"""
|
||||
|
||||
from monitors.cve_monitor import CVEMonitor
|
||||
from monitors.github_monitor import GithubMonitor
|
||||
|
||||
__all__ = ["CVEMonitor", "GithubMonitor"]
|
||||
@@ -0,0 +1,527 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from clients.github_client import GithubClient
|
||||
from database.repositories import CVERepo
|
||||
from database.session import SessionFactory, get_engine, get_session
|
||||
from utils.blacklist_manager import BlacklistManager
|
||||
from utils.logger import get_logger
|
||||
|
||||
logger = get_logger("monitors.cve_monitor")
|
||||
|
||||
|
||||
class CVEMonitor:
|
||||
"""CVE 监控器。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine: Engine | None = None,
|
||||
*,
|
||||
github_client: GithubClient | None = None,
|
||||
blacklist_manager: BlacklistManager | None = None,
|
||||
session_factory: Any | None = None,
|
||||
cve_quality_threshold: float = 0.25,
|
||||
) -> None:
|
||||
self.engine = engine or get_engine()
|
||||
self.session_factory = session_factory or SessionFactory(self.engine)
|
||||
self.github_client = github_client or GithubClient()
|
||||
self.blacklist_manager = blacklist_manager or BlacklistManager()
|
||||
self.cve_quality_threshold = min(max(float(cve_quality_threshold), 0.0), 1.0)
|
||||
|
||||
self.cve_pattern = re.compile(r"(?i)CVE[-_](\d{4})[-_](\d{4,7})")
|
||||
self.relevant_extensions = {
|
||||
".py",
|
||||
".sh",
|
||||
".java",
|
||||
".php",
|
||||
".js",
|
||||
".go",
|
||||
".txt",
|
||||
".md",
|
||||
".yaml",
|
||||
".yml",
|
||||
".json",
|
||||
".xml",
|
||||
}
|
||||
self.critical_patterns = {"poc", "exploit", "cve", "rce", "payload", "shell"}
|
||||
logger.info("初始化CVE监控器")
|
||||
logger.info("CVE质量阈值: %.2f", self.cve_quality_threshold)
|
||||
|
||||
def _is_blacklisted(self, repo_name: str, owner_login: str) -> bool:
|
||||
if self.blacklist_manager.is_blacklisted(repo_name, "repository"):
|
||||
return True
|
||||
if owner_login and self.blacklist_manager.is_blacklisted(owner_login, "user"):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _validate_cve_year(self, year: str) -> bool:
|
||||
try:
|
||||
year_int = int(year)
|
||||
return 1999 <= year_int <= datetime.now().year
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
def _extract_cve_ids(self, name: str, description: str) -> set[str]:
|
||||
matches = set()
|
||||
for text in [name or "", description or ""]:
|
||||
for year, number in self.cve_pattern.findall(text):
|
||||
if self._validate_cve_year(year):
|
||||
matches.add(f"CVE-{year}-{number}")
|
||||
return matches
|
||||
|
||||
def _decode_content(self, encoded: str) -> str:
|
||||
try:
|
||||
normalized = (encoded or "").replace("\n", "")
|
||||
if not normalized:
|
||||
return ""
|
||||
return base64.b64decode(normalized).decode("utf-8", errors="ignore")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def _calculate_file_score(self, file_path: str, content: str) -> float:
|
||||
path_lower = file_path.lower()
|
||||
score = 0.0
|
||||
if os.path.basename(path_lower) == "readme.md":
|
||||
return 1.0
|
||||
|
||||
ext = os.path.splitext(path_lower)[1]
|
||||
if ext in self.relevant_extensions:
|
||||
score += 0.4
|
||||
|
||||
if any(pattern in path_lower for pattern in self.critical_patterns):
|
||||
score += 0.4
|
||||
|
||||
if content:
|
||||
content_lower = content.lower()
|
||||
for pattern in self.critical_patterns:
|
||||
if pattern in content_lower:
|
||||
score += 0.05
|
||||
if "cve-" in content_lower:
|
||||
score += 0.2
|
||||
|
||||
return min(score, 1.0)
|
||||
|
||||
def _analyze_repository_files(self, repo_url: str) -> dict[str, Any]:
|
||||
if "github.com/" not in repo_url:
|
||||
return {"relevant_files": [], "code_contents": [], "has_readme": False}
|
||||
|
||||
repo_full_name = "/".join(repo_url.split("github.com/")[-1].split("/")[:2])
|
||||
contents = self.github_client.get_contents(repo_full_name) or []
|
||||
if not contents:
|
||||
return {"relevant_files": [], "code_contents": [], "has_readme": False}
|
||||
|
||||
relevant_files: list[str] = []
|
||||
code_contents: list[dict[str, Any]] = []
|
||||
has_readme = False
|
||||
|
||||
for item in contents[:40]:
|
||||
if item.get("type") != "file":
|
||||
continue
|
||||
file_name = item.get("name", "")
|
||||
file_path = item.get("path", file_name)
|
||||
item_url = item.get("url", "")
|
||||
if not file_name or not item_url:
|
||||
continue
|
||||
|
||||
item_detail = self.github_client.get_json_by_url(item_url)
|
||||
if not isinstance(item_detail, dict):
|
||||
continue
|
||||
content = self._decode_content(str(item_detail.get("content", "")))
|
||||
if not content:
|
||||
continue
|
||||
|
||||
if file_name.lower() == "readme.md":
|
||||
has_readme = True
|
||||
|
||||
score = self._calculate_file_score(file_path, content)
|
||||
if score < 0.3:
|
||||
continue
|
||||
|
||||
relevant_files.append(file_path)
|
||||
code_contents.append(
|
||||
{
|
||||
"file_name": file_name,
|
||||
"file_path": file_path,
|
||||
"content": content[:3000],
|
||||
"size": len(content),
|
||||
"score": score,
|
||||
}
|
||||
)
|
||||
|
||||
code_contents.sort(key=lambda x: x.get("score", 0), reverse=True)
|
||||
return {
|
||||
"relevant_files": relevant_files,
|
||||
"code_contents": code_contents[:20],
|
||||
"has_readme": has_readme,
|
||||
}
|
||||
|
||||
def _fetch_latest_commits(self, repo_full_name: str, max_commits: int = 5) -> list[dict[str, Any]]:
|
||||
commits = self.github_client.get_commits(repo_full_name, per_page=max_commits) or []
|
||||
processed: list[dict[str, Any]] = []
|
||||
|
||||
for commit in commits[:max_commits]:
|
||||
commit_url = str(commit.get("url", ""))
|
||||
sha = str(commit.get("sha", ""))
|
||||
if not commit_url and not sha:
|
||||
continue
|
||||
|
||||
detail = self.github_client.get_commit_detail(repo_full_name, commit_url or sha)
|
||||
if not detail:
|
||||
continue
|
||||
|
||||
files = []
|
||||
for changed_file in detail.get("files", []):
|
||||
if not isinstance(changed_file, dict):
|
||||
continue
|
||||
files.append(
|
||||
{
|
||||
"path": changed_file.get("filename", ""),
|
||||
"status": changed_file.get("status", "unknown"),
|
||||
"additions": int(changed_file.get("additions", 0) or 0),
|
||||
"deletions": int(changed_file.get("deletions", 0) or 0),
|
||||
"changes": int(changed_file.get("changes", 0) or 0),
|
||||
"patch": str(changed_file.get("patch", ""))[:1200],
|
||||
}
|
||||
)
|
||||
|
||||
commit_data = {
|
||||
"sha": detail.get("sha", sha),
|
||||
"message": detail.get("commit", {}).get("message", ""),
|
||||
"author": detail.get("commit", {}).get("author", {}).get("name", ""),
|
||||
"date": detail.get("commit", {}).get("author", {}).get("date", ""),
|
||||
"files": files,
|
||||
}
|
||||
processed.append(commit_data)
|
||||
|
||||
return processed
|
||||
|
||||
@staticmethod
|
||||
def _record_skip_reason(skip_stats: dict[str, int], reason: str) -> None:
|
||||
skip_stats[reason] = skip_stats.get(reason, 0) + 1
|
||||
|
||||
@staticmethod
|
||||
def _build_fingerprint(cve_id: str, repo_name: str, last_modified: datetime) -> str:
|
||||
minute_key = last_modified.strftime("%Y%m%d%H%M")
|
||||
return f"{cve_id}:{repo_name}:{minute_key}"
|
||||
|
||||
def _score_cve_candidate(
|
||||
self,
|
||||
*,
|
||||
repo_name: str,
|
||||
stars: int,
|
||||
repo_analysis: dict[str, Any],
|
||||
recent_commits: list[dict[str, Any]],
|
||||
) -> tuple[float, list[str]]:
|
||||
score = 0.0
|
||||
reasons: list[str] = []
|
||||
|
||||
repo_name_lower = repo_name.lower()
|
||||
name_hits = sum(1 for pattern in self.critical_patterns if pattern in repo_name_lower)
|
||||
if name_hits > 0:
|
||||
score += min(0.2, name_hits * 0.05)
|
||||
reasons.append("repo_name_signal")
|
||||
|
||||
if stars >= 20:
|
||||
score += 0.15
|
||||
reasons.append("high_stars")
|
||||
elif stars >= 5:
|
||||
score += 0.08
|
||||
reasons.append("mid_stars")
|
||||
elif stars > 0:
|
||||
score += 0.03
|
||||
reasons.append("low_stars")
|
||||
|
||||
relevant_files = repo_analysis.get("relevant_files", [])
|
||||
if isinstance(relevant_files, list) and relevant_files:
|
||||
score += min(0.35, len(relevant_files[:10]) * 0.05)
|
||||
reasons.append("relevant_files")
|
||||
|
||||
if repo_analysis.get("has_readme"):
|
||||
score += 0.05
|
||||
reasons.append("has_readme")
|
||||
|
||||
code_contents = repo_analysis.get("code_contents", [])
|
||||
if isinstance(code_contents, list):
|
||||
high_signal_files = sum(
|
||||
1
|
||||
for item in code_contents[:10]
|
||||
if isinstance(item, dict) and float(item.get("score", 0) or 0) >= 0.6
|
||||
)
|
||||
if high_signal_files > 0:
|
||||
score += min(0.2, high_signal_files * 0.05)
|
||||
reasons.append("high_signal_content")
|
||||
|
||||
if recent_commits:
|
||||
score += 0.15
|
||||
reasons.append("recent_commits")
|
||||
patch_hits = 0
|
||||
for commit in recent_commits[:5]:
|
||||
if not isinstance(commit, dict):
|
||||
continue
|
||||
for changed_file in commit.get("files", [])[:20]:
|
||||
if not isinstance(changed_file, dict):
|
||||
continue
|
||||
patch = str(changed_file.get("patch", "")).lower()
|
||||
if patch and any(pattern in patch for pattern in self.critical_patterns):
|
||||
patch_hits += 1
|
||||
if patch_hits > 0:
|
||||
score += min(0.15, patch_hits * 0.03)
|
||||
reasons.append("patch_signal")
|
||||
else:
|
||||
reasons.append("no_recent_commits")
|
||||
|
||||
return min(score, 1.0), reasons or ["insufficient_signal"]
|
||||
|
||||
def _should_filter_low_signal(
|
||||
self,
|
||||
*,
|
||||
repo_name: str,
|
||||
stars: int,
|
||||
repo_analysis: dict[str, Any],
|
||||
quality_score: float,
|
||||
) -> bool:
|
||||
relevant_files = repo_analysis.get("relevant_files", [])
|
||||
has_file_evidence = isinstance(relevant_files, list) and bool(relevant_files)
|
||||
if has_file_evidence:
|
||||
return False
|
||||
|
||||
repo_name_lower = repo_name.lower()
|
||||
has_keyword_signal = any(pattern in repo_name_lower for pattern in self.critical_patterns)
|
||||
if has_keyword_signal:
|
||||
return False
|
||||
|
||||
if stars >= 5:
|
||||
return False
|
||||
|
||||
return quality_score < self.cve_quality_threshold
|
||||
|
||||
def fetch_latest_cves(self) -> list[dict[str, Any]]:
|
||||
"""从 GitHub 获取最新 CVE 相关仓库并提取结构化数据。"""
|
||||
logger.info("开始获取最新CVE信息...")
|
||||
start_date = (datetime.now() - timedelta(days=5)).strftime("%Y-%m-%d")
|
||||
query = f"CVE-202 pushed:>={start_date}"
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
processed_pairs: set[str] = set()
|
||||
stats = {"processed": 0, "skipped": 0, "new": 0, "updated": 0, "accepted": 0}
|
||||
skip_reasons: dict[str, int] = {}
|
||||
|
||||
for page in range(1, 11):
|
||||
data = self.github_client.search_repos(query, page=page, per_page=50)
|
||||
if not data or not isinstance(data, dict):
|
||||
break
|
||||
|
||||
items = data.get("items")
|
||||
if not isinstance(items, list) or not items:
|
||||
break
|
||||
|
||||
with get_session(self.session_factory) as session:
|
||||
cve_repo = CVERepo(session)
|
||||
|
||||
for item_idx, item in enumerate(items, 1):
|
||||
if not isinstance(item, dict):
|
||||
stats["skipped"] += 1
|
||||
self._record_skip_reason(skip_reasons, "invalid_repo_payload")
|
||||
continue
|
||||
|
||||
stats["processed"] += 1
|
||||
owner_login = str(item.get("owner", {}).get("login", ""))
|
||||
repo_name = str(item.get("full_name", ""))
|
||||
if not repo_name or self._is_blacklisted(repo_name, owner_login):
|
||||
stats["skipped"] += 1
|
||||
self._record_skip_reason(skip_reasons, "blacklisted_or_invalid_repo")
|
||||
continue
|
||||
|
||||
pushed_at = str(item.get("pushed_at", ""))
|
||||
try:
|
||||
last_modified = datetime.strptime(pushed_at, "%Y-%m-%dT%H:%M:%SZ")
|
||||
except Exception:
|
||||
stats["skipped"] += 1
|
||||
self._record_skip_reason(skip_reasons, "invalid_pushed_at")
|
||||
continue
|
||||
|
||||
cve_ids = self._extract_cve_ids(str(item.get("name", "")), str(item.get("description", "")))
|
||||
if not cve_ids:
|
||||
stats["skipped"] += 1
|
||||
self._record_skip_reason(skip_reasons, "no_cve_id")
|
||||
continue
|
||||
|
||||
repo_url = str(item.get("html_url", ""))
|
||||
stars = int(item.get("stargazers_count", 0) or 0)
|
||||
logger.info("处理仓库 [%s/%s] (页%s): %s, CVEs=%s", item_idx, len(items), page, repo_name, cve_ids)
|
||||
repo_analysis = self._analyze_repository_files(repo_url) if repo_url else {}
|
||||
recent_commits = self._fetch_latest_commits(repo_name)
|
||||
quality_score, quality_reasons = self._score_cve_candidate(
|
||||
repo_name=repo_name,
|
||||
stars=stars,
|
||||
repo_analysis=repo_analysis,
|
||||
recent_commits=recent_commits,
|
||||
)
|
||||
|
||||
for cve_id in cve_ids:
|
||||
if self._should_filter_low_signal(
|
||||
repo_name=repo_name,
|
||||
stars=stars,
|
||||
repo_analysis=repo_analysis,
|
||||
quality_score=quality_score,
|
||||
):
|
||||
stats["skipped"] += 1
|
||||
self._record_skip_reason(skip_reasons, "low_signal_candidate")
|
||||
continue
|
||||
|
||||
pair_key = f"{cve_id}:{repo_name}"
|
||||
if pair_key in processed_pairs:
|
||||
stats["skipped"] += 1
|
||||
self._record_skip_reason(skip_reasons, "duplicate_pair")
|
||||
continue
|
||||
processed_pairs.add(pair_key)
|
||||
|
||||
existing = cve_repo.get_by_id(cve_id)
|
||||
if existing and existing.last_modified and existing.last_modified >= last_modified:
|
||||
stats["skipped"] += 1
|
||||
self._record_skip_reason(skip_reasons, "not_modified")
|
||||
continue
|
||||
|
||||
if existing:
|
||||
stats["updated"] += 1
|
||||
else:
|
||||
stats["new"] += 1
|
||||
stats["accepted"] += 1
|
||||
|
||||
results.append(
|
||||
{
|
||||
"cve_id": cve_id,
|
||||
"description": str(item.get("description", "") or ""),
|
||||
"published_date": datetime.strptime(pushed_at[:10], "%Y-%m-%d"),
|
||||
"last_modified": last_modified,
|
||||
"severity": "Unknown",
|
||||
"repository_url": repo_url,
|
||||
"stars": stars,
|
||||
"recent_commits": recent_commits,
|
||||
"repository_analysis": repo_analysis,
|
||||
"quality_score": round(quality_score, 3),
|
||||
"quality_reasons": quality_reasons,
|
||||
"skip_reason": None,
|
||||
"fingerprint": self._build_fingerprint(cve_id, repo_name, last_modified),
|
||||
}
|
||||
)
|
||||
|
||||
if len(items) < 50:
|
||||
break
|
||||
time.sleep(0.2)
|
||||
|
||||
top_skip_reasons = ", ".join(
|
||||
f"{reason}:{count}"
|
||||
for reason, count in sorted(skip_reasons.items(), key=lambda item: item[1], reverse=True)[:5]
|
||||
) or "none"
|
||||
logger.info(
|
||||
"CVE处理统计: 总处理=%s, 新增=%s, 更新=%s, 接受=%s, 跳过=%s, 跳过原因=%s",
|
||||
stats["processed"],
|
||||
stats["new"],
|
||||
stats["updated"],
|
||||
stats["accepted"],
|
||||
stats["skipped"],
|
||||
top_skip_reasons,
|
||||
)
|
||||
return results
|
||||
|
||||
def process_cve_data(self, cve_info_list: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""落库并返回需要分析的 CVE。"""
|
||||
if not cve_info_list:
|
||||
return []
|
||||
|
||||
to_analyze: list[dict[str, Any]] = []
|
||||
created = 0
|
||||
updated = 0
|
||||
skipped = 0
|
||||
|
||||
db_fields = {
|
||||
"cve_id",
|
||||
"description",
|
||||
"published_date",
|
||||
"last_modified",
|
||||
"repository_url",
|
||||
"severity",
|
||||
"ai_summary",
|
||||
"last_analyzed",
|
||||
"stars",
|
||||
"last_fetched",
|
||||
"created_at",
|
||||
}
|
||||
|
||||
with get_session(self.session_factory) as session:
|
||||
cve_repo = CVERepo(session)
|
||||
now = datetime.now()
|
||||
|
||||
for cve_info in cve_info_list:
|
||||
cve_id = str(cve_info.get("cve_id", ""))
|
||||
if not cve_id:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
latest_modified = cve_info.get("last_modified")
|
||||
existing = cve_repo.get_by_id(cve_id)
|
||||
|
||||
if existing:
|
||||
needs_analysis = cve_repo.needs_analysis(cve_id, latest_modified=latest_modified, interval_hours=1)
|
||||
if not needs_analysis:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
update_data = {
|
||||
key: value
|
||||
for key, value in cve_info.items()
|
||||
if key in db_fields and key != "last_analyzed"
|
||||
}
|
||||
update_data["last_fetched"] = now
|
||||
cve_repo.update(cve_id, update_data)
|
||||
updated += 1
|
||||
else:
|
||||
create_data = {
|
||||
key: value
|
||||
for key, value in cve_info.items()
|
||||
if key in db_fields
|
||||
}
|
||||
create_data["last_fetched"] = now
|
||||
cve_repo.create(create_data)
|
||||
created += 1
|
||||
|
||||
to_analyze.append(cve_info)
|
||||
|
||||
logger.info(
|
||||
"CVE入库统计: 输入=%s, 新增=%s, 更新=%s, 跳过=%s, 待分析=%s",
|
||||
len(cve_info_list),
|
||||
created,
|
||||
updated,
|
||||
skipped,
|
||||
len(to_analyze),
|
||||
)
|
||||
return to_analyze
|
||||
|
||||
def run(self, interval: int = 3600) -> None:
|
||||
logger.info("CVE监控器启动，监控间隔: %s秒", interval)
|
||||
while True:
|
||||
try:
|
||||
cve_info_list = self.fetch_latest_cves()
|
||||
if cve_info_list:
|
||||
cves_to_analyze = self.process_cve_data(cve_info_list)
|
||||
logger.info("本轮监控完成，处理了 %s 条待分析记录", len(cves_to_analyze))
|
||||
else:
|
||||
logger.info("本轮未获取到新的CVE信息")
|
||||
time.sleep(interval)
|
||||
except Exception as exc:
|
||||
logger.error("CVE监控运行失败: %s", str(exc), exc_info=True)
|
||||
time.sleep(300)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
monitor = CVEMonitor()
|
||||
monitor.run()
|
||||
@@ -0,0 +1,745 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from clients.github_client import GithubClient
|
||||
from config import GITHUB_KEYWORDS, WATCHED_REPOSITORIES
|
||||
from database.repositories import MonitorStateRepo, RepositoryRepo
|
||||
from database.session import SessionFactory, get_engine, get_session
|
||||
from utils.blacklist_manager import BlacklistManager
|
||||
from utils.logger import get_logger
|
||||
|
||||
logger = get_logger("monitors.github_monitor")
|
||||
|
||||
|
||||
class GithubMonitor:
|
||||
"""GitHub 仓库监控器。"""
|
||||
|
||||
RELEVANT_EXTENSIONS = {
|
||||
".yaml",
|
||||
".yml",
|
||||
".json",
|
||||
".md",
|
||||
".txt",
|
||||
".py",
|
||||
".go",
|
||||
".java",
|
||||
".php",
|
||||
".js",
|
||||
".sh",
|
||||
".rb",
|
||||
}
|
||||
SECURITY_KEYWORDS = {
|
||||
"cve",
|
||||
"rce",
|
||||
"exploit",
|
||||
"poc",
|
||||
"payload",
|
||||
"shell",
|
||||
"bypass",
|
||||
"inject",
|
||||
"漏洞",
|
||||
"利用",
|
||||
"安全",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine: Engine | None = None,
|
||||
*,
|
||||
github_client: GithubClient | None = None,
|
||||
blacklist_manager: BlacklistManager | None = None,
|
||||
session_factory: Any | None = None,
|
||||
keywords: list[str] | None = None,
|
||||
watched_repositories: list[str] | None = None,
|
||||
repo_quality_threshold: float = 0.35,
|
||||
) -> None:
|
||||
self.engine = engine or get_engine()
|
||||
self.session_factory = session_factory or SessionFactory(self.engine)
|
||||
self.github_client = github_client or GithubClient()
|
||||
self.blacklist_manager = blacklist_manager or BlacklistManager()
|
||||
self.keywords = list(keywords or GITHUB_KEYWORDS)
|
||||
self.watched_repositories = list(watched_repositories or WATCHED_REPOSITORIES)
|
||||
self.repo_quality_threshold = min(max(float(repo_quality_threshold), 0.0), 1.0)
|
||||
|
||||
logger.info("初始化GitHub监控器")
|
||||
logger.info("已加载 %s 个搜索关键词", len(self.keywords))
|
||||
logger.info("已加载 %s 个监控仓库", len(self.watched_repositories))
|
||||
logger.info("仓库质量阈值: %.2f", self.repo_quality_threshold)
|
||||
|
||||
@staticmethod
|
||||
def _truncate_patch(patch: str, max_length: int = 1200) -> str:
|
||||
if not patch:
|
||||
return ""
|
||||
return patch if len(patch) <= max_length else patch[:max_length] + "..."
|
||||
|
||||
@staticmethod
|
||||
def _record_skip_reason(skip_stats: dict[str, int] | None, reason: str) -> None:
|
||||
if skip_stats is None:
|
||||
return
|
||||
skip_stats[reason] = skip_stats.get(reason, 0) + 1
|
||||
|
||||
@staticmethod
|
||||
def _build_fingerprint(repo_name: str, last_update: datetime | None) -> str:
|
||||
if not repo_name:
|
||||
return ""
|
||||
minute_key = (last_update or datetime.now()).strftime("%Y%m%d%H%M")
|
||||
return f"{repo_name}:{minute_key}"
|
||||
|
||||
def _score_repository_metadata(self, repo: dict[str, Any]) -> tuple[float, list[str]]:
|
||||
score = 0.0
|
||||
reasons: list[str] = []
|
||||
|
||||
stars = int(repo.get("stars", 0) or 0)
|
||||
if stars >= 100:
|
||||
score += 0.25
|
||||
reasons.append("high_stars")
|
||||
elif stars >= 30:
|
||||
score += 0.18
|
||||
reasons.append("mid_stars")
|
||||
elif stars >= 10:
|
||||
score += 0.12
|
||||
reasons.append("basic_stars")
|
||||
elif stars > 0:
|
||||
score += 0.05
|
||||
reasons.append("low_stars")
|
||||
|
||||
repo_name = str(repo.get("name", "") or "").lower()
|
||||
description = str(repo.get("description", "") or "").lower()
|
||||
search_keyword = str(repo.get("search_keyword", "") or "").strip().lower()
|
||||
text = f"{repo_name} {description}".strip()
|
||||
|
||||
if search_keyword:
|
||||
normalized_keyword = search_keyword.replace("+", " ").strip()
|
||||
tokens = [token for token in normalized_keyword.split() if len(token) >= 3]
|
||||
if normalized_keyword and normalized_keyword in text:
|
||||
score += 0.22
|
||||
reasons.append("keyword_exact_match")
|
||||
elif any(token in text for token in tokens):
|
||||
score += 0.12
|
||||
reasons.append("keyword_partial_match")
|
||||
|
||||
keyword_hits = sum(1 for keyword in self.SECURITY_KEYWORDS if keyword in text)
|
||||
if keyword_hits > 0:
|
||||
score += min(0.20, keyword_hits * 0.05)
|
||||
reasons.append("security_context")
|
||||
|
||||
if repo.get("is_update"):
|
||||
score += 0.08
|
||||
reasons.append("updated_repository")
|
||||
if repo.get("is_watched"):
|
||||
score += 0.08
|
||||
reasons.append("watched_repository")
|
||||
|
||||
return min(score, 1.0), reasons
|
||||
|
||||
def _score_repository_commits(self, commits: list[dict[str, Any]]) -> tuple[float, list[str]]:
|
||||
if not commits:
|
||||
return 0.0, ["no_recent_commits"]
|
||||
|
||||
score = 0.15
|
||||
reasons = ["recent_commits"]
|
||||
file_signal = 0.0
|
||||
patch_signal_hits = 0
|
||||
|
||||
for commit in commits[:5]:
|
||||
if not isinstance(commit, dict):
|
||||
continue
|
||||
for changed_file in commit.get("files", [])[:20]:
|
||||
if not isinstance(changed_file, dict):
|
||||
continue
|
||||
path = str(changed_file.get("path", "")).lower()
|
||||
ext = os.path.splitext(path)[1]
|
||||
if ext in self.RELEVANT_EXTENSIONS:
|
||||
file_signal += 0.02
|
||||
patch = str(changed_file.get("patch", "")).lower()
|
||||
if patch and any(keyword in patch for keyword in self.SECURITY_KEYWORDS):
|
||||
patch_signal_hits += 1
|
||||
|
||||
if file_signal > 0:
|
||||
score += min(0.18, file_signal)
|
||||
reasons.append("relevant_file_changes")
|
||||
if patch_signal_hits > 0:
|
||||
score += min(0.15, patch_signal_hits * 0.03)
|
||||
reasons.append("security_patch_signal")
|
||||
|
||||
return min(score, 1.0), reasons
|
||||
|
||||
def _enrich_quality_info(
|
||||
self,
|
||||
repo_data: dict[str, Any],
|
||||
*,
|
||||
last_update: datetime,
|
||||
recent_commits: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
metadata_score, metadata_reasons = self._score_repository_metadata(repo_data)
|
||||
commit_score, commit_reasons = self._score_repository_commits(recent_commits or [])
|
||||
|
||||
result = dict(repo_data)
|
||||
result["quality_score"] = round(min(1.0, metadata_score + commit_score), 3)
|
||||
result["quality_reasons"] = [*metadata_reasons, *commit_reasons] or ["insufficient_signal"]
|
||||
result["fingerprint"] = self._build_fingerprint(str(result.get("name", "")), last_update)
|
||||
result["skip_reason"] = None
|
||||
return result
|
||||
|
||||
def _passes_quality_gate(self, repo_data: dict[str, Any], skip_stats: dict[str, int] | None = None) -> bool:
|
||||
score = float(repo_data.get("quality_score", 0.0) or 0.0)
|
||||
if repo_data.get("is_watched"):
|
||||
if score < self.repo_quality_threshold:
|
||||
reasons = list(repo_data.get("quality_reasons", []))
|
||||
reasons.append("watched_repository_bypass")
|
||||
repo_data["quality_reasons"] = reasons
|
||||
return True
|
||||
|
||||
if score < self.repo_quality_threshold:
|
||||
repo_data["skip_reason"] = "low_quality_score"
|
||||
self._record_skip_reason(skip_stats, "low_quality_score")
|
||||
return False
|
||||
return True
|
||||
|
||||
def _is_blacklisted(self, repo_name: str, owner_login: str) -> bool:
|
||||
if self.blacklist_manager.is_blacklisted(repo_name, "repository"):
|
||||
return True
|
||||
if owner_login and self.blacklist_manager.is_blacklisted(owner_login, "user"):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _fetch_readme_content(self, repo_name: str) -> str:
|
||||
return self.github_client.get_readme(repo_name) or ""
|
||||
|
||||
def _fetch_recent_commits(
|
||||
self,
|
||||
repo_name: str,
|
||||
since_time: datetime | None = None,
|
||||
max_commits: int = 10,
|
||||
) -> list[dict[str, Any]]:
|
||||
commits = self.github_client.get_commits(repo_name, since=since_time, per_page=max_commits) or []
|
||||
processed: list[dict[str, Any]] = []
|
||||
|
||||
for commit in commits[:max_commits]:
|
||||
commit_url = str(commit.get("url", ""))
|
||||
sha = str(commit.get("sha", ""))
|
||||
if not commit_url and not sha:
|
||||
continue
|
||||
|
||||
detail = self.github_client.get_commit_detail(repo_name, commit_url or sha)
|
||||
if not isinstance(detail, dict):
|
||||
continue
|
||||
|
||||
files: list[dict[str, Any]] = []
|
||||
for changed_file in detail.get("files", []):
|
||||
if not isinstance(changed_file, dict):
|
||||
continue
|
||||
|
||||
filename = str(changed_file.get("filename", ""))
|
||||
if not filename:
|
||||
continue
|
||||
|
||||
ext = os.path.splitext(filename)[1].lower()
|
||||
if ext not in self.RELEVANT_EXTENSIONS:
|
||||
continue
|
||||
|
||||
files.append(
|
||||
{
|
||||
"path": filename,
|
||||
"status": changed_file.get("status", "unknown"),
|
||||
"additions": int(changed_file.get("additions", 0) or 0),
|
||||
"deletions": int(changed_file.get("deletions", 0) or 0),
|
||||
"changes": int(changed_file.get("changes", 0) or 0),
|
||||
"patch": self._truncate_patch(str(changed_file.get("patch", ""))),
|
||||
}
|
||||
)
|
||||
|
||||
processed.append(
|
||||
{
|
||||
"sha": detail.get("sha", sha),
|
||||
"message": str(detail.get("commit", {}).get("message", "")).strip(),
|
||||
"author": str(detail.get("commit", {}).get("author", {}).get("name", "")).strip(),
|
||||
"date": str(detail.get("commit", {}).get("author", {}).get("date", "")),
|
||||
"files": files,
|
||||
}
|
||||
)
|
||||
|
||||
return processed
|
||||
|
||||
def search_repositories(
|
||||
self,
|
||||
keyword: str,
|
||||
*,
|
||||
seen_repo_names: set[str] | None = None,
|
||||
skip_stats: dict[str, int] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""按关键词搜索需要分析的仓库。"""
|
||||
repos: list[dict[str, Any]] = []
|
||||
per_page = 25
|
||||
max_pages = 2
|
||||
since_date = datetime.now().strftime("%Y-%m-%d")
|
||||
metadata_floor = max(0.15, self.repo_quality_threshold - 0.2)
|
||||
|
||||
processed_keyword = keyword
|
||||
if " AND " in keyword or " OR " in keyword:
|
||||
processed_keyword = keyword.replace(" AND ", "+").replace(" OR ", "+OR+")
|
||||
processed_keyword = processed_keyword.replace("(", "").replace(")", "")
|
||||
|
||||
with get_session(self.session_factory) as session:
|
||||
repo_repo = RepositoryRepo(session)
|
||||
|
||||
for page in range(1, max_pages + 1):
|
||||
query = f"{processed_keyword} pushed:>={since_date}"
|
||||
data = self.github_client.search_repos(query, page=page, per_page=per_page)
|
||||
if not data or not isinstance(data, dict):
|
||||
break
|
||||
|
||||
items = data.get("items")
|
||||
if not isinstance(items, list) or not items:
|
||||
break
|
||||
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
owner_login = str(item.get("owner", {}).get("login", ""))
|
||||
repo_name = str(item.get("full_name", ""))
|
||||
if not repo_name or self._is_blacklisted(repo_name, owner_login):
|
||||
self._record_skip_reason(skip_stats, "blacklisted_or_invalid_repo")
|
||||
continue
|
||||
|
||||
if seen_repo_names is not None and repo_name in seen_repo_names:
|
||||
self._record_skip_reason(skip_stats, "duplicate_repo_name")
|
||||
continue
|
||||
|
||||
pushed_at = str(item.get("pushed_at", ""))
|
||||
try:
|
||||
last_update = datetime.strptime(pushed_at, "%Y-%m-%dT%H:%M:%SZ")
|
||||
except Exception:
|
||||
self._record_skip_reason(skip_stats, "invalid_pushed_at")
|
||||
continue
|
||||
|
||||
existing = repo_repo.get_by_name(repo_name)
|
||||
if existing and existing.last_update:
|
||||
time_diff = abs((last_update - existing.last_update).total_seconds())
|
||||
if time_diff <= 60:
|
||||
self._record_skip_reason(skip_stats, "stale_update")
|
||||
continue
|
||||
|
||||
repo_info = {
|
||||
"name": repo_name,
|
||||
"url": item.get("html_url"),
|
||||
"description": str(item.get("description", "") or "")[:500],
|
||||
"last_update": last_update,
|
||||
"stars": int(item.get("stargazers_count", 0) or 0),
|
||||
"is_valuable": int(item.get("stargazers_count", 0) or 0) > 10,
|
||||
"search_keyword": keyword,
|
||||
"owner": owner_login,
|
||||
}
|
||||
lightweight_score, _ = self._score_repository_metadata(repo_info)
|
||||
if lightweight_score < metadata_floor:
|
||||
self._record_skip_reason(skip_stats, "low_metadata_signal")
|
||||
continue
|
||||
|
||||
repos.append(repo_info)
|
||||
if seen_repo_names is not None:
|
||||
seen_repo_names.add(repo_name)
|
||||
|
||||
if len(items) < per_page:
|
||||
break
|
||||
time.sleep(0.2)
|
||||
|
||||
return repos
|
||||
|
||||
def check_watched_repositories(self) -> list[dict[str, Any]]:
|
||||
"""检查固定监控仓库是否有更新。"""
|
||||
updates: list[dict[str, Any]] = []
|
||||
|
||||
with get_session(self.session_factory) as session:
|
||||
repo_repo = RepositoryRepo(session)
|
||||
|
||||
for repo_name in self.watched_repositories:
|
||||
try:
|
||||
info = self.github_client.get_repository(repo_name)
|
||||
if not info:
|
||||
continue
|
||||
|
||||
pushed_at = str(info.get("pushed_at", ""))
|
||||
last_update = datetime.strptime(pushed_at, "%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
existing = repo_repo.get_by_name(repo_name)
|
||||
if existing and existing.last_update and last_update <= existing.last_update:
|
||||
continue
|
||||
|
||||
since_time = existing.last_update if existing and existing.last_update else datetime.now() - timedelta(days=1)
|
||||
recent_commits = self._fetch_recent_commits(repo_name, since_time=since_time, max_commits=5)
|
||||
if not recent_commits:
|
||||
continue
|
||||
|
||||
if existing:
|
||||
repo_repo.update_analysis(
|
||||
repo_name,
|
||||
{
|
||||
"last_update": last_update,
|
||||
"description": str(info.get("description", "") or ""),
|
||||
"url": info.get("html_url"),
|
||||
},
|
||||
)
|
||||
else:
|
||||
repo_repo.create(
|
||||
{
|
||||
"name": repo_name,
|
||||
"description": str(info.get("description", "") or ""),
|
||||
"last_update": last_update,
|
||||
"url": info.get("html_url"),
|
||||
}
|
||||
)
|
||||
|
||||
updates.append(
|
||||
{
|
||||
"name": repo_name,
|
||||
"url": info.get("html_url"),
|
||||
"description": str(info.get("description", "") or ""),
|
||||
"stars": int(info.get("stargazers_count", 0) or 0),
|
||||
"last_update": last_update,
|
||||
"recent_commits": recent_commits,
|
||||
"is_update": bool(existing),
|
||||
"is_watched": True,
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("检查监控仓库失败 %s: %s", repo_name, str(exc))
|
||||
|
||||
return updates
|
||||
|
||||
def _process_watched_repository(
|
||||
self,
|
||||
repo: dict[str, Any],
|
||||
repo_name: str,
|
||||
repo_url: str,
|
||||
last_update: datetime,
|
||||
existing_repo: Any,
|
||||
repo_repo: RepositoryRepo,
|
||||
skip_stats: dict[str, int] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if existing_repo and existing_repo.last_update and last_update <= existing_repo.last_update:
|
||||
self._record_skip_reason(skip_stats, "watched_no_new_update")
|
||||
return None
|
||||
|
||||
since_time = existing_repo.last_update if existing_repo and existing_repo.last_update else None
|
||||
recent_commits = self._fetch_recent_commits(repo_name, since_time=since_time, max_commits=5)
|
||||
if not recent_commits:
|
||||
self._record_skip_reason(skip_stats, "watched_no_recent_commits")
|
||||
return None
|
||||
|
||||
if existing_repo:
|
||||
repo_repo.update_analysis(
|
||||
repo_name,
|
||||
{
|
||||
"last_update": last_update,
|
||||
"description": str(repo.get("description", "") or ""),
|
||||
"url": repo_url,
|
||||
},
|
||||
)
|
||||
else:
|
||||
repo_repo.create(
|
||||
{
|
||||
"name": repo_name,
|
||||
"description": str(repo.get("description", "") or ""),
|
||||
"last_update": last_update,
|
||||
"url": repo_url,
|
||||
}
|
||||
)
|
||||
|
||||
repo_data = {
|
||||
"name": repo_name,
|
||||
"url": repo_url,
|
||||
"description": str(repo.get("description", "") or ""),
|
||||
"stars": int(repo.get("stars", 0) or 0),
|
||||
"last_update": last_update,
|
||||
"recent_commits": recent_commits,
|
||||
"is_update": bool(existing_repo),
|
||||
"is_watched": True,
|
||||
}
|
||||
enriched = self._enrich_quality_info(repo_data, last_update=last_update, recent_commits=recent_commits)
|
||||
if not self._passes_quality_gate(enriched, skip_stats):
|
||||
return None
|
||||
return enriched
|
||||
|
||||
def _process_normal_repository(
|
||||
self,
|
||||
repo: dict[str, Any],
|
||||
repo_name: str,
|
||||
repo_url: str,
|
||||
last_update: datetime,
|
||||
existing_repo: Any,
|
||||
repo_repo: RepositoryRepo,
|
||||
skip_stats: dict[str, int] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if repo_name in self.watched_repositories:
|
||||
self._record_skip_reason(skip_stats, "already_in_watched_list")
|
||||
return None
|
||||
|
||||
if existing_repo:
|
||||
if existing_repo.last_update:
|
||||
try:
|
||||
time_diff = (last_update - existing_repo.last_update).total_seconds()
|
||||
except Exception:
|
||||
time_diff = float("inf")
|
||||
if time_diff < 60:
|
||||
self._record_skip_reason(skip_stats, "stale_update")
|
||||
return None
|
||||
|
||||
recent_commits = self._fetch_recent_commits(
|
||||
repo_name,
|
||||
since_time=existing_repo.last_update,
|
||||
max_commits=5,
|
||||
)
|
||||
if not recent_commits:
|
||||
self._record_skip_reason(skip_stats, "no_recent_commits")
|
||||
return None
|
||||
|
||||
repo_data = {
|
||||
"name": repo_name,
|
||||
"url": repo_url,
|
||||
"description": str(repo.get("description", "") or ""),
|
||||
"stars": int(repo.get("stars", 0) or 0),
|
||||
"last_update": last_update,
|
||||
"recent_commits": recent_commits,
|
||||
"search_keyword": repo.get("search_keyword"),
|
||||
"is_update": True,
|
||||
"is_watched": False,
|
||||
}
|
||||
enriched = self._enrich_quality_info(repo_data, last_update=last_update, recent_commits=recent_commits)
|
||||
if not self._passes_quality_gate(enriched, skip_stats):
|
||||
return None
|
||||
|
||||
repo_repo.update_analysis(
|
||||
repo_name,
|
||||
{
|
||||
"last_update": last_update,
|
||||
"description": str(repo.get("description", "") or ""),
|
||||
"url": repo_url,
|
||||
},
|
||||
)
|
||||
return enriched
|
||||
|
||||
recent_commits = repo.get("recent_commits") or self._fetch_recent_commits(repo_name, max_commits=5)
|
||||
if not recent_commits:
|
||||
self._record_skip_reason(skip_stats, "no_recent_commits")
|
||||
return None
|
||||
|
||||
repo_data = {
|
||||
"name": repo_name,
|
||||
"url": repo_url,
|
||||
"description": str(repo.get("description", "") or ""),
|
||||
"stars": int(repo.get("stars", 0) or 0),
|
||||
"recent_commits": recent_commits,
|
||||
"search_keyword": repo.get("search_keyword"),
|
||||
"is_update": False,
|
||||
"is_watched": False,
|
||||
"last_update": last_update,
|
||||
}
|
||||
enriched = self._enrich_quality_info(repo_data, last_update=last_update, recent_commits=recent_commits)
|
||||
if not self._passes_quality_gate(enriched, skip_stats):
|
||||
return None
|
||||
|
||||
readme_content = self._fetch_readme_content(repo_name)
|
||||
enriched["readme"] = readme_content
|
||||
|
||||
repo_repo.create(
|
||||
{
|
||||
"name": repo_name,
|
||||
"description": str(repo.get("description", "") or ""),
|
||||
"last_update": last_update,
|
||||
"url": repo_url,
|
||||
}
|
||||
)
|
||||
return enriched
|
||||
|
||||
def process_repository_data(
|
||||
self,
|
||||
repos: list[dict[str, Any]],
|
||||
is_watched: bool = False,
|
||||
*,
|
||||
seen_fingerprints: set[str] | None = None,
|
||||
skip_stats: dict[str, int] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""处理并存储仓库数据。"""
|
||||
processed: list[dict[str, Any]] = []
|
||||
|
||||
with get_session(self.session_factory) as session:
|
||||
repo_repo = RepositoryRepo(session)
|
||||
|
||||
for repo in repos:
|
||||
if not isinstance(repo, dict):
|
||||
self._record_skip_reason(skip_stats, "invalid_repo_payload")
|
||||
continue
|
||||
|
||||
repo_name = str(repo.get("full_name") or repo.get("name") or "")
|
||||
repo_url = str(repo.get("html_url") or repo.get("url") or "")
|
||||
if not repo_name or not repo_url:
|
||||
self._record_skip_reason(skip_stats, "missing_repo_fields")
|
||||
continue
|
||||
|
||||
owner = repo.get("owner", {})
|
||||
owner_name = owner.get("login") if isinstance(owner, dict) else repo.get("owner")
|
||||
if self._is_blacklisted(repo_name, str(owner_name or "")):
|
||||
self._record_skip_reason(skip_stats, "blacklisted_or_invalid_repo")
|
||||
continue
|
||||
|
||||
pushed_at = repo.get("pushed_at")
|
||||
if pushed_at:
|
||||
try:
|
||||
last_update = datetime.strptime(str(pushed_at), "%Y-%m-%dT%H:%M:%SZ")
|
||||
except Exception:
|
||||
self._record_skip_reason(skip_stats, "invalid_pushed_at")
|
||||
last_update = datetime.now()
|
||||
else:
|
||||
last_update = repo.get("last_update") if isinstance(repo.get("last_update"), datetime) else datetime.now()
|
||||
|
||||
existing = repo_repo.get_by_name(repo_name)
|
||||
if is_watched:
|
||||
repo_data = self._process_watched_repository(
|
||||
repo,
|
||||
repo_name,
|
||||
repo_url,
|
||||
last_update,
|
||||
existing,
|
||||
repo_repo,
|
||||
skip_stats,
|
||||
)
|
||||
else:
|
||||
repo_data = self._process_normal_repository(
|
||||
repo,
|
||||
repo_name,
|
||||
repo_url,
|
||||
last_update,
|
||||
existing,
|
||||
repo_repo,
|
||||
skip_stats,
|
||||
)
|
||||
|
||||
if repo_data:
|
||||
if seen_fingerprints is not None:
|
||||
fingerprint = str(repo_data.get("fingerprint", "") or "")
|
||||
if not fingerprint:
|
||||
self._record_skip_reason(skip_stats, "missing_fingerprint")
|
||||
continue
|
||||
if fingerprint in seen_fingerprints:
|
||||
self._record_skip_reason(skip_stats, "duplicate_fingerprint")
|
||||
continue
|
||||
seen_fingerprints.add(fingerprint)
|
||||
processed.append(repo_data)
|
||||
|
||||
return processed
|
||||
|
||||
def _update_state(self, status: str, error_message: str | None = None, success: bool = False) -> None:
|
||||
with get_session(self.session_factory) as session:
|
||||
state_repo = MonitorStateRepo(session)
|
||||
state_repo.update_state(
|
||||
"github",
|
||||
status=status,
|
||||
error_message=error_message,
|
||||
success=success,
|
||||
)
|
||||
|
||||
def run(self, interval: int = 3600) -> list[dict[str, Any]]:
|
||||
"""执行一轮 GitHub 监控并返回待分析仓库。"""
|
||||
logger.info("GitHub监控器启动")
|
||||
all_repos: list[dict[str, Any]] = []
|
||||
skip_stats: dict[str, int] = {}
|
||||
seen_repo_names: set[str] = set()
|
||||
seen_fingerprints: set[str] = set()
|
||||
|
||||
try:
|
||||
self._update_state("running")
|
||||
|
||||
watched_updates = self.check_watched_repositories()
|
||||
if watched_updates:
|
||||
accepted_watched = 0
|
||||
for repo in watched_updates:
|
||||
if not isinstance(repo, dict):
|
||||
self._record_skip_reason(skip_stats, "invalid_repo_payload")
|
||||
continue
|
||||
|
||||
repo_name = str(repo.get("name", "") or "")
|
||||
if not repo_name:
|
||||
self._record_skip_reason(skip_stats, "missing_repo_fields")
|
||||
continue
|
||||
if repo_name in seen_repo_names:
|
||||
self._record_skip_reason(skip_stats, "duplicate_repo_name")
|
||||
continue
|
||||
seen_repo_names.add(repo_name)
|
||||
|
||||
last_update = repo.get("last_update") if isinstance(repo.get("last_update"), datetime) else datetime.now()
|
||||
recent_commits = repo.get("recent_commits") if isinstance(repo.get("recent_commits"), list) else []
|
||||
enriched = self._enrich_quality_info(repo, last_update=last_update, recent_commits=recent_commits)
|
||||
if not self._passes_quality_gate(enriched, skip_stats):
|
||||
continue
|
||||
|
||||
fingerprint = str(enriched.get("fingerprint", "") or "")
|
||||
if not fingerprint:
|
||||
self._record_skip_reason(skip_stats, "missing_fingerprint")
|
||||
continue
|
||||
if fingerprint in seen_fingerprints:
|
||||
self._record_skip_reason(skip_stats, "duplicate_fingerprint")
|
||||
continue
|
||||
seen_fingerprints.add(fingerprint)
|
||||
|
||||
all_repos.append(enriched)
|
||||
accepted_watched += 1
|
||||
|
||||
logger.info("发现 %s 个特定仓库更新，入队 %s 个", len(watched_updates), accepted_watched)
|
||||
else:
|
||||
logger.info("特定仓库无更新")
|
||||
|
||||
for keyword in self.keywords:
|
||||
repos = self.search_repositories(
|
||||
keyword,
|
||||
seen_repo_names=seen_repo_names,
|
||||
skip_stats=skip_stats,
|
||||
)
|
||||
if not repos:
|
||||
continue
|
||||
|
||||
repos_to_analyze = self.process_repository_data(
|
||||
repos,
|
||||
is_watched=False,
|
||||
seen_fingerprints=seen_fingerprints,
|
||||
skip_stats=skip_stats,
|
||||
)
|
||||
if repos_to_analyze:
|
||||
all_repos.extend(repos_to_analyze)
|
||||
logger.info("关键词 %s 发现 %s 个仓库", keyword, len(repos_to_analyze))
|
||||
|
||||
total_skipped = sum(skip_stats.values())
|
||||
top_skip_reasons = ", ".join(
|
||||
f"{reason}:{count}"
|
||||
for reason, count in sorted(skip_stats.items(), key=lambda item: item[1], reverse=True)[:5]
|
||||
) or "none"
|
||||
|
||||
logger.info(
|
||||
"仓库筛选统计: accepted=%s, skipped=%s, top_skip_reasons=%s",
|
||||
len(all_repos),
|
||||
total_skipped,
|
||||
top_skip_reasons,
|
||||
)
|
||||
self._update_state("success", success=True)
|
||||
if all_repos:
|
||||
logger.info("本轮总计发现 %s 个需要分析的仓库", len(all_repos))
|
||||
else:
|
||||
logger.info("本轮未发现需要分析的仓库")
|
||||
return all_repos
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("GitHub监控失败: %s", str(exc), exc_info=True)
|
||||
self._update_state("failed", error_message=str(exc), success=False)
|
||||
time.sleep(min(interval, 300))
|
||||
return []
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
monitor = GithubMonitor()
|
||||
monitor.run()
|
||||
@@ -0,0 +1,60 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=68", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "cybersentinel-ai"
|
||||
version = "0.1.0"
|
||||
description = "Automated CVE and security repository monitoring with AI analysis"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
license = { text = "MIT" }
|
||||
authors = [
|
||||
{ name = "CyberSentinel AI contributors" }
|
||||
]
|
||||
keywords = ["security", "cve", "github", "monitoring", "ai", "threat-intelligence"]
|
||||
classifiers = [
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Intended Audience :: Information Technology",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Topic :: Security",
|
||||
]
|
||||
dependencies = [
|
||||
"sqlalchemy>=2.0.0",
|
||||
"requests>=2.31.0",
|
||||
"PyGithub>=2.1.1",
|
||||
"python-dateutil>=2.8.2",
|
||||
"beautifulsoup4>=4.12.2",
|
||||
"lxml>=4.9.3",
|
||||
"markdown>=3.5.1",
|
||||
"colorlog>=6.7.0",
|
||||
"aiohttp>=3.9.1",
|
||||
"feedparser>=6.0.10",
|
||||
"openai>=1.3.0",
|
||||
"pyyaml>=6.0.1",
|
||||
"schedule>=1.2.1",
|
||||
"python-dotenv>=1.0.0",
|
||||
"tqdm>=4.66.1",
|
||||
"google-genai",
|
||||
"GitPython",
|
||||
"pydantic>=2.6.0",
|
||||
"pydantic-settings>=2.2.1",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
cybersentinel-ai = "main:main"
|
||||
|
||||
[tool.setuptools]
|
||||
py-modules = ["config", "main"]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["ai*", "clients*", "database*", "monitors*", "reports*", "utils*"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
python_files = ["test_*.py"]
|
||||
addopts = "-q"
|
||||
@@ -0,0 +1,5 @@
|
||||
"""报告模块导出。"""
|
||||
|
||||
from reports.generator import ReportGenerator
|
||||
|
||||
__all__ = ["ReportGenerator"]
|
||||
@@ -0,0 +1,206 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from database.repositories import RepositoryRepo
|
||||
from database.session import get_session
|
||||
from utils.logger import get_logger
|
||||
from utils.markdown_parser import convert_md_to_html
|
||||
|
||||
logger = get_logger("reports.generator")
|
||||
|
||||
|
||||
class ReportGenerator:
|
||||
"""统一处理分析结果落盘与博客更新。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
session_factory: Any,
|
||||
blog_manager: Any,
|
||||
ai_analysis_dir: str = "results/ai_analysis",
|
||||
html_output_dir: str = "results/ai_html",
|
||||
) -> None:
|
||||
self.session_factory = session_factory
|
||||
self.blog_manager = blog_manager
|
||||
self.ai_analysis_dir = ai_analysis_dir
|
||||
self.html_output_dir = html_output_dir
|
||||
|
||||
os.makedirs(self.ai_analysis_dir, exist_ok=True)
|
||||
os.makedirs(self.html_output_dir, exist_ok=True)
|
||||
|
||||
def _check_repository_limit(self, repo_name: str) -> bool:
|
||||
with get_session(self.session_factory) as session:
|
||||
repo_repo = RepositoryRepo(session)
|
||||
return repo_repo.check_daily_limit(repo_name)
|
||||
|
||||
@staticmethod
|
||||
def _extract_repo_name(result: dict[str, Any], source_type: str) -> str | None:
|
||||
if source_type == "repository":
|
||||
return result.get("repository_name")
|
||||
if source_type == "cve":
|
||||
repo_url = result.get("repository_url", "")
|
||||
if "github.com/" in repo_url:
|
||||
return "/".join(repo_url.split("github.com/")[-1].split("/")[:2])
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _build_common_sections(result: dict[str, Any]) -> str:
|
||||
content = ""
|
||||
if result.get("summary"):
|
||||
content += f"\n#### 💡 分析概述\n\n{result.get('summary')}\n"
|
||||
|
||||
if result.get("key_points"):
|
||||
content += "\n#### 🔍 关键发现\n\n| 序号 | 发现内容 |\n|------|----------|\n"
|
||||
for idx, point in enumerate(result["key_points"], 1):
|
||||
if point and point != "Unknown":
|
||||
content += f"| {idx} | {point} |\n"
|
||||
|
||||
if result.get("technical_details"):
|
||||
content += "\n#### 🛠️ 技术细节\n\n"
|
||||
for detail in result["technical_details"]:
|
||||
if detail and detail != "Unknown":
|
||||
content += f"> {detail}\n\n"
|
||||
|
||||
if result.get("affected_components"):
|
||||
content += "\n#### 🎯 受影响组件\n\n```\n"
|
||||
for comp in result["affected_components"]:
|
||||
if comp and comp != "Unknown" and comp != "未指定受影响组件":
|
||||
content += f"• {comp}\n"
|
||||
content += "```\n"
|
||||
|
||||
if result.get("code_analysis"):
|
||||
content += "\n#### 💻 代码分析\n\n"
|
||||
for idx, analysis in enumerate(result["code_analysis"], 1):
|
||||
if analysis and analysis != "Unknown":
|
||||
content += f"**分析 {idx}**:\n> {analysis}\n\n"
|
||||
|
||||
if result.get("value_reason"):
|
||||
content += (
|
||||
"\n#### ⚡ 价值评估\n\n<details>\n<summary>展开查看详细评估</summary>\n\n"
|
||||
f"{result.get('value_reason')}\n</details>\n"
|
||||
)
|
||||
|
||||
if result.get("recent_commits"):
|
||||
content += "\n#### 📝 最近提交\n\n"
|
||||
for commit in result["recent_commits"][:5]:
|
||||
if isinstance(commit, dict):
|
||||
message = str(commit.get("message", "")).split("\n")[0]
|
||||
commit_date = commit.get("date", "")
|
||||
if message and commit_date:
|
||||
content += f"- **{commit_date}**: {message}\n"
|
||||
return content
|
||||
|
||||
def save_analysis_result(self, result: dict[str, Any], source_type: str) -> bool:
|
||||
"""保存分析结果到当日 Markdown 报告。"""
|
||||
try:
|
||||
repo_name = self._extract_repo_name(result, source_type)
|
||||
if repo_name and not self._check_repository_limit(repo_name):
|
||||
logger.info("仓库 %s 今日已记录，跳过写入", repo_name)
|
||||
return False
|
||||
|
||||
today = datetime.now()
|
||||
filename = os.path.join(self.ai_analysis_dir, f"Ai_{today.strftime('%Y_%m_%d')}.md")
|
||||
identifier = result.get("cve_id") if source_type == "cve" else result.get("repository_name")
|
||||
if not identifier:
|
||||
logger.warning("分析结果缺少唯一标识，跳过写入")
|
||||
return False
|
||||
|
||||
if not os.path.exists(filename):
|
||||
header = (
|
||||
f"# AI 安全分析日报 ({today.strftime('%Y-%m-%d')})\n\n"
|
||||
"本文档包含 AI 对安全相关内容的自动化分析结果。"
|
||||
"[概览](https://blog.897010.xyz/c/today)\n\n"
|
||||
)
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
f.write(header)
|
||||
|
||||
with open(filename, "r", encoding="utf-8") as f:
|
||||
existing_content = f.read()
|
||||
|
||||
if f"### {identifier}" in existing_content:
|
||||
logger.info("内容已存在，跳过: %s", identifier)
|
||||
return False
|
||||
|
||||
short_desc = str(result.get("short_description", "")).strip()
|
||||
desc_text = f" - {short_desc}" if short_desc else ""
|
||||
|
||||
if source_type == "cve":
|
||||
content = f"\n### {identifier}{desc_text}\n\n"
|
||||
content += "#### 📌 漏洞信息\n\n| 属性 | 详情 |\n|------|------|\n"
|
||||
content += f"| CVE编号 | {result.get('cve_id')} |\n"
|
||||
content += f"| 风险等级 | `{result.get('risk_level')}` |\n"
|
||||
content += f"| 利用状态 | `{result.get('exploit_status')}` |\n"
|
||||
if result.get("published_date"):
|
||||
content += f"| 发布时间 | {result.get('published_date')} |\n"
|
||||
if result.get("last_modified"):
|
||||
content += f"| 最后更新 | {result.get('last_modified')} |\n"
|
||||
if result.get("repository_url"):
|
||||
repo_url = result.get("repository_url")
|
||||
content += "\n#### 📦 相关仓库\n\n"
|
||||
content += f"- [{str(repo_url).split('/')[-1]}]({repo_url})\n"
|
||||
else:
|
||||
content = f"\n### {identifier}{desc_text}\n\n"
|
||||
content += "#### 📌 仓库信息\n\n| 属性 | 详情 |\n|------|------|\n"
|
||||
content += (
|
||||
f"| 仓库名称 | [{result.get('repository_name')}]({result.get('repository_url')}) |\n"
|
||||
)
|
||||
content += f"| 风险等级 | `{result.get('risk_level')}` |\n"
|
||||
content += f"| 安全类型 | `{result.get('security_type')}` |\n"
|
||||
if result.get("update_type"):
|
||||
content += f"| 更新类型 | `{result.get('update_type')}` |\n"
|
||||
if result.get("commits_analyzed") or result.get("files_changed"):
|
||||
content += "\n#### 📊 代码统计\n\n"
|
||||
if result.get("commits_analyzed"):
|
||||
content += f"- 分析提交数: **{result.get('commits_analyzed')}**\n"
|
||||
if result.get("files_changed"):
|
||||
content += f"- 变更文件数: **{result.get('files_changed')}**\n"
|
||||
|
||||
content += self._build_common_sections(result)
|
||||
content += "\n---\n"
|
||||
|
||||
with open(filename, "a", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
|
||||
logger.info("已保存分析结果到: %s", filename)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error("保存分析结果失败: %s", str(exc), exc_info=True)
|
||||
return False
|
||||
|
||||
def convert_and_update_blog(self) -> bool:
|
||||
"""将当天 Markdown 转 HTML 并同步到博客。"""
|
||||
try:
|
||||
today = datetime.now().strftime("%Y_%m_%d")
|
||||
md_file = os.path.join(self.ai_analysis_dir, f"Ai_{today}.md")
|
||||
html_file = os.path.join(self.html_output_dir, f"Ai_{today}.html")
|
||||
|
||||
if not os.path.exists(md_file):
|
||||
logger.info("今天没有AI分析结果，跳过博客更新")
|
||||
return False
|
||||
|
||||
with open(md_file, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
if not content.strip():
|
||||
logger.info("今天的AI分析结果为空，跳过博客更新")
|
||||
return False
|
||||
|
||||
convert_md_to_html(md_file)
|
||||
if not os.path.exists(html_file):
|
||||
logger.error("Markdown 转换 HTML 失败")
|
||||
return False
|
||||
|
||||
with open(html_file, "r", encoding="utf-8") as f:
|
||||
html_content = f.read()
|
||||
|
||||
success = self.blog_manager.update_html_content(html_content)
|
||||
if success:
|
||||
logger.info("成功更新博客的 HTML 内容")
|
||||
else:
|
||||
logger.error("更新博客 HTML 内容失败")
|
||||
return success
|
||||
except Exception as exc:
|
||||
logger.error("转换并更新博客失败: %s", str(exc), exc_info=True)
|
||||
return False
|
||||
@@ -0,0 +1,20 @@
|
||||
sqlalchemy>=2.0.0
|
||||
requests>=2.31.0
|
||||
PyGithub>=2.1.1
|
||||
python-dateutil>=2.8.2
|
||||
beautifulsoup4>=4.12.2
|
||||
lxml>=4.9.3
|
||||
markdown>=3.5.1
|
||||
colorlog>=6.7.0
|
||||
aiohttp>=3.9.1
|
||||
feedparser>=6.0.10
|
||||
openai>=1.3.0
|
||||
pyyaml>=6.0.1
|
||||
schedule>=1.2.1
|
||||
python-dotenv>=1.0.0
|
||||
tqdm>=4.66.1
|
||||
google-genai
|
||||
GitPython
|
||||
pydantic>=2.6.0
|
||||
pydantic-settings>=2.2.1
|
||||
pytest>=8.0.0
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -1,856 +0,0 @@
|
||||
|
||||
# 安全资讯日报 2025-02-24
|
||||
|
||||
> 本文由AI自动生成，基于对安全相关仓库、CVE和最新安全研究成果的自动化分析。
|
||||
>
|
||||
> 更新时间：2025-02-24 12:01:41
|
||||
|
||||
<!-- more -->
|
||||
|
||||
## 今日资讯
|
||||
|
||||
### 🔍 漏洞分析
|
||||
|
||||
* [安全圈马斯克DOGE网站数据库存在漏洞，任何人可随意篡改内容](https://mp.weixin.qq.com/s?__biz=MzIzMzE4NDU1OQ==&mid=2652068092&idx=1&sn=2aaad093096f88adba585911871847bd&chksm=f36e74bcc419fdaa044ff4055328159a66579d42cbbbda1577b8e0e97d0077812fe4f0f44877&scene=58&subscene=0#rd)
|
||||
* [安全圈Atos旗下Eviden公司紧急发布安全公告：IDPKI解决方案曝出高危漏洞](https://mp.weixin.qq.com/s?__biz=MzIzMzE4NDU1OQ==&mid=2652068092&idx=2&sn=07ceda3a43b852c9147f82cc1ed039fa&chksm=f36e74bcc419fdaac195c826bdb0acc22ee49a2e1a959765c9c3405ad72d8f3cb5509157f596&scene=58&subscene=0#rd)
|
||||
|
||||
### 🔬 安全研究
|
||||
|
||||
* [CISO专栏 | 企业如何应对后量子密码威胁](https://mp.weixin.qq.com/s?__biz=MzU0MzgyMzM2Nw==&mid=2247486268&idx=1&sn=768496b28b1a1134a77c378aa605a2ab&chksm=fb04c854cc734142726054d2ab71b4d282797aa374e46019c440fea183a417600b1f212b950d&scene=58&subscene=0#rd)
|
||||
* [.NET内网实战：通过伪造文件时间戳实现入侵阶段的痕迹清理](https://mp.weixin.qq.com/s?__biz=MzUyOTc3NTQ5MA==&mid=2247498989&idx=1&sn=03cb604d5a6470d7c9f0ab8755415024&chksm=fa595200cd2edb16d3f46d7488bc6f1fa567e951acdb9d1eda726d34bd9e5e295015e5172e3c&scene=58&subscene=0#rd)
|
||||
* [Linkedin 如何利用大语言模型赋能安全态势平台（SPP）](https://mp.weixin.qq.com/s?__biz=Mzg5NjAxNjc5OQ==&mid=2247484228&idx=1&sn=bdba90bfa8f158b5fb2260ad97bb20d4&chksm=c006cbb4f77142a22b6a6e438cfebb4de822e913c653a004e34c71b106209c8d72c0908d5124&scene=58&subscene=0#rd)
|
||||
|
||||
### 🎯 威胁情报
|
||||
|
||||
* [加密货币 APT 情报：揭秘 Lazarus Group 入侵手法](https://mp.weixin.qq.com/s?__biz=MzU4ODQ3NTM2OA==&mid=2247501259&idx=1&sn=0b2183929367aa5845f0a9e1d1a42e74&chksm=fddebb4ccaa9325a643092adbf32e184c96a1c95717f5f587e160eb60729319775c5ea708de0&scene=58&subscene=0#rd)
|
||||
|
||||
### 🛠️ 安全工具
|
||||
|
||||
* [Linux恶意软件自动化分析开源沙箱](https://mp.weixin.qq.com/s?__biz=MzA4ODEyODA3MQ==&mid=2247490656&idx=1&sn=f15e7959caf5f0e404b6abe35e5afe91&chksm=902fb348a7583a5e7fd0e9a9070584036e06a5534a2b06c7186e68fc41d169556b10b7012ba4&scene=58&subscene=0#rd)
|
||||
|
||||
### 🍉 吃瓜新闻
|
||||
|
||||
* [安全圈数据泄露警报拉响！2024 医疗行业成重灾区，远超金融行业](https://mp.weixin.qq.com/s?__biz=MzIzMzE4NDU1OQ==&mid=2652068092&idx=3&sn=feadbe596231da73575f5a1c17b7609a&chksm=f36e74bcc419fdaa8a359a5fc422574b26c9f63b4e291abe15ab5c71cd3fbacf913e6f32dff3&scene=58&subscene=0#rd)
|
||||
* [美计划推乔布斯 1 美元硬币；15 亿美元，加密币史上最大黑客盗窃；小米曝超大电池 AI PC | 极客早知道](https://mp.weixin.qq.com/s?__biz=MTMwNDMwODQ0MQ==&mid=2653074236&idx=1&sn=97eeb4aaef01b51b33db442af1cd28ef&chksm=7e57ce8a4920479c8b64adee91f539a84f302ec556587bf94c10b929c2f7310dcbafbbdc275a&scene=58&subscene=0#rd)
|
||||
|
||||
### 📌 其他
|
||||
|
||||
* [.NET 安全攻防知识交流社区](https://mp.weixin.qq.com/s?__biz=MzUyOTc3NTQ5MA==&mid=2247498989&idx=2&sn=ca5dde9acc7fc5418a50c1cb0c8250e7&chksm=fa595200cd2edb16a8dfaaba4bfe5cb5f39031cb1c789f9ed4c535844d9c2797fd6788a9b89e&scene=58&subscene=0#rd)
|
||||
* [.NET 总第 60 期红队武器库和资源汇总](https://mp.weixin.qq.com/s?__biz=MzUyOTc3NTQ5MA==&mid=2247498989&idx=3&sn=2c9827f541f8213a4b0c31708e2aa873&chksm=fa595200cd2edb164b0fff2297013795fb345e1bcd5f51c782da46bba2e311d64d8c32ed198c&scene=58&subscene=0#rd)
|
||||
* [直播预约：元方-原生安全大模型产品发布会](https://mp.weixin.qq.com/s?__biz=MzkxNzA3MTgyNg==&mid=2247537315&idx=1&sn=ccc6b58de029115efae4cfafb208eef2&chksm=c144201ef633a908a2eb78ca348aa1274354c2c8766f0217c8739ff85bfbcf9d09c362c292b6&scene=58&subscene=0#rd)
|
||||
* [掌握58门硬核课程，失业不再可能！卷到极致！](https://mp.weixin.qq.com/s?__biz=MzU0MzkzOTYzOQ==&mid=2247489705&idx=1&sn=2aac478313b608a5b19fd3ffddcc4ac9&chksm=fb0295f1cc751ce7cd8d74a4264cd20f0a3f365820efb060c7e8be43b3da2a2d86ef00db3e46&scene=58&subscene=0#rd)
|
||||
* [200小时黑客挑战：如何在BugCrowd上赚取2万美元？](https://mp.weixin.qq.com/s?__biz=MzU2NzcwNTY3Mg==&mid=2247485353&idx=1&sn=aa2044a30cc9dd73ceb51a58bde65afe&chksm=fc986e8ecbefe7987e15c22353288d0a1b7bc22f0ee6ee8a29edc795fc82367f74bccee82fc6&scene=58&subscene=0#rd)
|
||||
* [小伙子，你要“耗子尾汁”](https://mp.weixin.qq.com/s?__biz=MzIzMTIzNTM0MA==&mid=2247497182&idx=1&sn=8de0150bc807b15280bcbb46a26f33f8&chksm=e8a5ffbddfd276abdcbea8ab191e0283d495657aaf3a2e414a3b80811aff327722087d083a8b&scene=58&subscene=0#rd)
|
||||
* [特朗普系列总统行动：贸易优先和投资优先](https://mp.weixin.qq.com/s?__biz=MzIxODM0NDU4MQ==&mid=2247506533&idx=1&sn=360d6d956a59d56bf7010fc5d5733967&chksm=97e9678fa09eee9971c36e063f38c2ff6e3b5202cb79e6b385dd38d15a9375ad5ed7a985ca70&scene=58&subscene=0#rd)
|
||||
|
||||
## 安全分析
|
||||
(2025-02-24)
|
||||
|
||||
本文档包含 AI 对安全相关内容的自动化分析结果。[概览](https://blog.897010.xyz/c/today)
|
||||
|
||||
|
||||
### CVE-2024-49138 - Windows内核POC
|
||||
|
||||
#### 📌 漏洞信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| CVE编号 | CVE-2024-49138 |
|
||||
| 风险等级 | `HIGH` |
|
||||
| 利用状态 | `POC可用` |
|
||||
| 发布时间 | 2025-02-23 00:00:00 |
|
||||
| 最后更新 | 2025-02-23 16:14:28 |
|
||||
|
||||
#### 📦 相关仓库
|
||||
|
||||
- [CVE-2024-49138-POC](https://github.com/aspire20x/CVE-2024-49138-POC)
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
该CVE提供了一个Windows内核漏洞的POC。相关仓库中包含了POC利用代码，并在README中提供了下载链接和利用说明。最后一次提交更新了下载链接。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | Windows内核漏洞 |
|
||||
| 2 | POC代码已发布 |
|
||||
| 3 | 提供下载链接 |
|
||||
| 4 | MIT License |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> 漏洞涉及Windows内核，具体细节未知，但POC已发布
|
||||
|
||||
> POC提供下载，包含利用程序
|
||||
|
||||
> 修复方案取决于漏洞细节，目前未知
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• Windows Kernel
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
提供了针对Windows内核的POC，表明存在潜在的漏洞利用风险，虽然具体细节未知，但存在实际的利用代码。
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### CVE-2025-20029 - F5 BIG-IP TMSH CLI命令注入
|
||||
|
||||
#### 📌 漏洞信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| CVE编号 | CVE-2025-20029 |
|
||||
| 风险等级 | `CRITICAL` |
|
||||
| 利用状态 | `POC可用` |
|
||||
| 发布时间 | 2025-02-23 00:00:00 |
|
||||
| 最后更新 | 2025-02-23 18:53:24 |
|
||||
|
||||
#### 📦 相关仓库
|
||||
|
||||
- [CVE-2025-20029](https://github.com/mbadanoiu/CVE-2025-20029)
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
F5 BIG-IP 的TMSH CLI存在命令注入漏洞。经过身份验证的攻击者可以利用该漏洞绕过权限限制，注入任意命令并在目标系统上以root用户的身份执行代码。漏洞利用需要有效用户凭证，并能够发送请求到iControl REST组件或执行tmsh命令。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | TMSH CLI 命令注入漏洞 |
|
||||
| 2 | 需要有效用户凭证 |
|
||||
| 3 | 可执行tmsh命令或发送请求到iControl REST组件 |
|
||||
| 4 | 可绕过权限限制，执行任意命令，获取root权限 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> 漏洞位于TMSH CLI中。
|
||||
|
||||
> 攻击者通过构造恶意命令进行注入。
|
||||
|
||||
> 攻击者可提升至root权限。
|
||||
|
||||
> 利用方法参考提供的PDF文件。
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• F5 BIG-IP
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
漏洞影响F5 BIG-IP，属于关键基础设施。存在可用的POC，且可提升权限至root，因此是高价值漏洞。
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### microsoft-word-2021 - The repository description
|
||||
|
||||
#### 📌 仓库信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| 仓库名称 | [microsoft-word-2021](https://github.com/Nagy235/microsoft-word-2021) |
|
||||
| 风险等级 | `LOW` |
|
||||
| 安全类型 | `the-value` |
|
||||
| 更新类型 | `SECURITY_CRITICAL` |
|
||||
|
||||
#### 📊 代码统计
|
||||
|
||||
- 分析提交数: **1**
|
||||
- 变更文件数: **1**
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
内容描述
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | 仓库的主要功能 |
|
||||
| 2 | 更新的主要内容 |
|
||||
| 3 | 安全相关变更 |
|
||||
| 4 | 影响评估 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> 细节1
|
||||
|
||||
> 细节2
|
||||
|
||||
> 细节3
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• 部件列表
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
原因分析
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### poc_everyday - CVE漏洞PoC和Exp每日更新
|
||||
|
||||
#### 📌 仓库信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| 仓库名称 | [poc_everyday](https://github.com/126789t/poc_everyday) |
|
||||
| 风险等级 | `CRITICAL` |
|
||||
| 安全类型 | `POC更新/漏洞利用` |
|
||||
| 更新类型 | `SECURITY_CRITICAL` |
|
||||
|
||||
#### 📊 代码统计
|
||||
|
||||
- 分析提交数: **5**
|
||||
- 变更文件数: **1500**
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
该仓库每日更新Github上CVE相关的漏洞PoC和Exp信息，本次更新增加了多个CVE漏洞的描述和相关链接，其中包含一些高危漏洞，如CVE-2013-0230, CVE-2013-0229, CVE-2013-2596等，覆盖了广泛的软件和系统，具有很高的参考价值。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | 每日更新CVE相关的PoC和漏洞信息 |
|
||||
| 2 | 包含多个CVE漏洞的描述和漏洞利用代码链接 |
|
||||
| 3 | 覆盖广泛的软件和系统 |
|
||||
| 4 | 涉及多个高危漏洞 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> 该仓库通过自动化的方式收集和整理Github上的CVE相关信息，每次更新会添加最新的漏洞信息。
|
||||
|
||||
> 本次更新包含了多个CVE漏洞的描述，漏洞利用代码的链接，涉及多个高危漏洞。
|
||||
|
||||
> 更新的文件以年份和CVE编号进行组织，方便查找
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• Ruby
|
||||
• libpng
|
||||
• Java
|
||||
• Linksys WRT54GL
|
||||
• Firefox
|
||||
• Android
|
||||
• Palo Alto Networks PAN-OS
|
||||
• Ruby on Rails
|
||||
• JSON gem
|
||||
• MiniUPnP
|
||||
• Adobe Reader and Acrobat
|
||||
• Apache HTTP Server
|
||||
• Novell ZENworks Mobile Management
|
||||
• PolicyKit
|
||||
• Linux kernel
|
||||
• Qualcomm Android
|
||||
• MySecureShell
|
||||
• Microsoft Internet Explorer
|
||||
• SAP Netweaver
|
||||
• WordPress
|
||||
• Xen
|
||||
• Oracle Fusion Middleware
|
||||
• Technicolor TC7200
|
||||
• Oracle Database Server
|
||||
• Mozilla Firefox
|
||||
• Microsoft Windows
|
||||
• Windows XP
|
||||
• Windows Server
|
||||
• Windows Vista
|
||||
• Windows 7
|
||||
• Windows 8
|
||||
• Windows 8.1
|
||||
• Windows RT
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
该仓库每日更新，包含最新的CVE漏洞信息和PoC，为安全研究人员提供了重要的参考，价值极高。
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### PhoenixC2 - C2框架，DDoS攻击工具
|
||||
|
||||
#### 📌 仓库信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| 仓库名称 | [PhoenixC2](https://github.com/lilroniel/PhoenixC2) |
|
||||
| 风险等级 | `HIGH` |
|
||||
| 安全类型 | `安全研究` |
|
||||
| 更新类型 | `GENERAL_UPDATE` |
|
||||
|
||||
#### 📊 代码统计
|
||||
|
||||
- 分析提交数: **1**
|
||||
- 变更文件数: **1**
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
仓库更新内容增加了C2框架的下载链接，且提供了相关DDoS攻击的描述，可能涉及恶意软件或潜在的DDoS攻击。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | C2框架，潜在的DDoS攻击工具 |
|
||||
| 2 | 更新说明中提及了具体的DDoS攻击方法 |
|
||||
| 3 | 提供了C2框架的下载链接 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> C2框架，潜在的DDoS攻击工具，可能被用于恶意目的。
|
||||
|
||||
> 更新说明中提及了具体的DDoS攻击方法，增加了C2框架的价值，存在安全风险。
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• C2框架
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
C2框架往往用于攻击，存在安全风险。本更新提供了C2框架的下载链接以及相关的DDoS攻击的描述，意味着潜在的恶意用途。
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### TOP - POC和漏洞利用集合
|
||||
|
||||
#### 📌 仓库信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| 仓库名称 | [TOP](https://github.com/GhostTroops/TOP) |
|
||||
| 风险等级 | `HIGH` |
|
||||
| 安全类型 | `POC更新/漏洞利用` |
|
||||
| 更新类型 | `SECURITY_IMPROVEMENT` |
|
||||
|
||||
#### 📊 代码统计
|
||||
|
||||
- 分析提交数: **2**
|
||||
- 变更文件数: **2**
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
该仓库是一个POC和漏洞利用的集合，此次更新增加了多个CVE相关的POC，例如CVE-2025-21298, CVE-2025-0411, CVE-2025-21420, CVE-2025-0108。其中CVE-2025-21420展示了DLL侧加载的方法，CVE-2025-0108提供了批量检测脚本。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | 持续更新的漏洞POC和Exp集合。 |
|
||||
| 2 | 包含多个CVE漏洞的POC，如CVE-2025-21298, CVE-2025-0411, CVE-2025-21420, CVE-2025-0108 等。 |
|
||||
| 3 | 更新涉及多个项目，表明持续关注最新的安全漏洞。 |
|
||||
| 4 | 提供了关于CVE-2025-21420的DLL侧加载和CVE-2025-0108的批量检测脚本。 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> README.md 文件更新，增加了多个CVE漏洞的POC链接。CVE-2025-21420涉及到使用cleanmgr.exe进行DLL侧加载，CVE-2025-0108是Palo Alto Networks PAN-OS 身份验证绕过漏洞的批量检测脚本。
|
||||
|
||||
> 这些更新表明了对最新漏洞的关注，并提供了实际的漏洞利用或检测方法。DLL侧加载和身份验证绕过等漏洞具有较高的安全风险。
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• 多种软件和系统，具体取决于所列出的CVE漏洞
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
仓库更新了多个漏洞的POC，其中包含针对高危漏洞的利用方式和检测方法。这些POC对于安全研究人员和渗透测试人员来说具有重要价值。
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### CVE-2022-47522 - Wi-Fi网络帧拦截漏洞
|
||||
|
||||
#### 📌 漏洞信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| CVE编号 | CVE-2022-47522 |
|
||||
| 风险等级 | `HIGH` |
|
||||
| 利用状态 | `漏洞利用可用` |
|
||||
| 发布时间 | 2025-02-24 00:00:00 |
|
||||
| 最后更新 | 2025-02-24 02:12:40 |
|
||||
|
||||
#### 📦 相关仓库
|
||||
|
||||
- [CVE-2022-47522-PoC](https://github.com/toffeenutt/CVE-2022-47522-PoC)
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
该漏洞允许攻击者拦截Wi-Fi网络中发送给任意客户端的帧。该漏洞基于MAC地址欺骗和Deauthentication攻击。POC代码已发布，演示了拦截HTTP和ICMP请求。攻击需要配置wpa_supplicant和NetworkManager，并且需要禁用MFP。成功的攻击依赖于受害者和攻击者动作的同步，以及快速的连接时间。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | 攻击者能够拦截Wi-Fi网络中发送给任意客户端的帧。 |
|
||||
| 2 | 攻击涉及预连接、Deauth攻击和MAC地址欺骗。 |
|
||||
| 3 | 需要配置wpa_supplicant和NetworkManager，禁用MFP。 |
|
||||
| 4 | POC提供了HTTP和ICMP请求拦截的演示。 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> 攻击者伪装成受害者MAC地址连接到AP。
|
||||
|
||||
> 通过Deauthentication帧断开受害者和AP的连接。
|
||||
|
||||
> 攻击者需要配置两个无线网卡，一个用于监听模式。
|
||||
|
||||
> 成功拦截取决于时序，预连接有助于快速重新连接。
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• Wi-Fi网络
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
POC可用，详细描述了利用方法，可用于拦截网络流量。
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### sqlmap_gui - 图形化SQL注入工具
|
||||
|
||||
#### 📌 仓库信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| 仓库名称 | [sqlmap_gui](https://github.com/suqianjue/sqlmap_gui) |
|
||||
| 风险等级 | `MEDIUM` |
|
||||
| 安全类型 | `安全功能` |
|
||||
| 更新类型 | `GENERAL_UPDATE` |
|
||||
|
||||
#### 📊 代码统计
|
||||
|
||||
- 分析提交数: **2**
|
||||
- 变更文件数: **1**
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
更新了readme文件，明确了Java版本，说明了如何安装和配置SQLMap。修复了安全漏洞，并提供了java11版本的安装方式，并使用java11编译.
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | 修复了Java版本，明确了Java版本，要求使用java11编译 |
|
||||
| 2 | 修改了README.md，增加了软件的安装依赖 |
|
||||
| 3 | 安全风险相关的修复 |
|
||||
| 4 | 影响分析 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> 明确了Java版本需求
|
||||
|
||||
> 安全影响分析: 受影响的组件增加了Java版本
|
||||
|
||||
> Java版本兼容性问题
|
||||
|
||||
> 增加安全检测或防护功能
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• SQLMap工具
|
||||
• Java环境
|
||||
• SQLMap API
|
||||
• GUI程序
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
主要增加了readme文件，明确了java11版本。提高了安全性
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### dootask - 优化消息处理和文件读取
|
||||
|
||||
#### 📌 仓库信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| 仓库名称 | [dootask](https://github.com/kuaifan/dootask) |
|
||||
| 风险等级 | `LOW` |
|
||||
| 安全类型 | `安全功能` |
|
||||
| 更新类型 | `SECURITY_IMPROVEMENT` |
|
||||
|
||||
#### 📊 代码统计
|
||||
|
||||
- 分析提交数: **5**
|
||||
- 变更文件数: **80**
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
此次更新优化了消息处理逻辑，限制了文件读取大小，并增强了AI分析文件的功能。这些变更提高了系统的稳定性和安全性，避免了潜在的性能问题和安全风险。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | 优化了Markdown消息过长处理，通过截断和占位符处理特殊标记，避免Markdown转换错误。 |
|
||||
| 2 | 限制了getFileContent方法读取的最大文件大小，防止读取过大文件导致性能问题或潜在的安全风险。 |
|
||||
| 3 | 优化了AI分析指定文件的功能，提高了效率。 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> MsgTool.php: 优化了Markdown消息截断逻辑，使用占位符处理特殊标记。
|
||||
|
||||
> TextExtractor.php: 增加了getFileContent方法的文件大小限制，防止读取超大文件。
|
||||
|
||||
> FileContent.php: 优化了AI分析文件功能。
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• app/Module/MsgTool.php
|
||||
• app/Module/TextExtractor.php
|
||||
• app/Models/FileContent.php
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
这些改动虽然不是直接修复漏洞，但提高了系统的安全性和稳定性，降低了潜在的安全风险，例如拒绝服务攻击或信息泄露。
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### SkyFall-Pack - C2基础设施搭建
|
||||
|
||||
#### 📌 仓库信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| 仓库名称 | [SkyFall-Pack](https://github.com/nickvourd/SkyFall-Pack) |
|
||||
| 风险等级 | `MEDIUM` |
|
||||
| 安全类型 | `安全研究` |
|
||||
| 更新类型 | `GENERAL_UPDATE` |
|
||||
|
||||
#### 📊 代码统计
|
||||
|
||||
- 分析提交数: **5**
|
||||
- 变更文件数: **5**
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
该仓库是一个基础设施包，用于构建 C2 服务器。虽然更新内容主要是README文档的修改，但是鉴于其C2框架的特性，该项目仍然具有潜在的威胁。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | SkyFall Infrastructure Pack 旨在简化 C2 基础设施的搭建 |
|
||||
| 2 | 更新内容主要为 README 文件的修改，包含部署和配置说明 |
|
||||
| 3 | 该项目包含 Cloudflare Worker 相关配置 |
|
||||
| 4 | 项目可能用于构建恶意基础设施，存在安全风险 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> README 文件详细介绍了如何配置 Cloudflare Worker。
|
||||
|
||||
> 部署流程包括创建 Cloudflare 应用程序、配置 Cloudflare Worker，并使用 Wrangler 进行部署。
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• Cloudflare Worker
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
由于该项目明确用于搭建 C2 基础设施，虽然更新内容是README文件的修改，但是项目本身与网络安全强相关，具有潜在的威胁价值。
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### SAST-Test-Repo-e1c33015-c2f8-46de-b9e0-016044e80d9c - 包含C2 API接口及相关安全风险
|
||||
|
||||
#### 📌 仓库信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| 仓库名称 | [SAST-Test-Repo-e1c33015-c2f8-46de-b9e0-016044e80d9c](https://github.com/SAST-UP-PROD-app-mend/SAST-Test-Repo-e1c33015-c2f8-46de-b9e0-016044e80d9c) |
|
||||
| 风险等级 | `HIGH` |
|
||||
| 安全类型 | `安全研究` |
|
||||
| 更新类型 | `新增代码` |
|
||||
|
||||
#### 📊 代码统计
|
||||
|
||||
- 分析提交数: **3**
|
||||
- 变更文件数: **21**
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
该仓库包含多个Python文件，实现了一个Web应用程序。其中mod_api.py文件中定义了/key和/post API接口，与C2（Command and Control）通信模式相关。api_list.py, api_post.py， libapi.py相关文件，涉及API密钥的生成和使用，以及API接口的调用和实现。brute.py脚本展示了密码暴力破解的可能性，bad目录下的文件揭示了潜在的安全风险。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | 包含C2相关的API接口，可能被用于远程控制或数据窃取 |
|
||||
| 2 | 实现了一个简单的用户认证系统，存在潜在的安全漏洞 |
|
||||
| 3 | 存在密码暴力破解脚本，可能导致用户账户被攻破 |
|
||||
| 4 | 包含了与C2相关的API接口，与搜索关键词高度相关 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> mod_api.py定义了/key和/post API接口，/key接口用于生成API密钥，/post接口用于接收数据。libapi.py 实现了密钥的生成与验证。
|
||||
|
||||
> brute.py脚本展示了针对用户账户进行暴力破解的可能性
|
||||
|
||||
> libuser.py包含用户登录功能, db_init.py和db.py用于初始化和操作用户数据库。
|
||||
|
||||
> 该项目使用Flask框架，并使用SQLite作为数据库。
|
||||
|
||||
> 安全风险包括但不限于：API密钥泄露、暴力破解、SQL注入。
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• Flask
|
||||
• SQLite
|
||||
• Python Web Application
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
仓库与C2关键词高度相关，提供了API接口和用户认证功能，相关性评分HIGH。提供了技术实现细节，有安全研究价值。
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### C2IntelFeedsFGT - C2情报源，用于Fortigate防火墙
|
||||
|
||||
#### 📌 仓库信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| 仓库名称 | [C2IntelFeedsFGT](https://github.com/nwiecz/C2IntelFeedsFGT) |
|
||||
| 风险等级 | `MEDIUM` |
|
||||
| 安全类型 | `安全工具/安全研究` |
|
||||
| 更新类型 | `代码更新` |
|
||||
|
||||
#### 📊 代码统计
|
||||
|
||||
- 分析提交数: **5**
|
||||
- 变更文件数: **1**
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
该仓库自动化创建C2情报源，用于Fortigate的IP地址威胁源和域名威胁源。它基于Censys的数据，并为多个C2框架提供了对应的Censys搜索查询。更新主要集中在更新C2情报源的CSV文件。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | 自动生成C2情报订阅源，针对Fortigate防火墙优化 |
|
||||
| 2 | 基于Censys数据，涵盖IP和域名信息 |
|
||||
| 3 | 提供7天和30天C2活动数据，以及过滤后的域名列表 |
|
||||
| 4 | 包含针对多个C2框架（Sliver, Covenant, Brute Ratel C4, Mythic等）的Censys搜索查询 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> 使用Censys数据作为威胁情报来源，自动抓取C2相关信息。
|
||||
|
||||
> 提供IP和域名两种类型的C2情报，包括过滤后的域名列表。
|
||||
|
||||
> 使用Shell脚本 `stripper` 处理和生成 CSV 文件，用于提供给Fortigate防火墙。
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• Fortigate, Censys
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
该仓库与C2关键词高度相关，提供了C2情报源的自动化生成，并针对Fortigate进行了优化，具备一定的实用价值。虽然不包含漏洞利用代码，但为威胁情报的获取提供了便捷途径，对安全研究有参考价值。
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### PhD-DSO-Research - AI驱动的CVE扫描功能
|
||||
|
||||
#### 📌 仓库信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| 仓库名称 | [PhD-DSO-Research](https://github.com/yojimbo256/PhD-DSO-Research) |
|
||||
| 风险等级 | `MEDIUM` |
|
||||
| 安全类型 | `安全功能` |
|
||||
| 更新类型 | `SECURITY_IMPROVEMENT` |
|
||||
|
||||
#### 📊 代码统计
|
||||
|
||||
- 分析提交数: **5**
|
||||
- 变更文件数: **4**
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
该仓库新增了CVE扫描器的核心代码，`cve_scanner.py` 实现了从NVD获取CVE数据的基本功能。虽然目前只是实现了CVE数据的获取和解析，但为后续的漏洞扫描、安全检测和自动化合规提供了基础。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | 新增了CVE扫描器核心代码 |
|
||||
| 2 | cve_scanner.py实现了从NVD获取CVE数据的基本功能 |
|
||||
| 3 | 初步实现了CVE数据的获取和解析 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> `cve_scanner.py` 文件中，通过使用 `requests` 库从 NVD 的 API 获取最新的 CVE 数据。它定义了 `fetch_cve_data()` 函数，该函数向NVD API 发送请求并解析JSON响应。`scan_vulnerabilities()` 函数用于处理和打印 CVE 数据。目前仅打印了CVE ID和描述，后续可以扩展扫描模块，例如依赖扫描和漏洞利用。
|
||||
|
||||
> 代码通过调用NVD API获取CVE列表,并初步展示了CVE信息
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• cve_scanner.py
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
该更新为自动化CVE扫描提供了基础，对DevSecOps的自动化安全评估有实际价值，符合价值判断标准
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
### securityhub_soc2analysis - AI驱动的SOC 2合规分析
|
||||
|
||||
#### 📌 仓库信息
|
||||
|
||||
| 属性 | 详情 |
|
||||
|------|------|
|
||||
| 仓库名称 | [securityhub_soc2analysis](https://github.com/ajy0127/securityhub_soc2analysis) |
|
||||
| 风险等级 | `LOW` |
|
||||
| 安全类型 | `安全工具` |
|
||||
| 更新类型 | `新增` |
|
||||
|
||||
#### 📊 代码统计
|
||||
|
||||
- 分析提交数: **5**
|
||||
- 变更文件数: **5**
|
||||
|
||||
#### 💡 分析概述
|
||||
|
||||
该仓库提供了一个自动化工具，利用AWS SecurityHub和Amazon Bedrock，为SOC 2合规性提供AI驱动的分析和报告。它将SecurityHub的发现映射到SOC 2控制，并生成每日报告。代码更新主要集中在CI/CD流程的完善，包括添加lint检查和Git配置。
|
||||
|
||||
#### 🔍 关键发现
|
||||
|
||||
| 序号 | 发现内容 |
|
||||
|------|----------|
|
||||
| 1 | 自动化SOC 2合规分析 |
|
||||
| 2 | 利用AI进行安全发现分析 |
|
||||
| 3 | 生成SOC 2格式的报告 |
|
||||
| 4 | 与AWS Security Hub高度相关 |
|
||||
|
||||
#### 🛠️ 技术细节
|
||||
|
||||
> 使用Python 3.12编写的AWS Lambda函数
|
||||
|
||||
> 利用Amazon Bedrock的Claude 3 Sonnet模型进行AI分析
|
||||
|
||||
> 通过Amazon EventBridge定时触发分析
|
||||
|
||||
> 使用Amazon SES发送邮件报告
|
||||
|
||||
> 使用SAM进行部署
|
||||
|
||||
|
||||
#### 🎯 受影响组件
|
||||
|
||||
```
|
||||
• AWS Security Hub
|
||||
• Amazon Bedrock
|
||||
• AWS Lambda
|
||||
• Amazon SES
|
||||
• Amazon EventBridge
|
||||
```
|
||||
|
||||
#### ⚡ 价值评估
|
||||
|
||||
<details>
|
||||
<summary>展开查看详细评估</summary>
|
||||
|
||||
该项目与AI+Security高度相关，通过结合AI技术（Amazon Bedrock）实现了对AWS Security Hub数据的智能分析，并生成SOC 2合规报告，提供了创新的安全分析功能。虽然风险等级较低，但其应用场景具备一定的实用价值。
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
|
||||
## 免责声明
|
||||
本文内容由 AI 自动生成，仅供参考和学习交流。文章中的观点和建议不代表作者立场，使用本文信息需自行承担风险和责任。
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
Loaded 100 of 393 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user