fix: 修复 44 个代码审查问题 (CRITICAL/HIGH/MEDIUM/LOW)
Batch 1 — CRITICAL (1): - 提取 is_safe_path() 到 src/core/security.py 公共模块 - CLI 和 ingest_obsidian.py 统一添加路径遍历防护 Batch 2 — HIGH (13) + 架构重构: - CLI 复用 deps.py AppState, 消除 30 行重复代码 - AppState/get_state 添加线程安全锁 - serve 命令传递 --config 到 uvicorn (H1) - OpenAIEmbedder 懒创建+复用 HTTP 客户端 (H2) - DashscopeEmbedder import 移到模块顶部 (H3) - 路径检查改用 os.path.commonpath (H4) - embedder.embed() 返回值长度检查 (H5) - 健康检查不泄露内部错误详情 (H7) - /api/v1/collections 添加 API Key 认证 (H8) - API Key 使用 hmac.compare_digest 恒定时间比较 (H9) - 添加 CORS 中间件 (H10) - ServerConfig 支持 SSL 配置 (H11) - HF_ENDPOINT 修改添加详细注释 (H12) Batch 3 — MEDIUM (20) + Splitter Protocol: - 定义 Splitter(Protocol) 接口, DocumentIngestor 接受可选 splitter - DashScope 响应添加结构验证 (M2) - ingest_obsidian.py 支持 CLI 参数和 OBSIDIAN_DIRS 环境变量 (M6) - scripts/serve.py 添加废弃警告 (M7) - content 限制 500KB, collection 正则限制字符集 (M12-M14) - 默认监听地址 127.0.0.1 (M16) - 添加安全响应头中间件 (M17) - verify_api_key 认证失败记录日志 (M19) Batch 4 — LOW (10): - CLI emoji 清理为纯文本标记 (L5) - logging.basicConfig 移到 FastAPI lifespan (L1) - VectorDB 添加 write_guard() 上下文管理器 (L3) - IngestRequest file_path/content 互斥校验 (L10) - ingest_obsidian.py 注释修正 (L6) 测试: 46 → 70 (+24) - tests/test_security.py: 11 个路径安全测试 - tests/test_deps.py: 11 个依赖注入测试 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
+47
-15
@@ -1,5 +1,15 @@
|
||||
"""批量入库 Obsidian — 跳过超大文件 (>50KB CPU嵌入太慢)."""
|
||||
import sys, time
|
||||
"""批量入库 Obsidian 知识库.
|
||||
|
||||
用法:
|
||||
uv run python scripts/ingest_obsidian.py [目录1] [目录2] ...
|
||||
|
||||
若不传参数, 默认读取 OBSIDIAN_DIRS 环境变量
|
||||
(逗号分隔的目录列表), 例如:
|
||||
OBSIDIAN_DIRS="D:/Code/Obsidian/博客,D:/Code/Obsidian/其他" uv run python scripts/ingest_obsidian.py
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
@@ -7,8 +17,9 @@ from src.core.config import load_config
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
from src.core.security import is_safe_path
|
||||
|
||||
MAX_SIZE = 200_000 # GPU 嵌入,无需跳过
|
||||
MAX_SIZE = int(os.environ.get("INGEST_MAX_SIZE", "200000")) # 跳过超过此大小的文件 (字节), 默认 200KB
|
||||
|
||||
progress_file = Path(__file__).parent.parent / "ingest_progress.txt"
|
||||
def log(msg):
|
||||
@@ -23,15 +34,27 @@ db = VectorDB(persist_dir=cfg.chroma.persist_dir)
|
||||
embedder = create_embedder(cfg.embed)
|
||||
ingestor = DocumentIngestor(db, embedder, "obsidian_blog")
|
||||
|
||||
targets = [
|
||||
("博客", "D:/Code/Obsidian/博客"),
|
||||
("Club", "D:/Code/Obsidian/Club-Service-Guide"),
|
||||
("halo", "D:/Code/Obsidian/obsidian-halo"),
|
||||
]
|
||||
# 从 CLI 参数或环境变量获取目录列表
|
||||
if len(sys.argv) > 1:
|
||||
targets = [(Path(d).name, d) for d in sys.argv[1:]]
|
||||
else:
|
||||
env_dirs = os.environ.get("OBSIDIAN_DIRS", "")
|
||||
if env_dirs:
|
||||
dirs = [d.strip() for d in env_dirs.split(",") if d.strip()]
|
||||
targets = [(Path(d).name, d) for d in dirs]
|
||||
else:
|
||||
# 回退默认路径 (仅在本机可用)
|
||||
targets = [
|
||||
("博客", "D:/Code/Obsidian/博客"),
|
||||
("Club", "D:/Code/Obsidian/Club-Service-Guide"),
|
||||
("halo", "D:/Code/Obsidian/obsidian-halo"),
|
||||
]
|
||||
files = []
|
||||
skipped = []
|
||||
for label, d in targets:
|
||||
if not Path(d).exists(): continue
|
||||
if not Path(d).exists():
|
||||
log(f"跳过不存在的目录: {d}")
|
||||
continue
|
||||
for f in Path(d).rglob("*.md"):
|
||||
if any(p.startswith(".") for p in f.parts): continue
|
||||
if "node_modules" in f.parts: continue
|
||||
@@ -40,12 +63,18 @@ for label, d in targets:
|
||||
skipped.append((f.name, size))
|
||||
continue
|
||||
files.append((label, str(f)))
|
||||
for f in Path("D:/Code/Obsidian").glob("*.md"):
|
||||
sz = f.stat().st_size
|
||||
if sz > MAX_SIZE:
|
||||
skipped.append((f.name, sz))
|
||||
else:
|
||||
files.append(("顶层", str(f)))
|
||||
# 顶层 .md 文件
|
||||
for target_info in targets:
|
||||
d = target_info[1]
|
||||
p = Path(d)
|
||||
if not p.exists():
|
||||
continue
|
||||
for f in p.glob("*.md"):
|
||||
sz = f.stat().st_size
|
||||
if sz > MAX_SIZE:
|
||||
skipped.append((f.name, sz))
|
||||
else:
|
||||
files.append(("顶层/" + p.name, str(f)))
|
||||
|
||||
log(f"待处理: {len(files)} 个文件")
|
||||
if skipped:
|
||||
@@ -56,6 +85,9 @@ if skipped:
|
||||
total = 0
|
||||
for i, (label, fp) in enumerate(files, 1):
|
||||
name = Path(fp).name
|
||||
if not is_safe_path(fp):
|
||||
log(f"[{i}/{len(files)}] SKIP {label}/{name}: 不安全的路径 (路径遍历)")
|
||||
continue
|
||||
t1 = time.time()
|
||||
try:
|
||||
n = ingestor.ingest_file(fp)
|
||||
|
||||
+9
-2
@@ -1,5 +1,12 @@
|
||||
"""便捷启动脚本."""
|
||||
"""便捷启动脚本 — 已废弃, 请使用 `uv run md-vector-db serve`."""
|
||||
import uvicorn
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"scripts/serve.py 已废弃, 请使用 `uv run md-vector-db serve`",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run("src.server.app:app", host="0.0.0.0", port=8000, reload=True)
|
||||
uvicorn.run("src.server.app:app", host="127.0.0.1", port=8000, reload=True)
|
||||
|
||||
Reference in New Issue
Block a user