Serendipity
405303e82c
Batch 1 — CRITICAL (1): - 提取 is_safe_path() 到 src/core/security.py 公共模块 - CLI 和 ingest_obsidian.py 统一添加路径遍历防护 Batch 2 — HIGH (13) + 架构重构: - CLI 复用 deps.py AppState, 消除 30 行重复代码 - AppState/get_state 添加线程安全锁 - serve 命令传递 --config 到 uvicorn (H1) - OpenAIEmbedder 懒创建+复用 HTTP 客户端 (H2) - DashscopeEmbedder import 移到模块顶部 (H3) - 路径检查改用 os.path.commonpath (H4) - embedder.embed() 返回值长度检查 (H5) - 健康检查不泄露内部错误详情 (H7) - /api/v1/collections 添加 API Key 认证 (H8) - API Key 使用 hmac.compare_digest 恒定时间比较 (H9) - 添加 CORS 中间件 (H10) - ServerConfig 支持 SSL 配置 (H11) - HF_ENDPOINT 修改添加详细注释 (H12) Batch 3 — MEDIUM (20) + Splitter Protocol: - 定义 Splitter(Protocol) 接口, DocumentIngestor 接受可选 splitter - DashScope 响应添加结构验证 (M2) - ingest_obsidian.py 支持 CLI 参数和 OBSIDIAN_DIRS 环境变量 (M6) - scripts/serve.py 添加废弃警告 (M7) - content 限制 500KB, collection 正则限制字符集 (M12-M14) - 默认监听地址 127.0.0.1 (M16) - 添加安全响应头中间件 (M17) - verify_api_key 认证失败记录日志 (M19) Batch 4 — LOW (10): - CLI emoji 清理为纯文本标记 (L5) - logging.basicConfig 移到 FastAPI lifespan (L1) - VectorDB 添加 write_guard() 上下文管理器 (L3) - IngestRequest file_path/content 互斥校验 (L10) - ingest_obsidian.py 注释修正 (L6) 测试: 46 → 70 (+24) - tests/test_security.py: 11 个路径安全测试 - tests/test_deps.py: 11 个依赖注入测试 Co-Authored-By: Claude <noreply@anthropic.com>
102 lines
3.3 KiB
Python
102 lines
3.3 KiB
Python
"""批量入库 Obsidian 知识库.
|
|
|
|
用法:
|
|
uv run python scripts/ingest_obsidian.py [目录1] [目录2] ...
|
|
|
|
若不传参数, 默认读取 OBSIDIAN_DIRS 环境变量
|
|
(逗号分隔的目录列表), 例如:
|
|
OBSIDIAN_DIRS="D:/Code/Obsidian/博客,D:/Code/Obsidian/其他" uv run python scripts/ingest_obsidian.py
|
|
"""
|
|
import os
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
|
from src.core.config import load_config
|
|
from src.core.db import VectorDB
|
|
from src.core.embedder import create_embedder
|
|
from src.core.ingest import DocumentIngestor
|
|
from src.core.security import is_safe_path
|
|
|
|
MAX_SIZE = int(os.environ.get("INGEST_MAX_SIZE", "200000")) # 跳过超过此大小的文件 (字节), 默认 200KB
|
|
|
|
progress_file = Path(__file__).parent.parent / "ingest_progress.txt"
|
|
def log(msg):
|
|
print(msg, flush=True)
|
|
with open(progress_file, "a", encoding="utf-8") as f:
|
|
f.write(msg + "\n")
|
|
|
|
t0 = time.time()
|
|
log("初始化...")
|
|
cfg = load_config()
|
|
db = VectorDB(persist_dir=cfg.chroma.persist_dir)
|
|
embedder = create_embedder(cfg.embed)
|
|
ingestor = DocumentIngestor(db, embedder, "obsidian_blog")
|
|
|
|
# 从 CLI 参数或环境变量获取目录列表
|
|
if len(sys.argv) > 1:
|
|
targets = [(Path(d).name, d) for d in sys.argv[1:]]
|
|
else:
|
|
env_dirs = os.environ.get("OBSIDIAN_DIRS", "")
|
|
if env_dirs:
|
|
dirs = [d.strip() for d in env_dirs.split(",") if d.strip()]
|
|
targets = [(Path(d).name, d) for d in dirs]
|
|
else:
|
|
# 回退默认路径 (仅在本机可用)
|
|
targets = [
|
|
("博客", "D:/Code/Obsidian/博客"),
|
|
("Club", "D:/Code/Obsidian/Club-Service-Guide"),
|
|
("halo", "D:/Code/Obsidian/obsidian-halo"),
|
|
]
|
|
files = []
|
|
skipped = []
|
|
for label, d in targets:
|
|
if not Path(d).exists():
|
|
log(f"跳过不存在的目录: {d}")
|
|
continue
|
|
for f in Path(d).rglob("*.md"):
|
|
if any(p.startswith(".") for p in f.parts): continue
|
|
if "node_modules" in f.parts: continue
|
|
size = f.stat().st_size
|
|
if size > MAX_SIZE:
|
|
skipped.append((f.name, size))
|
|
continue
|
|
files.append((label, str(f)))
|
|
# 顶层 .md 文件
|
|
for target_info in targets:
|
|
d = target_info[1]
|
|
p = Path(d)
|
|
if not p.exists():
|
|
continue
|
|
for f in p.glob("*.md"):
|
|
sz = f.stat().st_size
|
|
if sz > MAX_SIZE:
|
|
skipped.append((f.name, sz))
|
|
else:
|
|
files.append(("顶层/" + p.name, str(f)))
|
|
|
|
log(f"待处理: {len(files)} 个文件")
|
|
if skipped:
|
|
log(f"跳过超大: {len(skipped)} 个")
|
|
for name, sz in sorted(skipped, key=lambda x: -x[1]):
|
|
log(f" SKIP {name} ({sz//1024}KB)")
|
|
|
|
total = 0
|
|
for i, (label, fp) in enumerate(files, 1):
|
|
name = Path(fp).name
|
|
if not is_safe_path(fp):
|
|
log(f"[{i}/{len(files)}] SKIP {label}/{name}: 不安全的路径 (路径遍历)")
|
|
continue
|
|
t1 = time.time()
|
|
try:
|
|
n = ingestor.ingest_file(fp)
|
|
total += n
|
|
dt = time.time() - t1
|
|
log(f"[{i}/{len(files)}] {n:>4d} chunks | {label}/{name} ({dt:.1f}s)")
|
|
except Exception as e:
|
|
log(f"[{i}/{len(files)}] ERROR {label}/{name}: {e}")
|
|
|
|
elapsed = time.time() - t0
|
|
log(f"[DONE] 完成: {len(files)} 文件, {total} chunks ({elapsed:.0f}s)")
|