fix: use path hash prefix for source_file names to prevent same-name collisions; add size filter to ingest script

2026-07-05 04:46:47 +08:00
parent d3cb88525c
commit d141baf964
4 changed files with 192 additions and 3 deletions
@@ -0,0 +1,69 @@
+"""批量入库 Obsidian — 跳过超大文件 (>50KB CPU嵌入太慢)."""
+import sys, time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+from src.core.config import load_config
+from src.core.db import VectorDB
+from src.core.embedder import create_embedder
+from src.core.ingest import DocumentIngestor
+
+MAX_SIZE = 50_000  # 跳过 >50KB 的文件
+
+progress_file = Path(__file__).parent.parent / "ingest_progress.txt"
+def log(msg):
+    print(msg, flush=True)
+    with open(progress_file, "a", encoding="utf-8") as f:
+        f.write(msg + "\n")
+
+t0 = time.time()
+log("初始化...")
+cfg = load_config()
+db = VectorDB(persist_dir=cfg.chroma.persist_dir)
+embedder = create_embedder(cfg.embed)
+ingestor = DocumentIngestor(db, embedder, "obsidian_blog")
+
+targets = [
+    ("博客", "D:/Code/Obsidian/博客"),
+    ("Club", "D:/Code/Obsidian/Club-Service-Guide"),
+    ("halo", "D:/Code/Obsidian/obsidian-halo"),
+]
+files = []
+skipped = []
+for label, d in targets:
+    if not Path(d).exists(): continue
+    for f in Path(d).rglob("*.md"):
+        if any(p.startswith(".") for p in f.parts): continue
+        if "node_modules" in f.parts: continue
+        size = f.stat().st_size
+        if size > MAX_SIZE:
+            skipped.append((f.name, size))
+            continue
+        files.append((label, str(f)))
+for f in Path("D:/Code/Obsidian").glob("*.md"):
+    sz = f.stat().st_size
+    if sz > MAX_SIZE:
+        skipped.append((f.name, sz))
+    else:
+        files.append(("顶层", str(f)))
+
+log(f"待处理: {len(files)} 个文件")
+if skipped:
+    log(f"跳过超大: {len(skipped)} 个")
+    for name, sz in sorted(skipped, key=lambda x: -x[1]):
+        log(f"  SKIP {name} ({sz//1024}KB)")
+
+total = 0
+for i, (label, fp) in enumerate(files, 1):
+    name = Path(fp).name
+    t1 = time.time()
+    try:
+        n = ingestor.ingest_file(fp)
+        total += n
+        dt = time.time() - t1
+        log(f"[{i}/{len(files)}] {n:>4d} chunks | {label}/{name} ({dt:.1f}s)")
+    except Exception as e:
+        log(f"[{i}/{len(files)}] ERROR {label}/{name}: {e}")
+
+elapsed = time.time() - t0
+log(f"[DONE] 完成: {len(files)} 文件, {total} chunks ({elapsed:.0f}s)")