70 lines
2.1 KiB
Python
70 lines
2.1 KiB
Python
"""批量入库 Obsidian — 跳过超大文件 (>50KB CPU嵌入太慢)."""
|
|
import sys, time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
|
from src.core.config import load_config
|
|
from src.core.db import VectorDB
|
|
from src.core.embedder import create_embedder
|
|
from src.core.ingest import DocumentIngestor
|
|
|
|
MAX_SIZE = 50_000 # 跳过 >50KB 的文件
|
|
|
|
progress_file = Path(__file__).parent.parent / "ingest_progress.txt"
|
|
def log(msg):
|
|
print(msg, flush=True)
|
|
with open(progress_file, "a", encoding="utf-8") as f:
|
|
f.write(msg + "\n")
|
|
|
|
t0 = time.time()
|
|
log("初始化...")
|
|
cfg = load_config()
|
|
db = VectorDB(persist_dir=cfg.chroma.persist_dir)
|
|
embedder = create_embedder(cfg.embed)
|
|
ingestor = DocumentIngestor(db, embedder, "obsidian_blog")
|
|
|
|
targets = [
|
|
("博客", "D:/Code/Obsidian/博客"),
|
|
("Club", "D:/Code/Obsidian/Club-Service-Guide"),
|
|
("halo", "D:/Code/Obsidian/obsidian-halo"),
|
|
]
|
|
files = []
|
|
skipped = []
|
|
for label, d in targets:
|
|
if not Path(d).exists(): continue
|
|
for f in Path(d).rglob("*.md"):
|
|
if any(p.startswith(".") for p in f.parts): continue
|
|
if "node_modules" in f.parts: continue
|
|
size = f.stat().st_size
|
|
if size > MAX_SIZE:
|
|
skipped.append((f.name, size))
|
|
continue
|
|
files.append((label, str(f)))
|
|
for f in Path("D:/Code/Obsidian").glob("*.md"):
|
|
sz = f.stat().st_size
|
|
if sz > MAX_SIZE:
|
|
skipped.append((f.name, sz))
|
|
else:
|
|
files.append(("顶层", str(f)))
|
|
|
|
log(f"待处理: {len(files)} 个文件")
|
|
if skipped:
|
|
log(f"跳过超大: {len(skipped)} 个")
|
|
for name, sz in sorted(skipped, key=lambda x: -x[1]):
|
|
log(f" SKIP {name} ({sz//1024}KB)")
|
|
|
|
total = 0
|
|
for i, (label, fp) in enumerate(files, 1):
|
|
name = Path(fp).name
|
|
t1 = time.time()
|
|
try:
|
|
n = ingestor.ingest_file(fp)
|
|
total += n
|
|
dt = time.time() - t1
|
|
log(f"[{i}/{len(files)}] {n:>4d} chunks | {label}/{name} ({dt:.1f}s)")
|
|
except Exception as e:
|
|
log(f"[{i}/{len(files)}] ERROR {label}/{name}: {e}")
|
|
|
|
elapsed = time.time() - t0
|
|
log(f"[DONE] 完成: {len(files)} 文件, {total} chunks ({elapsed:.0f}s)")
|