"""批量入库 Obsidian — 跳过超大文件 (>50KB CPU嵌入太慢)."""
import sys, time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
from src.core.config import load_config
from src.core.db import VectorDB
from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor

MAX_SIZE = 200_000  # GPU 嵌入,无需跳过

progress_file = Path(__file__).parent.parent / "ingest_progress.txt"
def log(msg):
    print(msg, flush=True)
    with open(progress_file, "a", encoding="utf-8") as f:
        f.write(msg + "\n")

t0 = time.time()
log("初始化...")
cfg = load_config()
db = VectorDB(persist_dir=cfg.chroma.persist_dir)
embedder = create_embedder(cfg.embed)
ingestor = DocumentIngestor(db, embedder, "obsidian_blog")

targets = [
    ("博客", "D:/Code/Obsidian/博客"),
    ("Club", "D:/Code/Obsidian/Club-Service-Guide"),
    ("halo", "D:/Code/Obsidian/obsidian-halo"),
]
files = []
skipped = []
for label, d in targets:
    if not Path(d).exists(): continue
    for f in Path(d).rglob("*.md"):
        if any(p.startswith(".") for p in f.parts): continue
        if "node_modules" in f.parts: continue
        size = f.stat().st_size
        if size > MAX_SIZE:
            skipped.append((f.name, size))
            continue
        files.append((label, str(f)))
for f in Path("D:/Code/Obsidian").glob("*.md"):
    sz = f.stat().st_size
    if sz > MAX_SIZE:
        skipped.append((f.name, sz))
    else:
        files.append(("顶层", str(f)))

log(f"待处理: {len(files)} 个文件")
if skipped:
    log(f"跳过超大: {len(skipped)} 个")
    for name, sz in sorted(skipped, key=lambda x: -x[1]):
        log(f"  SKIP {name} ({sz//1024}KB)")

total = 0
for i, (label, fp) in enumerate(files, 1):
    name = Path(fp).name
    t1 = time.time()
    try:
        n = ingestor.ingest_file(fp)
        total += n
        dt = time.time() - t1
        log(f"[{i}/{len(files)}] {n:>4d} chunks | {label}/{name} ({dt:.1f}s)")
    except Exception as e:
        log(f"[{i}/{len(files)}] ERROR {label}/{name}: {e}")

elapsed = time.time() - t0
log(f"[DONE] 完成: {len(files)} 文件, {total} chunks ({elapsed:.0f}s)")