fix: use path hash prefix for source_file names to prevent same-name collisions; add size filter to ingest script
This commit is contained in:
@@ -0,0 +1,56 @@
|
|||||||
|
初始化...
|
||||||
|
待处理: 46 个文件
|
||||||
|
跳过超大: 6 个
|
||||||
|
SKIP 人工智能协会内容创作通知.md (1915KB)
|
||||||
|
SKIP Git团队协作指南.md (94KB)
|
||||||
|
SKIP Git团队协作指南.sync-conflict-20260615-122209-R4RFYM7.md (94KB)
|
||||||
|
SKIP OpenClaw介绍.md (64KB)
|
||||||
|
SKIP Git内部原理详解.md (59KB)
|
||||||
|
SKIP Git团队协作指南大纲.md (58KB)
|
||||||
|
[1/46] 17 chunks | 博客/AI时代的Token战争-能源算力与我们每个人的选择.md (0.6s)
|
||||||
|
[2/46] 39 chunks | 博客/AI时代的VibeCoding与程序员进化.md (1.2s)
|
||||||
|
[3/46] 59 chunks | 博客/CLI在AI时代的浴火重生.md (1.7s)
|
||||||
|
[4/46] 68 chunks | 博客/DeepSeek-V4全面解析.md (1.1s)
|
||||||
|
[5/46] 16 chunks | 博客/大模型赋能架构设计.md (0.5s)
|
||||||
|
[6/46] 27 chunks | 博客/大语言模型算子详解.md (1.0s)
|
||||||
|
[7/46] 8 chunks | 博客/学习Agent，越学越像在重新理解操作系统.md (0.3s)
|
||||||
|
[8/46] 69 chunks | 博客/深入解析 Claude Code：Vibe Coding 时代的 AI 编程利器.md (1.5s)
|
||||||
|
[9/46] 12 chunks | 博客/自建Git服务-在NAS上部署Gitea.md (0.4s)
|
||||||
|
[10/46] 38 chunks | 博客/DeepSeek-V4博客大纲.md (0.4s)
|
||||||
|
[11/46] 110 chunks | 博客/uv工具推荐博客大纲.md (1.0s)
|
||||||
|
[12/46] 48 chunks | 博客/AI助你轻松上手LaTeX论文写作.md (1.1s)
|
||||||
|
[13/46] 15 chunks | 博客/SEO分析报告.md (0.4s)
|
||||||
|
[14/46] 18 chunks | 博客/博客爬取报告.md (0.3s)
|
||||||
|
[15/46] 15 chunks | 博客/大数据技术栈.md (0.3s)
|
||||||
|
[16/46] 7 chunks | 博客/摘要检查报告.md (0.3s)
|
||||||
|
[17/46] 27 chunks | 博客/文章列表.md (0.5s)
|
||||||
|
[18/46] 19 chunks | 博客/标签分类审查报告.md (0.7s)
|
||||||
|
[19/46] 13 chunks | 博客/LinearRegression线性回归.md (0.6s)
|
||||||
|
[20/46] 53 chunks | 博客/从全连接层到卷积.md (1.2s)
|
||||||
|
[21/46] 114 chunks | 博客/深度学习完全指南.md (1.6s)
|
||||||
|
[22/46] 43 chunks | 博客/视觉语言模型技术综述.md (1.4s)
|
||||||
|
[23/46] 52 chunks | 博客/Docker安装与入门指南.md (1.4s)
|
||||||
|
[24/46] 167 chunks | 博客/Docker部署完全指南.md (3.6s)
|
||||||
|
[25/46] 115 chunks | 博客/Git团队协作指南（精简版）.md (1.3s)
|
||||||
|
[26/46] 10 chunks | 博客/OpenClaw安装教程.md (0.1s)
|
||||||
|
[27/46] 247 chunks | 博客/uv工具推荐博客.md (2.8s)
|
||||||
|
[28/46] 6 chunks | Club/CLAUDE.md (0.1s)
|
||||||
|
[29/46] 7 chunks | Club/README.md (0.1s)
|
||||||
|
[30/46] 11 chunks | Club/仓库说明.md (0.4s)
|
||||||
|
[31/46] 9 chunks | Club/导航页.md (0.3s)
|
||||||
|
[32/46] 3 chunks | Club/说明.md (0.1s)
|
||||||
|
[33/46] 4 chunks | Club/说明.md (0.2s)
|
||||||
|
[34/46] 4 chunks | Club/MetaRL核心团队实施方案.md (0.3s)
|
||||||
|
[35/46] 3 chunks | Club/说明.md (0.1s)
|
||||||
|
[36/46] 20 chunks | Club/定量考核材料清单.md (0.7s)
|
||||||
|
[37/46] 14 chunks | Club/年度工作总结起草指南.md (0.6s)
|
||||||
|
[38/46] 3 chunks | Club/说明.md (0.2s)
|
||||||
|
[39/46] 3 chunks | Club/说明.md (0.1s)
|
||||||
|
[40/46] 7 chunks | Club/人工智能协会近期工作任务清单.md (0.3s)
|
||||||
|
[41/46] 3 chunks | Club/说明.md (0.2s)
|
||||||
|
[42/46] 7 chunks | halo/README.md (0.3s)
|
||||||
|
[43/46] 6 chunks | halo/README.zh-CN.md (0.3s)
|
||||||
|
[44/46] 21 chunks | halo/usage-guide.md (0.4s)
|
||||||
|
[45/46] 9 chunks | 顶层/CLAUDE.md (0.4s)
|
||||||
|
[46/46] 57 chunks | 顶层/temp-agent-os-v2.md (1.6s)
|
||||||
|
[DONE] 完成: 46 文件, 1623 chunks (45s)
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
初始化...
|
||||||
|
|
||||||
|
Loading weights: 0%| | 0/71 [00:00<?, ?it/s]
|
||||||
|
Loading weights: 100%|██████████| 71/71 [00:00<00:00, 12790.81it/s]
|
||||||
|
待处理: 46 个文件
|
||||||
|
跳过超大: 6 个
|
||||||
|
SKIP 人工智能协会内容创作通知.md (1915KB)
|
||||||
|
SKIP Git团队协作指南.md (94KB)
|
||||||
|
SKIP Git团队协作指南.sync-conflict-20260615-122209-R4RFYM7.md (94KB)
|
||||||
|
SKIP OpenClaw介绍.md (64KB)
|
||||||
|
SKIP Git内部原理详解.md (59KB)
|
||||||
|
SKIP Git团队协作指南大纲.md (58KB)
|
||||||
|
[1/46] 17 chunks | 博客/AI时代的Token战争-能源算力与我们每个人的选择.md (0.6s)
|
||||||
|
[2/46] 39 chunks | 博客/AI时代的VibeCoding与程序员进化.md (1.2s)
|
||||||
|
[3/46] 59 chunks | 博客/CLI在AI时代的浴火重生.md (1.7s)
|
||||||
|
[4/46] 68 chunks | 博客/DeepSeek-V4全面解析.md (1.1s)
|
||||||
|
[5/46] 16 chunks | 博客/大模型赋能架构设计.md (0.5s)
|
||||||
|
[6/46] 27 chunks | 博客/大语言模型算子详解.md (1.0s)
|
||||||
|
[7/46] 8 chunks | 博客/学习Agent，越学越像在重新理解操作系统.md (0.3s)
|
||||||
|
[8/46] 69 chunks | 博客/深入解析 Claude Code：Vibe Coding 时代的 AI 编程利器.md (1.5s)
|
||||||
|
[9/46] 12 chunks | 博客/自建Git服务-在NAS上部署Gitea.md (0.4s)
|
||||||
|
[10/46] 38 chunks | 博客/DeepSeek-V4博客大纲.md (0.4s)
|
||||||
|
[11/46] 110 chunks | 博客/uv工具推荐博客大纲.md (1.0s)
|
||||||
|
[12/46] 48 chunks | 博客/AI助你轻松上手LaTeX论文写作.md (1.1s)
|
||||||
|
[13/46] 15 chunks | 博客/SEO分析报告.md (0.4s)
|
||||||
|
[14/46] 18 chunks | 博客/博客爬取报告.md (0.3s)
|
||||||
|
[15/46] 15 chunks | 博客/大数据技术栈.md (0.3s)
|
||||||
|
[16/46] 7 chunks | 博客/摘要检查报告.md (0.3s)
|
||||||
|
[17/46] 27 chunks | 博客/文章列表.md (0.5s)
|
||||||
|
[18/46] 19 chunks | 博客/标签分类审查报告.md (0.7s)
|
||||||
|
[19/46] 13 chunks | 博客/LinearRegression线性回归.md (0.6s)
|
||||||
|
[20/46] 53 chunks | 博客/从全连接层到卷积.md (1.2s)
|
||||||
|
[21/46] 114 chunks | 博客/深度学习完全指南.md (1.6s)
|
||||||
|
[22/46] 43 chunks | 博客/视觉语言模型技术综述.md (1.4s)
|
||||||
|
[23/46] 52 chunks | 博客/Docker安装与入门指南.md (1.4s)
|
||||||
|
[24/46] 167 chunks | 博客/Docker部署完全指南.md (3.6s)
|
||||||
|
[25/46] 115 chunks | 博客/Git团队协作指南（精简版）.md (1.3s)
|
||||||
|
[26/46] 10 chunks | 博客/OpenClaw安装教程.md (0.1s)
|
||||||
|
[27/46] 247 chunks | 博客/uv工具推荐博客.md (2.8s)
|
||||||
|
[28/46] 6 chunks | Club/CLAUDE.md (0.1s)
|
||||||
|
[29/46] 7 chunks | Club/README.md (0.1s)
|
||||||
|
[30/46] 11 chunks | Club/仓库说明.md (0.4s)
|
||||||
|
[31/46] 9 chunks | Club/导航页.md (0.3s)
|
||||||
|
[32/46] 3 chunks | Club/说明.md (0.1s)
|
||||||
|
[33/46] 4 chunks | Club/说明.md (0.2s)
|
||||||
|
[34/46] 4 chunks | Club/MetaRL核心团队实施方案.md (0.3s)
|
||||||
|
[35/46] 3 chunks | Club/说明.md (0.1s)
|
||||||
|
[36/46] 20 chunks | Club/定量考核材料清单.md (0.7s)
|
||||||
|
[37/46] 14 chunks | Club/年度工作总结起草指南.md (0.6s)
|
||||||
|
[38/46] 3 chunks | Club/说明.md (0.2s)
|
||||||
|
[39/46] 3 chunks | Club/说明.md (0.1s)
|
||||||
|
[40/46] 7 chunks | Club/人工智能协会近期工作任务清单.md (0.3s)
|
||||||
|
[41/46] 3 chunks | Club/说明.md (0.2s)
|
||||||
|
[42/46] 7 chunks | halo/README.md (0.3s)
|
||||||
|
[43/46] 6 chunks | halo/README.zh-CN.md (0.3s)
|
||||||
|
[44/46] 21 chunks | halo/usage-guide.md (0.4s)
|
||||||
|
[45/46] 9 chunks | 顶层/CLAUDE.md (0.4s)
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
"""批量入库 Obsidian — 跳过超大文件 (>50KB CPU嵌入太慢)."""
|
||||||
|
import sys, time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||||
|
from src.core.config import load_config
|
||||||
|
from src.core.db import VectorDB
|
||||||
|
from src.core.embedder import create_embedder
|
||||||
|
from src.core.ingest import DocumentIngestor
|
||||||
|
|
||||||
|
MAX_SIZE = 50_000 # 跳过 >50KB 的文件
|
||||||
|
|
||||||
|
progress_file = Path(__file__).parent.parent / "ingest_progress.txt"
|
||||||
|
def log(msg):
|
||||||
|
print(msg, flush=True)
|
||||||
|
with open(progress_file, "a", encoding="utf-8") as f:
|
||||||
|
f.write(msg + "\n")
|
||||||
|
|
||||||
|
t0 = time.time()
|
||||||
|
log("初始化...")
|
||||||
|
cfg = load_config()
|
||||||
|
db = VectorDB(persist_dir=cfg.chroma.persist_dir)
|
||||||
|
embedder = create_embedder(cfg.embed)
|
||||||
|
ingestor = DocumentIngestor(db, embedder, "obsidian_blog")
|
||||||
|
|
||||||
|
targets = [
|
||||||
|
("博客", "D:/Code/Obsidian/博客"),
|
||||||
|
("Club", "D:/Code/Obsidian/Club-Service-Guide"),
|
||||||
|
("halo", "D:/Code/Obsidian/obsidian-halo"),
|
||||||
|
]
|
||||||
|
files = []
|
||||||
|
skipped = []
|
||||||
|
for label, d in targets:
|
||||||
|
if not Path(d).exists(): continue
|
||||||
|
for f in Path(d).rglob("*.md"):
|
||||||
|
if any(p.startswith(".") for p in f.parts): continue
|
||||||
|
if "node_modules" in f.parts: continue
|
||||||
|
size = f.stat().st_size
|
||||||
|
if size > MAX_SIZE:
|
||||||
|
skipped.append((f.name, size))
|
||||||
|
continue
|
||||||
|
files.append((label, str(f)))
|
||||||
|
for f in Path("D:/Code/Obsidian").glob("*.md"):
|
||||||
|
sz = f.stat().st_size
|
||||||
|
if sz > MAX_SIZE:
|
||||||
|
skipped.append((f.name, sz))
|
||||||
|
else:
|
||||||
|
files.append(("顶层", str(f)))
|
||||||
|
|
||||||
|
log(f"待处理: {len(files)} 个文件")
|
||||||
|
if skipped:
|
||||||
|
log(f"跳过超大: {len(skipped)} 个")
|
||||||
|
for name, sz in sorted(skipped, key=lambda x: -x[1]):
|
||||||
|
log(f" SKIP {name} ({sz//1024}KB)")
|
||||||
|
|
||||||
|
total = 0
|
||||||
|
for i, (label, fp) in enumerate(files, 1):
|
||||||
|
name = Path(fp).name
|
||||||
|
t1 = time.time()
|
||||||
|
try:
|
||||||
|
n = ingestor.ingest_file(fp)
|
||||||
|
total += n
|
||||||
|
dt = time.time() - t1
|
||||||
|
log(f"[{i}/{len(files)}] {n:>4d} chunks | {label}/{name} ({dt:.1f}s)")
|
||||||
|
except Exception as e:
|
||||||
|
log(f"[{i}/{len(files)}] ERROR {label}/{name}: {e}")
|
||||||
|
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
log(f"[DONE] 完成: {len(files)} 文件, {total} chunks ({elapsed:.0f}s)")
|
||||||
+10
-3
@@ -173,10 +173,17 @@ class DocumentIngestor:
|
|||||||
return self.db.get_or_create_collection(self.collection_name)
|
return self.db.get_or_create_collection(self.collection_name)
|
||||||
|
|
||||||
def ingest_file(self, file_path: str) -> int:
|
def ingest_file(self, file_path: str) -> int:
|
||||||
"""入库单个 Markdown 文件, 返回 chunk 数量."""
|
"""入库单个 Markdown 文件, 返回 chunk 数量.
|
||||||
path = Path(file_path)
|
|
||||||
|
使用文件路径的 SHA256 前 12 位 + 文件名作为唯一标识，
|
||||||
|
避免不同目录下同名文件冲突.
|
||||||
|
"""
|
||||||
|
import hashlib
|
||||||
|
path = Path(file_path).resolve()
|
||||||
content = path.read_text(encoding="utf-8")
|
content = path.read_text(encoding="utf-8")
|
||||||
file_name = path.name
|
# 用路径 hash 保证同名文件在不同目录下不冲突
|
||||||
|
path_hash = hashlib.sha256(str(path).encode()).hexdigest()[:12]
|
||||||
|
file_name = f"{path_hash}_{path.name}"
|
||||||
|
|
||||||
return self.ingest_content(content, file_name)
|
return self.ingest_content(content, file_name)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user