fix: use path hash prefix for source_file names to prevent same-name collisions; add size filter to ingest script
This commit is contained in:
+10
-3
@@ -173,10 +173,17 @@ class DocumentIngestor:
|
||||
return self.db.get_or_create_collection(self.collection_name)
|
||||
|
||||
def ingest_file(self, file_path: str) -> int:
|
||||
"""入库单个 Markdown 文件, 返回 chunk 数量."""
|
||||
path = Path(file_path)
|
||||
"""入库单个 Markdown 文件, 返回 chunk 数量.
|
||||
|
||||
使用文件路径的 SHA256 前 12 位 + 文件名作为唯一标识,
|
||||
避免不同目录下同名文件冲突.
|
||||
"""
|
||||
import hashlib
|
||||
path = Path(file_path).resolve()
|
||||
content = path.read_text(encoding="utf-8")
|
||||
file_name = path.name
|
||||
# 用路径 hash 保证同名文件在不同目录下不冲突
|
||||
path_hash = hashlib.sha256(str(path).encode()).hexdigest()[:12]
|
||||
file_name = f"{path_hash}_{path.name}"
|
||||
|
||||
return self.ingest_content(content, file_name)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user