refactor: 去重删除逻辑统一到 VectorDB.delete_by_source

This commit is contained in:
2026-07-10 15:27:52 +08:00
parent ebfae1f60c
commit 3308636086
6 changed files with 34 additions and 32 deletions
+2
View File
@@ -19,3 +19,5 @@ chunk:
server:
host: 0.0.0.0
port: 8000
# ssl_keyfile: "" # HTTPS 私钥路径(设置后启用 HTTPS)
# ssl_certfile: "" # HTTPS 证书路径(设置后启用 HTTPS)
+4 -5
View File
@@ -1,11 +1,10 @@
"""便捷启动脚本 — 已废弃, 请使用 `uv run md-vector-db serve`."""
import sys
import uvicorn
import warnings
warnings.warn(
"scripts/serve.py 已废弃, 请使用 `uv run md-vector-db serve`",
DeprecationWarning,
stacklevel=2,
print(
"[废弃] scripts/serve.py 已废弃, 请使用 `uv run md-vector-db serve`",
file=sys.stderr,
)
if __name__ == "__main__":
+19
View File
@@ -46,6 +46,25 @@ class VectorDB:
except ValueError:
pass # collection 不存在则忽略
def delete_by_source(self, collection_name: str, file_name: str) -> bool:
"""按 source_file 删除文档 (线程安全)."""
import logging
_logger = logging.getLogger("md-vector-db")
collection = self.get_or_create_collection(collection_name)
try:
with self._write_lock:
existing = collection.get(
where={"source_file": file_name}
)
if existing and existing["ids"]:
collection.delete(ids=existing["ids"])
return True
except ValueError:
pass
except Exception:
_logger.exception("删除文档失败: %s (collection=%s)", file_name, collection_name)
return False
def close(self) -> None:
"""释放数据库连接."""
self.client.close()
+2 -12
View File
@@ -111,15 +111,5 @@ class DocumentIngestor:
return results
def _remove_by_source(self, file_name: str) -> None:
"""按 source_file 删除已有 chunks."""
try:
with self.db.write_lock:
existing = self.collection.get(
where={"source_file": file_name}
)
if existing and existing["ids"]:
self.collection.delete(ids=existing["ids"])
except ValueError:
pass # collection 为空时 ChromaDB 抛 ValueError
except Exception:
logger.exception("去重检查失败: %s", file_name)
"""按 source_file 删除已有 chunks(委托 VectorDB."""
self.db.delete_by_source(self.collection_name, file_name)
+2 -14
View File
@@ -81,17 +81,5 @@ class Searcher:
return sorted(sources)
def delete_by_source(self, file_name: str) -> bool:
"""按文件名删除文档 (线程安全)."""
try:
with self.db.write_lock:
existing = self.collection.get(
where={"source_file": file_name}
)
if existing and existing["ids"]:
self.collection.delete(ids=existing["ids"])
return True
except ValueError:
pass # collection 为空时 ChromaDB 抛 ValueError
except Exception:
logger.exception("删除文档失败: %s", file_name)
return False
"""按文件名删除文档 (委托 VectorDB)."""
return self.db.delete_by_source(self.collection_name, file_name)
+5 -1
View File
@@ -21,7 +21,11 @@ _custom_registry: dict[str, type[Splitter]] = {}
def register_splitter(ext: str, splitter_cls: type[Splitter]) -> None:
"""注册自定义 Splitter 类."""
"""注册自定义 Splitter 类.
注意: 此函数非线程安全,请在程序启动时调用(单线程阶段)。
运行时动态注册需自行加锁。
"""
ext = ext.lower() if ext.startswith(".") else f".{ext}"
_custom_registry[ext] = splitter_cls