refactor: 去重删除逻辑统一到 VectorDB.delete_by_source
This commit is contained in:
@@ -19,3 +19,5 @@ chunk:
|
||||
server:
|
||||
host: 0.0.0.0
|
||||
port: 8000
|
||||
# ssl_keyfile: "" # HTTPS 私钥路径(设置后启用 HTTPS)
|
||||
# ssl_certfile: "" # HTTPS 证书路径(设置后启用 HTTPS)
|
||||
|
||||
+4
-5
@@ -1,11 +1,10 @@
|
||||
"""便捷启动脚本 — 已废弃, 请使用 `uv run md-vector-db serve`."""
|
||||
import sys
|
||||
import uvicorn
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"scripts/serve.py 已废弃, 请使用 `uv run md-vector-db serve`",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
print(
|
||||
"[废弃] scripts/serve.py 已废弃, 请使用 `uv run md-vector-db serve`",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -46,6 +46,25 @@ class VectorDB:
|
||||
except ValueError:
|
||||
pass # collection 不存在则忽略
|
||||
|
||||
def delete_by_source(self, collection_name: str, file_name: str) -> bool:
|
||||
"""按 source_file 删除文档 (线程安全)."""
|
||||
import logging
|
||||
_logger = logging.getLogger("md-vector-db")
|
||||
collection = self.get_or_create_collection(collection_name)
|
||||
try:
|
||||
with self._write_lock:
|
||||
existing = collection.get(
|
||||
where={"source_file": file_name}
|
||||
)
|
||||
if existing and existing["ids"]:
|
||||
collection.delete(ids=existing["ids"])
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
except Exception:
|
||||
_logger.exception("删除文档失败: %s (collection=%s)", file_name, collection_name)
|
||||
return False
|
||||
|
||||
def close(self) -> None:
|
||||
"""释放数据库连接."""
|
||||
self.client.close()
|
||||
|
||||
+2
-12
@@ -111,15 +111,5 @@ class DocumentIngestor:
|
||||
return results
|
||||
|
||||
def _remove_by_source(self, file_name: str) -> None:
|
||||
"""按 source_file 删除已有 chunks."""
|
||||
try:
|
||||
with self.db.write_lock:
|
||||
existing = self.collection.get(
|
||||
where={"source_file": file_name}
|
||||
)
|
||||
if existing and existing["ids"]:
|
||||
self.collection.delete(ids=existing["ids"])
|
||||
except ValueError:
|
||||
pass # collection 为空时 ChromaDB 抛 ValueError
|
||||
except Exception:
|
||||
logger.exception("去重检查失败: %s", file_name)
|
||||
"""按 source_file 删除已有 chunks(委托 VectorDB)."""
|
||||
self.db.delete_by_source(self.collection_name, file_name)
|
||||
|
||||
+2
-14
@@ -81,17 +81,5 @@ class Searcher:
|
||||
return sorted(sources)
|
||||
|
||||
def delete_by_source(self, file_name: str) -> bool:
|
||||
"""按文件名删除文档 (线程安全)."""
|
||||
try:
|
||||
with self.db.write_lock:
|
||||
existing = self.collection.get(
|
||||
where={"source_file": file_name}
|
||||
)
|
||||
if existing and existing["ids"]:
|
||||
self.collection.delete(ids=existing["ids"])
|
||||
return True
|
||||
except ValueError:
|
||||
pass # collection 为空时 ChromaDB 抛 ValueError
|
||||
except Exception:
|
||||
logger.exception("删除文档失败: %s", file_name)
|
||||
return False
|
||||
"""按文件名删除文档 (委托 VectorDB)."""
|
||||
return self.db.delete_by_source(self.collection_name, file_name)
|
||||
|
||||
@@ -21,7 +21,11 @@ _custom_registry: dict[str, type[Splitter]] = {}
|
||||
|
||||
|
||||
def register_splitter(ext: str, splitter_cls: type[Splitter]) -> None:
|
||||
"""注册自定义 Splitter 类."""
|
||||
"""注册自定义 Splitter 类.
|
||||
|
||||
注意: 此函数非线程安全,请在程序启动时调用(单线程阶段)。
|
||||
运行时动态注册需自行加锁。
|
||||
"""
|
||||
ext = ext.lower() if ext.startswith(".") else f".{ext}"
|
||||
_custom_registry[ext] = splitter_cls
|
||||
|
||||
|
||||
Reference in New Issue
Block a user