feat: Searcher 集成 HybridRetriever + SearchConfig 配置

This commit is contained in:
2026-07-11 19:43:49 +08:00
parent cb470e516b
commit 1d58a55a73
4 changed files with 115 additions and 4 deletions
+6
View File
@@ -21,3 +21,9 @@ server:
port: 8000
# ssl_keyfile: "" # HTTPS 私钥路径(设置后启用 HTTPS)
# ssl_certfile: "" # HTTPS 证书路径(设置后启用 HTTPS)
search:
mode: hybrid # hybrid | vector
bm25_weight: 0.3 # BM25 权重 (0=纯向量, 1=纯BM25)
candidate_multiplier: 3 # 向量检索候选倍数
enable_rerank: false # Cross-Encoder 重排序
+12
View File
@@ -65,6 +65,16 @@ class ServerConfig:
ssl_certfile: str = "" # HTTPS 证书文件路径 (空则使用 HTTP)
@dataclass
class SearchConfig:
"""检索配置."""
mode: str = "hybrid" # hybrid | vector
bm25_weight: float = 0.3 # BM25 权重 (0=纯向量, 1=纯BM25)
candidate_multiplier: int = 3 # 向量检索候选倍数
enable_rerank: bool = False # 是否启用 Cross-Encoder 重排序
@dataclass
class AppConfig:
"""应用总配置."""
@@ -73,6 +83,7 @@ class AppConfig:
embed: EmbedConfig = field(default_factory=EmbedConfig)
chunk: ChunkConfig = field(default_factory=ChunkConfig)
server: ServerConfig = field(default_factory=ServerConfig)
search: SearchConfig = field(default_factory=SearchConfig)
@classmethod
def from_dict(cls, data: dict) -> "AppConfig":
@@ -82,6 +93,7 @@ class AppConfig:
embed=EmbedConfig(**data.get("embed", {})),
chunk=ChunkConfig(**data.get("chunk", {})),
server=ServerConfig(**data.get("server", {})),
search=SearchConfig(**data.get("search", {})),
)
+93 -3
View File
@@ -1,30 +1,62 @@
"""语义检索模块."""
import csv
import io
import json as json_lib
import logging
from src.core.config import SearchConfig
from src.core.db import VectorDB
from src.core.embedder import Embedder
from src.core.retriever import HybridRetriever
logger = logging.getLogger("md-vector-db")
class Searcher:
"""向量检索器."""
"""向量检索器 — 支持纯向量或 HybridRetriever 混合检索."""
def __init__(self, db: VectorDB, embedder: Embedder, collection_name: str):
def __init__(
self,
db: VectorDB,
embedder: Embedder,
collection_name: str,
search_config: SearchConfig | None = None,
):
self.db = db
self.embedder = embedder
self.collection_name = collection_name
self._search_config = search_config or SearchConfig()
self._hybrid: HybridRetriever | None = None
@property
def collection(self):
return self.db.get_or_create_collection(self.collection_name)
def _get_hybrid(self) -> HybridRetriever:
if self._hybrid is None:
self._hybrid = HybridRetriever(
self.db,
self.embedder,
self.collection_name,
bm25_weight=self._search_config.bm25_weight,
vector_candidate_multiplier=self._search_config.candidate_multiplier,
)
return self._hybrid
def search(
self,
query: str,
top_k: int = 10,
source_file: str | None = None,
) -> list[dict]:
"""语义检索, 返回格式化结果列表."""
"""语义检索, 返回格式化结果列表.
根据 search.mode 配置自动选择 hybrid 或 vector 模式.
"""
if self._search_config.mode == "hybrid":
return self._get_hybrid().search(query, top_k=top_k, source_file=source_file)
# 纯向量模式(原有逻辑)
embeddings = self.embedder.embed([query])
if not embeddings:
raise RuntimeError("嵌入器返回空结果, 无法进行检索")
@@ -83,3 +115,61 @@ class Searcher:
def delete_by_source(self, file_name: str) -> bool:
"""按文件名删除文档 (委托 VectorDB)."""
return self.db.delete_by_source(self.collection_name, file_name)
def export_json(self, file_path: str | None = None) -> str:
"""导出 collection 所有 chunks 为 JSON.
Args:
file_path: 可选,写入文件路径。不传则返回 JSON 字符串。
Returns:
JSON 字符串
"""
all_data = self.collection.get(include=["documents", "metadatas"])
records = []
if all_data and all_data["ids"]:
for i, doc_id in enumerate(all_data["ids"]):
meta = all_data["metadatas"][i] if all_data["metadatas"] else {}
records.append({
"id": doc_id,
"content": all_data["documents"][i] if all_data["documents"] else "",
"source_file": meta.get("source_file", ""),
"section_title": meta.get("section_title", ""),
"heading_level": meta.get("heading_level", 0),
"chunk_index": meta.get("chunk_index", i),
})
json_str = json_lib.dumps(records, ensure_ascii=False, indent=2)
if file_path:
with open(file_path, "w", encoding="utf-8") as f:
f.write(json_str)
return json_str
def export_csv(self, file_path: str | None = None) -> str:
"""导出 collection 所有 chunks 为 CSV.
Args:
file_path: 可选,写入文件路径。不传则返回 CSV 字符串。
Returns:
CSV 字符串
"""
all_data = self.collection.get(include=["documents", "metadatas"])
output = io.StringIO()
writer = csv.writer(output)
writer.writerow(["id", "content", "source_file", "section_title", "heading_level", "chunk_index"])
if all_data and all_data["ids"]:
for i, doc_id in enumerate(all_data["ids"]):
meta = all_data["metadatas"][i] if all_data["metadatas"] else {}
writer.writerow([
doc_id,
all_data["documents"][i] if all_data["documents"] else "",
meta.get("source_file", ""),
meta.get("section_title", ""),
meta.get("heading_level", 0),
meta.get("chunk_index", i),
])
csv_str = output.getvalue()
if file_path:
with open(file_path, "w", encoding="utf-8", newline="") as f:
f.write(csv_str)
return csv_str
+4 -1
View File
@@ -38,7 +38,10 @@ class AppState:
name = collection or self.default_collection
with self._cache_lock:
if name not in self._searchers:
self._searchers[name] = Searcher(self.db, self.embedder, name)
self._searchers[name] = Searcher(
self.db, self.embedder, name,
search_config=self.config.search,
)
return self._searchers[name]
def get_ingestor(self, collection: str | None = None) -> DocumentIngestor: