fix: chunk 配置通过 DocumentIngestor 传递,不再被硬编码覆盖
- DocumentIngestor 新增 chunk_config 参数,类型 ChunkConfig - ingest_file 使用 self.chunk_config.max_size/overlap 代替硬编码 1000/100 - deps.py get_ingestor 传入 config.chunk 配置对象 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
+6
-1
@@ -3,6 +3,7 @@ import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from src.core.config import ChunkConfig
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import Embedder, batch_embed
|
||||
from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径
|
||||
@@ -21,11 +22,13 @@ class DocumentIngestor:
|
||||
embedder: Embedder,
|
||||
collection_name: str,
|
||||
splitter: Splitter | None = None,
|
||||
chunk_config: ChunkConfig | None = None,
|
||||
):
|
||||
self.db = db
|
||||
self.embedder = embedder
|
||||
self.collection_name = collection_name
|
||||
self.splitter = splitter or MarkdownSplitter()
|
||||
self.chunk_config = chunk_config or ChunkConfig()
|
||||
|
||||
@property
|
||||
def collection(self):
|
||||
@@ -43,7 +46,9 @@ class DocumentIngestor:
|
||||
file_name = f"{path_hash}_{path.name}"
|
||||
|
||||
splitter = self.splitter or get_splitter(
|
||||
file_path, max_size=1000, overlap=100
|
||||
file_path,
|
||||
max_size=self.chunk_config.max_size,
|
||||
overlap=self.chunk_config.overlap,
|
||||
)
|
||||
|
||||
# PDF/EPUB 二进制文件特殊处理:splitter 内部读取文件
|
||||
|
||||
+4
-1
@@ -45,7 +45,10 @@ class AppState:
|
||||
name = collection or self.default_collection
|
||||
with self._cache_lock:
|
||||
if name not in self._ingestors:
|
||||
self._ingestors[name] = DocumentIngestor(self.db, self.embedder, name)
|
||||
self._ingestors[name] = DocumentIngestor(
|
||||
self.db, self.embedder, name,
|
||||
chunk_config=self.config.chunk,
|
||||
)
|
||||
return self._ingestors[name]
|
||||
|
||||
def list_collections_with_stats(self) -> list[dict]:
|
||||
|
||||
Reference in New Issue
Block a user