fix: chunk 配置通过 DocumentIngestor 传递,不再被硬编码覆盖
- DocumentIngestor 新增 chunk_config 参数,类型 ChunkConfig - ingest_file 使用 self.chunk_config.max_size/overlap 代替硬编码 1000/100 - deps.py get_ingestor 传入 config.chunk 配置对象 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
+6
-1
@@ -3,6 +3,7 @@ import logging
|
|||||||
import re
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from src.core.config import ChunkConfig
|
||||||
from src.core.db import VectorDB
|
from src.core.db import VectorDB
|
||||||
from src.core.embedder import Embedder, batch_embed
|
from src.core.embedder import Embedder, batch_embed
|
||||||
from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径
|
from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径
|
||||||
@@ -21,11 +22,13 @@ class DocumentIngestor:
|
|||||||
embedder: Embedder,
|
embedder: Embedder,
|
||||||
collection_name: str,
|
collection_name: str,
|
||||||
splitter: Splitter | None = None,
|
splitter: Splitter | None = None,
|
||||||
|
chunk_config: ChunkConfig | None = None,
|
||||||
):
|
):
|
||||||
self.db = db
|
self.db = db
|
||||||
self.embedder = embedder
|
self.embedder = embedder
|
||||||
self.collection_name = collection_name
|
self.collection_name = collection_name
|
||||||
self.splitter = splitter or MarkdownSplitter()
|
self.splitter = splitter or MarkdownSplitter()
|
||||||
|
self.chunk_config = chunk_config or ChunkConfig()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def collection(self):
|
def collection(self):
|
||||||
@@ -43,7 +46,9 @@ class DocumentIngestor:
|
|||||||
file_name = f"{path_hash}_{path.name}"
|
file_name = f"{path_hash}_{path.name}"
|
||||||
|
|
||||||
splitter = self.splitter or get_splitter(
|
splitter = self.splitter or get_splitter(
|
||||||
file_path, max_size=1000, overlap=100
|
file_path,
|
||||||
|
max_size=self.chunk_config.max_size,
|
||||||
|
overlap=self.chunk_config.overlap,
|
||||||
)
|
)
|
||||||
|
|
||||||
# PDF/EPUB 二进制文件特殊处理:splitter 内部读取文件
|
# PDF/EPUB 二进制文件特殊处理:splitter 内部读取文件
|
||||||
|
|||||||
+4
-1
@@ -45,7 +45,10 @@ class AppState:
|
|||||||
name = collection or self.default_collection
|
name = collection or self.default_collection
|
||||||
with self._cache_lock:
|
with self._cache_lock:
|
||||||
if name not in self._ingestors:
|
if name not in self._ingestors:
|
||||||
self._ingestors[name] = DocumentIngestor(self.db, self.embedder, name)
|
self._ingestors[name] = DocumentIngestor(
|
||||||
|
self.db, self.embedder, name,
|
||||||
|
chunk_config=self.config.chunk,
|
||||||
|
)
|
||||||
return self._ingestors[name]
|
return self._ingestors[name]
|
||||||
|
|
||||||
def list_collections_with_stats(self) -> list[dict]:
|
def list_collections_with_stats(self) -> list[dict]:
|
||||||
|
|||||||
Reference in New Issue
Block a user