From b97cff8857dec5be00acf9d5521186f5520be05a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E8=88=AA=E5=AE=87?= <3364451258@qq.com> Date: Fri, 10 Jul 2026 15:23:14 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20chunk=20=E9=85=8D=E7=BD=AE=E9=80=9A?= =?UTF-8?q?=E8=BF=87=20DocumentIngestor=20=E4=BC=A0=E9=80=92=EF=BC=8C?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E8=A2=AB=E7=A1=AC=E7=BC=96=E7=A0=81=E8=A6=86?= =?UTF-8?q?=E7=9B=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DocumentIngestor 新增 chunk_config 参数,类型 ChunkConfig - ingest_file 使用 self.chunk_config.max_size/overlap 代替硬编码 1000/100 - deps.py get_ingestor 传入 config.chunk 配置对象 Co-Authored-By: Claude --- src/core/ingest.py | 7 ++++++- src/server/deps.py | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/core/ingest.py b/src/core/ingest.py index 520c409..e305d9b 100644 --- a/src/core/ingest.py +++ b/src/core/ingest.py @@ -3,6 +3,7 @@ import logging import re from pathlib import Path +from src.core.config import ChunkConfig from src.core.db import VectorDB from src.core.embedder import Embedder, batch_embed from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径 @@ -21,11 +22,13 @@ class DocumentIngestor: embedder: Embedder, collection_name: str, splitter: Splitter | None = None, + chunk_config: ChunkConfig | None = None, ): self.db = db self.embedder = embedder self.collection_name = collection_name self.splitter = splitter or MarkdownSplitter() + self.chunk_config = chunk_config or ChunkConfig() @property def collection(self): @@ -43,7 +46,9 @@ class DocumentIngestor: file_name = f"{path_hash}_{path.name}" splitter = self.splitter or get_splitter( - file_path, max_size=1000, overlap=100 + file_path, + max_size=self.chunk_config.max_size, + overlap=self.chunk_config.overlap, ) # PDF/EPUB 二进制文件特殊处理:splitter 内部读取文件 diff --git a/src/server/deps.py b/src/server/deps.py index 941b7a2..04899e6 100644 --- a/src/server/deps.py +++ b/src/server/deps.py @@ -45,7 +45,10 @@ class AppState: name = collection or self.default_collection with self._cache_lock: if name not in self._ingestors: - self._ingestors[name] = DocumentIngestor(self.db, self.embedder, name) + self._ingestors[name] = DocumentIngestor( + self.db, self.embedder, name, + chunk_config=self.config.chunk, + ) return self._ingestors[name] def list_collections_with_stats(self) -> list[dict]: