refactor: 修复 Embedder Protocol 标准写法和 PDF/EPUB Splitter 参数语义
This commit is contained in:
+15
-2
@@ -48,9 +48,22 @@ _PROVIDER_DEFAULTS: dict[str, dict[str, str | int]] = {
|
||||
# -- 接口 --
|
||||
class Embedder(Protocol):
|
||||
"""嵌入器接口."""
|
||||
|
||||
@property
|
||||
def dimension(self) -> int: ...
|
||||
def embed(self, texts: list[str]) -> list[list[float]]: ...
|
||||
def dimension(self) -> int:
|
||||
"""返回嵌入向量的维度."""
|
||||
...
|
||||
|
||||
def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
"""对文本列表进行嵌入.
|
||||
|
||||
Args:
|
||||
texts: 待嵌入的文本列表
|
||||
|
||||
Returns:
|
||||
嵌入向量列表,每个向量为 float 列表
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
# -- 本地模型 --
|
||||
|
||||
+1
-1
@@ -67,7 +67,7 @@ class DocumentIngestor:
|
||||
# PDF/EPUB 二进制文件特殊处理:splitter 内部读取文件
|
||||
suffix = path.suffix.lower()
|
||||
if suffix in (".pdf", ".epub"):
|
||||
chunks = splitter.split(str(path), source_file=file_name)
|
||||
chunks = splitter.split(source=str(path), source_file=file_name)
|
||||
result = self._add_chunks(chunks, file_name)
|
||||
else:
|
||||
content = path.read_text(encoding="utf-8")
|
||||
|
||||
@@ -10,17 +10,17 @@ class EPUBSplitter:
|
||||
"""EPUB 分块器:ebooklib 提取各章节文字 → TextSplitter 分块.
|
||||
|
||||
实现 Splitter Protocol,内部组合 TextSplitter 实例。
|
||||
split() 的 text 参数实际接收 EPUB 文件路径(非文本内容)。
|
||||
split() 的 source 参数接收 EPUB 文件路径(非文本内容)。
|
||||
"""
|
||||
|
||||
def __init__(self, max_size: int = 1000, overlap: int = 100):
|
||||
self._text_splitter = TextSplitter(max_size=max_size, overlap=overlap)
|
||||
|
||||
def split(self, text: str, source_file: str = "") -> list[dict]:
|
||||
def split(self, source: str, source_file: str = "") -> list[dict]:
|
||||
"""从 EPUB 文件提取各章节文字并分块.
|
||||
|
||||
Args:
|
||||
text: EPUB 文件路径(非文本内容,由 ingest_file 传入)
|
||||
source: EPUB 文件路径
|
||||
source_file: 来源文件名
|
||||
"""
|
||||
try:
|
||||
@@ -31,7 +31,7 @@ class EPUBSplitter:
|
||||
"EPUB 支持需要 ebooklib 库. 请执行: uv sync --extra epub"
|
||||
)
|
||||
|
||||
epub_path = text
|
||||
epub_path = source
|
||||
try:
|
||||
book = epub.read_epub(epub_path)
|
||||
except Exception as e:
|
||||
|
||||
@@ -10,17 +10,17 @@ class PDFSplitter:
|
||||
"""PDF 分块器:pymupdf 提取文字 → TextSplitter 分块.
|
||||
|
||||
实现 Splitter Protocol,内部组合 TextSplitter 实例。
|
||||
注意: split() 的 text 参数实际接收 PDF 文件路径(非文本内容)。
|
||||
split() 的 source 参数接收 PDF 文件路径(非文本内容)。
|
||||
"""
|
||||
|
||||
def __init__(self, max_size: int = 1000, overlap: int = 100):
|
||||
self._text_splitter = TextSplitter(max_size=max_size, overlap=overlap)
|
||||
|
||||
def split(self, text: str, source_file: str = "") -> list[dict]:
|
||||
def split(self, source: str, source_file: str = "") -> list[dict]:
|
||||
"""从 PDF 文件提取文字并分块.
|
||||
|
||||
Args:
|
||||
text: PDF 文件路径(非文本内容!由 ingest_file 传入)
|
||||
source: PDF 文件路径
|
||||
source_file: 来源文件名
|
||||
"""
|
||||
try:
|
||||
@@ -30,7 +30,7 @@ class PDFSplitter:
|
||||
"PDF 支持需要 pymupdf 库. 请执行: uv sync --extra pdf"
|
||||
)
|
||||
|
||||
pdf_path = text # text 参数实际是文件路径
|
||||
pdf_path = source
|
||||
extracted_pages = []
|
||||
try:
|
||||
with fitz.open(pdf_path) as doc:
|
||||
|
||||
Reference in New Issue
Block a user