feat: 添加 EPUBSplitter — ebooklib 提取章节文字后分块

- 新增 splitters/epub.py: ebooklib 读取 EPUB → BeautifulSoup 去标签 → TextSplitter 分块
- epub 作为可选依赖: uv sync --extra epub
- registry 新增 .epub 映射, ingest_file 处理 EPUB 二进制文件
- 测试: 3 个 EPUBSplitter 测试 (含空 EPUB/Protocol 合规)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-07-10 14:32:41 +08:00
parent eda080bc05
commit 3b8b585f31
6 changed files with 139 additions and 18 deletions
+2 -2
View File
@@ -46,9 +46,9 @@ class DocumentIngestor:
file_path, max_size=1000, overlap=100
)
# PDF 文件特殊处理:splitter 内部读取二进制内容
# PDF/EPUB 二进制文件特殊处理:splitter 内部读取文件
suffix = path.suffix.lower()
if suffix in (".pdf",):
if suffix in (".pdf", ".epub"):
chunks = splitter.split(str(path), source_file=file_name)
return self._add_chunks(chunks, file_name)
+3 -1
View File
@@ -1,9 +1,10 @@
"""文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML."""
"""文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML / EPUB."""
from src.core.splitters.base import Splitter, BaseTextSplitter
from src.core.splitters.markdown import MarkdownSplitter
from src.core.splitters.text import TextSplitter
from src.core.splitters.pdf import PDFSplitter
from src.core.splitters.epub import EPUBSplitter
from src.core.splitters.registry import get_splitter, register_splitter, SUPPORTED_SUFFIXES
__all__ = [
@@ -12,6 +13,7 @@ __all__ = [
"MarkdownSplitter",
"TextSplitter",
"PDFSplitter",
"EPUBSplitter",
"get_splitter",
"register_splitter",
"SUPPORTED_SUFFIXES",
+68
View File
@@ -0,0 +1,68 @@
"""EPUB 电子书分块器 — 使用 ebooklib 提取文字后委托 TextSplitter."""
import logging
from src.core.splitters.text import TextSplitter
logger = logging.getLogger("md-vector-db")
class EPUBSplitter:
"""EPUB 分块器:ebooklib 提取各章节文字 → TextSplitter 分块.
实现 Splitter Protocol,内部组合 TextSplitter 实例。
split() 的 text 参数实际接收 EPUB 文件路径(非文本内容)。
"""
def __init__(self, max_size: int = 1000, overlap: int = 100):
self._text_splitter = TextSplitter(max_size=max_size, overlap=overlap)
def split(self, text: str, source_file: str = "") -> list[dict]:
"""从 EPUB 文件提取各章节文字并分块.
Args:
text: EPUB 文件路径(非文本内容,由 ingest_file 传入)
source_file: 来源文件名
"""
try:
import ebooklib
from ebooklib import epub
except ImportError:
raise ImportError(
"EPUB 支持需要 ebooklib 库. 请执行: uv sync --extra epub"
)
epub_path = text
try:
book = epub.read_epub(epub_path)
except Exception as e:
logger.error("EPUB 解析失败: %s%s", epub_path, e)
raise ValueError(f"EPUB 解析失败: {e}") from e
extracted_chapters = []
for item in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
try:
# ebooklib 的 get_content() 返回 bytes
content = item.get_content().decode("utf-8")
except UnicodeDecodeError:
logger.warning("EPUB 跳过一个无法解码的章节: %s", item.get_name())
continue
# 用 BeautifulSoup 去标签(如果可用),否则保留原样
try:
from bs4 import BeautifulSoup
soup = BeautifulSoup(content, "html.parser")
for tag in soup(["script", "style"]):
tag.decompose()
text_content = soup.get_text(separator="\n")
except ImportError:
# 无 bs4 时手动去除简单标签
import re
text_content = re.sub(r"<[^>]+>", "", content)
if text_content.strip():
extracted_chapters.append(text_content)
if not extracted_chapters:
return []
full_text = "\n\n".join(extracted_chapters)
return self._text_splitter.split(full_text, source_file=source_file)
+5
View File
@@ -10,6 +10,7 @@ _DEFAULT_MAP: dict[str, str] = {
".pdf": "pdf",
".html": "html",
".htm": "html",
".epub": "epub",
}
# 所有支持的扩展名集合(供外部遍历文件使用)
@@ -64,6 +65,10 @@ def get_splitter(
from src.core.splitters.html import HTMLSplitter
return HTMLSplitter(max_size=max_size, overlap=overlap)
if kind == "epub":
from src.core.splitters.epub import EPUBSplitter
return EPUBSplitter(max_size=max_size, overlap=overlap)
# 回退
from src.core.splitters.text import TextSplitter
return TextSplitter(max_size=max_size, overlap=overlap)