feat: 添加 EPUBSplitter — ebooklib 提取章节文字后分块
- 新增 splitters/epub.py: ebooklib 读取 EPUB → BeautifulSoup 去标签 → TextSplitter 分块 - epub 作为可选依赖: uv sync --extra epub - registry 新增 .epub 映射, ingest_file 处理 EPUB 二进制文件 - 测试: 3 个 EPUBSplitter 测试 (含空 EPUB/Protocol 合规) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
+2
-2
@@ -46,9 +46,9 @@ class DocumentIngestor:
|
||||
file_path, max_size=1000, overlap=100
|
||||
)
|
||||
|
||||
# PDF 文件特殊处理:splitter 内部读取二进制内容
|
||||
# PDF/EPUB 二进制文件特殊处理:splitter 内部读取文件
|
||||
suffix = path.suffix.lower()
|
||||
if suffix in (".pdf",):
|
||||
if suffix in (".pdf", ".epub"):
|
||||
chunks = splitter.split(str(path), source_file=file_name)
|
||||
return self._add_chunks(chunks, file_name)
|
||||
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
"""文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML."""
|
||||
"""文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML / EPUB."""
|
||||
|
||||
from src.core.splitters.base import Splitter, BaseTextSplitter
|
||||
from src.core.splitters.markdown import MarkdownSplitter
|
||||
from src.core.splitters.text import TextSplitter
|
||||
from src.core.splitters.pdf import PDFSplitter
|
||||
from src.core.splitters.epub import EPUBSplitter
|
||||
from src.core.splitters.registry import get_splitter, register_splitter, SUPPORTED_SUFFIXES
|
||||
|
||||
__all__ = [
|
||||
@@ -12,6 +13,7 @@ __all__ = [
|
||||
"MarkdownSplitter",
|
||||
"TextSplitter",
|
||||
"PDFSplitter",
|
||||
"EPUBSplitter",
|
||||
"get_splitter",
|
||||
"register_splitter",
|
||||
"SUPPORTED_SUFFIXES",
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""EPUB 电子书分块器 — 使用 ebooklib 提取文字后委托 TextSplitter."""
|
||||
import logging
|
||||
from src.core.splitters.text import TextSplitter
|
||||
|
||||
logger = logging.getLogger("md-vector-db")
|
||||
|
||||
|
||||
class EPUBSplitter:
|
||||
"""EPUB 分块器:ebooklib 提取各章节文字 → TextSplitter 分块.
|
||||
|
||||
实现 Splitter Protocol,内部组合 TextSplitter 实例。
|
||||
split() 的 text 参数实际接收 EPUB 文件路径(非文本内容)。
|
||||
"""
|
||||
|
||||
def __init__(self, max_size: int = 1000, overlap: int = 100):
|
||||
self._text_splitter = TextSplitter(max_size=max_size, overlap=overlap)
|
||||
|
||||
def split(self, text: str, source_file: str = "") -> list[dict]:
|
||||
"""从 EPUB 文件提取各章节文字并分块.
|
||||
|
||||
Args:
|
||||
text: EPUB 文件路径(非文本内容,由 ingest_file 传入)
|
||||
source_file: 来源文件名
|
||||
"""
|
||||
try:
|
||||
import ebooklib
|
||||
from ebooklib import epub
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"EPUB 支持需要 ebooklib 库. 请执行: uv sync --extra epub"
|
||||
)
|
||||
|
||||
epub_path = text
|
||||
try:
|
||||
book = epub.read_epub(epub_path)
|
||||
except Exception as e:
|
||||
logger.error("EPUB 解析失败: %s — %s", epub_path, e)
|
||||
raise ValueError(f"EPUB 解析失败: {e}") from e
|
||||
|
||||
extracted_chapters = []
|
||||
for item in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
|
||||
try:
|
||||
# ebooklib 的 get_content() 返回 bytes
|
||||
content = item.get_content().decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
logger.warning("EPUB 跳过一个无法解码的章节: %s", item.get_name())
|
||||
continue
|
||||
|
||||
# 用 BeautifulSoup 去标签(如果可用),否则保留原样
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
for tag in soup(["script", "style"]):
|
||||
tag.decompose()
|
||||
text_content = soup.get_text(separator="\n")
|
||||
except ImportError:
|
||||
# 无 bs4 时手动去除简单标签
|
||||
import re
|
||||
text_content = re.sub(r"<[^>]+>", "", content)
|
||||
|
||||
if text_content.strip():
|
||||
extracted_chapters.append(text_content)
|
||||
|
||||
if not extracted_chapters:
|
||||
return []
|
||||
|
||||
full_text = "\n\n".join(extracted_chapters)
|
||||
return self._text_splitter.split(full_text, source_file=source_file)
|
||||
@@ -10,6 +10,7 @@ _DEFAULT_MAP: dict[str, str] = {
|
||||
".pdf": "pdf",
|
||||
".html": "html",
|
||||
".htm": "html",
|
||||
".epub": "epub",
|
||||
}
|
||||
|
||||
# 所有支持的扩展名集合(供外部遍历文件使用)
|
||||
@@ -64,6 +65,10 @@ def get_splitter(
|
||||
from src.core.splitters.html import HTMLSplitter
|
||||
return HTMLSplitter(max_size=max_size, overlap=overlap)
|
||||
|
||||
if kind == "epub":
|
||||
from src.core.splitters.epub import EPUBSplitter
|
||||
return EPUBSplitter(max_size=max_size, overlap=overlap)
|
||||
|
||||
# 回退
|
||||
from src.core.splitters.text import TextSplitter
|
||||
return TextSplitter(max_size=max_size, overlap=overlap)
|
||||
|
||||
Reference in New Issue
Block a user