feat: 新增 DocxSplitter — 支持 .docx 文档入库
This commit is contained in:
@@ -0,0 +1,53 @@
|
||||
"""DocxSplitter 测试."""
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("markitdown", reason="需要 markitdown 库")
|
||||
|
||||
from src.core.splitters.docx import DocxSplitter
|
||||
|
||||
|
||||
def test_docx_splitter_creates():
|
||||
"""创建 DocxSplitter 实例."""
|
||||
s = DocxSplitter(max_size=500, overlap=50)
|
||||
assert s is not None
|
||||
|
||||
|
||||
def test_docx_splitter_empty_text():
|
||||
"""空纯文本返回空列表."""
|
||||
s = DocxSplitter()
|
||||
result = s.split(" ", source_file="empty.docx")
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_docx_splitter_basic_text():
|
||||
"""基本文本文档分块."""
|
||||
s = DocxSplitter(max_size=200, overlap=20)
|
||||
text = "段落A。\n\n段落B。\n\n段落C。"
|
||||
result = s.split(text, source_file="test.docx")
|
||||
assert len(result) >= 1
|
||||
assert all("content" in r for r in result)
|
||||
assert all(r["source_file"] == "test.docx" for r in result)
|
||||
|
||||
|
||||
def test_docx_splitter_long_text():
|
||||
"""长文本分多块."""
|
||||
s = DocxSplitter(max_size=100, overlap=10)
|
||||
text = "这是一段非常长的文本。\n\n" * 50
|
||||
result = s.split(text, source_file="long.docx")
|
||||
assert len(result) >= 5
|
||||
|
||||
|
||||
def test_docx_splitter_source_file():
|
||||
"""source_file 正确传递到每个 chunk."""
|
||||
s = DocxSplitter(max_size=500, overlap=50)
|
||||
result = s.split("测试内容。", source_file="myfile.docx")
|
||||
assert all(r["source_file"] == "myfile.docx" for r in result)
|
||||
|
||||
|
||||
def test_docx_splitter_chunk_index():
|
||||
"""chunk_index 从 0 递增."""
|
||||
s = DocxSplitter(max_size=100, overlap=10)
|
||||
text = "chunk A。\n\n" * 20
|
||||
result = s.split(text, source_file="index.docx")
|
||||
indices = [r["chunk_index"] for r in result]
|
||||
assert indices == list(range(len(result)))
|
||||
Reference in New Issue
Block a user