122 lines
4.0 KiB
Python
122 lines
4.0 KiB
Python
"""文档入库测试."""
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from src.core.ingest import MarkdownSplitter, DocumentIngestor
|
|
|
|
|
|
class TestMarkdownSplitter:
|
|
"""Markdown 分块器测试."""
|
|
|
|
@pytest.fixture
|
|
def splitter(self):
|
|
return MarkdownSplitter(max_size=1000, overlap=100)
|
|
|
|
def test_split_simple_document(self, splitter):
|
|
"""简单文档按标题拆分."""
|
|
md = """# 标题一
|
|
这是第一段内容。
|
|
|
|
## 标题二
|
|
这是第二段内容。
|
|
|
|
# 标题三
|
|
这是第三段内容。"""
|
|
chunks = splitter.split(md, source_file="test.md")
|
|
assert len(chunks) >= 3
|
|
# 每个 chunk 有元数据
|
|
for chunk in chunks:
|
|
assert "content" in chunk
|
|
assert chunk["source_file"] == "test.md"
|
|
|
|
def test_chunk_has_heading_metadata(self, splitter):
|
|
"""chunk 附带标题元数据."""
|
|
md = "# 配置指南\n这里是配置说明。"
|
|
chunks = splitter.split(md, source_file="config.md")
|
|
assert len(chunks) >= 1
|
|
title = chunks[0]["section_title"]
|
|
assert "配置指南" in title or title == ""
|
|
|
|
def test_long_section_is_split(self, splitter):
|
|
"""超长章节被进一步拆分."""
|
|
# 创建一个超过 max_size 的段落
|
|
long_text = "这是很长的文本。" * 300 # ~3000 字符
|
|
md = f"# 长章节\n{long_text}"
|
|
small_splitter = MarkdownSplitter(max_size=500, overlap=50)
|
|
chunks = small_splitter.split(md, source_file="long.md")
|
|
assert len(chunks) > 1
|
|
|
|
def test_empty_document(self, splitter):
|
|
"""空文档返回空列表."""
|
|
chunks = splitter.split("", source_file="empty.md")
|
|
assert chunks == []
|
|
|
|
def test_code_blocks_preserved(self, splitter):
|
|
"""代码块不被拆分."""
|
|
md = """# 代码示例
|
|
```python
|
|
def hello():
|
|
print("world")
|
|
```
|
|
|
|
"""
|
|
chunks = splitter.split(md, source_file="code.md")
|
|
assert len(chunks) >= 1
|
|
# 代码块内容应在某个 chunk 中
|
|
all_content = " ".join(c["content"] for c in chunks)
|
|
assert "def hello()" in all_content
|
|
|
|
|
|
class TestDocumentIngestor:
|
|
"""文档入库器测试."""
|
|
|
|
@pytest.fixture
|
|
def temp_md_dir(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
# 创建测试 Markdown 文件
|
|
md_path = Path(d) / "test.md"
|
|
md_path.write_text("# 测试\n这是测试内容。", encoding="utf-8")
|
|
yield d
|
|
|
|
def test_read_markdown_file(self, temp_md_dir):
|
|
"""读取 Markdown 文件."""
|
|
content = Path(temp_md_dir + "/test.md").read_text(encoding="utf-8")
|
|
assert "测试" in content
|
|
assert "这是测试内容" in content
|
|
|
|
|
|
class TestIngestorIntegration:
|
|
"""入库器集成测试 (使用真实 embedder)."""
|
|
|
|
def test_ingest_content_real(self, tmp_path):
|
|
"""真实入库: 分块→嵌入→入库."""
|
|
from src.core.config import EmbedConfig
|
|
from src.core.db import VectorDB
|
|
from src.core.embedder import create_embedder
|
|
from src.core.ingest import DocumentIngestor
|
|
|
|
db = VectorDB(persist_dir=str(tmp_path))
|
|
embedder = create_embedder(EmbedConfig(mode="local"))
|
|
ingestor = DocumentIngestor(db, embedder, "test_integration")
|
|
|
|
count = ingestor.ingest_content("# Hello\nWorld.", "hello.md")
|
|
assert count > 0
|
|
assert ingestor.collection.count() == count
|
|
|
|
def test_ingest_deduplicates(self, tmp_path):
|
|
"""重复入库同一文件会去重."""
|
|
from src.core.config import EmbedConfig
|
|
from src.core.db import VectorDB
|
|
from src.core.embedder import create_embedder
|
|
from src.core.ingest import DocumentIngestor
|
|
|
|
db = VectorDB(persist_dir=str(tmp_path))
|
|
embedder = create_embedder(EmbedConfig(mode="local"))
|
|
ingestor = DocumentIngestor(db, embedder, "test_dedup")
|
|
|
|
c1 = ingestor.ingest_content("# A", "dup.md")
|
|
c2 = ingestor.ingest_content("# B", "dup.md")
|
|
assert ingestor.collection.count() == c2
|