"""文档入库测试.""" import tempfile from pathlib import Path import pytest from src.core.ingest import MarkdownSplitter, DocumentIngestor class TestMarkdownSplitter: """Markdown 分块器测试.""" @pytest.fixture def splitter(self): return MarkdownSplitter(max_size=1000, overlap=100) def test_split_simple_document(self, splitter): """简单文档按标题拆分.""" md = """# 标题一 这是第一段内容。 ## 标题二 这是第二段内容。 # 标题三 这是第三段内容。""" chunks = splitter.split(md, source_file="test.md") assert len(chunks) >= 3 # 每个 chunk 有元数据 for chunk in chunks: assert "content" in chunk assert chunk["source_file"] == "test.md" def test_chunk_has_heading_metadata(self, splitter): """chunk 附带标题元数据.""" md = "# 配置指南\n这里是配置说明。" chunks = splitter.split(md, source_file="config.md") assert len(chunks) >= 1 title = chunks[0]["section_title"] assert "配置指南" in title or title == "" def test_long_section_is_split(self, splitter): """超长章节被进一步拆分.""" # 创建一个超过 max_size 的段落 long_text = "这是很长的文本。" * 300 # ~3000 字符 md = f"# 长章节\n{long_text}" small_splitter = MarkdownSplitter(max_size=500, overlap=50) chunks = small_splitter.split(md, source_file="long.md") assert len(chunks) > 1 def test_empty_document(self, splitter): """空文档返回空列表.""" chunks = splitter.split("", source_file="empty.md") assert chunks == [] def test_code_blocks_preserved(self, splitter): """代码块不被拆分.""" md = """# 代码示例 ```python def hello(): print("world") ``` """ chunks = splitter.split(md, source_file="code.md") assert len(chunks) >= 1 # 代码块内容应在某个 chunk 中 all_content = " ".join(c["content"] for c in chunks) assert "def hello()" in all_content class TestDocumentIngestor: """文档入库器测试.""" @pytest.fixture def temp_md_dir(self): with tempfile.TemporaryDirectory() as d: # 创建测试 Markdown 文件 md_path = Path(d) / "test.md" md_path.write_text("# 测试\n这是测试内容。", encoding="utf-8") yield d def test_read_markdown_file(self, temp_md_dir): """读取 Markdown 文件.""" content = Path(temp_md_dir + "/test.md").read_text(encoding="utf-8") assert "测试" in content assert "这是测试内容" in content class TestIngestorIntegration: """入库器集成测试 (使用真实 embedder).""" def test_ingest_content_real(self, tmp_path): """真实入库: 分块→嵌入→入库.""" from src.core.config import EmbedConfig from src.core.db import VectorDB from src.core.embedder import create_embedder from src.core.ingest import DocumentIngestor db = VectorDB(persist_dir=str(tmp_path)) embedder = create_embedder(EmbedConfig(mode="local")) ingestor = DocumentIngestor(db, embedder, "test_integration") count = ingestor.ingest_content("# Hello\nWorld.", "hello.md") assert count > 0 assert ingestor.collection.count() == count def test_ingest_deduplicates(self, tmp_path): """重复入库同一文件会去重.""" from src.core.config import EmbedConfig from src.core.db import VectorDB from src.core.embedder import create_embedder from src.core.ingest import DocumentIngestor db = VectorDB(persist_dir=str(tmp_path)) embedder = create_embedder(EmbedConfig(mode="local")) ingestor = DocumentIngestor(db, embedder, "test_dedup") c1 = ingestor.ingest_content("# A", "dup.md") c2 = ingestor.ingest_content("# B", "dup.md") assert ingestor.collection.count() == c2