feat: DocumentIngestor 支持增量入库（FileTracker）

2026-07-11 19:41:21 +08:00
parent 3e2c61402a
commit 7b3c7d5323
6 changed files with 4604 additions and 4 deletions
@@ -221,3 +221,77 @@ class TestIngestDirectory:

        results = ingestor.ingest_directory(str(tmp_path))
        assert results == {}
+
+
+class TestIncrementalIngest:
+    """增量入库测试."""
+
+    def test_ingest_file_incremental_skips_unchanged(self, tmp_path):
+        """增量模式: 未修改的文件跳过入库."""
+        from src.core.config import EmbedConfig, ChunkConfig
+        from src.core.db import VectorDB
+        from src.core.embedder import create_embedder
+        from src.core.ingest import DocumentIngestor
+        from src.core.file_tracker import FileTracker
+
+        file = tmp_path / "stable.md"
+        file.write_text("# 稳定文档\n\n内容不变。", encoding="utf-8")
+        tracker_path = str(tmp_path / "tracker.json")
+        db = VectorDB(persist_dir=str(tmp_path / "db"))
+        embedder = create_embedder(EmbedConfig(mode="local"))
+        ingestor = DocumentIngestor(
+            db, embedder, "test_incr",
+            chunk_config=ChunkConfig(max_size=1000, overlap=100),
+            file_tracker=FileTracker(tracker_path),
+        )
+        count1 = ingestor.ingest_file(str(file), incremental=True)
+        assert count1 > 0
+        count2 = ingestor.ingest_file(str(file), incremental=True)
+        assert count2 == 0  # 跳过
+
+    def test_ingest_file_incremental_reingests_modified(self, tmp_path):
+        """增量模式: 修改后的文件重新入库."""
+        from src.core.config import EmbedConfig, ChunkConfig
+        from src.core.db import VectorDB
+        from src.core.embedder import create_embedder
+        from src.core.ingest import DocumentIngestor
+        from src.core.file_tracker import FileTracker
+
+        file = tmp_path / "changing.md"
+        file.write_text("# v1\n\n初始版本的内容段落。", encoding="utf-8")
+        tracker_path = str(tmp_path / "tracker2.json")
+        db = VectorDB(persist_dir=str(tmp_path / "db2"))
+        embedder = create_embedder(EmbedConfig(mode="local"))
+        ingestor = DocumentIngestor(
+            db, embedder, "test_incr2",
+            chunk_config=ChunkConfig(max_size=1000, overlap=100),
+            file_tracker=FileTracker(tracker_path),
+        )
+        count1 = ingestor.ingest_file(str(file), incremental=True)
+        assert count1 > 0
+        file.write_text("# v2\n\n新增段落，内容完全不同了。", encoding="utf-8")
+        count2 = ingestor.ingest_file(str(file), incremental=True)
+        assert count2 > 0
+
+    def test_ingest_file_force_mode_always_reingests(self, tmp_path):
+        """force=True 时始终重新入库（忽略 tracker）."""
+        from src.core.config import EmbedConfig, ChunkConfig
+        from src.core.db import VectorDB
+        from src.core.embedder import create_embedder
+        from src.core.ingest import DocumentIngestor
+        from src.core.file_tracker import FileTracker
+
+        file = tmp_path / "force.md"
+        file.write_text("# force test\n\n这是强制入库测试的内容。", encoding="utf-8")
+        tracker_path = str(tmp_path / "tracker3.json")
+        db = VectorDB(persist_dir=str(tmp_path / "db3"))
+        embedder = create_embedder(EmbedConfig(mode="local"))
+        ingestor = DocumentIngestor(
+            db, embedder, "test_force",
+            chunk_config=ChunkConfig(max_size=1000, overlap=100),
+            file_tracker=FileTracker(tracker_path),
+        )
+        count1 = ingestor.ingest_file(str(file), incremental=True)
+        count2 = ingestor.ingest_file(str(file), incremental=True, force=True)
+        assert count1 > 0
+        assert count2 > 0  # force 模式重新入库