test: 添加 PDFSplitter 和 HTMLSplitter 测试

2026-07-10 14:25:40 +08:00
parent 9db0c15dee
commit b8afc9043d
2 changed files with 91 additions and 0 deletions
@@ -0,0 +1,42 @@
+"""HTMLSplitter 测试."""
+import pytest
+
+bs4 = pytest.importorskip("bs4", reason="beautifulsoup4 未安装")
+
+
+class TestHTMLSplitter:
+    """HTMLSplitter 测试（需 beautifulsoup4）."""
+
+    def test_split_simple_html(self):
+        from src.core.splitters.html import HTMLSplitter
+        html = "<html><body><h1>标题</h1><p>这是段落内容。</p><p>第二段。</p></body></html>"
+        s = HTMLSplitter(max_size=500)
+        chunks = s.split(html, source_file="test.html")
+        assert len(chunks) >= 1
+        all_text = "".join(c["content"] for c in chunks)
+        assert "标题" in all_text
+        assert "段落内容" in all_text
+        assert "第二段" in all_text
+
+    def test_strips_script_and_style(self):
+        from src.core.splitters.html import HTMLSplitter
+        html = """<html><head><style>.a{color:red}</style><script>alert('xss')</script></head>
+<body><p>可见内容。</p></body></html>"""
+        s = HTMLSplitter()
+        chunks = s.split(html, source_file="test.html")
+        all_text = "".join(c["content"] for c in chunks)
+        assert "可见内容" in all_text
+        assert "alert" not in all_text
+        assert ".a{color:red}" not in all_text
+
+    def test_empty_html(self):
+        from src.core.splitters.html import HTMLSplitter
+        s = HTMLSplitter()
+        assert s.split("<html></html>") == []
+        assert s.split("") == []
+
+    def test_html_has_split_method(self):
+        """HTMLSplitter 遵循 Splitter Protocol."""
+        from src.core.splitters.html import HTMLSplitter
+        s = HTMLSplitter()
+        assert hasattr(s, "split")
@@ -0,0 +1,49 @@
+"""PDFSplitter 测试."""
+import pytest
+from pathlib import Path
+
+pymupdf = pytest.importorskip("fitz", reason="pymupdf 未安装")
+
+
+class TestPDFSplitter:
+    """PDFSplitter 测试（需 pymupdf）."""
+
+    def test_split_simple_pdf(self, tmp_path):
+        """用 pymupdf 创建一个简单 PDF 并测试分块."""
+        from src.core.splitters.pdf import PDFSplitter
+        import fitz
+
+        pdf_path = tmp_path / "test.pdf"
+        doc = fitz.open()
+        # 插入纯 ASCII 文本避免 CJK 字体编码问题
+        doc.new_page().insert_text((72, 72), "This is PDF document content.\n\nSecond paragraph text.")
+        doc.save(str(pdf_path))
+        doc.close()
+
+        s = PDFSplitter(max_size=500)
+        chunks = s.split(str(pdf_path), source_file="test.pdf")
+        assert len(chunks) >= 1
+        all_text = "".join(c["content"] for c in chunks)
+        assert "PDF document" in all_text
+        assert "Second paragraph" in all_text
+
+    def test_empty_pdf(self, tmp_path):
+        """空 PDF（有页但无文字）返回空列表."""
+        from src.core.splitters.pdf import PDFSplitter
+        import fitz
+
+        pdf_path = tmp_path / "empty.pdf"
+        doc = fitz.open()
+        doc.new_page()  # pymupdf 必须有至少一页才能保存
+        doc.save(str(pdf_path))
+        doc.close()
+
+        s = PDFSplitter()
+        chunks = s.split(str(pdf_path), source_file="empty.pdf")
+        assert chunks == []
+
+    def test_pdf_has_split_method(self):
+        """PDFSplitter 遵循 Splitter Protocol."""
+        from src.core.splitters.pdf import PDFSplitter
+        s = PDFSplitter()
+        assert hasattr(s, "split")