From d9b03ab5f085b8b13437b516c59ace4f271edde5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E8=88=AA=E5=AE=87?= <3364451258@qq.com> Date: Sat, 11 Jul 2026 19:55:03 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20DocxSplitter=20?= =?UTF-8?q?=E2=80=94=20=E6=94=AF=E6=8C=81=20.docx=20=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E5=85=A5=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...7-11-priority-2-experience-improvements.md | 7 ++ pyproject.toml | 3 +- src/core/splitters/__init__.py | 4 +- src/core/splitters/docx.py | 58 +++++++++ src/core/splitters/registry.py | 5 + tests/test_splitters_docx.py | 53 +++++++++ uv.lock | 111 ++++++++++++++++-- 7 files changed, 231 insertions(+), 10 deletions(-) create mode 100644 src/core/splitters/docx.py create mode 100644 tests/test_splitters_docx.py diff --git a/docs/superpowers/plans/2026-07-11-priority-2-experience-improvements.md b/docs/superpowers/plans/2026-07-11-priority-2-experience-improvements.md index 22ceff9..0c90339 100644 --- a/docs/superpowers/plans/2026-07-11-priority-2-experience-improvements.md +++ b/docs/superpowers/plans/2026-07-11-priority-2-experience-improvements.md @@ -48,6 +48,7 @@ ### Task 1: 修复 Embedder Protocol 和 Splitter 接口 **Files:** + - Modify: `src/core/embedder.py` - Modify: `src/core/splitters/pdf.py` - Modify: `src/core/splitters/epub.py` @@ -141,6 +142,7 @@ git commit -m "refactor: 修复 Embedder Protocol 标准写法和 PDF/EPUB Split ### Task 2: DocxSplitter — .docx 文档支持 **Files:** + - Create: `src/core/splitters/docx.py` - Create: `tests/test_splitters_docx.py` - Modify: `src/core/splitters/registry.py` @@ -336,6 +338,7 @@ git commit -m "feat: 新增 DocxSplitter — 支持 .docx 文档入库" ### Task 3: 数据导出功能 **Files:** + - Create: `tests/test_export.py` - Modify: `src/core/search.py` - Modify: `src/cli/main.py` @@ -581,6 +584,7 @@ git commit -m "feat: 新增数据导出功能（JSON/CSV）和 CLI export 命令 ### Task 4: Web 管理界面 **Files:** + - Create: `src/web/index.html` - Modify: `src/server/app.py` @@ -939,6 +943,7 @@ git commit -m "feat: 添加 Web 管理界面（Vue 3 SPA 单文件）" ### Task 5: 测试覆盖率提升到 80%+ **Files:** + - Modify: `tests/test_splitters_pdf.py` - Modify: `tests/test_splitters_html.py` - Modify: `tests/test_splitters_epub.py` @@ -1186,6 +1191,7 @@ git commit -m "test: 补充测试覆盖率至 80%+（PDF/HTML/EPUB/CLI/DB/Search ### Task 6: CORS 和安全配置修复 **Files:** + - Modify: `src/server/app.py` - [ ] **Step 1: 修复 CORS 配置** @@ -1227,6 +1233,7 @@ git commit -m "fix: 修复 CORS allow_credentials 配置，更新 .env.example" ### Task 7: 更新 README 和文档 **Files:** + - Modify: `README.md` - [ ] **Step 1: 更新 README 功能特性和新命令说明** diff --git a/pyproject.toml b/pyproject.toml index b1fd177..0b132de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,8 @@ dev = ["pytest>=8.0", "httpx>=0.27.0", "pytest-cov>=5.0", "ruff>=0.8.0", "mypy>= pdf = ["pymupdf>=1.24.0"] html = ["beautifulsoup4>=4.12.0"] epub = ["ebooklib>=0.18"] -all = ["md-vector-db[pdf,html,epub]", "requests>=2.31.0", "openai>=1.0.0"] +docx = ["markitdown>=0.1.0"] +all = ["md-vector-db[pdf,html,epub,docx]", "requests>=2.31.0", "openai>=1.0.0"] [build-system] requires = ["hatchling"] diff --git a/src/core/splitters/__init__.py b/src/core/splitters/__init__.py index 992ec64..3df5245 100644 --- a/src/core/splitters/__init__.py +++ b/src/core/splitters/__init__.py @@ -1,6 +1,7 @@ -"""文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML / EPUB.""" +"""文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML / EPUB / Docx.""" from src.core.splitters.base import BaseTextSplitter, Splitter +from src.core.splitters.docx import DocxSplitter from src.core.splitters.epub import EPUBSplitter from src.core.splitters.html import HTMLSplitter from src.core.splitters.markdown import MarkdownSplitter @@ -16,6 +17,7 @@ __all__ = [ "PDFSplitter", "HTMLSplitter", "EPUBSplitter", + "DocxSplitter", "get_splitter", "register_splitter", "SUPPORTED_SUFFIXES", diff --git a/src/core/splitters/docx.py b/src/core/splitters/docx.py new file mode 100644 index 0000000..b50cfc3 --- /dev/null +++ b/src/core/splitters/docx.py @@ -0,0 +1,58 @@ +""".docx Word 文档分块器 — 使用 markitdown 提取文字后委托 TextSplitter.""" +from __future__ import annotations + +import logging +from pathlib import Path + +from src.core.splitters.text import TextSplitter + +logger = logging.getLogger("md-vector-db") + + +class DocxSplitter: + """Docx 分块器：markitdown 提取文字 → TextSplitter 分块. + + 实现 Splitter Protocol，内部组合 TextSplitter 实例。 + split() 的 source 参数可接收文件路径或纯文本。 + """ + + def __init__(self, max_size: int = 1000, overlap: int = 100): + self._text_splitter = TextSplitter(max_size=max_size, overlap=overlap) + + def split(self, source: str, source_file: str = "") -> list[dict]: + """从 .docx 文件或纯文本提取文字并分块. + + Args: + source: .docx 文件路径或纯文本内容 + source_file: 来源文件名 + + Returns: + 分块后的 chunk 列表 + """ + path = Path(source) + if path.suffix.lower() == ".docx": + text = self._extract_from_docx(str(path)) + else: + text = source + + if not text.strip(): + return [] + + return self._text_splitter.split(text, source_file=source_file) + + def _extract_from_docx(self, file_path: str) -> str: + """使用 markitdown 从 .docx 文件中提取文字.""" + try: + from markitdown import MarkItDown + except ImportError: + raise ImportError( + "Docx 支持需要 markitdown 库. 请执行: uv sync --extra docx" + ) + + try: + md = MarkItDown() + result = md.convert(file_path) + return result.text_content + except Exception as e: + logger.error("Docx 解析失败: %s — %s", file_path, e) + raise ValueError(f"Docx 解析失败: {e}") from e diff --git a/src/core/splitters/registry.py b/src/core/splitters/registry.py index bd25961..f65f83a 100644 --- a/src/core/splitters/registry.py +++ b/src/core/splitters/registry.py @@ -12,6 +12,7 @@ _DEFAULT_MAP: dict[str, str] = { ".html": "html", ".htm": "html", ".epub": "epub", + ".docx": "docx", } # 所有支持的扩展名集合（供外部遍历文件使用） @@ -74,6 +75,10 @@ def get_splitter( from src.core.splitters.epub import EPUBSplitter return EPUBSplitter(max_size=max_size, overlap=overlap) + if kind == "docx": + from src.core.splitters.docx import DocxSplitter + return DocxSplitter(max_size=max_size, overlap=overlap) + # 回退 from src.core.splitters.text import TextSplitter return TextSplitter(max_size=max_size, overlap=overlap) diff --git a/tests/test_splitters_docx.py b/tests/test_splitters_docx.py new file mode 100644 index 0000000..9739b01 --- /dev/null +++ b/tests/test_splitters_docx.py @@ -0,0 +1,53 @@ +"""DocxSplitter 测试.""" +import pytest + +pytest.importorskip("markitdown", reason="需要 markitdown 库") + +from src.core.splitters.docx import DocxSplitter + + +def test_docx_splitter_creates(): + """创建 DocxSplitter 实例.""" + s = DocxSplitter(max_size=500, overlap=50) + assert s is not None + + +def test_docx_splitter_empty_text(): + """空纯文本返回空列表.""" + s = DocxSplitter() + result = s.split(" ", source_file="empty.docx") + assert result == [] + + +def test_docx_splitter_basic_text(): + """基本文本文档分块.""" + s = DocxSplitter(max_size=200, overlap=20) + text = "段落A。\n\n段落B。\n\n段落C。" + result = s.split(text, source_file="test.docx") + assert len(result) >= 1 + assert all("content" in r for r in result) + assert all(r["source_file"] == "test.docx" for r in result) + + +def test_docx_splitter_long_text(): + """长文本分多块.""" + s = DocxSplitter(max_size=100, overlap=10) + text = "这是一段非常长的文本。\n\n" * 50 + result = s.split(text, source_file="long.docx") + assert len(result) >= 5 + + +def test_docx_splitter_source_file(): + """source_file 正确传递到每个 chunk.""" + s = DocxSplitter(max_size=500, overlap=50) + result = s.split("测试内容。", source_file="myfile.docx") + assert all(r["source_file"] == "myfile.docx" for r in result) + + +def test_docx_splitter_chunk_index(): + """chunk_index 从 0 递增.""" + s = DocxSplitter(max_size=100, overlap=10) + text = "chunk A。\n\n" * 20 + result = s.split(text, source_file="index.docx") + indices = [r["chunk_index"] for r in result] + assert indices == list(range(len(result))) diff --git a/uv.lock b/uv.lock index 4145d4d..949c330 100644 --- a/uv.lock +++ b/uv.lock @@ -2,9 +2,12 @@ version = 1 revision = 3 requires-python = ">=3.13" resolution-markers = [ - "python_full_version >= '3.15'", - "python_full_version == '3.14.*'", - "python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", "(python_full_version < '3.14' and sys_platform == 'darwin') or (python_full_version < '3.14' and sys_platform == 'linux')", ] @@ -466,6 +469,15 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/52/30/21b2ad45959cd50e909e02ebac1e30b4ceb7162e91c11d4c570223a458b7/coverage-7.15.0-py3-none-any.whl", hash = "sha256:56da6a4cbe8f7e9e80bd072ca9cefe67d7106a440a7ec06519ec6507ac94ad19", size = 212632, upload-time = "2026-07-02T13:10:48.641Z" }, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -1030,6 +1042,50 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7f/2c/0f1e93c636720e8a3eb59af2bfda99d98b55891e1c53bc30c2e0e865f01b/lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c", size = 3817223, upload-time = "2026-05-19T19:22:56.823Z" }, ] +[[package]] +name = "magika" +version = "0.6.2" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", +] +dependencies = [ + { name = "click", marker = "sys_platform == 'win32'" }, + { name = "numpy", marker = "sys_platform == 'win32'" }, + { name = "onnxruntime", marker = "sys_platform == 'win32'" }, + { name = "python-dotenv", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/b6/8fdd991142ad3e037179a494b153f463024e5a211ef3ad948b955c26b4de/magika-0.6.2.tar.gz", hash = "sha256:37eb6ae8020f6e68f231bc06052c0a0cbe8e6fa27492db345e8dc867dbceb067", size = 3036634, upload-time = "2025-05-02T14:54:18.88Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c2/07/4f7748f34279f2852068256992377474f9700b6fbad6735d6be58605178f/magika-0.6.2-py3-none-any.whl", hash = "sha256:5ef72fbc07723029b3684ef81454bc224ac5f60986aa0fc5a28f4456eebcb5b2", size = 2967609, upload-time = "2025-05-02T14:54:09.696Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b0/1f/28e412d0ccedc068fbccdae6a6233faaa97ec3e5e2ffd242e49655b10064/magika-0.6.2-py3-none-win_amd64.whl", hash = "sha256:711f427a633e0182737dcc2074748004842f870643585813503ff2553b973b9f", size = 12385740, upload-time = "2025-05-02T14:54:14.096Z" }, +] + +[[package]] +name = "magika" +version = "0.6.3" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", + "(python_full_version < '3.14' and sys_platform == 'darwin') or (python_full_version < '3.14' and sys_platform == 'linux')", +] +dependencies = [ + { name = "click", marker = "sys_platform != 'win32'" }, + { name = "numpy", marker = "sys_platform != 'win32'" }, + { name = "onnxruntime", marker = "sys_platform != 'win32'" }, + { name = "python-dotenv", marker = "sys_platform != 'win32'" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a3/f3/3d1dcdd7b9c41d589f5cff252d32ed91cdf86ba84391cfc81d9d8773571d/magika-0.6.3.tar.gz", hash = "sha256:7cc52aa7359af861957043e2bf7265ed4741067251c104532765cd668c0c0cb1", size = 3042784, upload-time = "2025-10-30T15:22:34.499Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a2/e4/35c323beb3280482c94299d61626116856ac2d4ec16ecef50afc4fdd4291/magika-0.6.3-py3-none-any.whl", hash = "sha256:eda443d08006ee495e02083b32e51b98cb3696ab595a7d13900d8e2ef506ec9d", size = 2969474, upload-time = "2025-10-30T15:22:25.298Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/25/8f/132b0d7cd51c02c39fd52658a5896276c30c8cc2fd453270b19db8c40f7e/magika-0.6.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:86901e64b05dde5faff408c9b8245495b2e1fd4c226e3393d3d2a3fee65c504b", size = 13358841, upload-time = "2025-10-30T15:22:27.413Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c4/03/5ed859be502903a68b7b393b17ae0283bf34195cfcca79ce2dc25b9290e7/magika-0.6.3-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:3d9661eedbdf445ac9567e97e7ceefb93545d77a6a32858139ea966b5806fb64", size = 15367335, upload-time = "2025-10-30T15:22:29.907Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -1042,6 +1098,37 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, ] +[[package]] +name = "markdownify" +version = "1.2.3" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "six" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/92/ab/d1297139c0e2ceb151ae564c8c4f57ac0155d8f1f8b4cbd5d6523c82ea36/markdownify-1.2.3.tar.gz", hash = "sha256:1a176f05522c8a2cb1dd3ab9d307dcdadbed5c26ae717855bfc42b3b6d38d937", size = 18852, upload-time = "2026-06-30T20:27:39.06Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/04/10/fa543d484e8b1199243fe20eedd02cc5af050edebce98a7293a5773df592/markdownify-1.2.3-py3-none-any.whl", hash = "sha256:a189a0bedfd14009030fde5f85bb6f77c56897cb839b5c25315dd7d4e3e290ba", size = 15732, upload-time = "2026-06-30T20:27:38.094Z" }, +] + +[[package]] +name = "markitdown" +version = "0.1.6" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "charset-normalizer" }, + { name = "defusedxml" }, + { name = "magika", version = "0.6.2", source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }, marker = "sys_platform == 'win32'" }, + { name = "magika", version = "0.6.3", source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }, marker = "sys_platform != 'win32'" }, + { name = "markdownify" }, + { name = "requests" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/18/b7/91fe0e2df07107ab701a15c8ad3213135707e4d6206ae9bd8f457a7ad86a/markitdown-0.1.6.tar.gz", hash = "sha256:e5bdbaffd971b29598c7c39ef0e9afce2f08c0751fbfa4e4257678ebaf8cfc7e", size = 50795, upload-time = "2026-05-26T22:43:59.318Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9e/30/8031f183ee86ea8ac4e7ea1296bab4cca1bee2fd036a26df69764eb7ca74/markitdown-0.1.6-py3-none-any.whl", hash = "sha256:07b2d5bf87b5c53e13a9f2fdc440df8ccc85e26f40c1e557781727b700049775", size = 70032, upload-time = "2026-05-26T22:44:03.209Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -1114,6 +1201,7 @@ dependencies = [ all = [ { name = "beautifulsoup4" }, { name = "ebooklib" }, + { name = "markitdown" }, { name = "openai" }, { name = "pymupdf" }, { name = "requests" }, @@ -1125,6 +1213,9 @@ dev = [ { name = "pytest-cov" }, { name = "ruff" }, ] +docx = [ + { name = "markitdown" }, +] epub = [ { name = "ebooklib" }, ] @@ -1143,7 +1234,8 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.115.0" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27.0" }, { name = "markdown-it-py", specifier = ">=3.0.0" }, - { name = "md-vector-db", extras = ["pdf", "html", "epub"], marker = "extra == 'all'" }, + { name = "markitdown", marker = "extra == 'docx'", specifier = ">=0.1.0" }, + { name = "md-vector-db", extras = ["pdf", "html", "epub", "docx"], marker = "extra == 'all'" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.13" }, { name = "openai", marker = "extra == 'all'", specifier = ">=1.0.0" }, { name = "pymupdf", marker = "extra == 'pdf'", specifier = ">=1.24.0" }, @@ -1158,7 +1250,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.12.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" }, ] -provides-extras = ["dev", "pdf", "html", "epub", "all"] +provides-extras = ["dev", "pdf", "html", "epub", "docx", "all"] [[package]] name = "mdurl" @@ -2676,9 +2768,12 @@ name = "torch" version = "2.6.0+cu124" source = { registry = "D:/settings/Language/Python/库" } resolution-markers = [ - "python_full_version >= '3.15'", - "python_full_version == '3.14.*'", - "python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ { name = "filelock", marker = "(python_full_version >= '3.14' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },