Serendipity/md-vector-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 新增 DocxSplitter — 支持 .docx 文档入库

Browse Source
This commit is contained in:
刘航宇
2026-07-11 19:55:03 +08:00
parent 04f2f8a8a8
commit d9b03ab5f0
7 changed files with 231 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/superpowers/plans/2026-07-11-priority-2-experience-improvements.md
+7
View File
@@ -48,6 +48,7 @@
### Task 1: 修复 Embedder Protocol 和 Splitter 接口
**Files:**
- Modify: `src/core/embedder.py`
- Modify: `src/core/splitters/pdf.py`
- Modify: `src/core/splitters/epub.py`
@@ -141,6 +142,7 @@ git commit -m "refactor: 修复 Embedder Protocol 标准写法和 PDF/EPUB Split
### Task 2: DocxSplitter — .docx 文档支持
**Files:**
- Create: `src/core/splitters/docx.py`
- Create: `tests/test_splitters_docx.py`
- Modify: `src/core/splitters/registry.py`
@@ -336,6 +338,7 @@ git commit -m "feat: 新增 DocxSplitter — 支持 .docx 文档入库"
### Task 3: 数据导出功能
**Files:**
- Create: `tests/test_export.py`
- Modify: `src/core/search.py`
- Modify: `src/cli/main.py`
@@ -581,6 +584,7 @@ git commit -m "feat: 新增数据导出功能（JSON/CSV）和 CLI export 命令
### Task 4: Web 管理界面
**Files:**
- Create: `src/web/index.html`
- Modify: `src/server/app.py`
@@ -939,6 +943,7 @@ git commit -m "feat: 添加 Web 管理界面（Vue 3 SPA 单文件）"
### Task 5: 测试覆盖率提升到 80%+
**Files:**
- Modify: `tests/test_splitters_pdf.py`
- Modify: `tests/test_splitters_html.py`
- Modify: `tests/test_splitters_epub.py`
@@ -1186,6 +1191,7 @@ git commit -m "test: 补充测试覆盖率至 80%+（PDF/HTML/EPUB/CLI/DB/Search
### Task 6: CORS 和安全配置修复
**Files:**
- Modify: `src/server/app.py`
- [ ] **Step 1: 修复 CORS 配置**
@@ -1227,6 +1233,7 @@ git commit -m "fix: 修复 CORS allow_credentials 配置，更新 .env.example"
### Task 7: 更新 README 和文档
**Files:**
- Modify: `README.md`
- [ ] **Step 1: 更新 README 功能特性和新命令说明**
pyproject.toml
+2 -1
View File
@@ -23,7 +23,8 @@ dev = ["pytest>=8.0", "httpx>=0.27.0", "pytest-cov>=5.0", "ruff>=0.8.0", "mypy>=
pdf = ["pymupdf>=1.24.0"]
html = ["beautifulsoup4>=4.12.0"]
epub = ["ebooklib>=0.18"]
all = ["md-vector-db[pdf,html,epub]", "requests>=2.31.0", "openai>=1.0.0"]
docx = ["markitdown>=0.1.0"]
all = ["md-vector-db[pdf,html,epub,docx]", "requests>=2.31.0", "openai>=1.0.0"]
[build-system]
requires = ["hatchling"]
src/core/splitters/__init__.py
+3 -1
View File
@@ -1,6 +1,7 @@
"""文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML / EPUB."""
"""文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML / EPUB / Docx."""
from src.core.splitters.base import BaseTextSplitter, Splitter
from src.core.splitters.docx import DocxSplitter
from src.core.splitters.epub import EPUBSplitter
from src.core.splitters.html import HTMLSplitter
from src.core.splitters.markdown import MarkdownSplitter
@@ -16,6 +17,7 @@ __all__ = [
"PDFSplitter",
"HTMLSplitter",
"EPUBSplitter",
"DocxSplitter",
"get_splitter",
"register_splitter",
"SUPPORTED_SUFFIXES",
src/core/splitters/docx.py
+58
View File
@@ -0,0 +1,58 @@
""".docx Word 文档分块器 — 使用 markitdown 提取文字后委托 TextSplitter."""
from __future__ import annotations
import logging
from pathlib import Path
from src.core.splitters.text import TextSplitter
logger = logging.getLogger("md-vector-db")
class DocxSplitter:
"""Docx 分块器：markitdown 提取文字 → TextSplitter 分块.
实现 Splitter Protocol，内部组合 TextSplitter 实例。
split() 的 source 参数可接收文件路径或纯文本。
"""
def __init__(self, max_size: int = 1000, overlap: int = 100):
self._text_splitter = TextSplitter(max_size=max_size, overlap=overlap)
def split(self, source: str, source_file: str = "") -> list[dict]:
"""从 .docx 文件或纯文本提取文字并分块.
Args:
source: .docx 文件路径或纯文本内容
source_file: 来源文件名
Returns:
分块后的 chunk 列表
"""
path = Path(source)
if path.suffix.lower() == ".docx":
text = self._extract_from_docx(str(path))
else:
text = source
if not text.strip():
return []
return self._text_splitter.split(text, source_file=source_file)
def _extract_from_docx(self, file_path: str) -> str:
"""使用 markitdown 从 .docx 文件中提取文字."""
try:
from markitdown import MarkItDown
except ImportError:
raise ImportError(
"Docx 支持需要 markitdown 库. 请执行: uv sync --extra docx"
)
try:
md = MarkItDown()
result = md.convert(file_path)
return result.text_content
except Exception as e:
logger.error("Docx 解析失败: %s — %s", file_path, e)
raise ValueError(f"Docx 解析失败: {e}") from e
src/core/splitters/registry.py
+5
View File
@@ -12,6 +12,7 @@ _DEFAULT_MAP: dict[str, str] = {
".html": "html",
".htm": "html",
".epub": "epub",
".docx": "docx",
}
# 所有支持的扩展名集合（供外部遍历文件使用）
@@ -74,6 +75,10 @@ def get_splitter(
from src.core.splitters.epub import EPUBSplitter
return EPUBSplitter(max_size=max_size, overlap=overlap)
if kind == "docx":
from src.core.splitters.docx import DocxSplitter
return DocxSplitter(max_size=max_size, overlap=overlap)
# 回退
from src.core.splitters.text import TextSplitter
return TextSplitter(max_size=max_size, overlap=overlap)
tests/test_splitters_docx.py
+53
View File
@@ -0,0 +1,53 @@
"""DocxSplitter 测试."""
import pytest
pytest.importorskip("markitdown", reason="需要 markitdown 库")
from src.core.splitters.docx import DocxSplitter
def test_docx_splitter_creates():
"""创建 DocxSplitter 实例."""
s = DocxSplitter(max_size=500, overlap=50)
assert s is not None
def test_docx_splitter_empty_text():
"""空纯文本返回空列表."""
s = DocxSplitter()
result = s.split(" ", source_file="empty.docx")
assert result == []
def test_docx_splitter_basic_text():
"""基本文本文档分块."""
s = DocxSplitter(max_size=200, overlap=20)
text = "段落A。\n\n段落B。\n\n段落C。"
result = s.split(text, source_file="test.docx")
assert len(result) >= 1
assert all("content" in r for r in result)
assert all(r["source_file"] == "test.docx" for r in result)
def test_docx_splitter_long_text():
"""长文本分多块."""
s = DocxSplitter(max_size=100, overlap=10)
text = "这是一段非常长的文本。\n\n" * 50
result = s.split(text, source_file="long.docx")
assert len(result) >= 5
def test_docx_splitter_source_file():
"""source_file 正确传递到每个 chunk."""
s = DocxSplitter(max_size=500, overlap=50)
result = s.split("测试内容。", source_file="myfile.docx")
assert all(r["source_file"] == "myfile.docx" for r in result)
def test_docx_splitter_chunk_index():
"""chunk_index 从 0 递增."""
s = DocxSplitter(max_size=100, overlap=10)
text = "chunk A。\n\n" * 20
result = s.split(text, source_file="index.docx")
indices = [r["chunk_index"] for r in result]
assert indices == list(range(len(result)))
uv.lock
Generated
+103 -8
View File
@@ -2,9 +2,12 @@ version = 1
revision = 3
requires-python = ">=3.13"
resolution-markers = [
"python_full_version >= '3.15'",
"python_full_version == '3.14.*'",
"python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'",
"python_full_version >= '3.15' and sys_platform == 'win32'",
"python_full_version >= '3.15' and sys_platform != 'win32'",
"python_full_version == '3.14.*' and sys_platform == 'win32'",
"python_full_version == '3.14.*' and sys_platform != 'win32'",
"python_full_version < '3.14' and sys_platform == 'win32'",
"python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'",
"(python_full_version < '3.14' and sys_platform == 'darwin') or (python_full_version < '3.14' and sys_platform == 'linux')",
]
@@ -466,6 +469,15 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/52/30/21b2ad45959cd50e909e02ebac1e30b4ceb7162e91c11d4c570223a458b7/coverage-7.15.0-py3-none-any.whl", hash = "sha256:56da6a4cbe8f7e9e80bd072ca9cefe67d7106a440a7ec06519ec6507ac94ad19", size = 212632, upload-time = "2026-07-02T13:10:48.641Z" },
]
[[package]]
name = "defusedxml"
version = "0.7.1"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61" },
]
[[package]]
name = "distro"
version = "1.9.0"
@@ -1030,6 +1042,50 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/7f/2c/0f1e93c636720e8a3eb59af2bfda99d98b55891e1c53bc30c2e0e865f01b/lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c", size = 3817223, upload-time = "2026-05-19T19:22:56.823Z" },
]
[[package]]
name = "magika"
version = "0.6.2"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }
resolution-markers = [
"python_full_version >= '3.15' and sys_platform == 'win32'",
"python_full_version == '3.14.*' and sys_platform == 'win32'",
"python_full_version < '3.14' and sys_platform == 'win32'",
]
dependencies = [
{ name = "click", marker = "sys_platform == 'win32'" },
{ name = "numpy", marker = "sys_platform == 'win32'" },
{ name = "onnxruntime", marker = "sys_platform == 'win32'" },
{ name = "python-dotenv", marker = "sys_platform == 'win32'" },
]
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/b6/8fdd991142ad3e037179a494b153f463024e5a211ef3ad948b955c26b4de/magika-0.6.2.tar.gz", hash = "sha256:37eb6ae8020f6e68f231bc06052c0a0cbe8e6fa27492db345e8dc867dbceb067", size = 3036634, upload-time = "2025-05-02T14:54:18.88Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/c2/07/4f7748f34279f2852068256992377474f9700b6fbad6735d6be58605178f/magika-0.6.2-py3-none-any.whl", hash = "sha256:5ef72fbc07723029b3684ef81454bc224ac5f60986aa0fc5a28f4456eebcb5b2", size = 2967609, upload-time = "2025-05-02T14:54:09.696Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/b0/1f/28e412d0ccedc068fbccdae6a6233faaa97ec3e5e2ffd242e49655b10064/magika-0.6.2-py3-none-win_amd64.whl", hash = "sha256:711f427a633e0182737dcc2074748004842f870643585813503ff2553b973b9f", size = 12385740, upload-time = "2025-05-02T14:54:14.096Z" },
]
[[package]]
name = "magika"
version = "0.6.3"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }
resolution-markers = [
"python_full_version >= '3.15' and sys_platform != 'win32'",
"python_full_version == '3.14.*' and sys_platform != 'win32'",
"python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'",
"(python_full_version < '3.14' and sys_platform == 'darwin') or (python_full_version < '3.14' and sys_platform == 'linux')",
]
dependencies = [
{ name = "click", marker = "sys_platform != 'win32'" },
{ name = "numpy", marker = "sys_platform != 'win32'" },
{ name = "onnxruntime", marker = "sys_platform != 'win32'" },
{ name = "python-dotenv", marker = "sys_platform != 'win32'" },
]
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a3/f3/3d1dcdd7b9c41d589f5cff252d32ed91cdf86ba84391cfc81d9d8773571d/magika-0.6.3.tar.gz", hash = "sha256:7cc52aa7359af861957043e2bf7265ed4741067251c104532765cd668c0c0cb1", size = 3042784, upload-time = "2025-10-30T15:22:34.499Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/a2/e4/35c323beb3280482c94299d61626116856ac2d4ec16ecef50afc4fdd4291/magika-0.6.3-py3-none-any.whl", hash = "sha256:eda443d08006ee495e02083b32e51b98cb3696ab595a7d13900d8e2ef506ec9d", size = 2969474, upload-time = "2025-10-30T15:22:25.298Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/25/8f/132b0d7cd51c02c39fd52658a5896276c30c8cc2fd453270b19db8c40f7e/magika-0.6.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:86901e64b05dde5faff408c9b8245495b2e1fd4c226e3393d3d2a3fee65c504b", size = 13358841, upload-time = "2025-10-30T15:22:27.413Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/c4/03/5ed859be502903a68b7b393b17ae0283bf34195cfcca79ce2dc25b9290e7/magika-0.6.3-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:3d9661eedbdf445ac9567e97e7ceefb93545d77a6a32858139ea966b5806fb64", size = 15367335, upload-time = "2025-10-30T15:22:29.907Z" },
]
[[package]]
name = "markdown-it-py"
version = "4.2.0"
@@ -1042,6 +1098,37 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" },
]
[[package]]
name = "markdownify"
version = "1.2.3"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }
dependencies = [
{ name = "beautifulsoup4" },
{ name = "six" },
]
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/92/ab/d1297139c0e2ceb151ae564c8c4f57ac0155d8f1f8b4cbd5d6523c82ea36/markdownify-1.2.3.tar.gz", hash = "sha256:1a176f05522c8a2cb1dd3ab9d307dcdadbed5c26ae717855bfc42b3b6d38d937", size = 18852, upload-time = "2026-06-30T20:27:39.06Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/04/10/fa543d484e8b1199243fe20eedd02cc5af050edebce98a7293a5773df592/markdownify-1.2.3-py3-none-any.whl", hash = "sha256:a189a0bedfd14009030fde5f85bb6f77c56897cb839b5c25315dd7d4e3e290ba", size = 15732, upload-time = "2026-06-30T20:27:38.094Z" },
]
[[package]]
name = "markitdown"
version = "0.1.6"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }
dependencies = [
{ name = "beautifulsoup4" },
{ name = "charset-normalizer" },
{ name = "defusedxml" },
{ name = "magika", version = "0.6.2", source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }, marker = "sys_platform == 'win32'" },
{ name = "magika", version = "0.6.3", source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }, marker = "sys_platform != 'win32'" },
{ name = "markdownify" },
{ name = "requests" },
]
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/18/b7/91fe0e2df07107ab701a15c8ad3213135707e4d6206ae9bd8f457a7ad86a/markitdown-0.1.6.tar.gz", hash = "sha256:e5bdbaffd971b29598c7c39ef0e9afce2f08c0751fbfa4e4257678ebaf8cfc7e", size = 50795, upload-time = "2026-05-26T22:43:59.318Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/9e/30/8031f183ee86ea8ac4e7ea1296bab4cca1bee2fd036a26df69764eb7ca74/markitdown-0.1.6-py3-none-any.whl", hash = "sha256:07b2d5bf87b5c53e13a9f2fdc440df8ccc85e26f40c1e557781727b700049775", size = 70032, upload-time = "2026-05-26T22:44:03.209Z" },
]
[[package]]
name = "markupsafe"
version = "3.0.3"
@@ -1114,6 +1201,7 @@ dependencies = [
all = [
{ name = "beautifulsoup4" },
{ name = "ebooklib" },
{ name = "markitdown" },
{ name = "openai" },
{ name = "pymupdf" },
{ name = "requests" },
@@ -1125,6 +1213,9 @@ dev = [
{ name = "pytest-cov" },
{ name = "ruff" },
]
docx = [
{ name = "markitdown" },
]
epub = [
{ name = "ebooklib" },
]
@@ -1143,7 +1234,8 @@ requires-dist = [
{ name = "fastapi", specifier = ">=0.115.0" },
{ name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27.0" },
{ name = "markdown-it-py", specifier = ">=3.0.0" },
{ name = "md-vector-db", extras = ["pdf", "html", "epub"], marker = "extra == 'all'" },
{ name = "markitdown", marker = "extra == 'docx'", specifier = ">=0.1.0" },
{ name = "md-vector-db", extras = ["pdf", "html", "epub", "docx"], marker = "extra == 'all'" },
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.13" },
{ name = "openai", marker = "extra == 'all'", specifier = ">=1.0.0" },
{ name = "pymupdf", marker = "extra == 'pdf'", specifier = ">=1.24.0" },
@@ -1158,7 +1250,7 @@ requires-dist = [
{ name = "typer", specifier = ">=0.12.0" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" },
]
provides-extras = ["dev", "pdf", "html", "epub", "all"]
provides-extras = ["dev", "pdf", "html", "epub", "docx", "all"]
[[package]]
name = "mdurl"
@@ -2676,9 +2768,12 @@ name = "torch"
version = "2.6.0+cu124"
source = { registry = "D:/settings/Language/Python/库" }
resolution-markers = [
"python_full_version >= '3.15'",
"python_full_version == '3.14.*'",
"python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'",
"python_full_version >= '3.15' and sys_platform == 'win32'",
"python_full_version >= '3.15' and sys_platform != 'win32'",
"python_full_version == '3.14.*' and sys_platform == 'win32'",
"python_full_version == '3.14.*' and sys_platform != 'win32'",
"python_full_version < '3.14' and sys_platform == 'win32'",
"python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'",
]
dependencies = [
{ name = "filelock", marker = "(python_full_version >= '3.14' and sys_platform == 'darwin') or (python_full_version >= '3.14' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },