From 2d0e8c49971c9a046349c056a2f36cb347c42cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E8=88=AA=E5=AE=87?= <3364451258@qq.com> Date: Sat, 11 Jul 2026 19:50:35 +0800 Subject: [PATCH] =?UTF-8?q?chore:=20ruff=20lint=20=E4=BF=AE=E5=A4=8D=20+?= =?UTF-8?q?=20=E7=AC=AC=E4=B8=80=E4=BC=98=E5=85=88=E7=BA=A7=E5=85=A8?= =?UTF-8?q?=E9=83=A8=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .coverage | Bin 53248 -> 53248 bytes src/cli/main.py | 16 ++++++++++------ src/core/config.py | 3 +-- src/core/embedder.py | 2 +- src/core/ingest.py | 5 ++--- src/core/search.py | 4 +++- src/core/splitters/__init__.py | 12 ++++++------ src/core/splitters/epub.py | 1 + src/core/splitters/html.py | 1 + src/core/splitters/markdown.py | 1 + src/core/splitters/pdf.py | 1 + src/core/splitters/registry.py | 1 + src/server/app.py | 11 +++++------ src/server/auth.py | 6 +++--- src/server/deps.py | 2 +- tests/test_auth.py | 4 ++-- tests/test_cli.py | 1 + tests/test_config.py | 5 +---- tests/test_db.py | 1 - tests/test_deps.py | 3 --- tests/test_embedder.py | 8 ++++++-- tests/test_ingest.py | 19 +++++-------------- tests/test_reranker.py | 1 - tests/test_security.py | 1 - tests/test_splitters.py | 9 +++++++-- tests/test_splitters_epub.py | 3 ++- tests/test_splitters_markdown.py | 1 + tests/test_splitters_pdf.py | 8 +++++--- 28 files changed, 67 insertions(+), 63 deletions(-) diff --git a/.coverage b/.coverage index 8c5b91090a296967ca39ee5b285a7173319655a5..a2f9b3adef2c3b14e4bec93d203dba4e28999b53 100644 GIT binary patch delta 1153 zcmZ9JeQZ-z7{>2yZ*TACd2c^%+l$jhB!-x5UC13E9Im{ zaYj5KCPlJR@Mtdze3QX3PsljAaTOkIb>o@act)QJLWOerhf%6sB6@jmel^-E=${DnLq{VeU3n#5VT z26}|CLQUJSfDdhsvIZ`9)@oP84&BO;)Eg3O)GJd5arKtfc4+ax&;eYRxZS0j9ErRk z(dfP$*H6?KNh`BHoJ+)nf75#H#tzGRNY%<}wC^(AB(=Rjvbs z@);Z}t`tuXjYmo&qmQ56({z1Z~dyWS|D{&jc9z{#~2-yXha zKD_YUpZtWoZ>H|drb<3Q49LL>I-%E-zTi;(_6hnvYsE)>%%PItvj_eTUevs#TJGC7 zUV3!y$NsWa&pHH;J@j@w%?X-aX|@e)8@zJu(ean%!g|s&XibHe7-R0ks4EgfkS~>xHwM<5}!QQGLEIvPBeeb*%uXuQ2Vd;+PIhj!a zNgR8#culB-A7}4U%`Rs^B>rlp6rQ?pl&KBZq{mK`&$mzA=@*E8gE~J{_hjza=mP`6 zgSjIB*vj45=y%y8cQ5{_Vz6OH>HW##V#adw98nR^E_V{fT;-}G?v=k({Jpe}&9NWv z2j(CA-;cNH=K8HN)hZo=>cov?5 ze&~g_;2rn?lH?B>VDaP*Rx~J_Aqpc%;RsN8bPC;1;rCJaycAv!g{DyeP#{Dhs1yN( UA}CXYBnm~OPzAp5&2V+iKMQS3EC2ui delta 896 zcmX|-T}V@59LCSt=5{{c|Jlbm6%E27iNv`+FpY9WQ(`X4F8Wf^S<_YI##T-#zbGgO zk~$X=VoC+wRda=Xst`#8t-J_?re5@Q5xS&uscrSXr|9bUKhN{L|Mz)MN1wiB-_+lFRyb8za`Y9MhpV3WJjTEt5$O;eGw3ra;g~ePKH_ngn`P^o{8BVbM;1JK_H?*V12^fKXxDM@b1opyqa6tjAfF)o;88nSPpd=bZ zcThL3+JM?Eih*YH*oiCSMbqiMg}*{CKHL&ztl9Nh59^xEL`&>8{JBvXocMTZPup0@ zT&(wXaqw&FuxV1C{Z1RL*hM;;v0z%H`m|0oljF98>vC-=wLi*8#4tO~7$m^ca!53$ zAI(pdw)avr<444_IJW5+kxW(+6G#kQ%Y8iF!{3|#J+#ckV?`3@d6C0|o$*k`SB5D@ zkyw(H56<2mnEv)-XHPcY#$aW&Fj4qr!O`3qJMgZi?P!KmPj{_zS#q$IXKGscu!L$) zCL_URQz@eXYsg6F>M`9p*@8!=RqyRTQ-)1OrlFMT7>dq)qp0EZV;8N?bXlDW1SnX5 zRtUm(n1W9*2~QvZV=xLY;Xd4kt8f-BLm%9N3-An@@IPJ;0V7^cvx)}2tUPx# diff --git a/src/cli/main.py b/src/cli/main.py index 856721d..2cc2f91 100644 --- a/src/cli/main.py +++ b/src/cli/main.py @@ -1,10 +1,10 @@ """命令行工具入口 — 可作为 MCP tool 直接调用.""" -from pathlib import Path -import sys +import glob as _glob import io import json import os -import glob as _glob +import sys +from pathlib import Path from typing import Annotated import typer @@ -18,7 +18,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from src.core.config import DEFAULT_CONFIG_PATH from src.core.security import is_safe_cli_path -from src.server.deps import get_state, get_default_collection +from src.server.deps import get_default_collection, get_state app = typer.Typer( name="md-vector-db", @@ -63,8 +63,12 @@ def ingest( typer.Argument(help="Markdown 文件路径 (可多个, 或 - 从标准输入读取)"), ] = None, name: Annotated[str | None, typer.Option("--name", help="标准输入模式下的虚拟文件名")] = None, - incremental: Annotated[bool, typer.Option("--incremental", help="增量模式：跳过未变更文件")] = False, - force: Annotated[bool, typer.Option("--force", help="强制重新入库（忽略增量检查）")] = False, + incremental: Annotated[ + bool, typer.Option("--incremental", help="增量模式：跳过未变更文件") + ] = False, + force: Annotated[ + bool, typer.Option("--force", help="强制重新入库（忽略增量检查）") + ] = False, config: ConfigOpt = DEFAULT_CONFIG_PATH, collection: CollectionOpt = None, ): diff --git a/src/core/config.py b/src/core/config.py index f09bdb6..a2518de 100644 --- a/src/core/config.py +++ b/src/core/config.py @@ -5,9 +5,8 @@ import os from dataclasses import dataclass, field from pathlib import Path -from dotenv import load_dotenv - import yaml +from dotenv import load_dotenv # 加载 .env 文件 (若存在) load_dotenv() diff --git a/src/core/embedder.py b/src/core/embedder.py index 573dd78..bae6648 100644 --- a/src/core/embedder.py +++ b/src/core/embedder.py @@ -17,9 +17,9 @@ vectors = batch_embed(embedder, long_text_list) """ +import logging import os import threading -import logging from typing import Protocol from src.core.config import EmbedConfig diff --git a/src/core/ingest.py b/src/core/ingest.py index 527c0ff..1c586c6 100644 --- a/src/core/ingest.py +++ b/src/core/ingest.py @@ -1,14 +1,13 @@ """Markdown 文档解析与入库模块.""" import logging -import re from pathlib import Path from src.core.config import ChunkConfig from src.core.db import VectorDB from src.core.embedder import Embedder, batch_embed from src.core.file_tracker import FileTracker -from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径 from src.core.splitters.base import Splitter # 兼容旧 import 路径 +from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径 from src.core.splitters.registry import SUPPORTED_SUFFIXES, get_splitter logger = logging.getLogger("md-vector-db") @@ -40,7 +39,7 @@ class DocumentIngestor: def ingest_file(self, file_path: str, incremental: bool = False, force: bool = False) -> int: """入库单个文件, 返回 chunk 数量. - 根据文件扩展名自动选择 Splitter（.md→MarkdownSplitter, .txt→TextSplitter, .pdf→PDFSplitter 等）。 + 根据扩展名自动选择 Splitter（.md/.txt/.pdf/.html/.epub 等）。 使用文件路径的 SHA256 前 12 位 + 文件名作为唯一标识。 Args: diff --git a/src/core/search.py b/src/core/search.py index adb6040..f9ec5ba 100644 --- a/src/core/search.py +++ b/src/core/search.py @@ -161,7 +161,9 @@ class Searcher: all_data = self.collection.get(include=["documents", "metadatas"]) output = io.StringIO() writer = csv.writer(output) - writer.writerow(["id", "content", "source_file", "section_title", "heading_level", "chunk_index"]) + writer.writerow([ + "id", "content", "source_file", "section_title", "heading_level", "chunk_index", + ]) if all_data and all_data["ids"]: for i, doc_id in enumerate(all_data["ids"]): meta = all_data["metadatas"][i] if all_data["metadatas"] else {} diff --git a/src/core/splitters/__init__.py b/src/core/splitters/__init__.py index fe15eba..992ec64 100644 --- a/src/core/splitters/__init__.py +++ b/src/core/splitters/__init__.py @@ -1,12 +1,12 @@ """文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML / EPUB.""" -from src.core.splitters.base import Splitter, BaseTextSplitter -from src.core.splitters.markdown import MarkdownSplitter -from src.core.splitters.text import TextSplitter -from src.core.splitters.pdf import PDFSplitter -from src.core.splitters.html import HTMLSplitter +from src.core.splitters.base import BaseTextSplitter, Splitter from src.core.splitters.epub import EPUBSplitter -from src.core.splitters.registry import get_splitter, register_splitter, SUPPORTED_SUFFIXES +from src.core.splitters.html import HTMLSplitter +from src.core.splitters.markdown import MarkdownSplitter +from src.core.splitters.pdf import PDFSplitter +from src.core.splitters.registry import SUPPORTED_SUFFIXES, get_splitter, register_splitter +from src.core.splitters.text import TextSplitter __all__ = [ "Splitter", diff --git a/src/core/splitters/epub.py b/src/core/splitters/epub.py index b7f6f14..f42570d 100644 --- a/src/core/splitters/epub.py +++ b/src/core/splitters/epub.py @@ -1,5 +1,6 @@ """EPUB 电子书分块器 — 使用 ebooklib 提取文字后委托 TextSplitter.""" import logging + from src.core.splitters.text import TextSplitter logger = logging.getLogger("md-vector-db") diff --git a/src/core/splitters/html.py b/src/core/splitters/html.py index 97dfc3d..48dfbde 100644 --- a/src/core/splitters/html.py +++ b/src/core/splitters/html.py @@ -1,5 +1,6 @@ """HTML 文档分块器 — 使用 BeautifulSoup 去标签后委托 TextSplitter.""" import logging + from src.core.splitters.text import TextSplitter logger = logging.getLogger("md-vector-db") diff --git a/src/core/splitters/markdown.py b/src/core/splitters/markdown.py index d298189..1a5082d 100644 --- a/src/core/splitters/markdown.py +++ b/src/core/splitters/markdown.py @@ -1,5 +1,6 @@ """Markdown 文档分块器.""" import re + from src.core.splitters.base import BaseTextSplitter diff --git a/src/core/splitters/pdf.py b/src/core/splitters/pdf.py index 9ee745a..c378fd4 100644 --- a/src/core/splitters/pdf.py +++ b/src/core/splitters/pdf.py @@ -1,5 +1,6 @@ """PDF 文档分块器 — 使用 pymupdf 提取文字后委托 TextSplitter.""" import logging + from src.core.splitters.text import TextSplitter logger = logging.getLogger("md-vector-db") diff --git a/src/core/splitters/registry.py b/src/core/splitters/registry.py index 8e05707..bd25961 100644 --- a/src/core/splitters/registry.py +++ b/src/core/splitters/registry.py @@ -1,5 +1,6 @@ """Splitter 注册表 — 按文件扩展名自动选择分块器.""" from pathlib import Path + from src.core.splitters.base import Splitter # 扩展名 → Splitter 类名映射 diff --git a/src/server/app.py b/src/server/app.py index df909c6..b455be0 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -1,20 +1,19 @@ """FastAPI 服务层.""" +import logging import os import time import uuid -import logging +from contextlib import asynccontextmanager from pathlib import Path -from contextlib import asynccontextmanager - -from fastapi import FastAPI, HTTPException, Depends, Request, Query +from fastapi import Depends, FastAPI, HTTPException, Query, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import RedirectResponse from pydantic import BaseModel, Field, model_validator from src.core.security import is_path_within_workspace -from src.server.auth import verify_api_key, rate_limiter -from src.server.deps import get_state, AppState +from src.server.auth import rate_limiter, verify_api_key +from src.server.deps import AppState, get_state logger = logging.getLogger("md-vector-db") diff --git a/src/server/auth.py b/src/server/auth.py index 721d80a..3bd4cd4 100644 --- a/src/server/auth.py +++ b/src/server/auth.py @@ -1,9 +1,9 @@ """API 认证与安全中间件.""" import hmac -import os -import time -import threading import logging +import os +import threading +import time from collections import defaultdict from fastapi import Header, HTTPException, Request diff --git a/src/server/deps.py b/src/server/deps.py index 87f55aa..69f8ddb 100644 --- a/src/server/deps.py +++ b/src/server/deps.py @@ -1,7 +1,7 @@ """FastAPI 依赖注入 — 集中管理应用状态, 替代模块级全局变量.""" +import logging import os import threading -import logging from src.core.config import load_config from src.core.db import VectorDB diff --git a/tests/test_auth.py b/tests/test_auth.py index 0a79f60..0224778 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,11 +1,11 @@ """认证与速率限制测试.""" -import time import threading +import time import pytest from fastapi import HTTPException -from src.server.auth import verify_api_key, RateLimiter +from src.server.auth import RateLimiter, verify_api_key class TestVerifyApiKey: diff --git a/tests/test_cli.py b/tests/test_cli.py index 8f421d9..571b0c5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,6 @@ """CLI 命令测试.""" from typer.testing import CliRunner + from src.cli.main import app runner = CliRunner() diff --git a/tests/test_config.py b/tests/test_config.py index 1249125..d3d03b2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,11 +1,8 @@ """配置加载模块测试.""" -import os import tempfile from pathlib import Path -import pytest - -from src.core.config import AppConfig, EmbedConfig, ChunkConfig, load_config +from src.core.config import AppConfig, EmbedConfig, load_config class TestEmbedConfig: diff --git a/tests/test_db.py b/tests/test_db.py index f217e95..ea1ba8f 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1,5 +1,4 @@ """数据库层测试.""" -import gc import tempfile from pathlib import Path diff --git a/tests/test_deps.py b/tests/test_deps.py index aae47a6..2a1c81f 100644 --- a/tests/test_deps.py +++ b/tests/test_deps.py @@ -1,10 +1,7 @@ """AppState 和依赖注入测试.""" -import os -import tempfile import pytest -from src.core.config import load_config from src.server.deps import AppState diff --git a/tests/test_embedder.py b/tests/test_embedder.py index a4d9a43..1ce73e8 100644 --- a/tests/test_embedder.py +++ b/tests/test_embedder.py @@ -3,8 +3,12 @@ import pytest from src.core.config import EmbedConfig from src.core.embedder import ( - LocalEmbedder, OpenAIEmbedder, DashscopeEmbedder, - create_embedder, batch_embed, SUPPORTED_PROVIDERS, + SUPPORTED_PROVIDERS, + DashscopeEmbedder, + LocalEmbedder, + OpenAIEmbedder, + batch_embed, + create_embedder, ) diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 1a7234e..aaa74cb 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -4,8 +4,8 @@ from pathlib import Path import pytest -from src.core.splitters import MarkdownSplitter from src.core.ingest import DocumentIngestor +from src.core.splitters import MarkdownSplitter class TestMarkdownSplitter: @@ -122,7 +122,6 @@ class TestIngestorIntegration: from src.core.config import EmbedConfig from src.core.db import VectorDB from src.core.embedder import create_embedder - from src.core.ingest import DocumentIngestor db = VectorDB(persist_dir=str(tmp_path)) embedder = create_embedder(EmbedConfig(mode="local")) @@ -137,13 +136,12 @@ class TestIngestorIntegration: from src.core.config import EmbedConfig from src.core.db import VectorDB from src.core.embedder import create_embedder - from src.core.ingest import DocumentIngestor db = VectorDB(persist_dir=str(tmp_path)) embedder = create_embedder(EmbedConfig(mode="local")) ingestor = DocumentIngestor(db, embedder, "test_dedup") - c1 = ingestor.ingest_content("# A", "dup.md") + ingestor.ingest_content("# A", "dup.md") c2 = ingestor.ingest_content("# B", "dup.md") assert ingestor.collection.count() == c2 @@ -156,7 +154,6 @@ class TestIngestFile: from src.core.config import EmbedConfig from src.core.db import VectorDB from src.core.embedder import create_embedder - from src.core.ingest import DocumentIngestor md_file = tmp_path / "hello.md" md_file.write_text("# 测试\n这是测试内容。", encoding="utf-8") @@ -174,7 +171,6 @@ class TestIngestFile: from src.core.config import EmbedConfig from src.core.db import VectorDB from src.core.embedder import create_embedder - from src.core.ingest import DocumentIngestor txt_file = tmp_path / "notes.txt" txt_file.write_text("这是一段纯文本内容。\n\n第二段内容在这里。", encoding="utf-8") @@ -195,7 +191,6 @@ class TestIngestDirectory: from src.core.config import EmbedConfig from src.core.db import VectorDB from src.core.embedder import create_embedder - from src.core.ingest import DocumentIngestor (tmp_path / "a.md").write_text("# A\n内容 A", encoding="utf-8") (tmp_path / "b.txt").write_text("内容 B", encoding="utf-8") @@ -213,7 +208,6 @@ class TestIngestDirectory: from src.core.config import EmbedConfig from src.core.db import VectorDB from src.core.embedder import create_embedder - from src.core.ingest import DocumentIngestor db = VectorDB(persist_dir=str(tmp_path / "db")) embedder = create_embedder(EmbedConfig(mode="local")) @@ -228,10 +222,9 @@ class TestIncrementalIngest: def test_ingest_file_incremental_skips_unchanged(self, tmp_path): """增量模式: 未修改的文件跳过入库.""" - from src.core.config import EmbedConfig, ChunkConfig + from src.core.config import ChunkConfig, EmbedConfig from src.core.db import VectorDB from src.core.embedder import create_embedder - from src.core.ingest import DocumentIngestor from src.core.file_tracker import FileTracker file = tmp_path / "stable.md" @@ -251,10 +244,9 @@ class TestIncrementalIngest: def test_ingest_file_incremental_reingests_modified(self, tmp_path): """增量模式: 修改后的文件重新入库.""" - from src.core.config import EmbedConfig, ChunkConfig + from src.core.config import ChunkConfig, EmbedConfig from src.core.db import VectorDB from src.core.embedder import create_embedder - from src.core.ingest import DocumentIngestor from src.core.file_tracker import FileTracker file = tmp_path / "changing.md" @@ -275,10 +267,9 @@ class TestIncrementalIngest: def test_ingest_file_force_mode_always_reingests(self, tmp_path): """force=True 时始终重新入库（忽略 tracker）.""" - from src.core.config import EmbedConfig, ChunkConfig + from src.core.config import ChunkConfig, EmbedConfig from src.core.db import VectorDB from src.core.embedder import create_embedder - from src.core.ingest import DocumentIngestor from src.core.file_tracker import FileTracker file = tmp_path / "force.md" diff --git a/tests/test_reranker.py b/tests/test_reranker.py index ae0550e..5028c6f 100644 --- a/tests/test_reranker.py +++ b/tests/test_reranker.py @@ -1,5 +1,4 @@ """重排序器测试.""" -import pytest from src.core.reranker import Reranker diff --git a/tests/test_security.py b/tests/test_security.py index affcbd8..db0b2d5 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -1,5 +1,4 @@ """is_safe_path 路径遍历防护测试.""" -import pytest from src.core.security import is_safe_path diff --git a/tests/test_splitters.py b/tests/test_splitters.py index e73e240..cb5d583 100644 --- a/tests/test_splitters.py +++ b/tests/test_splitters.py @@ -1,6 +1,11 @@ """Splitter 注册表和 TextSplitter 测试.""" -import pytest -from src.core.splitters import TextSplitter, MarkdownSplitter, get_splitter, register_splitter, SUPPORTED_SUFFIXES +from src.core.splitters import ( + SUPPORTED_SUFFIXES, + MarkdownSplitter, + TextSplitter, + get_splitter, + register_splitter, +) class TestTextSplitter: diff --git a/tests/test_splitters_epub.py b/tests/test_splitters_epub.py index 9885c8f..e85b30d 100644 --- a/tests/test_splitters_epub.py +++ b/tests/test_splitters_epub.py @@ -28,9 +28,10 @@ class TestEPUBSplitter: def test_split_simple_epub(self, tmp_path): """用 ebooklib 创建一个简单 EPUB 并测试分块.""" - from src.core.splitters.epub import EPUBSplitter from ebooklib import epub + from src.core.splitters.epub import EPUBSplitter + epub_path = tmp_path / "test.epub" book = epub.EpubBook() diff --git a/tests/test_splitters_markdown.py b/tests/test_splitters_markdown.py index 513f462..9604113 100644 --- a/tests/test_splitters_markdown.py +++ b/tests/test_splitters_markdown.py @@ -1,5 +1,6 @@ """MarkdownSplitter 边界测试.""" import pytest + from src.core.splitters import MarkdownSplitter diff --git a/tests/test_splitters_pdf.py b/tests/test_splitters_pdf.py index ea0af2e..3a4f96a 100644 --- a/tests/test_splitters_pdf.py +++ b/tests/test_splitters_pdf.py @@ -1,6 +1,6 @@ """PDFSplitter 测试.""" + import pytest -from pathlib import Path pymupdf = pytest.importorskip("fitz", reason="pymupdf 未安装") @@ -10,9 +10,10 @@ class TestPDFSplitter: def test_split_simple_pdf(self, tmp_path): """用 pymupdf 创建一个简单 PDF 并测试分块.""" - from src.core.splitters.pdf import PDFSplitter import fitz + from src.core.splitters.pdf import PDFSplitter + pdf_path = tmp_path / "test.pdf" doc = fitz.open() # 插入纯 ASCII 文本避免 CJK 字体编码问题 @@ -29,9 +30,10 @@ class TestPDFSplitter: def test_empty_pdf(self, tmp_path): """空 PDF（有页但无文字）返回空列表.""" - from src.core.splitters.pdf import PDFSplitter import fitz + from src.core.splitters.pdf import PDFSplitter + pdf_path = tmp_path / "empty.pdf" doc = fitz.open() doc.new_page() # pymupdf 必须有至少一页才能保存