chore: ruff lint 修复 + 第一优先级全部完成

This commit is contained in:
2026-07-11 19:50:35 +08:00
parent 3e51e166c2
commit 2d0e8c4997
28 changed files with 67 additions and 63 deletions
BIN
View File
Binary file not shown.
+10 -6
View File
@@ -1,10 +1,10 @@
"""命令行工具入口 — 可作为 MCP tool 直接调用.""" """命令行工具入口 — 可作为 MCP tool 直接调用."""
from pathlib import Path import glob as _glob
import sys
import io import io
import json import json
import os import os
import glob as _glob import sys
from pathlib import Path
from typing import Annotated from typing import Annotated
import typer import typer
@@ -18,7 +18,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
from src.core.config import DEFAULT_CONFIG_PATH from src.core.config import DEFAULT_CONFIG_PATH
from src.core.security import is_safe_cli_path from src.core.security import is_safe_cli_path
from src.server.deps import get_state, get_default_collection from src.server.deps import get_default_collection, get_state
app = typer.Typer( app = typer.Typer(
name="md-vector-db", name="md-vector-db",
@@ -63,8 +63,12 @@ def ingest(
typer.Argument(help="Markdown 文件路径 (可多个, 或 - 从标准输入读取)"), typer.Argument(help="Markdown 文件路径 (可多个, 或 - 从标准输入读取)"),
] = None, ] = None,
name: Annotated[str | None, typer.Option("--name", help="标准输入模式下的虚拟文件名")] = None, name: Annotated[str | None, typer.Option("--name", help="标准输入模式下的虚拟文件名")] = None,
incremental: Annotated[bool, typer.Option("--incremental", help="增量模式:跳过未变更文件")] = False, incremental: Annotated[
force: Annotated[bool, typer.Option("--force", help="强制重新入库(忽略增量检查)")] = False, bool, typer.Option("--incremental", help="增量模式:跳过未变更文件")
] = False,
force: Annotated[
bool, typer.Option("--force", help="强制重新入库(忽略增量检查)")
] = False,
config: ConfigOpt = DEFAULT_CONFIG_PATH, config: ConfigOpt = DEFAULT_CONFIG_PATH,
collection: CollectionOpt = None, collection: CollectionOpt = None,
): ):
+1 -2
View File
@@ -5,9 +5,8 @@ import os
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from dotenv import load_dotenv
import yaml import yaml
from dotenv import load_dotenv
# 加载 .env 文件 (若存在) # 加载 .env 文件 (若存在)
load_dotenv() load_dotenv()
+1 -1
View File
@@ -17,9 +17,9 @@
vectors = batch_embed(embedder, long_text_list) vectors = batch_embed(embedder, long_text_list)
""" """
import logging
import os import os
import threading import threading
import logging
from typing import Protocol from typing import Protocol
from src.core.config import EmbedConfig from src.core.config import EmbedConfig
+2 -3
View File
@@ -1,14 +1,13 @@
"""Markdown 文档解析与入库模块.""" """Markdown 文档解析与入库模块."""
import logging import logging
import re
from pathlib import Path from pathlib import Path
from src.core.config import ChunkConfig from src.core.config import ChunkConfig
from src.core.db import VectorDB from src.core.db import VectorDB
from src.core.embedder import Embedder, batch_embed from src.core.embedder import Embedder, batch_embed
from src.core.file_tracker import FileTracker from src.core.file_tracker import FileTracker
from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径
from src.core.splitters.base import Splitter # 兼容旧 import 路径 from src.core.splitters.base import Splitter # 兼容旧 import 路径
from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径
from src.core.splitters.registry import SUPPORTED_SUFFIXES, get_splitter from src.core.splitters.registry import SUPPORTED_SUFFIXES, get_splitter
logger = logging.getLogger("md-vector-db") logger = logging.getLogger("md-vector-db")
@@ -40,7 +39,7 @@ class DocumentIngestor:
def ingest_file(self, file_path: str, incremental: bool = False, force: bool = False) -> int: def ingest_file(self, file_path: str, incremental: bool = False, force: bool = False) -> int:
"""入库单个文件, 返回 chunk 数量. """入库单个文件, 返回 chunk 数量.
根据文件扩展名自动选择 Splitter.md→MarkdownSplitter, .txt→TextSplitter, .pdf→PDFSplitter 等)。 根据扩展名自动选择 Splitter.md/.txt/.pdf/.html/.epub 等)。
使用文件路径的 SHA256 前 12 位 + 文件名作为唯一标识。 使用文件路径的 SHA256 前 12 位 + 文件名作为唯一标识。
Args: Args:
+3 -1
View File
@@ -161,7 +161,9 @@ class Searcher:
all_data = self.collection.get(include=["documents", "metadatas"]) all_data = self.collection.get(include=["documents", "metadatas"])
output = io.StringIO() output = io.StringIO()
writer = csv.writer(output) writer = csv.writer(output)
writer.writerow(["id", "content", "source_file", "section_title", "heading_level", "chunk_index"]) writer.writerow([
"id", "content", "source_file", "section_title", "heading_level", "chunk_index",
])
if all_data and all_data["ids"]: if all_data and all_data["ids"]:
for i, doc_id in enumerate(all_data["ids"]): for i, doc_id in enumerate(all_data["ids"]):
meta = all_data["metadatas"][i] if all_data["metadatas"] else {} meta = all_data["metadatas"][i] if all_data["metadatas"] else {}
+6 -6
View File
@@ -1,12 +1,12 @@
"""文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML / EPUB.""" """文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML / EPUB."""
from src.core.splitters.base import Splitter, BaseTextSplitter from src.core.splitters.base import BaseTextSplitter, Splitter
from src.core.splitters.markdown import MarkdownSplitter
from src.core.splitters.text import TextSplitter
from src.core.splitters.pdf import PDFSplitter
from src.core.splitters.html import HTMLSplitter
from src.core.splitters.epub import EPUBSplitter from src.core.splitters.epub import EPUBSplitter
from src.core.splitters.registry import get_splitter, register_splitter, SUPPORTED_SUFFIXES from src.core.splitters.html import HTMLSplitter
from src.core.splitters.markdown import MarkdownSplitter
from src.core.splitters.pdf import PDFSplitter
from src.core.splitters.registry import SUPPORTED_SUFFIXES, get_splitter, register_splitter
from src.core.splitters.text import TextSplitter
__all__ = [ __all__ = [
"Splitter", "Splitter",
+1
View File
@@ -1,5 +1,6 @@
"""EPUB 电子书分块器 — 使用 ebooklib 提取文字后委托 TextSplitter.""" """EPUB 电子书分块器 — 使用 ebooklib 提取文字后委托 TextSplitter."""
import logging import logging
from src.core.splitters.text import TextSplitter from src.core.splitters.text import TextSplitter
logger = logging.getLogger("md-vector-db") logger = logging.getLogger("md-vector-db")
+1
View File
@@ -1,5 +1,6 @@
"""HTML 文档分块器 — 使用 BeautifulSoup 去标签后委托 TextSplitter.""" """HTML 文档分块器 — 使用 BeautifulSoup 去标签后委托 TextSplitter."""
import logging import logging
from src.core.splitters.text import TextSplitter from src.core.splitters.text import TextSplitter
logger = logging.getLogger("md-vector-db") logger = logging.getLogger("md-vector-db")
+1
View File
@@ -1,5 +1,6 @@
"""Markdown 文档分块器.""" """Markdown 文档分块器."""
import re import re
from src.core.splitters.base import BaseTextSplitter from src.core.splitters.base import BaseTextSplitter
+1
View File
@@ -1,5 +1,6 @@
"""PDF 文档分块器 — 使用 pymupdf 提取文字后委托 TextSplitter.""" """PDF 文档分块器 — 使用 pymupdf 提取文字后委托 TextSplitter."""
import logging import logging
from src.core.splitters.text import TextSplitter from src.core.splitters.text import TextSplitter
logger = logging.getLogger("md-vector-db") logger = logging.getLogger("md-vector-db")
+1
View File
@@ -1,5 +1,6 @@
"""Splitter 注册表 — 按文件扩展名自动选择分块器.""" """Splitter 注册表 — 按文件扩展名自动选择分块器."""
from pathlib import Path from pathlib import Path
from src.core.splitters.base import Splitter from src.core.splitters.base import Splitter
# 扩展名 → Splitter 类名映射 # 扩展名 → Splitter 类名映射
+5 -6
View File
@@ -1,20 +1,19 @@
"""FastAPI 服务层.""" """FastAPI 服务层."""
import logging
import os import os
import time import time
import uuid import uuid
import logging from contextlib import asynccontextmanager
from pathlib import Path from pathlib import Path
from contextlib import asynccontextmanager from fastapi import Depends, FastAPI, HTTPException, Query, Request
from fastapi import FastAPI, HTTPException, Depends, Request, Query
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import RedirectResponse from fastapi.responses import RedirectResponse
from pydantic import BaseModel, Field, model_validator from pydantic import BaseModel, Field, model_validator
from src.core.security import is_path_within_workspace from src.core.security import is_path_within_workspace
from src.server.auth import verify_api_key, rate_limiter from src.server.auth import rate_limiter, verify_api_key
from src.server.deps import get_state, AppState from src.server.deps import AppState, get_state
logger = logging.getLogger("md-vector-db") logger = logging.getLogger("md-vector-db")
+3 -3
View File
@@ -1,9 +1,9 @@
"""API 认证与安全中间件.""" """API 认证与安全中间件."""
import hmac import hmac
import os
import time
import threading
import logging import logging
import os
import threading
import time
from collections import defaultdict from collections import defaultdict
from fastapi import Header, HTTPException, Request from fastapi import Header, HTTPException, Request
+1 -1
View File
@@ -1,7 +1,7 @@
"""FastAPI 依赖注入 — 集中管理应用状态, 替代模块级全局变量.""" """FastAPI 依赖注入 — 集中管理应用状态, 替代模块级全局变量."""
import logging
import os import os
import threading import threading
import logging
from src.core.config import load_config from src.core.config import load_config
from src.core.db import VectorDB from src.core.db import VectorDB
+2 -2
View File
@@ -1,11 +1,11 @@
"""认证与速率限制测试.""" """认证与速率限制测试."""
import time
import threading import threading
import time
import pytest import pytest
from fastapi import HTTPException from fastapi import HTTPException
from src.server.auth import verify_api_key, RateLimiter from src.server.auth import RateLimiter, verify_api_key
class TestVerifyApiKey: class TestVerifyApiKey:
+1
View File
@@ -1,5 +1,6 @@
"""CLI 命令测试.""" """CLI 命令测试."""
from typer.testing import CliRunner from typer.testing import CliRunner
from src.cli.main import app from src.cli.main import app
runner = CliRunner() runner = CliRunner()
+1 -4
View File
@@ -1,11 +1,8 @@
"""配置加载模块测试.""" """配置加载模块测试."""
import os
import tempfile import tempfile
from pathlib import Path from pathlib import Path
import pytest from src.core.config import AppConfig, EmbedConfig, load_config
from src.core.config import AppConfig, EmbedConfig, ChunkConfig, load_config
class TestEmbedConfig: class TestEmbedConfig:
-1
View File
@@ -1,5 +1,4 @@
"""数据库层测试.""" """数据库层测试."""
import gc
import tempfile import tempfile
from pathlib import Path from pathlib import Path
-3
View File
@@ -1,10 +1,7 @@
"""AppState 和依赖注入测试.""" """AppState 和依赖注入测试."""
import os
import tempfile
import pytest import pytest
from src.core.config import load_config
from src.server.deps import AppState from src.server.deps import AppState
+6 -2
View File
@@ -3,8 +3,12 @@ import pytest
from src.core.config import EmbedConfig from src.core.config import EmbedConfig
from src.core.embedder import ( from src.core.embedder import (
LocalEmbedder, OpenAIEmbedder, DashscopeEmbedder, SUPPORTED_PROVIDERS,
create_embedder, batch_embed, SUPPORTED_PROVIDERS, DashscopeEmbedder,
LocalEmbedder,
OpenAIEmbedder,
batch_embed,
create_embedder,
) )
+5 -14
View File
@@ -4,8 +4,8 @@ from pathlib import Path
import pytest import pytest
from src.core.splitters import MarkdownSplitter
from src.core.ingest import DocumentIngestor from src.core.ingest import DocumentIngestor
from src.core.splitters import MarkdownSplitter
class TestMarkdownSplitter: class TestMarkdownSplitter:
@@ -122,7 +122,6 @@ class TestIngestorIntegration:
from src.core.config import EmbedConfig from src.core.config import EmbedConfig
from src.core.db import VectorDB from src.core.db import VectorDB
from src.core.embedder import create_embedder from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
db = VectorDB(persist_dir=str(tmp_path)) db = VectorDB(persist_dir=str(tmp_path))
embedder = create_embedder(EmbedConfig(mode="local")) embedder = create_embedder(EmbedConfig(mode="local"))
@@ -137,13 +136,12 @@ class TestIngestorIntegration:
from src.core.config import EmbedConfig from src.core.config import EmbedConfig
from src.core.db import VectorDB from src.core.db import VectorDB
from src.core.embedder import create_embedder from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
db = VectorDB(persist_dir=str(tmp_path)) db = VectorDB(persist_dir=str(tmp_path))
embedder = create_embedder(EmbedConfig(mode="local")) embedder = create_embedder(EmbedConfig(mode="local"))
ingestor = DocumentIngestor(db, embedder, "test_dedup") ingestor = DocumentIngestor(db, embedder, "test_dedup")
c1 = ingestor.ingest_content("# A", "dup.md") ingestor.ingest_content("# A", "dup.md")
c2 = ingestor.ingest_content("# B", "dup.md") c2 = ingestor.ingest_content("# B", "dup.md")
assert ingestor.collection.count() == c2 assert ingestor.collection.count() == c2
@@ -156,7 +154,6 @@ class TestIngestFile:
from src.core.config import EmbedConfig from src.core.config import EmbedConfig
from src.core.db import VectorDB from src.core.db import VectorDB
from src.core.embedder import create_embedder from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
md_file = tmp_path / "hello.md" md_file = tmp_path / "hello.md"
md_file.write_text("# 测试\n这是测试内容。", encoding="utf-8") md_file.write_text("# 测试\n这是测试内容。", encoding="utf-8")
@@ -174,7 +171,6 @@ class TestIngestFile:
from src.core.config import EmbedConfig from src.core.config import EmbedConfig
from src.core.db import VectorDB from src.core.db import VectorDB
from src.core.embedder import create_embedder from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
txt_file = tmp_path / "notes.txt" txt_file = tmp_path / "notes.txt"
txt_file.write_text("这是一段纯文本内容。\n\n第二段内容在这里。", encoding="utf-8") txt_file.write_text("这是一段纯文本内容。\n\n第二段内容在这里。", encoding="utf-8")
@@ -195,7 +191,6 @@ class TestIngestDirectory:
from src.core.config import EmbedConfig from src.core.config import EmbedConfig
from src.core.db import VectorDB from src.core.db import VectorDB
from src.core.embedder import create_embedder from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
(tmp_path / "a.md").write_text("# A\n内容 A", encoding="utf-8") (tmp_path / "a.md").write_text("# A\n内容 A", encoding="utf-8")
(tmp_path / "b.txt").write_text("内容 B", encoding="utf-8") (tmp_path / "b.txt").write_text("内容 B", encoding="utf-8")
@@ -213,7 +208,6 @@ class TestIngestDirectory:
from src.core.config import EmbedConfig from src.core.config import EmbedConfig
from src.core.db import VectorDB from src.core.db import VectorDB
from src.core.embedder import create_embedder from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
db = VectorDB(persist_dir=str(tmp_path / "db")) db = VectorDB(persist_dir=str(tmp_path / "db"))
embedder = create_embedder(EmbedConfig(mode="local")) embedder = create_embedder(EmbedConfig(mode="local"))
@@ -228,10 +222,9 @@ class TestIncrementalIngest:
def test_ingest_file_incremental_skips_unchanged(self, tmp_path): def test_ingest_file_incremental_skips_unchanged(self, tmp_path):
"""增量模式: 未修改的文件跳过入库.""" """增量模式: 未修改的文件跳过入库."""
from src.core.config import EmbedConfig, ChunkConfig from src.core.config import ChunkConfig, EmbedConfig
from src.core.db import VectorDB from src.core.db import VectorDB
from src.core.embedder import create_embedder from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
from src.core.file_tracker import FileTracker from src.core.file_tracker import FileTracker
file = tmp_path / "stable.md" file = tmp_path / "stable.md"
@@ -251,10 +244,9 @@ class TestIncrementalIngest:
def test_ingest_file_incremental_reingests_modified(self, tmp_path): def test_ingest_file_incremental_reingests_modified(self, tmp_path):
"""增量模式: 修改后的文件重新入库.""" """增量模式: 修改后的文件重新入库."""
from src.core.config import EmbedConfig, ChunkConfig from src.core.config import ChunkConfig, EmbedConfig
from src.core.db import VectorDB from src.core.db import VectorDB
from src.core.embedder import create_embedder from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
from src.core.file_tracker import FileTracker from src.core.file_tracker import FileTracker
file = tmp_path / "changing.md" file = tmp_path / "changing.md"
@@ -275,10 +267,9 @@ class TestIncrementalIngest:
def test_ingest_file_force_mode_always_reingests(self, tmp_path): def test_ingest_file_force_mode_always_reingests(self, tmp_path):
"""force=True 时始终重新入库(忽略 tracker).""" """force=True 时始终重新入库(忽略 tracker)."""
from src.core.config import EmbedConfig, ChunkConfig from src.core.config import ChunkConfig, EmbedConfig
from src.core.db import VectorDB from src.core.db import VectorDB
from src.core.embedder import create_embedder from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
from src.core.file_tracker import FileTracker from src.core.file_tracker import FileTracker
file = tmp_path / "force.md" file = tmp_path / "force.md"
-1
View File
@@ -1,5 +1,4 @@
"""重排序器测试.""" """重排序器测试."""
import pytest
from src.core.reranker import Reranker from src.core.reranker import Reranker
-1
View File
@@ -1,5 +1,4 @@
"""is_safe_path 路径遍历防护测试.""" """is_safe_path 路径遍历防护测试."""
import pytest
from src.core.security import is_safe_path from src.core.security import is_safe_path
+7 -2
View File
@@ -1,6 +1,11 @@
"""Splitter 注册表和 TextSplitter 测试.""" """Splitter 注册表和 TextSplitter 测试."""
import pytest from src.core.splitters import (
from src.core.splitters import TextSplitter, MarkdownSplitter, get_splitter, register_splitter, SUPPORTED_SUFFIXES SUPPORTED_SUFFIXES,
MarkdownSplitter,
TextSplitter,
get_splitter,
register_splitter,
)
class TestTextSplitter: class TestTextSplitter:
+2 -1
View File
@@ -28,9 +28,10 @@ class TestEPUBSplitter:
def test_split_simple_epub(self, tmp_path): def test_split_simple_epub(self, tmp_path):
"""用 ebooklib 创建一个简单 EPUB 并测试分块.""" """用 ebooklib 创建一个简单 EPUB 并测试分块."""
from src.core.splitters.epub import EPUBSplitter
from ebooklib import epub from ebooklib import epub
from src.core.splitters.epub import EPUBSplitter
epub_path = tmp_path / "test.epub" epub_path = tmp_path / "test.epub"
book = epub.EpubBook() book = epub.EpubBook()
+1
View File
@@ -1,5 +1,6 @@
"""MarkdownSplitter 边界测试.""" """MarkdownSplitter 边界测试."""
import pytest import pytest
from src.core.splitters import MarkdownSplitter from src.core.splitters import MarkdownSplitter
+5 -3
View File
@@ -1,6 +1,6 @@
"""PDFSplitter 测试.""" """PDFSplitter 测试."""
import pytest import pytest
from pathlib import Path
pymupdf = pytest.importorskip("fitz", reason="pymupdf 未安装") pymupdf = pytest.importorskip("fitz", reason="pymupdf 未安装")
@@ -10,9 +10,10 @@ class TestPDFSplitter:
def test_split_simple_pdf(self, tmp_path): def test_split_simple_pdf(self, tmp_path):
"""用 pymupdf 创建一个简单 PDF 并测试分块.""" """用 pymupdf 创建一个简单 PDF 并测试分块."""
from src.core.splitters.pdf import PDFSplitter
import fitz import fitz
from src.core.splitters.pdf import PDFSplitter
pdf_path = tmp_path / "test.pdf" pdf_path = tmp_path / "test.pdf"
doc = fitz.open() doc = fitz.open()
# 插入纯 ASCII 文本避免 CJK 字体编码问题 # 插入纯 ASCII 文本避免 CJK 字体编码问题
@@ -29,9 +30,10 @@ class TestPDFSplitter:
def test_empty_pdf(self, tmp_path): def test_empty_pdf(self, tmp_path):
"""空 PDF(有页但无文字)返回空列表.""" """空 PDF(有页但无文字)返回空列表."""
from src.core.splitters.pdf import PDFSplitter
import fitz import fitz
from src.core.splitters.pdf import PDFSplitter
pdf_path = tmp_path / "empty.pdf" pdf_path = tmp_path / "empty.pdf"
doc = fitz.open() doc = fitz.open()
doc.new_page() # pymupdf 必须有至少一页才能保存 doc.new_page() # pymupdf 必须有至少一页才能保存