chore: ruff lint 修复 + 第一优先级全部完成

This commit is contained in:
2026-07-11 19:50:35 +08:00
parent 3e51e166c2
commit 2d0e8c4997
28 changed files with 67 additions and 63 deletions
BIN
View File
Binary file not shown.
+10 -6
View File
@@ -1,10 +1,10 @@
"""命令行工具入口 — 可作为 MCP tool 直接调用."""
from pathlib import Path
import sys
import glob as _glob
import io
import json
import os
import glob as _glob
import sys
from pathlib import Path
from typing import Annotated
import typer
@@ -18,7 +18,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
from src.core.config import DEFAULT_CONFIG_PATH
from src.core.security import is_safe_cli_path
from src.server.deps import get_state, get_default_collection
from src.server.deps import get_default_collection, get_state
app = typer.Typer(
name="md-vector-db",
@@ -63,8 +63,12 @@ def ingest(
typer.Argument(help="Markdown 文件路径 (可多个, 或 - 从标准输入读取)"),
] = None,
name: Annotated[str | None, typer.Option("--name", help="标准输入模式下的虚拟文件名")] = None,
incremental: Annotated[bool, typer.Option("--incremental", help="增量模式:跳过未变更文件")] = False,
force: Annotated[bool, typer.Option("--force", help="强制重新入库(忽略增量检查)")] = False,
incremental: Annotated[
bool, typer.Option("--incremental", help="增量模式:跳过未变更文件")
] = False,
force: Annotated[
bool, typer.Option("--force", help="强制重新入库(忽略增量检查)")
] = False,
config: ConfigOpt = DEFAULT_CONFIG_PATH,
collection: CollectionOpt = None,
):
+1 -2
View File
@@ -5,9 +5,8 @@ import os
from dataclasses import dataclass, field
from pathlib import Path
from dotenv import load_dotenv
import yaml
from dotenv import load_dotenv
# 加载 .env 文件 (若存在)
load_dotenv()
+1 -1
View File
@@ -17,9 +17,9 @@
vectors = batch_embed(embedder, long_text_list)
"""
import logging
import os
import threading
import logging
from typing import Protocol
from src.core.config import EmbedConfig
+2 -3
View File
@@ -1,14 +1,13 @@
"""Markdown 文档解析与入库模块."""
import logging
import re
from pathlib import Path
from src.core.config import ChunkConfig
from src.core.db import VectorDB
from src.core.embedder import Embedder, batch_embed
from src.core.file_tracker import FileTracker
from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径
from src.core.splitters.base import Splitter # 兼容旧 import 路径
from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径
from src.core.splitters.registry import SUPPORTED_SUFFIXES, get_splitter
logger = logging.getLogger("md-vector-db")
@@ -40,7 +39,7 @@ class DocumentIngestor:
def ingest_file(self, file_path: str, incremental: bool = False, force: bool = False) -> int:
"""入库单个文件, 返回 chunk 数量.
根据文件扩展名自动选择 Splitter.md→MarkdownSplitter, .txt→TextSplitter, .pdf→PDFSplitter 等)。
根据扩展名自动选择 Splitter.md/.txt/.pdf/.html/.epub 等)。
使用文件路径的 SHA256 前 12 位 + 文件名作为唯一标识。
Args:
+3 -1
View File
@@ -161,7 +161,9 @@ class Searcher:
all_data = self.collection.get(include=["documents", "metadatas"])
output = io.StringIO()
writer = csv.writer(output)
writer.writerow(["id", "content", "source_file", "section_title", "heading_level", "chunk_index"])
writer.writerow([
"id", "content", "source_file", "section_title", "heading_level", "chunk_index",
])
if all_data and all_data["ids"]:
for i, doc_id in enumerate(all_data["ids"]):
meta = all_data["metadatas"][i] if all_data["metadatas"] else {}
+6 -6
View File
@@ -1,12 +1,12 @@
"""文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML / EPUB."""
from src.core.splitters.base import Splitter, BaseTextSplitter
from src.core.splitters.markdown import MarkdownSplitter
from src.core.splitters.text import TextSplitter
from src.core.splitters.pdf import PDFSplitter
from src.core.splitters.html import HTMLSplitter
from src.core.splitters.base import BaseTextSplitter, Splitter
from src.core.splitters.epub import EPUBSplitter
from src.core.splitters.registry import get_splitter, register_splitter, SUPPORTED_SUFFIXES
from src.core.splitters.html import HTMLSplitter
from src.core.splitters.markdown import MarkdownSplitter
from src.core.splitters.pdf import PDFSplitter
from src.core.splitters.registry import SUPPORTED_SUFFIXES, get_splitter, register_splitter
from src.core.splitters.text import TextSplitter
__all__ = [
"Splitter",
+1
View File
@@ -1,5 +1,6 @@
"""EPUB 电子书分块器 — 使用 ebooklib 提取文字后委托 TextSplitter."""
import logging
from src.core.splitters.text import TextSplitter
logger = logging.getLogger("md-vector-db")
+1
View File
@@ -1,5 +1,6 @@
"""HTML 文档分块器 — 使用 BeautifulSoup 去标签后委托 TextSplitter."""
import logging
from src.core.splitters.text import TextSplitter
logger = logging.getLogger("md-vector-db")
+1
View File
@@ -1,5 +1,6 @@
"""Markdown 文档分块器."""
import re
from src.core.splitters.base import BaseTextSplitter
+1
View File
@@ -1,5 +1,6 @@
"""PDF 文档分块器 — 使用 pymupdf 提取文字后委托 TextSplitter."""
import logging
from src.core.splitters.text import TextSplitter
logger = logging.getLogger("md-vector-db")
+1
View File
@@ -1,5 +1,6 @@
"""Splitter 注册表 — 按文件扩展名自动选择分块器."""
from pathlib import Path
from src.core.splitters.base import Splitter
# 扩展名 → Splitter 类名映射
+5 -6
View File
@@ -1,20 +1,19 @@
"""FastAPI 服务层."""
import logging
import os
import time
import uuid
import logging
from contextlib import asynccontextmanager
from pathlib import Path
from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException, Depends, Request, Query
from fastapi import Depends, FastAPI, HTTPException, Query, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import RedirectResponse
from pydantic import BaseModel, Field, model_validator
from src.core.security import is_path_within_workspace
from src.server.auth import verify_api_key, rate_limiter
from src.server.deps import get_state, AppState
from src.server.auth import rate_limiter, verify_api_key
from src.server.deps import AppState, get_state
logger = logging.getLogger("md-vector-db")
+3 -3
View File
@@ -1,9 +1,9 @@
"""API 认证与安全中间件."""
import hmac
import os
import time
import threading
import logging
import os
import threading
import time
from collections import defaultdict
from fastapi import Header, HTTPException, Request
+1 -1
View File
@@ -1,7 +1,7 @@
"""FastAPI 依赖注入 — 集中管理应用状态, 替代模块级全局变量."""
import logging
import os
import threading
import logging
from src.core.config import load_config
from src.core.db import VectorDB
+2 -2
View File
@@ -1,11 +1,11 @@
"""认证与速率限制测试."""
import time
import threading
import time
import pytest
from fastapi import HTTPException
from src.server.auth import verify_api_key, RateLimiter
from src.server.auth import RateLimiter, verify_api_key
class TestVerifyApiKey:
+1
View File
@@ -1,5 +1,6 @@
"""CLI 命令测试."""
from typer.testing import CliRunner
from src.cli.main import app
runner = CliRunner()
+1 -4
View File
@@ -1,11 +1,8 @@
"""配置加载模块测试."""
import os
import tempfile
from pathlib import Path
import pytest
from src.core.config import AppConfig, EmbedConfig, ChunkConfig, load_config
from src.core.config import AppConfig, EmbedConfig, load_config
class TestEmbedConfig:
-1
View File
@@ -1,5 +1,4 @@
"""数据库层测试."""
import gc
import tempfile
from pathlib import Path
-3
View File
@@ -1,10 +1,7 @@
"""AppState 和依赖注入测试."""
import os
import tempfile
import pytest
from src.core.config import load_config
from src.server.deps import AppState
+6 -2
View File
@@ -3,8 +3,12 @@ import pytest
from src.core.config import EmbedConfig
from src.core.embedder import (
LocalEmbedder, OpenAIEmbedder, DashscopeEmbedder,
create_embedder, batch_embed, SUPPORTED_PROVIDERS,
SUPPORTED_PROVIDERS,
DashscopeEmbedder,
LocalEmbedder,
OpenAIEmbedder,
batch_embed,
create_embedder,
)
+5 -14
View File
@@ -4,8 +4,8 @@ from pathlib import Path
import pytest
from src.core.splitters import MarkdownSplitter
from src.core.ingest import DocumentIngestor
from src.core.splitters import MarkdownSplitter
class TestMarkdownSplitter:
@@ -122,7 +122,6 @@ class TestIngestorIntegration:
from src.core.config import EmbedConfig
from src.core.db import VectorDB
from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
db = VectorDB(persist_dir=str(tmp_path))
embedder = create_embedder(EmbedConfig(mode="local"))
@@ -137,13 +136,12 @@ class TestIngestorIntegration:
from src.core.config import EmbedConfig
from src.core.db import VectorDB
from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
db = VectorDB(persist_dir=str(tmp_path))
embedder = create_embedder(EmbedConfig(mode="local"))
ingestor = DocumentIngestor(db, embedder, "test_dedup")
c1 = ingestor.ingest_content("# A", "dup.md")
ingestor.ingest_content("# A", "dup.md")
c2 = ingestor.ingest_content("# B", "dup.md")
assert ingestor.collection.count() == c2
@@ -156,7 +154,6 @@ class TestIngestFile:
from src.core.config import EmbedConfig
from src.core.db import VectorDB
from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
md_file = tmp_path / "hello.md"
md_file.write_text("# 测试\n这是测试内容。", encoding="utf-8")
@@ -174,7 +171,6 @@ class TestIngestFile:
from src.core.config import EmbedConfig
from src.core.db import VectorDB
from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
txt_file = tmp_path / "notes.txt"
txt_file.write_text("这是一段纯文本内容。\n\n第二段内容在这里。", encoding="utf-8")
@@ -195,7 +191,6 @@ class TestIngestDirectory:
from src.core.config import EmbedConfig
from src.core.db import VectorDB
from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
(tmp_path / "a.md").write_text("# A\n内容 A", encoding="utf-8")
(tmp_path / "b.txt").write_text("内容 B", encoding="utf-8")
@@ -213,7 +208,6 @@ class TestIngestDirectory:
from src.core.config import EmbedConfig
from src.core.db import VectorDB
from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
db = VectorDB(persist_dir=str(tmp_path / "db"))
embedder = create_embedder(EmbedConfig(mode="local"))
@@ -228,10 +222,9 @@ class TestIncrementalIngest:
def test_ingest_file_incremental_skips_unchanged(self, tmp_path):
"""增量模式: 未修改的文件跳过入库."""
from src.core.config import EmbedConfig, ChunkConfig
from src.core.config import ChunkConfig, EmbedConfig
from src.core.db import VectorDB
from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
from src.core.file_tracker import FileTracker
file = tmp_path / "stable.md"
@@ -251,10 +244,9 @@ class TestIncrementalIngest:
def test_ingest_file_incremental_reingests_modified(self, tmp_path):
"""增量模式: 修改后的文件重新入库."""
from src.core.config import EmbedConfig, ChunkConfig
from src.core.config import ChunkConfig, EmbedConfig
from src.core.db import VectorDB
from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
from src.core.file_tracker import FileTracker
file = tmp_path / "changing.md"
@@ -275,10 +267,9 @@ class TestIncrementalIngest:
def test_ingest_file_force_mode_always_reingests(self, tmp_path):
"""force=True 时始终重新入库(忽略 tracker)."""
from src.core.config import EmbedConfig, ChunkConfig
from src.core.config import ChunkConfig, EmbedConfig
from src.core.db import VectorDB
from src.core.embedder import create_embedder
from src.core.ingest import DocumentIngestor
from src.core.file_tracker import FileTracker
file = tmp_path / "force.md"
-1
View File
@@ -1,5 +1,4 @@
"""重排序器测试."""
import pytest
from src.core.reranker import Reranker
-1
View File
@@ -1,5 +1,4 @@
"""is_safe_path 路径遍历防护测试."""
import pytest
from src.core.security import is_safe_path
+7 -2
View File
@@ -1,6 +1,11 @@
"""Splitter 注册表和 TextSplitter 测试."""
import pytest
from src.core.splitters import TextSplitter, MarkdownSplitter, get_splitter, register_splitter, SUPPORTED_SUFFIXES
from src.core.splitters import (
SUPPORTED_SUFFIXES,
MarkdownSplitter,
TextSplitter,
get_splitter,
register_splitter,
)
class TestTextSplitter:
+2 -1
View File
@@ -28,9 +28,10 @@ class TestEPUBSplitter:
def test_split_simple_epub(self, tmp_path):
"""用 ebooklib 创建一个简单 EPUB 并测试分块."""
from src.core.splitters.epub import EPUBSplitter
from ebooklib import epub
from src.core.splitters.epub import EPUBSplitter
epub_path = tmp_path / "test.epub"
book = epub.EpubBook()
+1
View File
@@ -1,5 +1,6 @@
"""MarkdownSplitter 边界测试."""
import pytest
from src.core.splitters import MarkdownSplitter
+5 -3
View File
@@ -1,6 +1,6 @@
"""PDFSplitter 测试."""
import pytest
from pathlib import Path
pymupdf = pytest.importorskip("fitz", reason="pymupdf 未安装")
@@ -10,9 +10,10 @@ class TestPDFSplitter:
def test_split_simple_pdf(self, tmp_path):
"""用 pymupdf 创建一个简单 PDF 并测试分块."""
from src.core.splitters.pdf import PDFSplitter
import fitz
from src.core.splitters.pdf import PDFSplitter
pdf_path = tmp_path / "test.pdf"
doc = fitz.open()
# 插入纯 ASCII 文本避免 CJK 字体编码问题
@@ -29,9 +30,10 @@ class TestPDFSplitter:
def test_empty_pdf(self, tmp_path):
"""空 PDF(有页但无文字)返回空列表."""
from src.core.splitters.pdf import PDFSplitter
import fitz
from src.core.splitters.pdf import PDFSplitter
pdf_path = tmp_path / "empty.pdf"
doc = fitz.open()
doc.new_page() # pymupdf 必须有至少一页才能保存