chore: ruff lint 修复 + 第一优先级全部完成
This commit is contained in:
+10
-6
@@ -1,10 +1,10 @@
|
||||
"""命令行工具入口 — 可作为 MCP tool 直接调用."""
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import glob as _glob
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import glob as _glob
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
@@ -18,7 +18,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.core.config import DEFAULT_CONFIG_PATH
|
||||
from src.core.security import is_safe_cli_path
|
||||
from src.server.deps import get_state, get_default_collection
|
||||
from src.server.deps import get_default_collection, get_state
|
||||
|
||||
app = typer.Typer(
|
||||
name="md-vector-db",
|
||||
@@ -63,8 +63,12 @@ def ingest(
|
||||
typer.Argument(help="Markdown 文件路径 (可多个, 或 - 从标准输入读取)"),
|
||||
] = None,
|
||||
name: Annotated[str | None, typer.Option("--name", help="标准输入模式下的虚拟文件名")] = None,
|
||||
incremental: Annotated[bool, typer.Option("--incremental", help="增量模式:跳过未变更文件")] = False,
|
||||
force: Annotated[bool, typer.Option("--force", help="强制重新入库(忽略增量检查)")] = False,
|
||||
incremental: Annotated[
|
||||
bool, typer.Option("--incremental", help="增量模式:跳过未变更文件")
|
||||
] = False,
|
||||
force: Annotated[
|
||||
bool, typer.Option("--force", help="强制重新入库(忽略增量检查)")
|
||||
] = False,
|
||||
config: ConfigOpt = DEFAULT_CONFIG_PATH,
|
||||
collection: CollectionOpt = None,
|
||||
):
|
||||
|
||||
+1
-2
@@ -5,9 +5,8 @@ import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
import yaml
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# 加载 .env 文件 (若存在)
|
||||
load_dotenv()
|
||||
|
||||
@@ -17,9 +17,9 @@
|
||||
|
||||
vectors = batch_embed(embedder, long_text_list)
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import logging
|
||||
from typing import Protocol
|
||||
|
||||
from src.core.config import EmbedConfig
|
||||
|
||||
+2
-3
@@ -1,14 +1,13 @@
|
||||
"""Markdown 文档解析与入库模块."""
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from src.core.config import ChunkConfig
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import Embedder, batch_embed
|
||||
from src.core.file_tracker import FileTracker
|
||||
from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径
|
||||
from src.core.splitters.base import Splitter # 兼容旧 import 路径
|
||||
from src.core.splitters.markdown import MarkdownSplitter # 兼容旧 import 路径
|
||||
from src.core.splitters.registry import SUPPORTED_SUFFIXES, get_splitter
|
||||
|
||||
logger = logging.getLogger("md-vector-db")
|
||||
@@ -40,7 +39,7 @@ class DocumentIngestor:
|
||||
def ingest_file(self, file_path: str, incremental: bool = False, force: bool = False) -> int:
|
||||
"""入库单个文件, 返回 chunk 数量.
|
||||
|
||||
根据文件扩展名自动选择 Splitter(.md→MarkdownSplitter, .txt→TextSplitter, .pdf→PDFSplitter 等)。
|
||||
根据扩展名自动选择 Splitter(.md/.txt/.pdf/.html/.epub 等)。
|
||||
使用文件路径的 SHA256 前 12 位 + 文件名作为唯一标识。
|
||||
|
||||
Args:
|
||||
|
||||
+3
-1
@@ -161,7 +161,9 @@ class Searcher:
|
||||
all_data = self.collection.get(include=["documents", "metadatas"])
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow(["id", "content", "source_file", "section_title", "heading_level", "chunk_index"])
|
||||
writer.writerow([
|
||||
"id", "content", "source_file", "section_title", "heading_level", "chunk_index",
|
||||
])
|
||||
if all_data and all_data["ids"]:
|
||||
for i, doc_id in enumerate(all_data["ids"]):
|
||||
meta = all_data["metadatas"][i] if all_data["metadatas"] else {}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
"""文档分块器包 — 支持 Markdown / 纯文本 / PDF / HTML / EPUB."""
|
||||
|
||||
from src.core.splitters.base import Splitter, BaseTextSplitter
|
||||
from src.core.splitters.markdown import MarkdownSplitter
|
||||
from src.core.splitters.text import TextSplitter
|
||||
from src.core.splitters.pdf import PDFSplitter
|
||||
from src.core.splitters.html import HTMLSplitter
|
||||
from src.core.splitters.base import BaseTextSplitter, Splitter
|
||||
from src.core.splitters.epub import EPUBSplitter
|
||||
from src.core.splitters.registry import get_splitter, register_splitter, SUPPORTED_SUFFIXES
|
||||
from src.core.splitters.html import HTMLSplitter
|
||||
from src.core.splitters.markdown import MarkdownSplitter
|
||||
from src.core.splitters.pdf import PDFSplitter
|
||||
from src.core.splitters.registry import SUPPORTED_SUFFIXES, get_splitter, register_splitter
|
||||
from src.core.splitters.text import TextSplitter
|
||||
|
||||
__all__ = [
|
||||
"Splitter",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""EPUB 电子书分块器 — 使用 ebooklib 提取文字后委托 TextSplitter."""
|
||||
import logging
|
||||
|
||||
from src.core.splitters.text import TextSplitter
|
||||
|
||||
logger = logging.getLogger("md-vector-db")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""HTML 文档分块器 — 使用 BeautifulSoup 去标签后委托 TextSplitter."""
|
||||
import logging
|
||||
|
||||
from src.core.splitters.text import TextSplitter
|
||||
|
||||
logger = logging.getLogger("md-vector-db")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Markdown 文档分块器."""
|
||||
import re
|
||||
|
||||
from src.core.splitters.base import BaseTextSplitter
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""PDF 文档分块器 — 使用 pymupdf 提取文字后委托 TextSplitter."""
|
||||
import logging
|
||||
|
||||
from src.core.splitters.text import TextSplitter
|
||||
|
||||
logger = logging.getLogger("md-vector-db")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Splitter 注册表 — 按文件扩展名自动选择分块器."""
|
||||
from pathlib import Path
|
||||
|
||||
from src.core.splitters.base import Splitter
|
||||
|
||||
# 扩展名 → Splitter 类名映射
|
||||
|
||||
+5
-6
@@ -1,20 +1,19 @@
|
||||
"""FastAPI 服务层."""
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Depends, Request, Query
|
||||
from fastapi import Depends, FastAPI, HTTPException, Query, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import RedirectResponse
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from src.core.security import is_path_within_workspace
|
||||
from src.server.auth import verify_api_key, rate_limiter
|
||||
from src.server.deps import get_state, AppState
|
||||
from src.server.auth import rate_limiter, verify_api_key
|
||||
from src.server.deps import AppState, get_state
|
||||
|
||||
logger = logging.getLogger("md-vector-db")
|
||||
|
||||
|
||||
+3
-3
@@ -1,9 +1,9 @@
|
||||
"""API 认证与安全中间件."""
|
||||
import hmac
|
||||
import os
|
||||
import time
|
||||
import threading
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
|
||||
from fastapi import Header, HTTPException, Request
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
"""FastAPI 依赖注入 — 集中管理应用状态, 替代模块级全局变量."""
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import logging
|
||||
|
||||
from src.core.config import load_config
|
||||
from src.core.db import VectorDB
|
||||
|
||||
+2
-2
@@ -1,11 +1,11 @@
|
||||
"""认证与速率限制测试."""
|
||||
import time
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from src.server.auth import verify_api_key, RateLimiter
|
||||
from src.server.auth import RateLimiter, verify_api_key
|
||||
|
||||
|
||||
class TestVerifyApiKey:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""CLI 命令测试."""
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from src.cli.main import app
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
"""配置加载模块测试."""
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.config import AppConfig, EmbedConfig, ChunkConfig, load_config
|
||||
from src.core.config import AppConfig, EmbedConfig, load_config
|
||||
|
||||
|
||||
class TestEmbedConfig:
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
"""数据库层测试."""
|
||||
import gc
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
"""AppState 和依赖注入测试."""
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.config import load_config
|
||||
from src.server.deps import AppState
|
||||
|
||||
|
||||
|
||||
@@ -3,8 +3,12 @@ import pytest
|
||||
|
||||
from src.core.config import EmbedConfig
|
||||
from src.core.embedder import (
|
||||
LocalEmbedder, OpenAIEmbedder, DashscopeEmbedder,
|
||||
create_embedder, batch_embed, SUPPORTED_PROVIDERS,
|
||||
SUPPORTED_PROVIDERS,
|
||||
DashscopeEmbedder,
|
||||
LocalEmbedder,
|
||||
OpenAIEmbedder,
|
||||
batch_embed,
|
||||
create_embedder,
|
||||
)
|
||||
|
||||
|
||||
|
||||
+5
-14
@@ -4,8 +4,8 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.splitters import MarkdownSplitter
|
||||
from src.core.ingest import DocumentIngestor
|
||||
from src.core.splitters import MarkdownSplitter
|
||||
|
||||
|
||||
class TestMarkdownSplitter:
|
||||
@@ -122,7 +122,6 @@ class TestIngestorIntegration:
|
||||
from src.core.config import EmbedConfig
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
|
||||
db = VectorDB(persist_dir=str(tmp_path))
|
||||
embedder = create_embedder(EmbedConfig(mode="local"))
|
||||
@@ -137,13 +136,12 @@ class TestIngestorIntegration:
|
||||
from src.core.config import EmbedConfig
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
|
||||
db = VectorDB(persist_dir=str(tmp_path))
|
||||
embedder = create_embedder(EmbedConfig(mode="local"))
|
||||
ingestor = DocumentIngestor(db, embedder, "test_dedup")
|
||||
|
||||
c1 = ingestor.ingest_content("# A", "dup.md")
|
||||
ingestor.ingest_content("# A", "dup.md")
|
||||
c2 = ingestor.ingest_content("# B", "dup.md")
|
||||
assert ingestor.collection.count() == c2
|
||||
|
||||
@@ -156,7 +154,6 @@ class TestIngestFile:
|
||||
from src.core.config import EmbedConfig
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
|
||||
md_file = tmp_path / "hello.md"
|
||||
md_file.write_text("# 测试\n这是测试内容。", encoding="utf-8")
|
||||
@@ -174,7 +171,6 @@ class TestIngestFile:
|
||||
from src.core.config import EmbedConfig
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
|
||||
txt_file = tmp_path / "notes.txt"
|
||||
txt_file.write_text("这是一段纯文本内容。\n\n第二段内容在这里。", encoding="utf-8")
|
||||
@@ -195,7 +191,6 @@ class TestIngestDirectory:
|
||||
from src.core.config import EmbedConfig
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
|
||||
(tmp_path / "a.md").write_text("# A\n内容 A", encoding="utf-8")
|
||||
(tmp_path / "b.txt").write_text("内容 B", encoding="utf-8")
|
||||
@@ -213,7 +208,6 @@ class TestIngestDirectory:
|
||||
from src.core.config import EmbedConfig
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
|
||||
db = VectorDB(persist_dir=str(tmp_path / "db"))
|
||||
embedder = create_embedder(EmbedConfig(mode="local"))
|
||||
@@ -228,10 +222,9 @@ class TestIncrementalIngest:
|
||||
|
||||
def test_ingest_file_incremental_skips_unchanged(self, tmp_path):
|
||||
"""增量模式: 未修改的文件跳过入库."""
|
||||
from src.core.config import EmbedConfig, ChunkConfig
|
||||
from src.core.config import ChunkConfig, EmbedConfig
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
from src.core.file_tracker import FileTracker
|
||||
|
||||
file = tmp_path / "stable.md"
|
||||
@@ -251,10 +244,9 @@ class TestIncrementalIngest:
|
||||
|
||||
def test_ingest_file_incremental_reingests_modified(self, tmp_path):
|
||||
"""增量模式: 修改后的文件重新入库."""
|
||||
from src.core.config import EmbedConfig, ChunkConfig
|
||||
from src.core.config import ChunkConfig, EmbedConfig
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
from src.core.file_tracker import FileTracker
|
||||
|
||||
file = tmp_path / "changing.md"
|
||||
@@ -275,10 +267,9 @@ class TestIncrementalIngest:
|
||||
|
||||
def test_ingest_file_force_mode_always_reingests(self, tmp_path):
|
||||
"""force=True 时始终重新入库(忽略 tracker)."""
|
||||
from src.core.config import EmbedConfig, ChunkConfig
|
||||
from src.core.config import ChunkConfig, EmbedConfig
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
from src.core.file_tracker import FileTracker
|
||||
|
||||
file = tmp_path / "force.md"
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
"""重排序器测试."""
|
||||
import pytest
|
||||
|
||||
from src.core.reranker import Reranker
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
"""is_safe_path 路径遍历防护测试."""
|
||||
import pytest
|
||||
from src.core.security import is_safe_path
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
"""Splitter 注册表和 TextSplitter 测试."""
|
||||
import pytest
|
||||
from src.core.splitters import TextSplitter, MarkdownSplitter, get_splitter, register_splitter, SUPPORTED_SUFFIXES
|
||||
from src.core.splitters import (
|
||||
SUPPORTED_SUFFIXES,
|
||||
MarkdownSplitter,
|
||||
TextSplitter,
|
||||
get_splitter,
|
||||
register_splitter,
|
||||
)
|
||||
|
||||
|
||||
class TestTextSplitter:
|
||||
|
||||
@@ -28,9 +28,10 @@ class TestEPUBSplitter:
|
||||
|
||||
def test_split_simple_epub(self, tmp_path):
|
||||
"""用 ebooklib 创建一个简单 EPUB 并测试分块."""
|
||||
from src.core.splitters.epub import EPUBSplitter
|
||||
from ebooklib import epub
|
||||
|
||||
from src.core.splitters.epub import EPUBSplitter
|
||||
|
||||
epub_path = tmp_path / "test.epub"
|
||||
|
||||
book = epub.EpubBook()
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""MarkdownSplitter 边界测试."""
|
||||
import pytest
|
||||
|
||||
from src.core.splitters import MarkdownSplitter
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""PDFSplitter 测试."""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
|
||||
pymupdf = pytest.importorskip("fitz", reason="pymupdf 未安装")
|
||||
|
||||
@@ -10,9 +10,10 @@ class TestPDFSplitter:
|
||||
|
||||
def test_split_simple_pdf(self, tmp_path):
|
||||
"""用 pymupdf 创建一个简单 PDF 并测试分块."""
|
||||
from src.core.splitters.pdf import PDFSplitter
|
||||
import fitz
|
||||
|
||||
from src.core.splitters.pdf import PDFSplitter
|
||||
|
||||
pdf_path = tmp_path / "test.pdf"
|
||||
doc = fitz.open()
|
||||
# 插入纯 ASCII 文本避免 CJK 字体编码问题
|
||||
@@ -29,9 +30,10 @@ class TestPDFSplitter:
|
||||
|
||||
def test_empty_pdf(self, tmp_path):
|
||||
"""空 PDF(有页但无文字)返回空列表."""
|
||||
from src.core.splitters.pdf import PDFSplitter
|
||||
import fitz
|
||||
|
||||
from src.core.splitters.pdf import PDFSplitter
|
||||
|
||||
pdf_path = tmp_path / "empty.pdf"
|
||||
doc = fitz.open()
|
||||
doc.new_page() # pymupdf 必须有至少一页才能保存
|
||||
|
||||
Reference in New Issue
Block a user