fix: 修复 11 个代码架构审计问题
H1: AppConfig 自定义 __init__ 改用 from_dict() 类方法 H2: list_collections_with_stats 改为从 ChromaDB 直接查询 H3: RateLimiter 过期 key 自动清理, 防止内存泄漏 M1: delete_by_source 区分 ValueError 与真实异常, 记日志 M2: VectorDB 新增 list_collections() 封装方法 M3: _remove_by_source 异常记日志, 不再静默吞掉 M4: CLI 集合回退支持 MD_VECTOR_DB_COLLECTION 环境变量 L1: DEFAULT_CONFIG_PATH 自动从项目根目录解析 L2: ingest 命令内重复 import 移至模块顶部 L3: 新增死循环回归测试 + 密集分隔符分块测试 L4: 统一 logger 名称为 md-vector-db 删除旧版审计文档 测试: 48 passed
This commit is contained in:
+7
-4
@@ -3,6 +3,8 @@ from pathlib import Path
|
||||
import sys
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import glob as _glob
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
@@ -45,7 +47,10 @@ def _init_shared(config_path: str = DEFAULT_CONFIG_PATH):
|
||||
|
||||
|
||||
def _get_default_collection() -> str:
|
||||
return _cfg.chroma.collection_name if _cfg else "markdown_docs"
|
||||
return os.environ.get(
|
||||
"MD_VECTOR_DB_COLLECTION",
|
||||
_cfg.chroma.collection_name if _cfg else "markdown_docs",
|
||||
)
|
||||
|
||||
|
||||
def _resolve_collection(collection: str | None) -> str:
|
||||
@@ -94,10 +99,8 @@ def ingest(
|
||||
total = 0
|
||||
for fp in file_paths:
|
||||
# 支持通配符 (shell 展开或 Python glob)
|
||||
from pathlib import Path as _Path
|
||||
p = _Path(fp)
|
||||
p = Path(fp)
|
||||
if "*" in fp or "?" in fp:
|
||||
import glob as _glob
|
||||
matches = _glob.glob(fp, recursive=True)
|
||||
for m in matches:
|
||||
c = ingestor.ingest_file(m)
|
||||
|
||||
+19
-8
@@ -72,19 +72,30 @@ class AppConfig:
|
||||
chunk: ChunkConfig = field(default_factory=ChunkConfig)
|
||||
server: ServerConfig = field(default_factory=ServerConfig)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.chroma = ChromaConfig(**kwargs.get("chroma", {}))
|
||||
self.embed = EmbedConfig(**kwargs.get("embed", {}))
|
||||
self.chunk = ChunkConfig(**kwargs.get("chunk", {}))
|
||||
self.server = ServerConfig(**kwargs.get("server", {}))
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "AppConfig":
|
||||
"""从字典构造 (用于 YAML 加载)."""
|
||||
return cls(
|
||||
chroma=ChromaConfig(**data.get("chroma", {})),
|
||||
embed=EmbedConfig(**data.get("embed", {})),
|
||||
chunk=ChunkConfig(**data.get("chunk", {})),
|
||||
server=ServerConfig(**data.get("server", {})),
|
||||
)
|
||||
|
||||
|
||||
def load_config(path: str | None = None) -> AppConfig:
|
||||
"""从 YAML 文件加载配置, 若文件不存在则返回默认配置."""
|
||||
config_path = path or DEFAULT_CONFIG_PATH
|
||||
if not Path(config_path).exists():
|
||||
config_file = Path(config_path)
|
||||
# 相对路径 → 从项目根目录解析
|
||||
if not config_file.is_absolute():
|
||||
root = Path(__file__).resolve().parent.parent.parent
|
||||
_try = root / config_file
|
||||
if _try.exists():
|
||||
config_file = _try
|
||||
if not config_file.exists():
|
||||
return AppConfig()
|
||||
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
with open(config_file, "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
return AppConfig(**data)
|
||||
return AppConfig.from_dict(data)
|
||||
|
||||
@@ -20,6 +20,10 @@ class VectorDB:
|
||||
"""获取或创建 collection."""
|
||||
return self.client.get_or_create_collection(name=name)
|
||||
|
||||
def list_collections(self) -> list[Collection]:
|
||||
"""列出所有 collection."""
|
||||
return self.client.list_collections()
|
||||
|
||||
def delete_collection(self, name: str) -> None:
|
||||
"""删除 collection (线程安全)."""
|
||||
with self._write_lock:
|
||||
|
||||
+6
-1
@@ -1,4 +1,5 @@
|
||||
"""Markdown 文档解析与入库模块."""
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
@@ -6,6 +7,8 @@ from pathlib import Path
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import Embedder, batch_embed
|
||||
|
||||
logger = logging.getLogger("md-vector-db")
|
||||
|
||||
|
||||
class MarkdownSplitter:
|
||||
"""Markdown 混合分块器:先按标题拆,超长再按段落拆."""
|
||||
@@ -242,5 +245,7 @@ class DocumentIngestor:
|
||||
)
|
||||
if existing and existing["ids"]:
|
||||
self.collection.delete(ids=existing["ids"])
|
||||
except ValueError:
|
||||
pass # collection 为空时 ChromaDB 抛 ValueError
|
||||
except Exception:
|
||||
pass # collection 为空时 get 可能抛异常
|
||||
logger.exception("去重检查失败: %s", file_name)
|
||||
|
||||
+6
-1
@@ -1,7 +1,10 @@
|
||||
"""语义检索模块."""
|
||||
import logging
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import Embedder
|
||||
|
||||
logger = logging.getLogger("md-vector-db")
|
||||
|
||||
|
||||
class Searcher:
|
||||
"""向量检索器."""
|
||||
@@ -84,6 +87,8 @@ class Searcher:
|
||||
if existing and existing["ids"]:
|
||||
self.collection.delete(ids=existing["ids"])
|
||||
return True
|
||||
except ValueError:
|
||||
pass # collection 为空时 ChromaDB 抛 ValueError
|
||||
except Exception:
|
||||
pass
|
||||
logger.exception("删除文档失败: %s", file_name)
|
||||
return False
|
||||
|
||||
@@ -32,6 +32,8 @@ class RateLimiter:
|
||||
with self._lock:
|
||||
records = self._store[client_id]
|
||||
records[:] = [t for t in records if now - t < self.window]
|
||||
if not records:
|
||||
del self._store[client_id] # 清理空 key,防止内存泄漏
|
||||
if len(records) >= self.max_requests:
|
||||
return False
|
||||
records.append(now)
|
||||
|
||||
+7
-10
@@ -8,7 +8,7 @@ from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
from src.core.search import Searcher
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger = logging.getLogger("md-vector-db")
|
||||
|
||||
|
||||
class AppState:
|
||||
@@ -45,16 +45,13 @@ class AppState:
|
||||
return self._ingestors[name]
|
||||
|
||||
def list_collections_with_stats(self) -> list[dict]:
|
||||
"""列出所有 collection 及其统计."""
|
||||
"""列出所有 collection 及其统计(直接从 ChromaDB 查询)."""
|
||||
result = []
|
||||
all_names = set(self._searchers.keys()) | set(self._ingestors.keys())
|
||||
all_names.add(self.default_collection)
|
||||
for name in sorted(all_names):
|
||||
try:
|
||||
coll = self.db.get_or_create_collection(name)
|
||||
result.append({"name": name, "count": coll.count()})
|
||||
except Exception:
|
||||
result.append({"name": name, "count": 0})
|
||||
try:
|
||||
for coll in self.db.client.list_collections():
|
||||
result.append({"name": coll.name, "count": coll.count()})
|
||||
except Exception:
|
||||
logger.exception("列出集合失败")
|
||||
return result
|
||||
|
||||
def is_healthy(self) -> dict:
|
||||
|
||||
Reference in New Issue
Block a user