fix: 修复 MarkdownSplitter 死循环 + 添加 GPU 自动检测
- ingest.py: _split_single_paragraph 中分隔符距 start 小于 overlap 时 start 会回退为负数，Python rfind 负索引导致无限循环 修复: start = max(start + 1, next_start) 确保始终前进 - embedder.py: LocalEmbedder 自动检测 CUDA，优先使用 GPU - ingest_obsidian.py: MAX_SIZE 调整为 200KB 适配 GPU 嵌入 - 清理临时日志文件
This commit is contained in:
@@ -1,56 +0,0 @@
|
||||
初始化...
|
||||
待处理: 46 个文件
|
||||
跳过超大: 6 个
|
||||
SKIP 人工智能协会内容创作通知.md (1915KB)
|
||||
SKIP Git团队协作指南.md (94KB)
|
||||
SKIP Git团队协作指南.sync-conflict-20260615-122209-R4RFYM7.md (94KB)
|
||||
SKIP OpenClaw介绍.md (64KB)
|
||||
SKIP Git内部原理详解.md (59KB)
|
||||
SKIP Git团队协作指南大纲.md (58KB)
|
||||
[1/46] 17 chunks | 博客/AI时代的Token战争-能源算力与我们每个人的选择.md (0.6s)
|
||||
[2/46] 39 chunks | 博客/AI时代的VibeCoding与程序员进化.md (1.2s)
|
||||
[3/46] 59 chunks | 博客/CLI在AI时代的浴火重生.md (1.7s)
|
||||
[4/46] 68 chunks | 博客/DeepSeek-V4全面解析.md (1.1s)
|
||||
[5/46] 16 chunks | 博客/大模型赋能架构设计.md (0.5s)
|
||||
[6/46] 27 chunks | 博客/大语言模型算子详解.md (1.0s)
|
||||
[7/46] 8 chunks | 博客/学习Agent，越学越像在重新理解操作系统.md (0.3s)
|
||||
[8/46] 69 chunks | 博客/深入解析 Claude Code：Vibe Coding 时代的 AI 编程利器.md (1.5s)
|
||||
[9/46] 12 chunks | 博客/自建Git服务-在NAS上部署Gitea.md (0.4s)
|
||||
[10/46] 38 chunks | 博客/DeepSeek-V4博客大纲.md (0.4s)
|
||||
[11/46] 110 chunks | 博客/uv工具推荐博客大纲.md (1.0s)
|
||||
[12/46] 48 chunks | 博客/AI助你轻松上手LaTeX论文写作.md (1.1s)
|
||||
[13/46] 15 chunks | 博客/SEO分析报告.md (0.4s)
|
||||
[14/46] 18 chunks | 博客/博客爬取报告.md (0.3s)
|
||||
[15/46] 15 chunks | 博客/大数据技术栈.md (0.3s)
|
||||
[16/46] 7 chunks | 博客/摘要检查报告.md (0.3s)
|
||||
[17/46] 27 chunks | 博客/文章列表.md (0.5s)
|
||||
[18/46] 19 chunks | 博客/标签分类审查报告.md (0.7s)
|
||||
[19/46] 13 chunks | 博客/LinearRegression线性回归.md (0.6s)
|
||||
[20/46] 53 chunks | 博客/从全连接层到卷积.md (1.2s)
|
||||
[21/46] 114 chunks | 博客/深度学习完全指南.md (1.6s)
|
||||
[22/46] 43 chunks | 博客/视觉语言模型技术综述.md (1.4s)
|
||||
[23/46] 52 chunks | 博客/Docker安装与入门指南.md (1.4s)
|
||||
[24/46] 167 chunks | 博客/Docker部署完全指南.md (3.6s)
|
||||
[25/46] 115 chunks | 博客/Git团队协作指南（精简版）.md (1.3s)
|
||||
[26/46] 10 chunks | 博客/OpenClaw安装教程.md (0.1s)
|
||||
[27/46] 247 chunks | 博客/uv工具推荐博客.md (2.8s)
|
||||
[28/46] 6 chunks | Club/CLAUDE.md (0.1s)
|
||||
[29/46] 7 chunks | Club/README.md (0.1s)
|
||||
[30/46] 11 chunks | Club/仓库说明.md (0.4s)
|
||||
[31/46] 9 chunks | Club/导航页.md (0.3s)
|
||||
[32/46] 3 chunks | Club/说明.md (0.1s)
|
||||
[33/46] 4 chunks | Club/说明.md (0.2s)
|
||||
[34/46] 4 chunks | Club/MetaRL核心团队实施方案.md (0.3s)
|
||||
[35/46] 3 chunks | Club/说明.md (0.1s)
|
||||
[36/46] 20 chunks | Club/定量考核材料清单.md (0.7s)
|
||||
[37/46] 14 chunks | Club/年度工作总结起草指南.md (0.6s)
|
||||
[38/46] 3 chunks | Club/说明.md (0.2s)
|
||||
[39/46] 3 chunks | Club/说明.md (0.1s)
|
||||
[40/46] 7 chunks | Club/人工智能协会近期工作任务清单.md (0.3s)
|
||||
[41/46] 3 chunks | Club/说明.md (0.2s)
|
||||
[42/46] 7 chunks | halo/README.md (0.3s)
|
||||
[43/46] 6 chunks | halo/README.zh-CN.md (0.3s)
|
||||
[44/46] 21 chunks | halo/usage-guide.md (0.4s)
|
||||
[45/46] 9 chunks | 顶层/CLAUDE.md (0.4s)
|
||||
[46/46] 57 chunks | 顶层/temp-agent-os-v2.md (1.6s)
|
||||
[DONE] 完成: 46 文件, 1623 chunks (45s)
|
||||
@@ -1,57 +0,0 @@
|
||||
初始化...
|
||||
|
||||
Loading weights: 0%| | 0/71 [00:00<?, ?it/s]
|
||||
Loading weights: 100%|██████████| 71/71 [00:00<00:00, 12790.81it/s]
|
||||
待处理: 46 个文件
|
||||
跳过超大: 6 个
|
||||
SKIP 人工智能协会内容创作通知.md (1915KB)
|
||||
SKIP Git团队协作指南.md (94KB)
|
||||
SKIP Git团队协作指南.sync-conflict-20260615-122209-R4RFYM7.md (94KB)
|
||||
SKIP OpenClaw介绍.md (64KB)
|
||||
SKIP Git内部原理详解.md (59KB)
|
||||
SKIP Git团队协作指南大纲.md (58KB)
|
||||
[1/46] 17 chunks | 博客/AI时代的Token战争-能源算力与我们每个人的选择.md (0.6s)
|
||||
[2/46] 39 chunks | 博客/AI时代的VibeCoding与程序员进化.md (1.2s)
|
||||
[3/46] 59 chunks | 博客/CLI在AI时代的浴火重生.md (1.7s)
|
||||
[4/46] 68 chunks | 博客/DeepSeek-V4全面解析.md (1.1s)
|
||||
[5/46] 16 chunks | 博客/大模型赋能架构设计.md (0.5s)
|
||||
[6/46] 27 chunks | 博客/大语言模型算子详解.md (1.0s)
|
||||
[7/46] 8 chunks | 博客/学习Agent，越学越像在重新理解操作系统.md (0.3s)
|
||||
[8/46] 69 chunks | 博客/深入解析 Claude Code：Vibe Coding 时代的 AI 编程利器.md (1.5s)
|
||||
[9/46] 12 chunks | 博客/自建Git服务-在NAS上部署Gitea.md (0.4s)
|
||||
[10/46] 38 chunks | 博客/DeepSeek-V4博客大纲.md (0.4s)
|
||||
[11/46] 110 chunks | 博客/uv工具推荐博客大纲.md (1.0s)
|
||||
[12/46] 48 chunks | 博客/AI助你轻松上手LaTeX论文写作.md (1.1s)
|
||||
[13/46] 15 chunks | 博客/SEO分析报告.md (0.4s)
|
||||
[14/46] 18 chunks | 博客/博客爬取报告.md (0.3s)
|
||||
[15/46] 15 chunks | 博客/大数据技术栈.md (0.3s)
|
||||
[16/46] 7 chunks | 博客/摘要检查报告.md (0.3s)
|
||||
[17/46] 27 chunks | 博客/文章列表.md (0.5s)
|
||||
[18/46] 19 chunks | 博客/标签分类审查报告.md (0.7s)
|
||||
[19/46] 13 chunks | 博客/LinearRegression线性回归.md (0.6s)
|
||||
[20/46] 53 chunks | 博客/从全连接层到卷积.md (1.2s)
|
||||
[21/46] 114 chunks | 博客/深度学习完全指南.md (1.6s)
|
||||
[22/46] 43 chunks | 博客/视觉语言模型技术综述.md (1.4s)
|
||||
[23/46] 52 chunks | 博客/Docker安装与入门指南.md (1.4s)
|
||||
[24/46] 167 chunks | 博客/Docker部署完全指南.md (3.6s)
|
||||
[25/46] 115 chunks | 博客/Git团队协作指南（精简版）.md (1.3s)
|
||||
[26/46] 10 chunks | 博客/OpenClaw安装教程.md (0.1s)
|
||||
[27/46] 247 chunks | 博客/uv工具推荐博客.md (2.8s)
|
||||
[28/46] 6 chunks | Club/CLAUDE.md (0.1s)
|
||||
[29/46] 7 chunks | Club/README.md (0.1s)
|
||||
[30/46] 11 chunks | Club/仓库说明.md (0.4s)
|
||||
[31/46] 9 chunks | Club/导航页.md (0.3s)
|
||||
[32/46] 3 chunks | Club/说明.md (0.1s)
|
||||
[33/46] 4 chunks | Club/说明.md (0.2s)
|
||||
[34/46] 4 chunks | Club/MetaRL核心团队实施方案.md (0.3s)
|
||||
[35/46] 3 chunks | Club/说明.md (0.1s)
|
||||
[36/46] 20 chunks | Club/定量考核材料清单.md (0.7s)
|
||||
[37/46] 14 chunks | Club/年度工作总结起草指南.md (0.6s)
|
||||
[38/46] 3 chunks | Club/说明.md (0.2s)
|
||||
[39/46] 3 chunks | Club/说明.md (0.1s)
|
||||
[40/46] 7 chunks | Club/人工智能协会近期工作任务清单.md (0.3s)
|
||||
[41/46] 3 chunks | Club/说明.md (0.2s)
|
||||
[42/46] 7 chunks | halo/README.md (0.3s)
|
||||
[43/46] 6 chunks | halo/README.zh-CN.md (0.3s)
|
||||
[44/46] 21 chunks | halo/usage-guide.md (0.4s)
|
||||
[45/46] 9 chunks | 顶层/CLAUDE.md (0.4s)
|
||||
@@ -8,7 +8,7 @@ from src.core.db import VectorDB
|
||||
from src.core.embedder import create_embedder
|
||||
from src.core.ingest import DocumentIngestor
|
||||
|
||||
MAX_SIZE = 50_000 # 跳过 >50KB 的文件
|
||||
MAX_SIZE = 200_000 # GPU 嵌入,无需跳过
|
||||
|
||||
progress_file = Path(__file__).parent.parent / "ingest_progress.txt"
|
||||
def log(msg):
|
||||
|
||||
+15
-3
@@ -52,21 +52,33 @@ class Embedder(Protocol):
|
||||
|
||||
# -- 本地模型 --
|
||||
class LocalEmbedder:
|
||||
"""sentence-transformers 本地模型嵌入器."""
|
||||
"""sentence-transformers 本地模型嵌入器. 自动检测 GPU."""
|
||||
|
||||
def __init__(self, config: EmbedConfig):
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
# 自动检测 GPU
|
||||
try:
|
||||
import torch
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
if device == "cuda":
|
||||
logger.info("检测到 GPU: %s", torch.cuda.get_device_name(0))
|
||||
except ImportError:
|
||||
device = "cpu"
|
||||
|
||||
self._config = config
|
||||
try:
|
||||
self._model = SentenceTransformer(
|
||||
config.local_model, local_files_only=True
|
||||
config.local_model, local_files_only=True, device=device
|
||||
)
|
||||
except Exception:
|
||||
logger.info("模型未缓存, 通过镜像下载 %s", config.local_model)
|
||||
old_endpoint = os.environ.get("HF_ENDPOINT")
|
||||
os.environ["HF_ENDPOINT"] = _HF_MIRROR
|
||||
try:
|
||||
self._model = SentenceTransformer(config.local_model)
|
||||
self._model = SentenceTransformer(
|
||||
config.local_model, device=device
|
||||
)
|
||||
finally:
|
||||
if old_endpoint is not None:
|
||||
os.environ["HF_ENDPOINT"] = old_endpoint
|
||||
|
||||
+3
-1
@@ -155,7 +155,9 @@ class MarkdownSplitter:
|
||||
part = text[start:break_point].strip()
|
||||
if part:
|
||||
parts.append(part)
|
||||
start = break_point - self.overlap if self.overlap > 0 else break_point
|
||||
# 确保 start 始终前进（避免分隔符距 start 小于 overlap 时 start 回退导致死循环）
|
||||
next_start = break_point - self.overlap if self.overlap > 0 else break_point
|
||||
start = max(start + 1, next_start)
|
||||
return parts
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user