diff --git a/ingest_progress.txt b/ingest_progress.txt deleted file mode 100644 index 7f31f0a..0000000 --- a/ingest_progress.txt +++ /dev/null @@ -1,56 +0,0 @@ -初始化... -待处理: 46 个文件 -跳过超大: 6 个 - SKIP 人工智能协会内容创作通知.md (1915KB) - SKIP Git团队协作指南.md (94KB) - SKIP Git团队协作指南.sync-conflict-20260615-122209-R4RFYM7.md (94KB) - SKIP OpenClaw介绍.md (64KB) - SKIP Git内部原理详解.md (59KB) - SKIP Git团队协作指南大纲.md (58KB) -[1/46] 17 chunks | 博客/AI时代的Token战争-能源算力与我们每个人的选择.md (0.6s) -[2/46] 39 chunks | 博客/AI时代的VibeCoding与程序员进化.md (1.2s) -[3/46] 59 chunks | 博客/CLI在AI时代的浴火重生.md (1.7s) -[4/46] 68 chunks | 博客/DeepSeek-V4全面解析.md (1.1s) -[5/46] 16 chunks | 博客/大模型赋能架构设计.md (0.5s) -[6/46] 27 chunks | 博客/大语言模型算子详解.md (1.0s) -[7/46] 8 chunks | 博客/学习Agent,越学越像在重新理解操作系统.md (0.3s) -[8/46] 69 chunks | 博客/深入解析 Claude Code:Vibe Coding 时代的 AI 编程利器.md (1.5s) -[9/46] 12 chunks | 博客/自建Git服务-在NAS上部署Gitea.md (0.4s) -[10/46] 38 chunks | 博客/DeepSeek-V4博客大纲.md (0.4s) -[11/46] 110 chunks | 博客/uv工具推荐博客大纲.md (1.0s) -[12/46] 48 chunks | 博客/AI助你轻松上手LaTeX论文写作.md (1.1s) -[13/46] 15 chunks | 博客/SEO分析报告.md (0.4s) -[14/46] 18 chunks | 博客/博客爬取报告.md (0.3s) -[15/46] 15 chunks | 博客/大数据技术栈.md (0.3s) -[16/46] 7 chunks | 博客/摘要检查报告.md (0.3s) -[17/46] 27 chunks | 博客/文章列表.md (0.5s) -[18/46] 19 chunks | 博客/标签分类审查报告.md (0.7s) -[19/46] 13 chunks | 博客/LinearRegression线性回归.md (0.6s) -[20/46] 53 chunks | 博客/从全连接层到卷积.md (1.2s) -[21/46] 114 chunks | 博客/深度学习完全指南.md (1.6s) -[22/46] 43 chunks | 博客/视觉语言模型技术综述.md (1.4s) -[23/46] 52 chunks | 博客/Docker安装与入门指南.md (1.4s) -[24/46] 167 chunks | 博客/Docker部署完全指南.md (3.6s) -[25/46] 115 chunks | 博客/Git团队协作指南(精简版).md (1.3s) -[26/46] 10 chunks | 博客/OpenClaw安装教程.md (0.1s) -[27/46] 247 chunks | 博客/uv工具推荐博客.md (2.8s) -[28/46] 6 chunks | Club/CLAUDE.md (0.1s) -[29/46] 7 chunks | Club/README.md (0.1s) -[30/46] 11 chunks | Club/仓库说明.md (0.4s) -[31/46] 9 chunks | Club/导航页.md (0.3s) -[32/46] 3 chunks | Club/说明.md (0.1s) -[33/46] 4 chunks | Club/说明.md (0.2s) -[34/46] 4 chunks | Club/MetaRL核心团队实施方案.md (0.3s) -[35/46] 3 chunks | Club/说明.md (0.1s) -[36/46] 20 chunks | Club/定量考核材料清单.md (0.7s) -[37/46] 14 chunks | Club/年度工作总结起草指南.md (0.6s) -[38/46] 3 chunks | Club/说明.md (0.2s) -[39/46] 3 chunks | Club/说明.md (0.1s) -[40/46] 7 chunks | Club/人工智能协会近期工作任务清单.md (0.3s) -[41/46] 3 chunks | Club/说明.md (0.2s) -[42/46] 7 chunks | halo/README.md (0.3s) -[43/46] 6 chunks | halo/README.zh-CN.md (0.3s) -[44/46] 21 chunks | halo/usage-guide.md (0.4s) -[45/46] 9 chunks | 顶层/CLAUDE.md (0.4s) -[46/46] 57 chunks | 顶层/temp-agent-os-v2.md (1.6s) -[DONE] 完成: 46 文件, 1623 chunks (45s) diff --git a/ingest_stdout.txt b/ingest_stdout.txt deleted file mode 100644 index 4b23c91..0000000 --- a/ingest_stdout.txt +++ /dev/null @@ -1,57 +0,0 @@ -ʼ... - Loading weights: 0%| | 0/71 [00:0050KB 的文件 +MAX_SIZE = 200_000 # GPU 嵌入,无需跳过 progress_file = Path(__file__).parent.parent / "ingest_progress.txt" def log(msg): diff --git a/src/core/embedder.py b/src/core/embedder.py index 1132a62..d999c75 100644 --- a/src/core/embedder.py +++ b/src/core/embedder.py @@ -52,21 +52,33 @@ class Embedder(Protocol): # -- 本地模型 -- class LocalEmbedder: - """sentence-transformers 本地模型嵌入器.""" + """sentence-transformers 本地模型嵌入器. 自动检测 GPU.""" def __init__(self, config: EmbedConfig): from sentence_transformers import SentenceTransformer + + # 自动检测 GPU + try: + import torch + device = "cuda" if torch.cuda.is_available() else "cpu" + if device == "cuda": + logger.info("检测到 GPU: %s", torch.cuda.get_device_name(0)) + except ImportError: + device = "cpu" + self._config = config try: self._model = SentenceTransformer( - config.local_model, local_files_only=True + config.local_model, local_files_only=True, device=device ) except Exception: logger.info("模型未缓存, 通过镜像下载 %s", config.local_model) old_endpoint = os.environ.get("HF_ENDPOINT") os.environ["HF_ENDPOINT"] = _HF_MIRROR try: - self._model = SentenceTransformer(config.local_model) + self._model = SentenceTransformer( + config.local_model, device=device + ) finally: if old_endpoint is not None: os.environ["HF_ENDPOINT"] = old_endpoint diff --git a/src/core/ingest.py b/src/core/ingest.py index a9e7d58..d439915 100644 --- a/src/core/ingest.py +++ b/src/core/ingest.py @@ -155,7 +155,9 @@ class MarkdownSplitter: part = text[start:break_point].strip() if part: parts.append(part) - start = break_point - self.overlap if self.overlap > 0 else break_point + # 确保 start 始终前进(避免分隔符距 start 小于 overlap 时 start 回退导致死循环) + next_start = break_point - self.overlap if self.overlap > 0 else break_point + start = max(start + 1, next_start) return parts