refactor: strategy-pattern embedder (Local/API), remove global env vars, add batch embedding
This commit is contained in:
+3
-3
@@ -4,7 +4,7 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from src.core.db import VectorDB
|
||||
from src.core.embedder import Embedder
|
||||
from src.core.embedder import Embedder, batch_embed
|
||||
|
||||
|
||||
class MarkdownSplitter:
|
||||
@@ -190,9 +190,9 @@ class DocumentIngestor:
|
||||
if not chunks:
|
||||
return 0
|
||||
|
||||
# 嵌入
|
||||
# 分批嵌入 (避免大文档 OOM)
|
||||
texts = [c["content"] for c in chunks]
|
||||
embeddings = self.embedder.embed(texts)
|
||||
embeddings = batch_embed(self.embedder, texts)
|
||||
|
||||
# 入库
|
||||
ids = [f"{file_name}_{i}" for i in range(len(chunks))]
|
||||
|
||||
Reference in New Issue
Block a user