fix: auth 测试改用 monkeypatch.setattr,MarkdownSplitter 跳过代码块内标题识别
This commit is contained in:
@@ -31,8 +31,20 @@ class MarkdownSplitter(BaseTextSplitter):
|
||||
return chunks
|
||||
|
||||
def _split_by_headings(self, text: str) -> list[dict]:
|
||||
# 识别代码块范围,跳过其中的假标题
|
||||
code_block_ranges = []
|
||||
fence_re = re.compile(r"^```", re.MULTILINE)
|
||||
fences = [m.start() for m in fence_re.finditer(text)]
|
||||
for i in range(0, len(fences), 2):
|
||||
if i + 1 < len(fences):
|
||||
code_block_ranges.append((fences[i], fences[i + 1]))
|
||||
|
||||
def in_code_block(pos: int) -> bool:
|
||||
return any(start <= pos <= end for start, end in code_block_ranges)
|
||||
|
||||
heading_pattern = re.compile(r"^(#{1,6})\s+(.+)$", re.MULTILINE)
|
||||
matches = list(heading_pattern.finditer(text))
|
||||
all_matches = list(heading_pattern.finditer(text))
|
||||
matches = [m for m in all_matches if not in_code_block(m.start())]
|
||||
|
||||
if not matches:
|
||||
return [{
|
||||
|
||||
Reference in New Issue
Block a user