fix(chunker): preserve Markdown source lines after front matter

This commit is contained in:
cangen185-sudo 2026-10-15 14:08:21 +08:00
parent d865776edd
commit ab5bcdfb10
2 changed files with 51 additions and 0 deletions

View file

@ -98,6 +98,12 @@ class DefaultFileChunker(BaseFileChunker):
links = []
chunks = self.chunk_content(content, rel_path, parse_links=is_markdown)
line_offset = text[: len(text) - len(content)].count("\n") if is_markdown else 0
if line_offset:
for chunk in chunks:
chunk.start_line += line_offset
chunk.end_line += line_offset
chunk.set_hash_id()
chunk_ids = [c.id for c in chunks]
return (
FileNode(

View file

@ -4,12 +4,57 @@ import asyncio
import os
import tempfile
import pytest
from reme.components.file_chunker import DefaultFileChunker
from reme.schema import FileChunk
from reme.utils.wikilink_handler import WikilinkHandler
# Add parent path for import
@pytest.mark.parametrize(
"prefix,start_line",
[("", 1), ("---\nname: t\n---\n", 4), ("---\nname: t\n---\n\n\n", 6)],
)
def test_markdown_front_matter_preserves_source_lines(tmp_path, prefix, start_line):
"""Byte chunking must cite the original Markdown body and refresh its ID."""
body = "# H\nline 1\nline 2\n"
path = tmp_path / "note.md"
path.write_text(prefix + body, encoding="utf-8")
node, chunks = asyncio.run(DefaultFileChunker().chunk(path))
assert len(chunks) == 1
chunk = chunks[0]
assert (chunk.start_line, chunk.end_line) == (start_line, start_line + 2)
assert chunk.text == body
expected = FileChunk(path=chunk.path, start_line=start_line, end_line=start_line + 2, text=body).set_hash_id()
assert chunk.id == expected.id
assert node.chunk_ids == [expected.id]
def test_markdown_front_matter_offsets_every_chunk(tmp_path):
"""All overlapping chunks retain their original file ranges and hashes."""
body = "".join(f"line {i:02d}\n" for i in range(40))
path = tmp_path / "note.md"
path.write_text("---\nname: t\n---\n" + body, encoding="utf-8")
chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=8)
node, chunks = asyncio.run(chunker.chunk(path))
assert [(chunk.start_line, chunk.end_line) for chunk in chunks] == [(4, 16), (15, 27), (27, 39), (38, 43)]
for chunk in chunks:
expected = FileChunk(
path=chunk.path,
start_line=chunk.start_line,
end_line=chunk.end_line,
text=chunk.text,
).set_hash_id()
assert chunk.id == expected.id
assert node.chunk_ids == [chunk.id for chunk in chunks]
def test_parse_empty_file():
"""Test parsing an empty file."""