mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-10-23 05:35:30 +00:00
fix(chunker): preserve Markdown source lines after front matter
This commit is contained in:
parent
d865776edd
commit
ab5bcdfb10
2 changed files with 51 additions and 0 deletions
|
|
@ -98,6 +98,12 @@ class DefaultFileChunker(BaseFileChunker):
|
|||
links = []
|
||||
|
||||
chunks = self.chunk_content(content, rel_path, parse_links=is_markdown)
|
||||
line_offset = text[: len(text) - len(content)].count("\n") if is_markdown else 0
|
||||
if line_offset:
|
||||
for chunk in chunks:
|
||||
chunk.start_line += line_offset
|
||||
chunk.end_line += line_offset
|
||||
chunk.set_hash_id()
|
||||
chunk_ids = [c.id for c in chunks]
|
||||
return (
|
||||
FileNode(
|
||||
|
|
|
|||
|
|
@ -4,12 +4,57 @@ import asyncio
|
|||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
from reme.components.file_chunker import DefaultFileChunker
|
||||
from reme.schema import FileChunk
|
||||
from reme.utils.wikilink_handler import WikilinkHandler
|
||||
|
||||
# Add parent path for import
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prefix,start_line",
|
||||
[("", 1), ("---\nname: t\n---\n", 4), ("---\nname: t\n---\n\n\n", 6)],
|
||||
)
|
||||
def test_markdown_front_matter_preserves_source_lines(tmp_path, prefix, start_line):
|
||||
"""Byte chunking must cite the original Markdown body and refresh its ID."""
|
||||
body = "# H\nline 1\nline 2\n"
|
||||
path = tmp_path / "note.md"
|
||||
path.write_text(prefix + body, encoding="utf-8")
|
||||
|
||||
node, chunks = asyncio.run(DefaultFileChunker().chunk(path))
|
||||
|
||||
assert len(chunks) == 1
|
||||
chunk = chunks[0]
|
||||
assert (chunk.start_line, chunk.end_line) == (start_line, start_line + 2)
|
||||
assert chunk.text == body
|
||||
expected = FileChunk(path=chunk.path, start_line=start_line, end_line=start_line + 2, text=body).set_hash_id()
|
||||
assert chunk.id == expected.id
|
||||
assert node.chunk_ids == [expected.id]
|
||||
|
||||
|
||||
def test_markdown_front_matter_offsets_every_chunk(tmp_path):
|
||||
"""All overlapping chunks retain their original file ranges and hashes."""
|
||||
body = "".join(f"line {i:02d}\n" for i in range(40))
|
||||
path = tmp_path / "note.md"
|
||||
path.write_text("---\nname: t\n---\n" + body, encoding="utf-8")
|
||||
chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=8)
|
||||
|
||||
node, chunks = asyncio.run(chunker.chunk(path))
|
||||
|
||||
assert [(chunk.start_line, chunk.end_line) for chunk in chunks] == [(4, 16), (15, 27), (27, 39), (38, 43)]
|
||||
for chunk in chunks:
|
||||
expected = FileChunk(
|
||||
path=chunk.path,
|
||||
start_line=chunk.start_line,
|
||||
end_line=chunk.end_line,
|
||||
text=chunk.text,
|
||||
).set_hash_id()
|
||||
assert chunk.id == expected.id
|
||||
assert node.chunk_ids == [chunk.id for chunk in chunks]
|
||||
|
||||
|
||||
def test_parse_empty_file():
|
||||
"""Test parsing an empty file."""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue