fix(chunker): treat default file chunks as plain text (#624)

* fix(chunker): preserve Markdown source lines after front matter

* fix(chunker): keep default file chunks as plain text

---------

Co-authored-by: cangen185-sudo <234582933+cangen185-sudo@users.noreply.github.com>

reme/components/file_chunker/default_file_chunker.py

@@ -5,11 +5,10 @@ from pathlib import Path
 from typing import Literal
 
 import aiofiles
-import yaml
 
 from .base_file_chunker import BaseFileChunker
 from ..component_registry import R
-from ...schema import FileChunk, FileFrontMatter, FileNode
+from ...schema import FileChunk, FileNode
 from ...utils.wikilink_handler import WikilinkHandler
 
 InvalidEncodingPolicy = Literal["replace", "strict"]
@@ -17,7 +16,7 @@ InvalidEncodingPolicy = Literal["replace", "strict"]
 
 @R.register("default")
 class DefaultFileChunker(BaseFileChunker):
-    """Default chunker that splits files into byte-based overlapping chunks."""
+    """Split complete files into plain-text byte chunks, regardless of extension."""
 
     def __init__(
         self,
@@ -38,21 +37,6 @@ class DefaultFileChunker(BaseFileChunker):
         self.chunk_byte_size = max(100, chunk_byte_size)
         self.overlap_byte_size = max(4, overlap_byte_size)
 
-    @staticmethod
-    def _parse_front_matter(text: str) -> tuple[FileFrontMatter, str]:
-        """Parse YAML front matter delimited by ---, return (front_matter, remaining)."""
-        if not text.startswith("---"):
-            return FileFrontMatter(), text
-        end_idx = text.find("\n---", 3)
-        if end_idx == -1:
-            return FileFrontMatter(), text
-        try:
-            data = yaml.safe_load(text[3:end_idx].strip()) or {}
-            front_matter = FileFrontMatter(**(data if isinstance(data, dict) else {}))
-        except yaml.YAMLError:
-            front_matter = FileFrontMatter()
-        return front_matter, text[end_idx + 4 :].lstrip("\n")
-
     async def _read_text_for_indexing(self, file_path: Path) -> str:
         """Decode text for a derived index without modifying the source file."""
         async with aiofiles.open(file_path, "rb") as f:
@@ -86,25 +70,12 @@ class DefaultFileChunker(BaseFileChunker):
         if not text:
             return FileNode(path=rel_path, st_mtime=stat.st_mtime), []
 
-        is_markdown = file_path.suffix.lower() == ".md"
-        if is_markdown:
-            front_matter, content = self._parse_front_matter(text)
-            if not content:
-                return FileNode(path=rel_path, st_mtime=stat.st_mtime, front_matter=front_matter), []
-            links = WikilinkHandler.extract_links(content, rel_path)
-        else:
-            front_matter = FileFrontMatter()
-            content = text
-            links = []
-
-        chunks = self.chunk_content(content, rel_path, parse_links=is_markdown)
+        chunks = self.chunk_content(text, rel_path, parse_links=False)
         chunk_ids = [c.id for c in chunks]
         return (
             FileNode(
                 path=rel_path,
                 st_mtime=stat.st_mtime,
-                front_matter=front_matter,
-                links=links,
                 chunk_ids=chunk_ids,
             ),
             chunks,
@@ -139,7 +110,8 @@ class DefaultFileChunker(BaseFileChunker):
         """Split content into overlapping byte-range chunks, avoiding cuts inside wikilinks.
 
         When ``parse_links`` is False, skip wikilink span computation and boundary checks
-        — used for non-markdown files where wikilink semantics don't apply.
+        — used by the default chunker for all files. Markdown fallback explicitly
+        enables link-aware chunking.
         """
         content_bytes = content.encode(self.encoding)
         n = len(content_bytes)

tests/unit/test_default_file_chunker.py

@@ -4,12 +4,97 @@ import asyncio
 import os
 import tempfile
 
+import pytest
+
 from reme.components.file_chunker import DefaultFileChunker
+from reme.schema import FileChunk, FileFrontMatter
 from reme.utils.wikilink_handler import WikilinkHandler
 
 # Add parent path for import
 
 
+@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"])
+@pytest.mark.parametrize(
+    "content",
+    [
+        "",
+        "# H\nline 1\nline 2\n",
+        "---\nname: t\n---\n# H\nline 1\nline 2\n",
+        "---\nname: t\n---\n\n\n# H\nline 1\nline 2\n",
+        "---\nname: t\n---\n",
+        "\n\n",
+        "---\nname: [invalid\n---\n[[target.md]]\n",
+        "---\nname: unclosed\n[[target.md]]",
+    ],
+)
+def test_file_extensions_preserve_identical_plain_text(tmp_path, content, newline):
+    """The default chunker keeps the whole file without interpreting Markdown."""
+    results = []
+    source = content.replace("\n", newline).encode("utf-8")
+    for suffix in (".md", ".MD", ".txt"):
+        path = tmp_path / f"note{suffix}"
+        path.write_bytes(source)
+        node, chunks = asyncio.run(DefaultFileChunker().chunk(path))
+
+        assert path.read_bytes() == source
+        assert node.front_matter == FileFrontMatter()
+        assert node.links == []
+        assert node.chunk_ids == [chunk.id for chunk in chunks]
+        if content:
+            assert len(chunks) == 1
+            chunk = chunks[0]
+            end_line = content.count("\n") + (not content.endswith("\n"))
+            assert (chunk.start_line, chunk.end_line, chunk.text) == (1, end_line, content)
+            expected = FileChunk(path=chunk.path, start_line=1, end_line=end_line, text=content).set_hash_id()
+            assert chunk.id == expected.id
+        else:
+            assert chunks == []
+        results.append([(chunk.text, chunk.start_line, chunk.end_line) for chunk in chunks])
+    assert results[0] == results[1] == results[2]
+
+
+def test_plain_text_overlaps_preserve_source_ranges_and_ids(tmp_path):
+    """Every overlapping chunk includes its unmodified source bytes and range."""
+    content = "---\nname: t\n---\n\n\n" + "".join(f"line {i:02d}\n" for i in range(40))
+    chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=8)
+    results = []
+    for suffix in (".md", ".txt"):
+        path = tmp_path / f"note{suffix}"
+        path.write_bytes(content.encode("utf-8"))
+        node, chunks = asyncio.run(chunker.chunk(path))
+
+        starts = list(range(0, len(content) - chunker.overlap_byte_size, 92))
+        assert len(chunks) == len(starts) > 1
+        for start, chunk in zip(starts, chunks):
+            end = min(start + 100, len(content))
+            assert chunk.text == content[start:end]
+            assert chunk.start_line == content[:start].count("\n") + 1
+            assert chunk.end_line == content[:end].count("\n") + (not content[:end].endswith("\n"))
+            expected = FileChunk(
+                path=chunk.path,
+                start_line=chunk.start_line,
+                end_line=chunk.end_line,
+                text=chunk.text,
+            ).set_hash_id()
+            assert chunk.id == expected.id
+        assert node.chunk_ids == [chunk.id for chunk in chunks]
+        results.append([(chunk.text, chunk.start_line, chunk.end_line) for chunk in chunks])
+    assert results[0] == results[1]
+
+
+@pytest.mark.parametrize("suffix", [".md", ".txt"])
+def test_wikilinks_do_not_change_plain_text_boundaries(tmp_path, suffix):
+    """The default path may cut a literal wikilink just like any other text."""
+    content = "x" * 90 + "[[a-very-long-target]]" + "y" * 90
+    path = tmp_path / f"note{suffix}"
+    path.write_bytes(content.encode("utf-8"))
+    node, chunks = asyncio.run(DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=10).chunk(path))
+
+    assert [chunk.text for chunk in chunks] == [content[0:100], content[90:190], content[180:]]
+    assert node.links == []
+    assert node.front_matter == FileFrontMatter()
+
+
 def test_parse_empty_file():
     """Test parsing an empty file."""
 
@@ -255,7 +340,7 @@ def test_parse_links_no_match():
 
 
 def test_parse_links_in_file():
-    """Integration: parse() populates FileNode.links from file content."""
+    """Literal wikilinks and YAML remain text without populating node metadata."""
 
     async def run():
         content = (
@@ -273,10 +358,11 @@ def test_parse_links_in_file():
 
         try:
             chunker = DefaultFileChunker()
-            file_node, _ = await chunker.chunk(temp_path)
-            pairs = {(link.target_path, link.target_anchor) for link in file_node.links}
-            assert pairs == {("alpha", None), ("beta", "h2"), ("Alice", None), ("paper", "chapter 1")}
-            assert all(link.source_path == file_node.path for link in file_node.links)
+            file_node, chunks = await chunker.chunk(temp_path)
+            assert file_node.links == []
+            assert file_node.front_matter == FileFrontMatter()
+            assert len(chunks) == 1
+            assert chunks[0].text == content
             print("✓ test_parse_links_in_file passed")
         finally:
             os.unlink(temp_path)
@@ -310,91 +396,6 @@ def test_parse_links_empty_when_no_content():
     asyncio.run(run())
 
 
-def test_chunk_does_not_split_wikilink_at_boundary():
-    """A wikilink straddling the chunk_byte_size boundary should be retreated to its start."""
-
-    async def run():
-        # Pre-link filler is 90 bytes, link itself is 19 bytes ("[[a-very-long-target]]"=22).
-        # With chunk_byte_size=100, the boundary lands inside the link.
-        prefix = "x" * 90
-        link = "[[a-very-long-target]]"  # 22 bytes
-        suffix = "y" * 90
-        content = f"{prefix}{link}{suffix}"
-        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
-            f.write(content)
-            temp_path = f.name
-
-        try:
-            chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=10)
-            _, chunks = await chunker.chunk(temp_path)
-            # The first chunk must NOT contain a partial link.
-            first = chunks[0].text
-            assert "[[" not in first or "]]" in first, f"first chunk has dangling '[[': {first!r}"
-            # And the link should appear intact in some chunk.
-            assert any(link in c.text for c in chunks), "link was split across all chunks"
-            print("✓ test_chunk_does_not_split_wikilink_at_boundary passed")
-        finally:
-            os.unlink(temp_path)
-
-    asyncio.run(run())
-
-
-def test_chunk_does_not_split_wikilink_in_overlap():
-    """A wikilink landing inside the overlap region should be advanced past."""
-
-    async def run():
-        # 200-byte content, chunk=100, overlap=20. First chunk ends near byte 100,
-        # next start = 80. Place a link straddling byte 80 to land in the overlap.
-        prefix = "a" * 75
-        link = "[[overlap-target]]"  # 18 bytes; spans bytes 75..93
-        suffix = "b" * 110
-        content = f"{prefix}{link}{suffix}"
-        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
-            f.write(content)
-            temp_path = f.name
-
-        try:
-            chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=20)
-            _, chunks = await chunker.chunk(temp_path)
-            # No chunk should start mid-link.
-            for c in chunks:
-                t = c.text
-                if "]]" in t and "[[" not in t.split("]]", 1)[0]:
-                    raise AssertionError(f"chunk starts mid-link: {t[:40]!r}")
-            assert any(link in c.text for c in chunks)
-            print("✓ test_chunk_does_not_split_wikilink_in_overlap passed")
-        finally:
-            os.unlink(temp_path)
-
-    asyncio.run(run())
-
-
-def test_chunk_falls_back_for_oversize_link():
-    """If a single link exceeds half the chunk size, the parser hard-cuts to make progress."""
-
-    async def run():
-        # chunk=100, link is 80 bytes, surrounded by short filler.
-        # Retreating would leave a tiny chunk (< 50), so the fallback kicks in.
-        prefix = "x" * 30
-        link = "[[" + ("L" * 76) + "]]"  # 80 bytes total
-        suffix = "y" * 200
-        content = f"{prefix}{link}{suffix}"
-        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
-            f.write(content)
-            temp_path = f.name
-
-        try:
-            chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=10)
-            _, chunks = await chunker.chunk(temp_path)
-            # Must terminate (not hang) and cover the whole file.
-            assert len(chunks) >= 2
-            print("✓ test_chunk_falls_back_for_oversize_link passed")
-        finally:
-            os.unlink(temp_path)
-
-    asyncio.run(run())
-
-
 def test_min_chunk_and_overlap_size():
     """Test that minimum chunk and overlap sizes are enforced."""
 
@@ -435,8 +436,5 @@ if __name__ == "__main__":
     test_parse_links_no_match()
     test_parse_links_in_file()
     test_parse_links_empty_when_no_content()
-    test_chunk_does_not_split_wikilink_at_boundary()
-    test_chunk_does_not_split_wikilink_in_overlap()
-    test_chunk_falls_back_for_oversize_link()
     test_min_chunk_and_overlap_size()
     print("\n所有测试通过!")

tests/unit/test_markdown_file_chunker.py

@@ -13,6 +13,8 @@ import os
 import tempfile
 from unittest.mock import patch
 
+import pytest
+
 from reme.components.file_chunker import DefaultFileChunker, MarkdownFileChunker
 
 
@@ -171,6 +173,26 @@ def test_parse_frontmatter_only():
     asyncio.run(run())
 
 
+@pytest.mark.parametrize("max_ast_sections", [0, None])
+def test_markdown_source_lines_and_metadata_remain_separate(tmp_path, max_ast_sections):
+    """AST and inherited byte fallback retain Markdown-specific semantics."""
+    path = tmp_path / "note.md"
+    source = b"---\nname: t\n---\n# H\nline 1\n[[target.md]]\n"
+    path.write_bytes(source)
+    chunker = MarkdownFileChunker(max_ast_sections=max_ast_sections, include_frontmatter_in_metadata=True)
+
+    node, chunks = asyncio.run(chunker.chunk(path))
+
+    assert len(chunks) == 1
+    assert (chunks[0].start_line, chunks[0].end_line) == (4, 6)
+    assert "name: t" not in chunks[0].text
+    assert chunks[0].metadata == {"name": "t"}
+    assert node.front_matter.name == "t"
+    assert [link.target_path for link in node.links] == ["target.md"]
+    assert node.chunk_ids == [chunks[0].id]
+    assert path.read_bytes() == source
+
+
 def test_parse_frontmatter_metadata_is_opt_in():
     """Chunk metadata preserves the old empty default unless explicitly enabled."""
 
@@ -510,7 +532,7 @@ def test_parse_excessive_sections_uses_plain_text_without_ast():
                     ) as chunk_content,
                 ):
                     node, chunks = await chunker.chunk(path)
-                assert chunk_content.call_count == 1
+                chunk_content.assert_called_once_with(sections, path, parse_links=True)
             finally:
                 await chunker.close()
 
@@ -699,6 +721,91 @@ def test_parse_bad_frontmatter_does_not_abort_chunking():
     asyncio.run(run())
 
 
+def test_chunk_does_not_split_wikilink_at_boundary():
+    """A wikilink straddling the chunk_byte_size boundary should be retreated to its start."""
+
+    async def run():
+        # Pre-link filler is 90 bytes, link itself is 19 bytes ("[[a-very-long-target]]"=22).
+        # With chunk_byte_size=100, the boundary lands inside the link.
+        prefix = "x" * 90
+        link = "[[a-very-long-target]]"  # 22 bytes
+        suffix = "y" * 90
+        content = f"# H\n{prefix}{link}{suffix}"
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
+            f.write(content)
+            temp_path = f.name
+
+        try:
+            chunker = MarkdownFileChunker(chunk_byte_size=100, overlap_byte_size=10, max_ast_sections=0)
+            _, chunks = await chunker.chunk(temp_path)
+            # The first chunk must NOT contain a partial link.
+            first = chunks[0].text
+            assert "[[" not in first or "]]" in first, f"first chunk has dangling '[[': {first!r}"
+            # And the link should appear intact in some chunk.
+            assert any(link in c.text for c in chunks), "link was split across all chunks"
+            print("✓ test_chunk_does_not_split_wikilink_at_boundary passed")
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
+def test_chunk_does_not_split_wikilink_in_overlap():
+    """A wikilink landing inside the overlap region should be advanced past."""
+
+    async def run():
+        # 200-byte content, chunk=100, overlap=20. First chunk ends near byte 100,
+        # next start = 80. Place a link straddling byte 80 to land in the overlap.
+        prefix = "a" * 75
+        link = "[[overlap-target]]"  # 18 bytes; spans bytes 75..93
+        suffix = "b" * 110
+        content = f"# H\n{prefix}{link}{suffix}"
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
+            f.write(content)
+            temp_path = f.name
+
+        try:
+            chunker = MarkdownFileChunker(chunk_byte_size=100, overlap_byte_size=20, max_ast_sections=0)
+            _, chunks = await chunker.chunk(temp_path)
+            # No chunk should start mid-link.
+            for c in chunks:
+                t = c.text
+                if "]]" in t and "[[" not in t.split("]]", 1)[0]:
+                    raise AssertionError(f"chunk starts mid-link: {t[:40]!r}")
+            assert any(link in c.text for c in chunks)
+            print("✓ test_chunk_does_not_split_wikilink_in_overlap passed")
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
+def test_chunk_falls_back_for_oversize_link():
+    """If a single link exceeds half the chunk size, the parser hard-cuts to make progress."""
+
+    async def run():
+        # chunk=100, link is 80 bytes, surrounded by short filler.
+        # Retreating would leave a tiny chunk (< 50), so the fallback kicks in.
+        prefix = "x" * 30
+        link = "[[" + ("L" * 76) + "]]"  # 80 bytes total
+        suffix = "y" * 200
+        content = f"# H\n{prefix}{link}{suffix}"
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f:
+            f.write(content)
+            temp_path = f.name
+
+        try:
+            chunker = MarkdownFileChunker(chunk_byte_size=100, overlap_byte_size=10, max_ast_sections=0)
+            _, chunks = await chunker.chunk(temp_path)
+            # Must terminate (not hang) and cover the whole file.
+            assert len(chunks) >= 2
+            print("✓ test_chunk_falls_back_for_oversize_link passed")
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
 if __name__ == "__main__":
     print("\n=== MarkdownFileChunker tests ===")
     test_parse_empty_file()