diff --git a/reme/components/file_chunker/default_file_chunker.py b/reme/components/file_chunker/default_file_chunker.py index 4fddc632..615e1171 100644 --- a/reme/components/file_chunker/default_file_chunker.py +++ b/reme/components/file_chunker/default_file_chunker.py @@ -5,11 +5,10 @@ from pathlib import Path from typing import Literal import aiofiles -import yaml from .base_file_chunker import BaseFileChunker from ..component_registry import R -from ...schema import FileChunk, FileFrontMatter, FileNode +from ...schema import FileChunk, FileNode from ...utils.wikilink_handler import WikilinkHandler InvalidEncodingPolicy = Literal["replace", "strict"] @@ -17,7 +16,7 @@ InvalidEncodingPolicy = Literal["replace", "strict"] @R.register("default") class DefaultFileChunker(BaseFileChunker): - """Default chunker that splits files into byte-based overlapping chunks.""" + """Split complete files into plain-text byte chunks, regardless of extension.""" def __init__( self, @@ -38,21 +37,6 @@ class DefaultFileChunker(BaseFileChunker): self.chunk_byte_size = max(100, chunk_byte_size) self.overlap_byte_size = max(4, overlap_byte_size) - @staticmethod - def _parse_front_matter(text: str) -> tuple[FileFrontMatter, str]: - """Parse YAML front matter delimited by ---, return (front_matter, remaining).""" - if not text.startswith("---"): - return FileFrontMatter(), text - end_idx = text.find("\n---", 3) - if end_idx == -1: - return FileFrontMatter(), text - try: - data = yaml.safe_load(text[3:end_idx].strip()) or {} - front_matter = FileFrontMatter(**(data if isinstance(data, dict) else {})) - except yaml.YAMLError: - front_matter = FileFrontMatter() - return front_matter, text[end_idx + 4 :].lstrip("\n") - async def _read_text_for_indexing(self, file_path: Path) -> str: """Decode text for a derived index without modifying the source file.""" async with aiofiles.open(file_path, "rb") as f: @@ -86,25 +70,12 @@ class DefaultFileChunker(BaseFileChunker): if not text: return FileNode(path=rel_path, st_mtime=stat.st_mtime), [] - is_markdown = file_path.suffix.lower() == ".md" - if is_markdown: - front_matter, content = self._parse_front_matter(text) - if not content: - return FileNode(path=rel_path, st_mtime=stat.st_mtime, front_matter=front_matter), [] - links = WikilinkHandler.extract_links(content, rel_path) - else: - front_matter = FileFrontMatter() - content = text - links = [] - - chunks = self.chunk_content(content, rel_path, parse_links=is_markdown) + chunks = self.chunk_content(text, rel_path, parse_links=False) chunk_ids = [c.id for c in chunks] return ( FileNode( path=rel_path, st_mtime=stat.st_mtime, - front_matter=front_matter, - links=links, chunk_ids=chunk_ids, ), chunks, @@ -139,7 +110,8 @@ class DefaultFileChunker(BaseFileChunker): """Split content into overlapping byte-range chunks, avoiding cuts inside wikilinks. When ``parse_links`` is False, skip wikilink span computation and boundary checks - — used for non-markdown files where wikilink semantics don't apply. + — used by the default chunker for all files. Markdown fallback explicitly + enables link-aware chunking. """ content_bytes = content.encode(self.encoding) n = len(content_bytes) diff --git a/tests/unit/test_default_file_chunker.py b/tests/unit/test_default_file_chunker.py index aebd19ba..17665318 100644 --- a/tests/unit/test_default_file_chunker.py +++ b/tests/unit/test_default_file_chunker.py @@ -4,12 +4,97 @@ import asyncio import os import tempfile +import pytest + from reme.components.file_chunker import DefaultFileChunker +from reme.schema import FileChunk, FileFrontMatter from reme.utils.wikilink_handler import WikilinkHandler # Add parent path for import +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"]) +@pytest.mark.parametrize( + "content", + [ + "", + "# H\nline 1\nline 2\n", + "---\nname: t\n---\n# H\nline 1\nline 2\n", + "---\nname: t\n---\n\n\n# H\nline 1\nline 2\n", + "---\nname: t\n---\n", + "\n\n", + "---\nname: [invalid\n---\n[[target.md]]\n", + "---\nname: unclosed\n[[target.md]]", + ], +) +def test_file_extensions_preserve_identical_plain_text(tmp_path, content, newline): + """The default chunker keeps the whole file without interpreting Markdown.""" + results = [] + source = content.replace("\n", newline).encode("utf-8") + for suffix in (".md", ".MD", ".txt"): + path = tmp_path / f"note{suffix}" + path.write_bytes(source) + node, chunks = asyncio.run(DefaultFileChunker().chunk(path)) + + assert path.read_bytes() == source + assert node.front_matter == FileFrontMatter() + assert node.links == [] + assert node.chunk_ids == [chunk.id for chunk in chunks] + if content: + assert len(chunks) == 1 + chunk = chunks[0] + end_line = content.count("\n") + (not content.endswith("\n")) + assert (chunk.start_line, chunk.end_line, chunk.text) == (1, end_line, content) + expected = FileChunk(path=chunk.path, start_line=1, end_line=end_line, text=content).set_hash_id() + assert chunk.id == expected.id + else: + assert chunks == [] + results.append([(chunk.text, chunk.start_line, chunk.end_line) for chunk in chunks]) + assert results[0] == results[1] == results[2] + + +def test_plain_text_overlaps_preserve_source_ranges_and_ids(tmp_path): + """Every overlapping chunk includes its unmodified source bytes and range.""" + content = "---\nname: t\n---\n\n\n" + "".join(f"line {i:02d}\n" for i in range(40)) + chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=8) + results = [] + for suffix in (".md", ".txt"): + path = tmp_path / f"note{suffix}" + path.write_bytes(content.encode("utf-8")) + node, chunks = asyncio.run(chunker.chunk(path)) + + starts = list(range(0, len(content) - chunker.overlap_byte_size, 92)) + assert len(chunks) == len(starts) > 1 + for start, chunk in zip(starts, chunks): + end = min(start + 100, len(content)) + assert chunk.text == content[start:end] + assert chunk.start_line == content[:start].count("\n") + 1 + assert chunk.end_line == content[:end].count("\n") + (not content[:end].endswith("\n")) + expected = FileChunk( + path=chunk.path, + start_line=chunk.start_line, + end_line=chunk.end_line, + text=chunk.text, + ).set_hash_id() + assert chunk.id == expected.id + assert node.chunk_ids == [chunk.id for chunk in chunks] + results.append([(chunk.text, chunk.start_line, chunk.end_line) for chunk in chunks]) + assert results[0] == results[1] + + +@pytest.mark.parametrize("suffix", [".md", ".txt"]) +def test_wikilinks_do_not_change_plain_text_boundaries(tmp_path, suffix): + """The default path may cut a literal wikilink just like any other text.""" + content = "x" * 90 + "[[a-very-long-target]]" + "y" * 90 + path = tmp_path / f"note{suffix}" + path.write_bytes(content.encode("utf-8")) + node, chunks = asyncio.run(DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=10).chunk(path)) + + assert [chunk.text for chunk in chunks] == [content[0:100], content[90:190], content[180:]] + assert node.links == [] + assert node.front_matter == FileFrontMatter() + + def test_parse_empty_file(): """Test parsing an empty file.""" @@ -255,7 +340,7 @@ def test_parse_links_no_match(): def test_parse_links_in_file(): - """Integration: parse() populates FileNode.links from file content.""" + """Literal wikilinks and YAML remain text without populating node metadata.""" async def run(): content = ( @@ -273,10 +358,11 @@ def test_parse_links_in_file(): try: chunker = DefaultFileChunker() - file_node, _ = await chunker.chunk(temp_path) - pairs = {(link.target_path, link.target_anchor) for link in file_node.links} - assert pairs == {("alpha", None), ("beta", "h2"), ("Alice", None), ("paper", "chapter 1")} - assert all(link.source_path == file_node.path for link in file_node.links) + file_node, chunks = await chunker.chunk(temp_path) + assert file_node.links == [] + assert file_node.front_matter == FileFrontMatter() + assert len(chunks) == 1 + assert chunks[0].text == content print("✓ test_parse_links_in_file passed") finally: os.unlink(temp_path) @@ -310,91 +396,6 @@ def test_parse_links_empty_when_no_content(): asyncio.run(run()) -def test_chunk_does_not_split_wikilink_at_boundary(): - """A wikilink straddling the chunk_byte_size boundary should be retreated to its start.""" - - async def run(): - # Pre-link filler is 90 bytes, link itself is 19 bytes ("[[a-very-long-target]]"=22). - # With chunk_byte_size=100, the boundary lands inside the link. - prefix = "x" * 90 - link = "[[a-very-long-target]]" # 22 bytes - suffix = "y" * 90 - content = f"{prefix}{link}{suffix}" - with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f: - f.write(content) - temp_path = f.name - - try: - chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=10) - _, chunks = await chunker.chunk(temp_path) - # The first chunk must NOT contain a partial link. - first = chunks[0].text - assert "[[" not in first or "]]" in first, f"first chunk has dangling '[[': {first!r}" - # And the link should appear intact in some chunk. - assert any(link in c.text for c in chunks), "link was split across all chunks" - print("✓ test_chunk_does_not_split_wikilink_at_boundary passed") - finally: - os.unlink(temp_path) - - asyncio.run(run()) - - -def test_chunk_does_not_split_wikilink_in_overlap(): - """A wikilink landing inside the overlap region should be advanced past.""" - - async def run(): - # 200-byte content, chunk=100, overlap=20. First chunk ends near byte 100, - # next start = 80. Place a link straddling byte 80 to land in the overlap. - prefix = "a" * 75 - link = "[[overlap-target]]" # 18 bytes; spans bytes 75..93 - suffix = "b" * 110 - content = f"{prefix}{link}{suffix}" - with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f: - f.write(content) - temp_path = f.name - - try: - chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=20) - _, chunks = await chunker.chunk(temp_path) - # No chunk should start mid-link. - for c in chunks: - t = c.text - if "]]" in t and "[[" not in t.split("]]", 1)[0]: - raise AssertionError(f"chunk starts mid-link: {t[:40]!r}") - assert any(link in c.text for c in chunks) - print("✓ test_chunk_does_not_split_wikilink_in_overlap passed") - finally: - os.unlink(temp_path) - - asyncio.run(run()) - - -def test_chunk_falls_back_for_oversize_link(): - """If a single link exceeds half the chunk size, the parser hard-cuts to make progress.""" - - async def run(): - # chunk=100, link is 80 bytes, surrounded by short filler. - # Retreating would leave a tiny chunk (< 50), so the fallback kicks in. - prefix = "x" * 30 - link = "[[" + ("L" * 76) + "]]" # 80 bytes total - suffix = "y" * 200 - content = f"{prefix}{link}{suffix}" - with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f: - f.write(content) - temp_path = f.name - - try: - chunker = DefaultFileChunker(chunk_byte_size=100, overlap_byte_size=10) - _, chunks = await chunker.chunk(temp_path) - # Must terminate (not hang) and cover the whole file. - assert len(chunks) >= 2 - print("✓ test_chunk_falls_back_for_oversize_link passed") - finally: - os.unlink(temp_path) - - asyncio.run(run()) - - def test_min_chunk_and_overlap_size(): """Test that minimum chunk and overlap sizes are enforced.""" @@ -435,8 +436,5 @@ if __name__ == "__main__": test_parse_links_no_match() test_parse_links_in_file() test_parse_links_empty_when_no_content() - test_chunk_does_not_split_wikilink_at_boundary() - test_chunk_does_not_split_wikilink_in_overlap() - test_chunk_falls_back_for_oversize_link() test_min_chunk_and_overlap_size() print("\n所有测试通过!") diff --git a/tests/unit/test_markdown_file_chunker.py b/tests/unit/test_markdown_file_chunker.py index fbeeffaf..62dec8d2 100644 --- a/tests/unit/test_markdown_file_chunker.py +++ b/tests/unit/test_markdown_file_chunker.py @@ -13,6 +13,8 @@ import os import tempfile from unittest.mock import patch +import pytest + from reme.components.file_chunker import DefaultFileChunker, MarkdownFileChunker @@ -171,6 +173,26 @@ def test_parse_frontmatter_only(): asyncio.run(run()) +@pytest.mark.parametrize("max_ast_sections", [0, None]) +def test_markdown_source_lines_and_metadata_remain_separate(tmp_path, max_ast_sections): + """AST and inherited byte fallback retain Markdown-specific semantics.""" + path = tmp_path / "note.md" + source = b"---\nname: t\n---\n# H\nline 1\n[[target.md]]\n" + path.write_bytes(source) + chunker = MarkdownFileChunker(max_ast_sections=max_ast_sections, include_frontmatter_in_metadata=True) + + node, chunks = asyncio.run(chunker.chunk(path)) + + assert len(chunks) == 1 + assert (chunks[0].start_line, chunks[0].end_line) == (4, 6) + assert "name: t" not in chunks[0].text + assert chunks[0].metadata == {"name": "t"} + assert node.front_matter.name == "t" + assert [link.target_path for link in node.links] == ["target.md"] + assert node.chunk_ids == [chunks[0].id] + assert path.read_bytes() == source + + def test_parse_frontmatter_metadata_is_opt_in(): """Chunk metadata preserves the old empty default unless explicitly enabled.""" @@ -510,7 +532,7 @@ def test_parse_excessive_sections_uses_plain_text_without_ast(): ) as chunk_content, ): node, chunks = await chunker.chunk(path) - assert chunk_content.call_count == 1 + chunk_content.assert_called_once_with(sections, path, parse_links=True) finally: await chunker.close() @@ -699,6 +721,91 @@ def test_parse_bad_frontmatter_does_not_abort_chunking(): asyncio.run(run()) +def test_chunk_does_not_split_wikilink_at_boundary(): + """A wikilink straddling the chunk_byte_size boundary should be retreated to its start.""" + + async def run(): + # Pre-link filler is 90 bytes, link itself is 19 bytes ("[[a-very-long-target]]"=22). + # With chunk_byte_size=100, the boundary lands inside the link. + prefix = "x" * 90 + link = "[[a-very-long-target]]" # 22 bytes + suffix = "y" * 90 + content = f"# H\n{prefix}{link}{suffix}" + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f: + f.write(content) + temp_path = f.name + + try: + chunker = MarkdownFileChunker(chunk_byte_size=100, overlap_byte_size=10, max_ast_sections=0) + _, chunks = await chunker.chunk(temp_path) + # The first chunk must NOT contain a partial link. + first = chunks[0].text + assert "[[" not in first or "]]" in first, f"first chunk has dangling '[[': {first!r}" + # And the link should appear intact in some chunk. + assert any(link in c.text for c in chunks), "link was split across all chunks" + print("✓ test_chunk_does_not_split_wikilink_at_boundary passed") + finally: + os.unlink(temp_path) + + asyncio.run(run()) + + +def test_chunk_does_not_split_wikilink_in_overlap(): + """A wikilink landing inside the overlap region should be advanced past.""" + + async def run(): + # 200-byte content, chunk=100, overlap=20. First chunk ends near byte 100, + # next start = 80. Place a link straddling byte 80 to land in the overlap. + prefix = "a" * 75 + link = "[[overlap-target]]" # 18 bytes; spans bytes 75..93 + suffix = "b" * 110 + content = f"# H\n{prefix}{link}{suffix}" + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f: + f.write(content) + temp_path = f.name + + try: + chunker = MarkdownFileChunker(chunk_byte_size=100, overlap_byte_size=20, max_ast_sections=0) + _, chunks = await chunker.chunk(temp_path) + # No chunk should start mid-link. + for c in chunks: + t = c.text + if "]]" in t and "[[" not in t.split("]]", 1)[0]: + raise AssertionError(f"chunk starts mid-link: {t[:40]!r}") + assert any(link in c.text for c in chunks) + print("✓ test_chunk_does_not_split_wikilink_in_overlap passed") + finally: + os.unlink(temp_path) + + asyncio.run(run()) + + +def test_chunk_falls_back_for_oversize_link(): + """If a single link exceeds half the chunk size, the parser hard-cuts to make progress.""" + + async def run(): + # chunk=100, link is 80 bytes, surrounded by short filler. + # Retreating would leave a tiny chunk (< 50), so the fallback kicks in. + prefix = "x" * 30 + link = "[[" + ("L" * 76) + "]]" # 80 bytes total + suffix = "y" * 200 + content = f"# H\n{prefix}{link}{suffix}" + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".md") as f: + f.write(content) + temp_path = f.name + + try: + chunker = MarkdownFileChunker(chunk_byte_size=100, overlap_byte_size=10, max_ast_sections=0) + _, chunks = await chunker.chunk(temp_path) + # Must terminate (not hang) and cover the whole file. + assert len(chunks) >= 2 + print("✓ test_chunk_falls_back_for_oversize_link passed") + finally: + os.unlink(temp_path) + + asyncio.run(run()) + + if __name__ == "__main__": print("\n=== MarkdownFileChunker tests ===") test_parse_empty_file()