up

reme2/component/file_parser/__init__.py

@@ -2,12 +2,10 @@
 
 from .base_file_parser import BaseFileParser
 from .default_file_parser import DefaultFileParser
-from .md_file_parser import MdFileParser
-from .text_file_parser import TextFileParser
+from .linked_file_parser import LinkedFileParser
 
 __all__ = [
     "BaseFileParser",
     "DefaultFileParser",
-    "MdFileParser",
-    "TextFileParser",
+    "LinkedFileParser",
 ]

reme2/component/file_parser/linked_file_parser.py

@@ -36,7 +36,7 @@ from ...utils import hash_text
 
 
 @R.register("md")
-class MdFileParser(BaseFileParser):
+class LinkedFileParser(BaseFileParser):
     """Parser for Markdown files with YAML frontmatter and wikilink support."""
 
     suffixes = [FileSuffixEnum.MD, FileSuffixEnum.MARKDOWN]

reme2/component/file_parser/text_file_parser.py

@@ -1,117 +0,0 @@
-"""Text file parser with built-in chunking."""
-
-import hashlib
-from pathlib import Path
-
-import aiofiles
-
-from .base_file_parser import BaseFileParser
-from ..component_registry import R
-from ...enumeration import FileSuffixEnum
-from ...schema import FileChunk, ParsedFile
-
-
-@R.register("text")
-class TextFileParser(BaseFileParser):
-    """Parser for text files with built-in chunking support."""
-
-    suffixes = [FileSuffixEnum.TXT]
-
-    def __init__(self, encoding: str = "utf-8", chunk_tokens: int = 400, chunk_overlap: int = 80, **kwargs):
-        super().__init__(**kwargs)
-        self.encoding = encoding
-        self.chunk_size = max(32, chunk_tokens * 4)
-        self.overlap_size = max(0, chunk_overlap * 4)
-
-    async def parse(
-        self,
-        path: str,
-        existing_chunks: list[FileChunk] | None = None,
-    ) -> ParsedFile:
-        file_path = Path(path)
-        stat = file_path.stat()
-        absolute_path = str(file_path.absolute())
-
-        try:
-            async with aiofiles.open(file_path, encoding=self.encoding) as f:
-                content = await f.read()
-        except UnicodeDecodeError:
-            async with aiofiles.open(file_path, encoding=self.encoding, errors="ignore") as f:
-                content = await f.read()
-        except Exception:
-            content = ""
-
-        chunks = self._chunk(content, absolute_path) if content else []
-        dirty = self._hash_diff_attach(chunks, existing_chunks)
-        if dirty:
-            await self._embed_chunks(dirty)
-
-        return ParsedFile(
-            file=file_path.stem,
-            path=absolute_path,
-            st_mtime=stat.st_mtime,
-            chunks=chunks,
-        )
-
-    def _chunk(self, text: str, path: str) -> list[FileChunk]:
-        """Split text into chunks with overlap."""
-        if not text.strip():
-            return []
-
-        lines = text.split("\n")
-        chunks: list[FileChunk] = []
-        buf: list[tuple[str, int]] = []  # (line_content, line_no)
-        buf_chars = 0
-
-        for line_no, line in enumerate(lines, 1):
-            # Split long lines into segments
-            for start in range(0, max(1, len(line)), self.chunk_size):
-                seg = line[start : start + self.chunk_size]
-                seg_chars = len(seg) + 1  # +1 for newline
-
-                # Flush when buffer would exceed limit
-                if buf and buf_chars + seg_chars > self.chunk_size:
-                    self._flush_chunk(chunks, buf, path)
-                    buf, buf_chars = self._carry_overlap(buf)
-
-                buf.append((seg, line_no))
-                buf_chars += seg_chars
-
-        if buf:
-            self._flush_chunk(chunks, buf, path)
-
-        return [c for c in chunks if c.text.strip()]
-
-    @staticmethod
-    def _flush_chunk(chunks: list[FileChunk], buf: list[tuple[str, int]], path: str):
-        """Create a chunk from buffer and append to chunks list."""
-        chunk_text = "\n".join(content for content, _ in buf)
-        start_line, end_line = buf[0][1], buf[-1][1]
-        h = hashlib.sha256(chunk_text.encode()).hexdigest()
-        chunk_id = hashlib.sha256(f"{path}:{start_line}:{end_line}:{h}:{len(chunks)}".encode()).hexdigest()
-
-        chunks.append(
-            FileChunk(
-                id=chunk_id,
-                path=path,
-                start_line=start_line,
-                end_line=end_line,
-                text=chunk_text,
-                hash=h,
-            ),
-        )
-
-    def _carry_overlap(self, buf: list[tuple[str, int]]) -> tuple[list[tuple[str, int]], int]:
-        """Keep overlapping lines from the end of buffer."""
-        if self.overlap_size <= 0 or not buf:
-            return [], 0
-
-        acc, kept = 0, []
-        for content, line_no in reversed(buf):
-            acc += len(content) + 1
-            kept.insert(0, (content, line_no))
-            if acc >= self.overlap_size:
-                break
-
-        buf_chars = sum(len(c) + 1 for c, _ in kept)
-        return kept, buf_chars

reme2/schema/__init__.py

@@ -6,7 +6,7 @@ from .emb_node import EmbNode
 from .chunk_filter import ChunkFilter
 from .file_chunk import FileChunk
 from .file_edge import FileEdge
-from .file_node import FileNode
+from .file_node import FileNode, FileMetadata
 from .parsed_file import ParsedFile
 from .request import Request
 from .response import Response
@@ -23,6 +23,7 @@ __all__ = [
     "FileChunk",
     "FileEdge",
     "FileNode",
+    "FileMetadata",
     "ParsedFile",
     "Request",
     "Response",

reme2/schema/file_node.py

@@ -1,6 +1,6 @@
-from pydantic import BaseModel, Field, ConfigDict
+from __future__ import annotations
 
-from reme.core.schema import FileMetadata
+from pydantic import BaseModel, Field, ConfigDict
 
 
 class FileNode(BaseModel):
@@ -8,21 +8,26 @@ class FileNode(BaseModel):
 
     path: str = Field(...)
     st_mtime: float = Field(...)
-    edges: list[FileEdge] = Field(default_factory=list)
+    edges: list["FileEdge"] = Field(default_factory=list)
 
     @property
     def file(self):
         ...
 
 
-
 class FileMetadata(BaseModel):
     # FileNode -> FileMetadata
     path: str = Field(...)
 
-    edges: list[FileEdge] = Field(default_factory=list)
+    edges: list["FileEdge"] = Field(default_factory=list)
 
     title: str = Field(default="")
     description: str = Field(default="")
     tags: list[str] = Field(default_factory=list)
 
+
+# Rebuild models after FileEdge is defined
+from .file_edge import FileEdge
+
+FileNode.model_rebuild()
+FileMetadata.model_rebuild()

reme2/schema/parsed_file.py

@@ -20,7 +20,7 @@ from pydantic import Field
 
 from .file_chunk import FileChunk
 from .file_edge import FileEdge
-from .file_metadata import FileMetadata
+from .file_node import FileMetadata
 
 
 class ParsedFile(FileMetadata):

test/reme2/test_default_file_parser.py

@@ -0,0 +1,209 @@
+"""Tests for DefaultFileParser."""
+
+import asyncio
+import tempfile
+import os
+from pathlib import Path
+
+# Add parent path for import
+import sys
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from reme2.component.file_parser import DefaultFileParser
+
+
+def test_parse_empty_file():
+    """Test parsing an empty file."""
+
+    async def run():
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+            temp_path = f.name
+
+        try:
+            parser = DefaultFileParser()
+            file_node, chunks = await parser.parse(temp_path)
+            assert file_node.path == temp_path
+            assert len(chunks) == 0
+            print("✓ test_parse_empty_file passed")
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
+def test_parse_small_file():
+    """Test parsing a file smaller than chunk size."""
+
+    async def run():
+        content = "Hello World\nThis is a test\nLine 3"
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+            f.write(content)
+            temp_path = f.name
+
+        try:
+            parser = DefaultFileParser(chunk_byte_size=10000)
+            file_node, chunks = await parser.parse(temp_path)
+            assert len(chunks) == 1
+            assert chunks[0].start_line == 1
+            assert chunks[0].end_line == 3
+            assert chunks[0].text == content
+            print("✓ test_parse_small_file passed")
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
+def test_parse_multiline_file():
+    """Test parsing a file with multiple lines."""
+
+    async def run():
+        lines = ["Line 1", "Line 2", "Line 3", "Line 4", "Line 5"]
+        content = "\n".join(lines)
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+            f.write(content)
+            temp_path = f.name
+
+        try:
+            parser = DefaultFileParser(chunk_byte_size=10000)
+            file_node, chunks = await parser.parse(temp_path)
+            assert len(chunks) == 1
+            assert chunks[0].start_line == 1
+            assert chunks[0].end_line == 5
+            print("✓ test_parse_multiline_file passed")
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
+def test_parse_chunked_file():
+    """Test parsing a file that requires multiple chunks."""
+
+    async def run():
+        # Create content larger than chunk size
+        lines = ["A" * 100 for _ in range(200)]  # ~20200 bytes
+        content = "\n".join(lines)
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+            f.write(content)
+            temp_path = f.name
+
+        try:
+            parser = DefaultFileParser(chunk_byte_size=5000, overlap_byte_size=100)
+            file_node, chunks = await parser.parse(temp_path)
+            assert len(chunks) > 1, f"Expected multiple chunks, got {len(chunks)}"
+            # Verify overlap by checking that consecutive chunks share some content
+            print(f"  Created {len(chunks)} chunks")
+            print("✓ test_parse_chunked_file passed")
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
+def test_parse_with_custom_encoding():
+    """Test parsing a file with different encodings."""
+
+    async def run():
+        content = "你好世界\n测试内容"
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt", encoding="utf-8") as f:
+            f.write(content)
+            temp_path = f.name
+
+        try:
+            parser = DefaultFileParser(encoding="utf-8")
+            file_node, chunks = await parser.parse(temp_path)
+            assert len(chunks) >= 1
+            assert "你好世界" in chunks[0].text
+            print("✓ test_parse_with_custom_encoding passed")
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
+def test_file_node_properties():
+    """Test FileNode has correct properties."""
+
+    async def run():
+        content = "test content"
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+            f.write(content)
+            temp_path = f.name
+
+        try:
+            parser = DefaultFileParser()
+            file_node, chunks = await parser.parse(temp_path)
+            assert hasattr(file_node, "path")
+            assert hasattr(file_node, "st_mtime")
+            assert file_node.st_mtime > 0
+            print("✓ test_file_node_properties passed")
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
+def test_file_chunk_properties():
+    """Test FileChunk has correct properties."""
+
+    async def run():
+        content = "test content for chunk"
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+            f.write(content)
+            temp_path = f.name
+
+        try:
+            parser = DefaultFileParser()
+            file_node, chunks = await parser.parse(temp_path)
+            chunk = chunks[0]
+            assert hasattr(chunk, "path")
+            assert hasattr(chunk, "start_line")
+            assert hasattr(chunk, "end_line")
+            assert hasattr(chunk, "text")
+            assert hasattr(chunk, "id")
+            assert hasattr(chunk, "hash")
+            assert chunk.start_line >= 1
+            assert chunk.end_line >= chunk.start_line
+            print("✓ test_file_chunk_properties passed")
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
+def test_min_chunk_and_overlap_size():
+    """Test that minimum chunk and overlap sizes are enforced."""
+
+    async def run():
+        # These values should be clamped to minimums
+        parser = DefaultFileParser(chunk_byte_size=1, overlap_byte_size=0)
+        assert parser.chunk_byte_size == 100  # minimum
+        assert parser.overlap_byte_size == 4  # minimum
+
+        content = "test"
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as f:
+            f.write(content)
+            temp_path = f.name
+
+        try:
+            file_node, chunks = await parser.parse(temp_path)
+            assert len(chunks) == 1
+            print("✓ test_min_chunk_and_overlap_size passed")
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
+if __name__ == "__main__":
+    test_parse_empty_file()
+    test_parse_small_file()
+    test_parse_multiline_file()
+    test_parse_chunked_file()
+    test_parse_with_custom_encoding()
+    test_file_node_properties()
+    test_file_chunk_properties()
+    test_min_chunk_and_overlap_size()
+    print("\n所有测试通过!")