up

reme2/component/file_parser/__init__.py

@@ -1,12 +1,10 @@
 """File parser implementations for different file formats."""
 
 from .base_file_parser import BaseFileParser
+from .default_file_parser import DefaultFileParser
 from .md_file_parser import MdFileParser
 from .text_file_parser import TextFileParser
 
-# Backwards-compat alias.
-DefaultFileParser = TextFileParser
-
 __all__ = [
     "BaseFileParser",
     "DefaultFileParser",

reme2/component/file_parser/base_file_parser.py

@@ -1,6 +1,7 @@
 """Base file parser interface."""
 
 from abc import abstractmethod
+from pathlib import Path
 
 from ..base_component import BaseComponent
 from ...enumeration import ComponentEnum
@@ -15,7 +16,31 @@ class BaseFileParser(BaseComponent):
 
     component_type = ComponentEnum.FILE_PARSER
 
-    async def parse(self, path: str, cache_chunks: list[FileChunk] | None = None) -> tuple[FileNode, list[FileChunk]]:
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        if self.app_context is not None:
+            self.working_dir = self.app_context.app_config.working_dir
+        else:
+            self.working_dir = str(Path.cwd())
+
+    def _get_relative_path(self, path: str | Path) -> str:
+        """Get path relative to working_dir.
+
+        Args:
+            path: Absolute or relative path to the file.
+
+        Returns:
+            Path relative to working_dir.
+        """
+        file_path = Path(path).absolute()
+        working_path = Path(self.working_dir).absolute()
+        try:
+            return str(file_path.relative_to(working_path))
+        except ValueError:
+            return str(file_path)
+
+    async def parse(self, path: str | Path, cache_chunks: list[FileChunk] | None = None) -> tuple[
+        FileNode, list[FileChunk]]:
         """Parse a file and optionally reuse cached chunks by hash.
 
         Args:
@@ -35,7 +60,7 @@ class BaseFileParser(BaseComponent):
         return file_node, chunks
 
     @abstractmethod
-    async def _parse(self, path: str) -> tuple[FileNode, list[FileChunk]]:
+    async def _parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
         """Parse a file into metadata and chunks. Subclasses must implement this.
 
         Args:

reme2/component/file_parser/default_file_parser.py

@@ -0,0 +1,61 @@
+"""Default file parser with byte-based chunking."""
+
+import hashlib
+from pathlib import Path
+
+import aiofiles
+
+from .base_file_parser import BaseFileParser
+from ..component_registry import R
+from ...schema import FileChunk, FileNode
+
+
+@R.register("default")
+class DefaultFileParser(BaseFileParser):
+    """Parser for files using byte-based chunking."""
+
+    def __init__(self, encoding: str = "utf-8", chunk_byte_sizes: int = 10000, overlap_byte_size: int = 100, **kwargs):
+        super().__init__(**kwargs)
+        self.encoding = encoding
+        self.chunk_byte_size = max(100, chunk_byte_sizes)
+        # overlap >= 4 ensures truncated multibyte UTF-8 chars decode correctly in next chunk
+        self.overlap_byte_size = max(4, overlap_byte_size)
+
+    async def _parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
+        file_path = Path(path)
+        stat = file_path.stat()
+        relative_path = self._get_relative_path(path)
+
+        async with aiofiles.open(file_path, "rb") as f:
+            data = await f.read()
+
+        if not data:
+            return FileNode(path=relative_path, st_mtime=stat.st_mtime), []
+
+        # Find all newline byte positions
+        newline_positions = [i for i, b in enumerate(data) if b == ord(b"\n")]
+
+        chunks: list[FileChunk] = []
+        step = self.chunk_byte_size - self.overlap_byte_size
+        start_byte = 0
+
+        while start_byte < len(data):
+            end_byte = min(start_byte + self.chunk_byte_size, len(data))
+            chunk_data = data[start_byte:end_byte]
+            text = chunk_data.decode(self.encoding, errors="ignore")
+
+            # Calculate line numbers from byte positions
+            start_line = sum(1 for p in newline_positions if p < start_byte) + 1
+            end_line = sum(1 for p in newline_positions if p < end_byte) + 1
+
+            h = hashlib.sha256(text.encode()).hexdigest()
+            chunks.append(FileChunk(
+                path=relative_path,
+                start_line=start_line,
+                end_line=end_line,
+                text=text,
+            ))
+
+            start_byte += step if end_byte < len(data) else len(data)
+
+        return FileNode(path=relative_path, st_mtime=stat.st_mtime), chunks