fix(index): tolerate invalid text encoding (#490)

* fix(index): tolerate invalid text encoding

* fix(index): preserve text chunker compatibility

docs/en/memory_as_file.md

@@ -388,3 +388,8 @@ This lets the agent see not only an isolated paragraph but also its structural p
 
 Non-Markdown files use `DefaultFileChunker` by default. It splits by byte size and preserves a small overlap. For
 Markdown, the chunker also avoids cutting `[[wikilinks]]` in the middle.
+
+`DefaultFileChunker` and `MarkdownFileChunker` decode files with their configured `encoding` and normalize platform
+newlines to LF before indexing. Their default `invalid_encoding_policy: replace` keeps decodable content searchable
+when a source contains invalid bytes, without modifying the source file. Set `invalid_encoding_policy: strict` on a
+chunker component to reject such files instead.

docs/zh/memory_as_file.md

@@ -363,3 +363,7 @@ FileChunk[]
 这样检索命中时，Agent 不只看到孤立段落，还能看到它在原文件中的结构位置。
 
 非 Markdown 默认走 `DefaultFileChunker`：按字节大小切分，并保留少量 overlap；对 Markdown 则会避免把 `[[wikilink]]` 从中间切开。
+
+`DefaultFileChunker` 和 `MarkdownFileChunker` 使用各自配置的 `encoding` 解码文件，并在索引前将平台换行符统一为
+LF。默认的 `invalid_encoding_policy: replace` 会在源文件含无效字节时保留其中可解码的内容用于检索，但不会修改源
+文件；如需拒绝此类文件，可在 chunker 组件上设置 `invalid_encoding_policy: strict`。

reme/components/file_chunker/default_file_chunker.py

@@ -2,6 +2,7 @@
 
 from bisect import bisect_right
 from pathlib import Path
+from typing import Literal
 
 import aiofiles
 import yaml
@@ -11,14 +12,29 @@ from ..component_registry import R
 from ...schema import FileChunk, FileFrontMatter, FileNode
 from ...utils.wikilink_handler import WikilinkHandler
 
+InvalidEncodingPolicy = Literal["replace", "strict"]
+
 
 @R.register("default")
 class DefaultFileChunker(BaseFileChunker):
     """Default chunker that splits files into byte-based overlapping chunks."""
 
-    def __init__(self, encoding: str = "utf-8", chunk_byte_size: int = 10000, overlap_byte_size: int = 100, **kwargs):
+    def __init__(
+        self,
+        encoding: str = "utf-8",
+        chunk_byte_size: int = 10000,
+        overlap_byte_size: int = 100,
+        *,
+        invalid_encoding_policy: InvalidEncodingPolicy = "replace",
+        **kwargs,
+    ):
         super().__init__(**kwargs)
+        if invalid_encoding_policy not in {"replace", "strict"}:
+            raise ValueError(
+                f"invalid_encoding_policy must be 'replace' or 'strict', got {invalid_encoding_policy!r}",
+            )
         self.encoding = encoding
+        self.invalid_encoding_policy = invalid_encoding_policy
         self.chunk_byte_size = max(100, chunk_byte_size)
         self.overlap_byte_size = max(4, overlap_byte_size)
 
@@ -37,13 +53,35 @@ class DefaultFileChunker(BaseFileChunker):
             front_matter = FileFrontMatter()
         return front_matter, text[end_idx + 4 :].lstrip("\n")
 
+    async def _read_text_for_indexing(self, file_path: Path) -> str:
+        """Decode text for a derived index without modifying the source file."""
+        async with aiofiles.open(file_path, "rb") as f:
+            data = await f.read()
+        try:
+            text = data.decode(self.encoding)
+        except UnicodeDecodeError as exc:
+            if self.invalid_encoding_policy == "strict":
+                raise
+            invalid_bytes = data[exc.start : exc.end].hex(" ")
+            self.logger.warning(
+                f"Invalid {self.encoding} in {file_path} at byte {exc.start} (bytes: {invalid_bytes}); "
+                "indexed with replacement characters; source file unchanged",
+            )
+            text = data.decode(self.encoding, errors="replace")
+            # Some codecs (for example ASCII) cannot encode U+FFFD. Convert the
+            # decoded fallback to that codec's own replacement representation so
+            # later byte-based chunking remains safe.
+            text = text.encode(self.encoding, errors="replace").decode(self.encoding)
+
+        # Match the universal-newline behavior of the previous text-mode reads.
+        return text.replace("\r\n", "\n").replace("\r", "\n")
+
     async def chunk(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
         file_path = Path(path)
         stat = file_path.stat()
         rel_path = self.to_workspace_relative(path)
 
-        async with aiofiles.open(file_path, encoding=self.encoding) as f:
-            text = await f.read()
+        text = await self._read_text_for_indexing(file_path)
 
         if not text:
             return FileNode(path=rel_path, st_mtime=stat.st_mtime), []

reme/components/file_chunker/markdown_file_chunker.py

@@ -24,7 +24,7 @@ import yaml
 from pydantic import ValidationError
 
 
-from .default_file_chunker import DefaultFileChunker
+from .default_file_chunker import DefaultFileChunker, InvalidEncodingPolicy
 from ..component_registry import R
 from ...schema import (
     FileChunk,
@@ -108,9 +108,16 @@ class MarkdownFileChunker(DefaultFileChunker):
         max_ast_sections: int | None = 100,
         include_frontmatter_in_metadata: bool = False,
         include_frontmatter_keys_in_metadata: list[str] | None = None,
+        *,
+        invalid_encoding_policy: InvalidEncodingPolicy = "replace",
         **kwargs,
     ):
-        super().__init__(encoding=encoding, chunk_byte_size=chunk_byte_size, **kwargs)
+        super().__init__(
+            encoding=encoding,
+            invalid_encoding_policy=invalid_encoding_policy,
+            chunk_byte_size=chunk_byte_size,
+            **kwargs,
+        )
         self.embed_toc = embed_toc
         self.max_ast_sections = max(0, max_ast_sections) if max_ast_sections is not None else None
         self.include_frontmatter_in_metadata = include_frontmatter_in_metadata
@@ -119,7 +126,8 @@ class MarkdownFileChunker(DefaultFileChunker):
     async def chunk(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
         file_path = Path(path)
         rel_path = self.to_workspace_relative(path)
-        front_matter, content, line_offset = self._parse_front_matter(file_path.read_text(encoding=self.encoding))
+        text = await self._read_text_for_indexing(file_path)
+        front_matter, content, line_offset = self._parse_front_matter(text)
 
         chunks: list[FileChunk] = []
         if content and content.strip():

tests/unit/test_default_file_chunker.py

@@ -97,6 +97,69 @@ def test_parse_with_custom_encoding():
     asyncio.run(run())
 
 
+def test_invalid_encoding_policy_is_validated():
+    """Reject misspelled policies instead of silently changing decoding behavior."""
+    try:
+        DefaultFileChunker(invalid_encoding_policy="ignore")
+    except ValueError as exc:
+        assert "invalid_encoding_policy" in str(exc)
+    else:
+        raise AssertionError("invalid encoding policy must be rejected")
+
+
+def test_constructor_preserves_positional_arguments():
+    """New decoding options must not reinterpret the established positional API."""
+    chunker = DefaultFileChunker("utf-8", 5000, 100)
+
+    assert chunker.encoding == "utf-8"
+    assert chunker.chunk_byte_size == 5000
+    assert chunker.overlap_byte_size == 100
+
+
+def test_newlines_are_normalized_before_chunking():
+    """Binary reads retain the universal-newline behavior of the old text reader."""
+
+    async def run():
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f:
+            f.write(b"alpha\r\nbeta\rgamma\n")
+            temp_path = f.name
+
+        try:
+            chunker = DefaultFileChunker()
+            _, original_chunks = await chunker.chunk(temp_path)
+            with open(temp_path, "wb") as f:
+                f.write(b"alpha\nbeta\ngamma\n")
+            _, normalized_chunks = await chunker.chunk(temp_path)
+
+            assert [chunk.text for chunk in original_chunks] == ["alpha\nbeta\ngamma\n"]
+            assert [chunk.id for chunk in original_chunks] == [chunk.id for chunk in normalized_chunks]
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
+def test_ascii_replacement_remains_encodable():
+    """Replacement mode must survive byte-based chunking with a single-byte codec."""
+
+    async def run():
+        source = b"valid\xffinvalid"
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f:
+            f.write(source)
+            temp_path = f.name
+
+        try:
+            _, chunks = await DefaultFileChunker(encoding="ascii").chunk(temp_path)
+
+            assert [chunk.text for chunk in chunks] == ["valid?invalid"]
+            with open(temp_path, "rb") as f:
+                assert f.read() == source
+        finally:
+            os.unlink(temp_path)
+
+    asyncio.run(run())
+
+
 def test_parse_links_bare():
     """Bare wikilink: [[target]]."""
     links = WikilinkHandler.extract_links("see [[note]]", "src.md")

tests/unit/test_markdown_file_chunker.py

@@ -57,6 +57,104 @@ def test_parse_empty_file():
     asyncio.run(run())
 
 
+def test_invalid_utf8_is_replaced_without_modifying_source():
+    """Bad source bytes degrade the derived index but remain untouched on disk."""
+
+    async def run():
+        with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+            source = b"# valid\ncontent before \xcd content after\n"
+            path = os.path.join(tmp, "invalid.md")
+            with open(path, "wb") as f:
+                f.write(source)
+
+            chunker = MarkdownFileChunker()
+            with patch.object(chunker.logger, "warning") as warning:
+                node, chunks = await chunker.chunk("invalid.md")
+
+            assert node.path == "invalid.md"
+            assert "content before \ufffd content after" in "\n".join(chunk.text for chunk in chunks)
+            with open(path, "rb") as f:
+                assert f.read() == source
+            warning.assert_called_once_with(
+                "Invalid utf-8 in invalid.md at byte 23 (bytes: cd); "
+                "indexed with replacement characters; source file unchanged",
+            )
+
+    asyncio.run(run())
+
+
+def test_invalid_utf8_strict_policy_still_raises():
+    """Strict mode remains available when callers require exact decoding."""
+
+    async def run():
+        with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+            with open(os.path.join(tmp, "invalid.md"), "wb") as f:
+                f.write(b"valid\xcdinvalid")
+
+            chunker = MarkdownFileChunker(invalid_encoding_policy="strict")
+            try:
+                await chunker.chunk("invalid.md")
+            except UnicodeDecodeError as exc:
+                assert exc.start == 5
+            else:
+                raise AssertionError("strict policy must reject invalid UTF-8")
+
+    asyncio.run(run())
+
+
+def test_constructor_preserves_positional_arguments():
+    """The decoding policy must not shift the established positional parameters."""
+    chunker = MarkdownFileChunker("utf-8", 5000, False, 10, True, ["name"])
+
+    assert chunker.encoding == "utf-8"
+    assert chunker.chunk_byte_size == 5000
+    assert chunker.embed_toc is False
+    assert chunker.max_ast_sections == 10
+    assert chunker.include_frontmatter_in_metadata is True
+    assert chunker.include_frontmatter_keys_in_metadata == ["name"]
+
+
+def test_plain_text_fallback_normalizes_newlines():
+    """Markdown fallback chunks stay stable for equivalent platform newlines."""
+
+    async def run():
+        with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+            path = os.path.join(tmp, "fallback.md")
+            with open(path, "wb") as f:
+                f.write(b"# One\r\nbody\r# Two\r\nbody\r\n")
+
+            chunker = MarkdownFileChunker(max_ast_sections=0)
+            _, original_chunks = await chunker.chunk("fallback.md")
+            with open(path, "wb") as f:
+                f.write(b"# One\nbody\n# Two\nbody\n")
+            _, normalized_chunks = await chunker.chunk("fallback.md")
+
+            assert [chunk.text for chunk in original_chunks] == [chunk.text for chunk in normalized_chunks]
+            assert [chunk.id for chunk in original_chunks] == [chunk.id for chunk in normalized_chunks]
+
+    asyncio.run(run())
+
+
+def test_invalid_ascii_is_replaced_for_markdown():
+    """Markdown byte accounting accepts the configured codec's replacement text."""
+
+    async def run():
+        with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
+            source = b"# valid\ncontent before \xff content after\n"
+            path = os.path.join(tmp, "invalid.md")
+            with open(path, "wb") as f:
+                f.write(source)
+
+            chunker = MarkdownFileChunker(encoding="ascii")
+            _, chunks = await chunker.chunk("invalid.md")
+
+            assert "content before ? content after" in "\n".join(chunk.text for chunk in chunks)
+            with open(path, "rb") as f:
+                assert f.read() == source
+
+    asyncio.run(run())
+
+
 def test_parse_frontmatter_only():
     """A file with only frontmatter (no body) → no chunks, no links."""