From bce64e76d04580c158294e81a26af5e862b3da58 Mon Sep 17 00:00:00 2001 From: cangen185-sudo <234582933+cangen185-sudo@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:08:50 +0000 Subject: [PATCH] fix(read): normalize newlines before selecting source lines --- reme/steps/file_io/read.py | 2 ++ tests/unit/test_read_newlines.py | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 tests/unit/test_read_newlines.py diff --git a/reme/steps/file_io/read.py b/reme/steps/file_io/read.py index 48f53aea..d8d1b23f 100644 --- a/reme/steps/file_io/read.py +++ b/reme/steps/file_io/read.py @@ -156,6 +156,8 @@ class ReadStep(BaseStep): if content is None: return None + # Match text-mode streaming reads and the chunker's source line numbers. + content = content.replace("\r\n", "\n").replace("\r", "\n") all_lines = content.split("\n") if content.endswith("\n"): # split() yields a trailing empty element for the final newline; diff --git a/tests/unit/test_read_newlines.py b/tests/unit/test_read_newlines.py new file mode 100644 index 00000000..32131c33 --- /dev/null +++ b/tests/unit/test_read_newlines.py @@ -0,0 +1,56 @@ +"""Keep ranged reads aligned with universal-newline chunking and large-file reads.""" + +import asyncio + +import pytest + +from reme.components import ApplicationContext +from reme.components.file_chunker import DefaultFileChunker +from reme.steps.file_io import read as read_module + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-16"]) +@pytest.mark.parametrize("trailing_newline", [False, True]) +@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r", "mixed"]) +def test_read_line_ranges_match_chunker_and_streaming_path(tmp_path, monkeypatch, encoding, trailing_newline, newline): + """The same bytes must produce the same line ranges on both read paths.""" + logical_lines = ["first\u2028inline", "second 中文", "third\x0binline", "fourth"] + separators = ["\r\n", "\r", "\n"] if newline == "mixed" else [newline] * 3 + text = logical_lines[0] + "".join(separator + line for separator, line in zip(separators, logical_lines[1:])) + if trailing_newline: + text += "\r" if newline == "mixed" else newline + path = tmp_path / "note.md" + source = text.encode(encoding) + path.write_bytes(source) + app_context = ApplicationContext(workspace_dir=str(tmp_path)) + + async def run(): + # UTF-8 is the normal indexer configuration. UTF-16 still exercises the + # read helper's BOM detection without asserting byte-chunker geometry. + if encoding == "utf-8": + _, chunks = await DefaultFileChunker(encoding=encoding, app_context=app_context).chunk(path) + assert chunks[0].start_line == 1 + assert chunks[0].end_line == len(logical_lines) + + responses = [] + for threshold in (len(source) + 1, 1): + monkeypatch.setattr(read_module, "MAX_FILE_READ_BYTES", threshold) + step = read_module.ReadStep(app_context=app_context) + await step(path="note.md", start_line=2, end_line=3) + assert step.context.response.success is True + responses.append(step.context.response.answer) + + complete = read_module.ReadStep(app_context=app_context) + await complete(path="note.md") + assert complete.context.response.success is True + assert complete.context.response.answer == "\n".join(logical_lines) + + past_end = read_module.ReadStep(app_context=app_context) + await past_end(path="note.md", start_line=5) + assert past_end.context.response.success is False + assert "exceeds file length (4 lines)" in past_end.context.response.answer + + assert responses == ["\n".join(logical_lines[1:3])] * 2 + assert path.read_bytes() == source + + asyncio.run(run())