mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-18 09:42:31 +00:00
fix(file-io): avoid backtracking in multiline edit searches
This commit is contained in:
parent
a1318e39ea
commit
ce2bd522af
2 changed files with 100 additions and 6 deletions
|
|
@ -11,6 +11,44 @@ from ..base_step import BaseStep
|
|||
from ...components import R
|
||||
|
||||
|
||||
def _replace_normalized_newlines(body: str, old: str, new: str) -> tuple[str, int]:
|
||||
"""Find literal LF text, mapping its spans back without normalizing untouched text."""
|
||||
normalized = body.replace("\r\n", "\n").replace("\r", "\n")
|
||||
position = normalized.find(old)
|
||||
if position < 0:
|
||||
return body, 0
|
||||
|
||||
# Only CRLF pairs change offsets. Consume them once as match boundaries advance;
|
||||
# no per-character offset array or regex search for the user-supplied needle.
|
||||
crlf_starts = (match.start() for match in re.finditer("\r\n", body))
|
||||
next_crlf = next(crlf_starts, len(body))
|
||||
removed = 0
|
||||
source_cursor = 0
|
||||
count = 0
|
||||
parts = []
|
||||
while position >= 0:
|
||||
source_start = position + removed
|
||||
while next_crlf < source_start:
|
||||
removed += 1
|
||||
source_start += 1
|
||||
next_crlf = next(crlf_starts, len(body))
|
||||
|
||||
end = position + len(old)
|
||||
source_end = end + removed
|
||||
while next_crlf < source_end:
|
||||
removed += 1
|
||||
source_end += 1
|
||||
next_crlf = next(crlf_starts, len(body))
|
||||
|
||||
parts.extend((body[source_cursor:source_start], new))
|
||||
source_cursor = source_end
|
||||
count += 1
|
||||
position = normalized.find(old, end)
|
||||
|
||||
parts.append(body[source_cursor:])
|
||||
return "".join(parts), count
|
||||
|
||||
|
||||
@R.register("edit_step")
|
||||
class EditStep(BaseStep):
|
||||
"""Replace every occurrence of ``old`` with ``new`` inside the file body.
|
||||
|
|
@ -91,12 +129,8 @@ class EditStep(BaseStep):
|
|||
body = raw_text
|
||||
not_found_msg = f"text to replace was not found in {target}"
|
||||
|
||||
if "\n" in old_str and "\r" not in old_str:
|
||||
# Match the LF view returned by read without rewriting unmatched text.
|
||||
# A CRLF pair is one line break, even when matching consecutive breaks.
|
||||
newline = r"(?:\r\n|\r(?!\n)|(?<!\r)\n)"
|
||||
pattern = newline.join(re.escape(part) for part in old_str.split("\n"))
|
||||
new_body, count = re.subn(pattern, lambda _: new_str, body)
|
||||
if "\r" in body and "\n" in old_str and "\r" not in old_str:
|
||||
new_body, count = _replace_normalized_newlines(body, old_str, new_str)
|
||||
else:
|
||||
count = body.count(old_str)
|
||||
new_body = body.replace(old_str, new_str)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""Keep ranged reads aligned with universal-newline chunking and large-file reads."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -10,6 +11,65 @@ from reme.steps.file_io import edit as edit_module
|
|||
from reme.steps.file_io import read as read_module
|
||||
|
||||
|
||||
@pytest.mark.parametrize("newline,suffix", [("\n", ".md"), ("\r\n", ".txt")])
|
||||
@pytest.mark.parametrize("found", [False, True])
|
||||
def test_edit_repeated_prefix_search(tmp_path, newline, suffix, found):
|
||||
"""A small stale edit must not spend seconds retrying a long shared prefix."""
|
||||
path = tmp_path / ("note" + suffix)
|
||||
source = ("x" + newline) * 100_000
|
||||
if found:
|
||||
source += "z"
|
||||
path.write_bytes(source.encode("utf-8"))
|
||||
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
|
||||
async def run():
|
||||
editor = edit_module.EditStep(app_context=app_context)
|
||||
started = time.process_time()
|
||||
await editor(path=path.name, old="x\n" * 2_000 + "z", new="replacement")
|
||||
cpu_seconds = time.process_time() - started
|
||||
assert editor.context.response.success is found
|
||||
if found:
|
||||
expected = ("x" + newline) * 98_000 + "replacement" + ("\n" if suffix == ".md" else "")
|
||||
assert "Replaced 1 occurrence(s)" in editor.context.response.answer
|
||||
assert path.read_bytes() == expected.encode("utf-8")
|
||||
else:
|
||||
assert "not found" in editor.context.response.answer
|
||||
assert path.read_bytes() == source.encode("utf-8")
|
||||
# CPU time excludes delays from other processes on shared CI runners.
|
||||
# The literal-search implementation is orders of magnitude below this bound.
|
||||
assert cpu_seconds < 1.0, f"repeated-prefix search used {cpu_seconds:.3f}s CPU"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source,old,expected",
|
||||
[
|
||||
("\r\na\r\nb\r\n", "a\nb", "\r\nR\r\n"),
|
||||
("\r\na\r\nb", "\na", "R\r\nb"),
|
||||
("a\r\nb\r\n", "b\n", "a\r\nR"),
|
||||
("\r\n\r\n", "\n", "RR"),
|
||||
("a\r\nb\rc\nd", "\n", "aRbRcRd"),
|
||||
("a\r\n\r\nb", "\n\n", "aRb"),
|
||||
("a\r\na\na\ra", "a\na", "R\nR"),
|
||||
("\r\na\u2028b\x0bc\r\n", "a\u2028b\x0bc\n", "\r\nR"),
|
||||
],
|
||||
)
|
||||
def test_edit_normalized_match_boundaries(tmp_path, source, old, expected):
|
||||
"""Mapped spans may start/end at a newline and must remain non-overlapping."""
|
||||
path = tmp_path / "note.txt"
|
||||
path.write_bytes(source.encode("utf-8"))
|
||||
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
|
||||
async def run():
|
||||
editor = edit_module.EditStep(app_context=app_context)
|
||||
await editor(path=path.name, old=old, new="R")
|
||||
assert editor.context.response.success is True
|
||||
assert path.read_bytes() == expected.encode("utf-8")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["utf-8", "utf-16"])
|
||||
@pytest.mark.parametrize("trailing_newline", [False, True])
|
||||
@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r", "mixed"])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue