mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-20 10:02:58 +00:00
fix(file-io): match normalized multiline reads during edit
This commit is contained in:
parent
bce64e76d0
commit
a1318e39ea
2 changed files with 145 additions and 7 deletions
|
|
@ -1,5 +1,7 @@
|
|||
"""Find-and-replace text in a markdown file body (front matter is preserved)."""
|
||||
|
||||
import re
|
||||
|
||||
import frontmatter
|
||||
import yaml
|
||||
|
||||
|
|
@ -89,13 +91,20 @@ class EditStep(BaseStep):
|
|||
body = raw_text
|
||||
not_found_msg = f"text to replace was not found in {target}"
|
||||
|
||||
if old_str not in body:
|
||||
if "\n" in old_str and "\r" not in old_str:
|
||||
# Match the LF view returned by read without rewriting unmatched text.
|
||||
# A CRLF pair is one line break, even when matching consecutive breaks.
|
||||
newline = r"(?:\r\n|\r(?!\n)|(?<!\r)\n)"
|
||||
pattern = newline.join(re.escape(part) for part in old_str.split("\n"))
|
||||
new_body, count = re.subn(pattern, lambda _: new_str, body)
|
||||
else:
|
||||
count = body.count(old_str)
|
||||
new_body = body.replace(old_str, new_str)
|
||||
|
||||
if count == 0:
|
||||
self._fail(not_found_msg, path=str(target))
|
||||
return None
|
||||
|
||||
count = body.count(old_str)
|
||||
new_body = body.replace(old_str, new_str)
|
||||
|
||||
if is_md and post is not None:
|
||||
post.content = new_body
|
||||
# Re-serialize: keep front matter when present, otherwise emit body alone
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import pytest
|
|||
|
||||
from reme.components import ApplicationContext
|
||||
from reme.components.file_chunker import DefaultFileChunker
|
||||
from reme.steps.file_io import edit as edit_module
|
||||
from reme.steps.file_io import read as read_module
|
||||
|
||||
|
||||
|
|
@ -32,13 +33,12 @@ def test_read_line_ranges_match_chunker_and_streaming_path(tmp_path, monkeypatch
|
|||
assert chunks[0].start_line == 1
|
||||
assert chunks[0].end_line == len(logical_lines)
|
||||
|
||||
responses = []
|
||||
for threshold in (len(source) + 1, 1):
|
||||
monkeypatch.setattr(read_module, "MAX_FILE_READ_BYTES", threshold)
|
||||
step = read_module.ReadStep(app_context=app_context)
|
||||
await step(path="note.md", start_line=2, end_line=3)
|
||||
assert step.context.response.success is True
|
||||
responses.append(step.context.response.answer)
|
||||
assert step.context.response.answer == "\n".join(logical_lines[1:3])
|
||||
|
||||
complete = read_module.ReadStep(app_context=app_context)
|
||||
await complete(path="note.md")
|
||||
|
|
@ -50,7 +50,136 @@ def test_read_line_ranges_match_chunker_and_streaming_path(tmp_path, monkeypatch
|
|||
assert past_end.context.response.success is False
|
||||
assert "exceeds file length (4 lines)" in past_end.context.response.answer
|
||||
|
||||
assert responses == ["\n".join(logical_lines[1:3])] * 2
|
||||
assert path.read_bytes() == source
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["utf-8", "utf-16"])
|
||||
@pytest.mark.parametrize("trailing_newline", [False, True])
|
||||
@pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"])
|
||||
@pytest.mark.parametrize("suffix", [".md", ".txt"])
|
||||
def test_read_then_edit_multiline(tmp_path, encoding, trailing_newline, newline, suffix):
|
||||
"""Read output remains usable as an edit needle without changing write rules."""
|
||||
text = f"first\u2028inline{newline}second\x0binline"
|
||||
if trailing_newline:
|
||||
text += newline
|
||||
path = tmp_path / ("note" + suffix)
|
||||
source = text.encode(encoding)
|
||||
path.write_bytes(source)
|
||||
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
|
||||
async def run():
|
||||
reader = read_module.ReadStep(app_context=app_context)
|
||||
await reader(path=path.name)
|
||||
assert reader.context.response.success is True
|
||||
assert reader.context.response.answer == "first\u2028inline\nsecond\x0binline"
|
||||
assert path.read_bytes() == source
|
||||
|
||||
editor = edit_module.EditStep(app_context=app_context)
|
||||
await editor(path=path.name, old=reader.context.response.answer, new="replacement")
|
||||
assert editor.context.response.success is True
|
||||
assert "Replaced 1 occurrence(s)" in editor.context.response.answer
|
||||
ending = "\n" if suffix == ".md" else newline if trailing_newline else ""
|
||||
assert path.read_bytes() == ("replacement" + ending).encode(encoding)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("newline", ["\r\n", "\r"])
|
||||
@pytest.mark.parametrize("filename,encoding", [("note.txt", "utf-8"), ("note.txt", "utf-16"), ("note.csv", "gbk")])
|
||||
def test_ranged_read_then_edit_preserves_unmatched_bytes(tmp_path, newline, filename, encoding):
|
||||
"""Only the matched span changes; encoding, other newlines and replacement literals survive."""
|
||||
prefix = "开头" + newline
|
||||
target = "first [1].*" + newline + r"second (2)\path"
|
||||
suffix = newline + "结尾" + newline
|
||||
path = tmp_path / filename
|
||||
path.write_bytes((prefix + target + suffix).encode(encoding))
|
||||
replacement = r"\1 replacement" + "\n" + r"next\g<0>"
|
||||
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
|
||||
async def run():
|
||||
reader = read_module.ReadStep(app_context=app_context)
|
||||
await reader(path=filename, start_line=2, end_line=3)
|
||||
assert reader.context.response.success is True
|
||||
editor = edit_module.EditStep(app_context=app_context)
|
||||
await editor(path=filename, old=reader.context.response.answer, new=replacement)
|
||||
assert editor.context.response.success is True
|
||||
assert path.read_bytes() == (prefix + replacement + suffix).encode(encoding)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_edit_normalized_needle_replaces_all_newline_styles(tmp_path):
|
||||
"""A mixed-newline body must not leave non-LF matches behind."""
|
||||
path = tmp_path / "note.txt"
|
||||
path.write_bytes(b"prefix\r\nfirst\r\nsecond\nmiddle\rfirst\nsecond\r\nfirst\rsecond\r\nsuffix")
|
||||
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
|
||||
async def run():
|
||||
editor = edit_module.EditStep(app_context=app_context)
|
||||
await editor(path=path.name, old="first\nsecond", new="done")
|
||||
assert editor.context.response.success is True
|
||||
assert "Replaced 3 occurrence(s)" in editor.context.response.answer
|
||||
assert path.read_bytes() == b"prefix\r\ndone\nmiddle\rdone\r\ndone\r\nsuffix"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("separator", ["\r\n", "\r\n\r\n", "\r\r", "\n\n"])
|
||||
def test_edit_does_not_treat_crlf_as_two_line_breaks(tmp_path, separator):
|
||||
"""A normalized blank line cannot match a single CRLF separator."""
|
||||
path = tmp_path / "note.txt"
|
||||
source = ("first" + separator + "second").encode("utf-8")
|
||||
path.write_bytes(source)
|
||||
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
|
||||
async def run():
|
||||
editor = edit_module.EditStep(app_context=app_context)
|
||||
await editor(path=path.name, old="first\n\nsecond", new="wrong")
|
||||
if separator == "\r\n":
|
||||
assert editor.context.response.success is False
|
||||
assert "not found" in editor.context.response.answer
|
||||
assert path.read_bytes() == source
|
||||
else:
|
||||
assert editor.context.response.success is True
|
||||
assert path.read_bytes() == b"wrong"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_edit_normalized_needle_preserves_frontmatter(tmp_path):
|
||||
"""Universal-newline matching stays inside the Markdown body."""
|
||||
path = tmp_path / "note.md"
|
||||
path.write_bytes(b"---\nname: first\ndescription: second\n---\nfirst\rsecond\rsuffix")
|
||||
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
|
||||
async def run():
|
||||
editor = edit_module.EditStep(app_context=app_context)
|
||||
await editor(path=path.name, old="first\nsecond", new="done")
|
||||
assert editor.context.response.success is True
|
||||
assert path.read_bytes() == b"---\ndescription: second\nname: first\n---\n\ndone\rsuffix\n"
|
||||
|
||||
before = path.read_bytes()
|
||||
await editor(path=path.name, old="description: second\nname: first", new="wrong")
|
||||
assert editor.context.response.success is False
|
||||
assert path.read_bytes() == before
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("old", ["first", "first\rsecond", "first\r\nsecond"])
|
||||
def test_edit_literal_needles_preserve_existing_matching(tmp_path, old):
|
||||
"""Single-line and explicitly raw-newline needles retain exact replacement behavior."""
|
||||
path = tmp_path / "note.txt"
|
||||
source = "prefix\r\nfirst\rsecond / first\r\nsecond / first\nsecond\r\nsuffix"
|
||||
path.write_bytes(source.encode("utf-8"))
|
||||
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
|
||||
async def run():
|
||||
editor = edit_module.EditStep(app_context=app_context)
|
||||
await editor(path=path.name, old=old, new="done")
|
||||
assert editor.context.response.success is True
|
||||
assert path.read_bytes() == source.replace(old, "done").encode("utf-8")
|
||||
|
||||
asyncio.run(run())
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue