From dc8eab56a127708e22752dfe1202421744ffd137 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Thu, 26 Mar 2026 12:10:24 +0800 Subject: [PATCH] refactor(core): replace text truncation utilities with new marker system (#179) * refactor(core): replace text truncation utilities with new marker system - Remove old truncate_text_utils module and its exports - Replace TRUNCATION_MARKER_START with _TRUNCATION_NOTICE_MARKER constant - Update as_msg_stat.py to split content using new marker format - Modify FileIO tool to use TRUNCATION_NOTICE_MARKER for continuation hints - Change is_truncated function checks to use marker presence detection - Move transformers dependency from main deps to light extra dependencies - Update tool result compactor tests to verify marker instead of is_truncated calls * feat(file_io): enhance file operations with path resolution and append functionality - Add expanduser() to resolve file paths with ~ symbol - Implement proper file existence and type validation in update_file - Add new append_file method to append content to files - Update truncation notice format for better readability - Fix typo in error message from "provide" to "provided" - Update transformers dependency in pyproject.toml - Remove duplicate transformers dependency from light extras * refactor(file_io): disable pylint too-many-return-statements warning * perf(file_watcher): increase default polling delay and optimize watcher configuration - Increased default poll_delay_ms from 1000ms to 2000ms to reduce CPU usage - Removed force_polling parameter as it's no longer needed with updated polling strategy - Simplified async watch configuration by removing conditional force_polling logic - Reduced overall system resource consumption during file watching operations * refactor(memory): update conversation log documentation in memory summary - Changed "Raw conversation logs" to "Earlier conversation logs" for clarity - Added warning note about potentially large dialog file sizes - Improved formatting with additional line break for better readability - Maintained existing compressed summary integration unchanged * feat(memory): add long-term memory support to file-based memory system - Initialize _long_term_memory attribute as empty string - Add memories section to content when long-term memory exists - Consolidate summary and memories into single user message - Format memories with markdown header # Memories - Maintain existing compressed summary functionality - Join multiple content parts with double newlines --- pyproject.toml | 2 +- reme/core/file_watcher/base_file_watcher.py | 5 +- reme/core/schema/as_msg_stat.py | 5 +- reme/core/utils/__init__.py | 5 - reme/core/utils/truncate_text_utils.py | 82 ------------- .../file_based/reme_in_memory_memory.py | 26 ++-- reme/memory/file_based/tools/file_io.py | 115 +++++++++++++++--- reme/memory/file_based/utils/file_utils.py | 24 ++-- tests/light/test_tool_result_compactor.py | 12 +- 9 files changed, 136 insertions(+), 140 deletions(-) delete mode 100644 reme/core/utils/truncate_text_utils.py diff --git a/pyproject.toml b/pyproject.toml index ececfe6d..36b12e98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,11 +80,11 @@ full = [ litellm = [ "litellm==1.80.0", - "flowllm[reme]>=0.2.0.10", ] light = [ "agentscope==1.0.17", + "flowllm[reme]>=0.2.0.10", ] [tool.setuptools.packages.find] diff --git a/reme/core/file_watcher/base_file_watcher.py b/reme/core/file_watcher/base_file_watcher.py index 383d5294..b4644dd7 100644 --- a/reme/core/file_watcher/base_file_watcher.py +++ b/reme/core/file_watcher/base_file_watcher.py @@ -35,7 +35,7 @@ class BaseFileWatcher: file_store: BaseFileStore | None = None, callback: Callable[[set[tuple[Change, str]]], None | Coroutine[Any, Any, None]] | None = None, rebuild_index_on_start: bool = True, - poll_delay_ms: int = 1000, + poll_delay_ms: int = 2000, **kwargs, ): """ @@ -181,15 +181,12 @@ class BaseFileWatcher: try: logger.info(f"Starting watch on valid paths: {valid_paths}") - # Enable force_polling if poll_delay_ms > default 300ms to reduce CPU usage - force_polling = self.poll_delay_ms > 300 async for changes in awatch( *valid_paths, watch_filter=self.watch_filter, recursive=self.recursive, debounce=self.debounce, poll_delay_ms=self.poll_delay_ms, - force_polling=force_polling, stop_event=self._stop_event, ): if self._stop_event.is_set(): diff --git a/reme/core/schema/as_msg_stat.py b/reme/core/schema/as_msg_stat.py index b502dd02..a8d2daaa 100644 --- a/reme/core/schema/as_msg_stat.py +++ b/reme/core/schema/as_msg_stat.py @@ -2,6 +2,8 @@ from pydantic import BaseModel, Field +_TRUNCATION_NOTICE_MARKER = "<<>>" + _DEFAULT_MAX_BLOCK_TEXT_PREVIEW_LENGTH = 100 _DEFAULT_MAX_FORMATTER_TEXT_LENGTH = 1000 @@ -61,7 +63,8 @@ class AsBlockStat(BaseModel): if self.block_type == "tool_result": if not self.tool_output: return "" - content = f"{self.tool_name} output={self._truncate(self.tool_output, max_length)}" + display_output = self.tool_output.split(_TRUNCATION_NOTICE_MARKER)[0] + content = f"{self.tool_name} output={self._truncate(display_output, max_length)}" return f"[tool_result]: {content}" return "" diff --git a/reme/core/utils/__init__.py b/reme/core/utils/__init__.py index b51936c7..26ca2827 100644 --- a/reme/core/utils/__init__.py +++ b/reme/core/utils/__init__.py @@ -19,7 +19,6 @@ from .pydantic_utils import create_pydantic_model from .singleton import singleton from .time import timer, get_now_time from .hf_token_counter_utils import get_hf_token_counter -from .truncate_text_utils import truncate_text, truncate_text_head, is_truncated, TRUNCATION_MARKER_START __all__ = [ "convert_dashscope_to_agentscope", @@ -51,8 +50,4 @@ __all__ = [ "timer", "get_now_time", "get_hf_token_counter", - "truncate_text", - "truncate_text_head", - "is_truncated", - "TRUNCATION_MARKER_START", ] diff --git a/reme/core/utils/truncate_text_utils.py b/reme/core/utils/truncate_text_utils.py deleted file mode 100644 index 1f60d19f..00000000 --- a/reme/core/utils/truncate_text_utils.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Utility functions for truncating long text strings.""" - -from .std_logger import get_logger - -logger = get_logger() - -TRUNCATION_MARKER_START = "<<>>" -TRUNCATION_MARKER_END = "<<>>" - - -def truncate_text(text: str, max_length: int) -> str: - """Truncate text to max length, keeping head and tail portions. - - Args: - text: The text to truncate - max_length: Maximum allowed length - - Returns: - Truncated text with unique markers indicating truncation - """ - text = str(text) if text else "" - if not text: - return text - - if len(text) <= max_length: - return text - - half_length = max_length // 2 - truncated_chars = len(text) - max_length - logger.debug( - "Text truncated: original %d chars, kept head %d + tail %d, removed %d chars.", - len(text), - half_length, - half_length, - truncated_chars, - ) - return ( - f"{text[:half_length]}\n\n{TRUNCATION_MARKER_START} " - f"({truncated_chars} characters omitted) " - f"{TRUNCATION_MARKER_END}\n\n{text[-half_length:]}" - ) - - -def truncate_text_head(text: str, max_length: int) -> str: - """Truncate text from the beginning, keeping only the head portion. - - Args: - text: The text to truncate - max_length: Maximum allowed length - - Returns: - Truncated text with marker indicating truncation at the end - """ - text = str(text) if text else "" - if not text: - return text - - if len(text) <= max_length: - return text - - truncated_chars = len(text) - max_length - logger.debug( - "Text truncated from head: original %d chars, kept %d, removed %d chars from tail.", - len(text), - max_length, - truncated_chars, - ) - return f"{text[:max_length]}{TRUNCATION_MARKER_START}" - - -def is_truncated(text: str) -> bool: - """Check if the text has been truncated (contains truncation marker). - - Args: - text: The text to check - - Returns: - bool: True if text contains truncation marker, False otherwise - """ - if not text: - return False - return TRUNCATION_MARKER_START in text diff --git a/reme/memory/file_based/reme_in_memory_memory.py b/reme/memory/file_based/reme_in_memory_memory.py index 1ee687fd..d8293e0e 100644 --- a/reme/memory/file_based/reme_in_memory_memory.py +++ b/reme/memory/file_based/reme_in_memory_memory.py @@ -34,6 +34,7 @@ class ReMeInMemoryMemory(InMemoryMemory): self._token_counter: HuggingFaceTokenCounter = token_counter self._msg_handler: AsMsgHandler = AsMsgHandler(token_counter) self._dialog_path: Path | None = Path(dialog_path) if dialog_path else None + self._long_term_memory: str = "" def _append_messages_to_dialog(self, messages: list[Msg]) -> int: """Append messages to dialog storage file. @@ -124,21 +125,20 @@ class ReMeInMemoryMemory(InMemoryMemory): """ filtered_content = [(msg, marks) for msg, marks in self.content if _MemoryMark.COMPRESSED not in marks] + parts = [] + if self._long_term_memory: + parts.append(f"# Memories\n\n{self._long_term_memory}") if prepend_summary and self._compressed_summary: - previous_summary = f""" -Raw conversation logs are in dialog/YYYY-MM-DD.jsonl (or nearby date files). -Entries are chronological; read from the end for recent history. -{self._compressed_summary} -The above is a summary of previous conversation, use it as context to maintain continuity.""".strip() + parts.append( + f"# Summary of previous conversation\n\n" + f"Previous conversation logs are offloaded to dialog/YYYY-MM-DD.jsonl (or nearby date files). " + "Here is the summary:\n\n" + f"{self._compressed_summary}\n" + f"The above is a summary of previous conversation, use it as context to maintain continuity.", + ) - return [ - Msg( - "user", - previous_summary, - "user", - ), - *[msg for msg, _ in filtered_content], - ] + if parts: + return [Msg("user", "\n\n".join(parts), "user"), *[msg for msg, _ in filtered_content]] return [msg for msg, _ in filtered_content] diff --git a/reme/memory/file_based/tools/file_io.py b/reme/memory/file_based/tools/file_io.py index 6f4ea9cf..d876407c 100644 --- a/reme/memory/file_based/tools/file_io.py +++ b/reme/memory/file_based/tools/file_io.py @@ -7,7 +7,7 @@ from typing import Optional from agentscope.message import TextBlock from agentscope.tool import ToolResponse -from ..utils import read_file_safe, truncate_text_output +from ..utils import read_file_safe, truncate_text_output, TRUNCATION_NOTICE_MARKER class FileIO: @@ -32,7 +32,7 @@ class FileIO: Returns: The resolved absolute file path as string. """ - path = Path(file_path) + path = Path(file_path).expanduser() if path.is_absolute(): return str(path) else: @@ -147,13 +147,18 @@ class FileIO: file_path=file_path, ) - # Add continuation hint if partial read without truncation + # Add continuation hint if partial read without truncation. + # Use TRUNCATION_NOTICE_MARKER format so ToolResultCompactor can + # re-truncate with the correct start_line when compacting old messages. if text == selected_content and e < total: - remaining = total - e - text = ( - f"{file_path} (lines {s}-{e} of {total})\n{text}\n\n" - f"[{remaining} more lines. Use start_line={e + 1} to continue.]" + content_bytes = len(text.encode("utf-8")) + notice = ( + TRUNCATION_NOTICE_MARKER + + f"\nFile: {file_path}\nStarting at start_line={s}, next {content_bytes} bytes." + f"\nTotal lines: {total}" + f"\nUse start_line={e + 1} to continue." ) + text = text + notice return ToolResponse( content=[TextBlock(type="text", text=text)], @@ -187,7 +192,7 @@ class FileIO: content=[ TextBlock( type="text", - text="Error: No `file_path` provide.", + text="Error: No `file_path` provided.", ), ], ) @@ -215,6 +220,7 @@ class FileIO: ], ) + # pylint: disable=too-many-return-statements async def edit_file( self, file_path: str, @@ -232,22 +238,50 @@ class FileIO: new_text (`str`): Replacement text. """ - response = await self.read_file(file_path=file_path) - if response.content and len(response.content) > 0: - error_text = response.content[0].get("text", "") - if error_text.startswith("Error:"): - return response - if not response.content or len(response.content) == 0: + if not file_path: return ToolResponse( content=[ TextBlock( type="text", - text=f"Error: Failed to read file {file_path}.", + text="Error: No `file_path` provided.", + ), + ], + ) + + resolved_path = self._resolve_file_path(file_path) + + if not os.path.exists(resolved_path): + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Error: The file {resolved_path} does not exist.", + ), + ], + ) + + if not os.path.isfile(resolved_path): + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Error: The path {resolved_path} is not a file.", + ), + ], + ) + + try: + content = read_file_safe(resolved_path) + except Exception as e: + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Error: Read file failed due to \n{e}", ), ], ) - content = response.content[0].get("text", "") if old_text not in content: return ToolResponse( content=[ @@ -259,7 +293,7 @@ class FileIO: ) new_content = content.replace(old_text, new_text) - write_response = await self.write_file(file_path=file_path, content=new_content) + write_response = await self.write_file(file_path=resolved_path, content=new_content) if write_response.content and len(write_response.content) > 0: write_text = write_response.content[0].get("text", "") @@ -274,3 +308,50 @@ class FileIO: ), ], ) + + async def append_file( + self, + file_path: str, + content: str, + ) -> ToolResponse: + """Append content to the end of a file. Relative paths resolve from + working_dir. + + Args: + file_path (`str`): + Path to the file. + content (`str`): + Content to append. + """ + if not file_path: + return ToolResponse( + content=[ + TextBlock( + type="text", + text="Error: No `file_path` provided.", + ), + ], + ) + + file_path = self._resolve_file_path(file_path) + + try: + with open(file_path, "a", encoding="utf-8") as file: + file.write(content) + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Appended {len(content)} bytes to {file_path}.", + ), + ], + ) + except Exception as e: + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Error: Append file failed due to \n{e}", + ), + ], + ) diff --git a/reme/memory/file_based/utils/file_utils.py b/reme/memory/file_based/utils/file_utils.py index f6324296..ee556bba 100644 --- a/reme/memory/file_based/utils/file_utils.py +++ b/reme/memory/file_based/utils/file_utils.py @@ -14,12 +14,14 @@ DEFAULT_MAX_BYTES = 100 * 1024 MAX_FILE_READ_BYTES = 1024 * 1024 * 1024 # Marker prepended to every truncation notice. -# Format: <<>> -# File: file_path -# Content from start_line=X, next N bytes. -# total_lines=Z -# Use start_line=Y to continue. -# Split on this to recover the original (un-truncated) portion: +# Format: +# <<>> +# File: +# Starting at start_line=X, next N bytes. +# Total lines: Z +# Use start_line=Y to continue. +# +# Split output on this marker to recover the original (untruncated) portion: # original = output.split(TRUNCATION_NOTICE_MARKER)[0] TRUNCATION_NOTICE_MARKER = "<<>>" @@ -72,8 +74,8 @@ def truncate_text_output( return text # Parse start_line and total_lines from notice; return text unchanged if not found - start_match = re.search(r"start_line=(\d+),", old_notice) - total_match = re.search(r"total_lines=(\d+)", old_notice) + start_match = re.search(r"Starting at start_line=(\d+)", old_notice) + total_match = re.search(r"Total lines: (\d+)", old_notice) if not start_match or not total_match: return text start_line_parsed = int(start_match.group(1)) @@ -92,7 +94,7 @@ def truncate_text_output( if has_continuation: new_notice = re.sub(r"Use start_line=\d+", f"Use start_line={next_line}", new_notice) elif next_line <= total_lines_parsed: - new_notice = re.sub(r"(total_lines=\d+)", f"\\1\nUse start_line={next_line} to continue.", new_notice) + new_notice = re.sub(r"(Total lines: \d+)", f"\\1\nUse start_line={next_line} to continue.", new_notice) return result + TRUNCATION_NOTICE_MARKER + new_notice @@ -112,8 +114,8 @@ def truncate_text_output( continuation = f"\nUse start_line={next_line} to continue." if next_line <= total_lines else "" notice = ( TRUNCATION_NOTICE_MARKER - + f"\n\nFile: {file_path or ''}\nContent from start_line={start_line}, next {max_bytes} bytes." - f"\ntotal_lines={total_lines}{continuation}" + + f"\nFile: {file_path or ''}\nStarting at start_line={start_line}, next {max_bytes} bytes." + f"\nTotal lines: {total_lines}{continuation}" ) return result + notice diff --git a/tests/light/test_tool_result_compactor.py b/tests/light/test_tool_result_compactor.py index ac812c35..b05dad29 100644 --- a/tests/light/test_tool_result_compactor.py +++ b/tests/light/test_tool_result_compactor.py @@ -7,8 +7,8 @@ from pathlib import Path from agentscope.message import Msg -from reme.core.utils import is_truncated from reme.memory.file_based.components import ToolResultCompactor +from reme.memory.file_based.utils import TRUNCATION_NOTICE_MARKER def create_tool_result_msg(output: str | list, tool_name: str = "test_tool") -> Msg: @@ -52,7 +52,7 @@ class TestToolResultCompactor: _ = asyncio.run(op.call(messages=messages)) output = messages[0].content[0]["output"] - assert is_truncated(output) + assert TRUNCATION_NOTICE_MARKER in output assert "[Full content saved to:" in output # Verify file was created @@ -87,7 +87,7 @@ class TestToolResultCompactor: asyncio.run(op.call(messages=messages)) text_block = messages[0].content[0]["output"][0] - assert is_truncated(text_block["text"]) + assert TRUNCATION_NOTICE_MARKER in text_block["text"] assert len(list(Path(tmpdir).glob("*.txt"))) == 1 def test_list_output_no_truncation_when_short(self): @@ -116,9 +116,9 @@ class TestToolResultCompactor: asyncio.run(op.call(messages=messages)) output = messages[0].content[0]["output"] - assert is_truncated(output[0]["text"]) + assert TRUNCATION_NOTICE_MARKER in output[0]["text"] assert output[1]["text"] == "short" # unchanged - assert is_truncated(output[2]["text"]) + assert TRUNCATION_NOTICE_MARKER in output[2]["text"] assert len(list(Path(tmpdir).glob("*.txt"))) == 2 def test_list_output_mixed_block_types(self): @@ -134,7 +134,7 @@ class TestToolResultCompactor: asyncio.run(op.call(messages=messages)) output = messages[0].content[0]["output"] - assert is_truncated(output[0]["text"]) + assert TRUNCATION_NOTICE_MARKER in output[0]["text"] assert output[1] == {"type": "image", "source": {"type": "url", "url": "http://example.com/img.png"}} assert len(list(Path(tmpdir).glob("*.txt"))) == 1