From 6dd987a1d24c860816d9b61516d90fc0bd0a9640 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:53:32 +0800 Subject: [PATCH] refactor(core): update text truncation utilities and tool result handling (#168) * refactor(memory): remove mark filtering parameters and simplify get_memory logic * feat(core): bump version to 0.3.1.0 * refactor(memory): update comment to clarify dialog storage persistence * refactor(core): update text truncation utilities and tool result handling - Add new truncate_text_head function for head-based truncation - Introduce TRUNCATION_MARKER_START constant for truncation detection - Replace tail-based truncation with head-based truncation in tool result compaction - Remove tool_result_threshold and retention_days parameters from RemeLight initialization - Update ToolResultCompactor to use configurable thresholds for recent vs old messages - Modify compact_tool_result method to accept multiple threshold parameters - Adjust cleanup logic to use default compactor configuration - Simplify is_truncated function to check only start marker * ``` feat(memory): add long line splitting in tool result compaction - Added _split_long_lines function to break oversized lines at 10000 characters - Implemented line splitting before saving tool results to files - Prevents extremely long lines from breaking file-based storage - Maintains compatibility with existing tool result format - Preserves original content integrity through chunked processing ``` --- reme/__init__.py | 2 +- reme/core/utils/__init__.py | 4 +- reme/core/utils/truncate_text_utils.py | 33 +++++++++- .../components/tool_result_compactor.py | 64 +++++++++++++++---- .../file_based/reme_in_memory_memory.py | 24 +------ reme/reme_light.py | 46 ++++++------- 6 files changed, 111 insertions(+), 62 deletions(-) diff --git a/reme/__init__.py b/reme/__init__.py index 22374421..874836a7 100644 --- a/reme/__init__.py +++ b/reme/__init__.py @@ -6,7 +6,7 @@ from . import extension from . import memory from .reme import ReMe -__version__ = "0.3.0.9" +__version__ = "0.3.1.0" __all__ = [ "config", diff --git a/reme/core/utils/__init__.py b/reme/core/utils/__init__.py index 9dbc59dc..b51936c7 100644 --- a/reme/core/utils/__init__.py +++ b/reme/core/utils/__init__.py @@ -19,7 +19,7 @@ from .pydantic_utils import create_pydantic_model from .singleton import singleton from .time import timer, get_now_time from .hf_token_counter_utils import get_hf_token_counter -from .truncate_text_utils import truncate_text, is_truncated +from .truncate_text_utils import truncate_text, truncate_text_head, is_truncated, TRUNCATION_MARKER_START __all__ = [ "convert_dashscope_to_agentscope", @@ -52,5 +52,7 @@ __all__ = [ "get_now_time", "get_hf_token_counter", "truncate_text", + "truncate_text_head", "is_truncated", + "TRUNCATION_MARKER_START", ] diff --git a/reme/core/utils/truncate_text_utils.py b/reme/core/utils/truncate_text_utils.py index da0c473a..1f60d19f 100644 --- a/reme/core/utils/truncate_text_utils.py +++ b/reme/core/utils/truncate_text_utils.py @@ -41,15 +41,42 @@ def truncate_text(text: str, max_length: int) -> str: ) +def truncate_text_head(text: str, max_length: int) -> str: + """Truncate text from the beginning, keeping only the head portion. + + Args: + text: The text to truncate + max_length: Maximum allowed length + + Returns: + Truncated text with marker indicating truncation at the end + """ + text = str(text) if text else "" + if not text: + return text + + if len(text) <= max_length: + return text + + truncated_chars = len(text) - max_length + logger.debug( + "Text truncated from head: original %d chars, kept %d, removed %d chars from tail.", + len(text), + max_length, + truncated_chars, + ) + return f"{text[:max_length]}{TRUNCATION_MARKER_START}" + + def is_truncated(text: str) -> bool: - """Check if the text has been truncated (contains truncation markers). + """Check if the text has been truncated (contains truncation marker). Args: text: The text to check Returns: - bool: True if text contains truncation markers, False otherwise + bool: True if text contains truncation marker, False otherwise """ if not text: return False - return TRUNCATION_MARKER_START in text and TRUNCATION_MARKER_END in text + return TRUNCATION_MARKER_START in text diff --git a/reme/memory/file_based/components/tool_result_compactor.py b/reme/memory/file_based/components/tool_result_compactor.py index c1c6fe00..002bb8ae 100644 --- a/reme/memory/file_based/components/tool_result_compactor.py +++ b/reme/memory/file_based/components/tool_result_compactor.py @@ -8,10 +8,26 @@ from agentscope.message import Msg from ....core.op import BaseOp from ....core.utils import get_logger -from ....core.utils import truncate_text, is_truncated +from ....core.utils import truncate_text_head, TRUNCATION_MARKER_START logger = get_logger() +MAX_LINE_LENGTH = 10000 + + +def _split_long_lines(text: str, max_len: int = MAX_LINE_LENGTH) -> str: + """Split lines that exceed max_len by inserting newlines.""" + lines = text.split("\n") + result = [] + for line in lines: + if len(line) <= max_len: + result.append(line) + else: + # Split line into chunks of max_len + for i in range(0, len(line), max_len): + result.append(line[i : i + max_len]) + return "\n".join(result) + class ToolResultCompactor(BaseOp): """Truncate large tool_result outputs and save full content to files.""" @@ -19,43 +35,59 @@ class ToolResultCompactor(BaseOp): def __init__( self, tool_result_dir: str | Path, - tool_result_threshold: int, retention_days: int = 7, + recent_n: int = 1, + old_threshold: int = 500, + recent_threshold: int = 30000, **kwargs, ): super().__init__(**kwargs) self.tool_result_dir = Path(tool_result_dir) - self.tool_result_threshold = tool_result_threshold self.retention_days = retention_days + self.recent_n = recent_n + self.old_threshold = old_threshold + self.recent_threshold = recent_threshold - def _save_and_truncate(self, content: str, tool_name: str) -> str: + def _save_and_truncate(self, content: str, tool_name: str, threshold: int) -> str: """Save full content to file and return truncated version with file reference.""" - if not content or is_truncated(content) or len(content) <= self.tool_result_threshold: + if not content: return content - # Save full content + # Check if content was previously truncated + if TRUNCATION_MARKER_START in content: + parts = content.split(TRUNCATION_MARKER_START, 1) + if len(parts[0]) <= threshold: + return content + return f"{truncate_text_head(parts[0], threshold)}{parts[1]}" + + # Not truncated before + if len(content) <= threshold: + return content + + # Save full content with long lines split self.tool_result_dir.mkdir(parents=True, exist_ok=True) file_path = self.tool_result_dir / f"{uuid.uuid4().hex}.txt" created_at = datetime.now().isoformat() + processed_content = _split_long_lines(content) file_path.write_text( - f"# tool_name: {tool_name}\n# created_at: {created_at}\n# ---\n{content}", + f"# tool_name: {tool_name}\n# created_at: {created_at}\n# ---\n{processed_content}", encoding="utf-8", ) logger.debug("Saved tool result to %s (len=%d)", file_path, len(content)) # Return truncated with file reference - return f"{truncate_text(content, self.tool_result_threshold)}\n\n[Full content saved to: {file_path}]" + return f"{truncate_text_head(content, threshold)}\n\n[Full content saved to: {file_path}]" - def _process_output(self, output: str | list[dict], tool_name: str) -> str | list[dict]: + def _process_output(self, output: str | list[dict], tool_name: str, threshold: int) -> str | list[dict]: """Process tool result output, truncating if necessary.""" if isinstance(output, str): - return self._save_and_truncate(output, tool_name) + return self._save_and_truncate(output, tool_name, threshold) if isinstance(output, list): return [ ( - {**b, "text": self._save_and_truncate(b.get("text", ""), tool_name)} + {**b, "text": self._save_and_truncate(b.get("text", ""), tool_name, threshold)} if isinstance(b, dict) and b.get("type") == "text" else b ) @@ -69,15 +101,21 @@ class ToolResultCompactor(BaseOp): if not messages: return messages - for msg in messages: + # Split messages into old and recent parts + split_index = max(0, len(messages) - self.recent_n) + + for idx, msg in enumerate(messages): if not isinstance(msg.content, list): continue + # Determine threshold based on message position + threshold = self.recent_threshold if idx >= split_index else self.old_threshold + for block in msg.content: if isinstance(block, dict) and block.get("type") == "tool_result": output = block.get("output") if output: - block["output"] = self._process_output(output, block.get("name", "unknown")) + block["output"] = self._process_output(output, block.get("name", "unknown"), threshold) return messages diff --git a/reme/memory/file_based/reme_in_memory_memory.py b/reme/memory/file_based/reme_in_memory_memory.py index 4fcec627..1c53c314 100644 --- a/reme/memory/file_based/reme_in_memory_memory.py +++ b/reme/memory/file_based/reme_in_memory_memory.py @@ -110,34 +110,19 @@ class ReMeInMemoryMemory(InMemoryMemory): async def get_memory( self, - mark: str | None = None, - exclude_mark: str | None = _MemoryMark.COMPRESSED, prepend_summary: bool = True, **_kwargs, ) -> list[Msg]: """Get the messages from the memory by mark (if provided). Args: - mark: Optional mark to filter messages - exclude_mark: Optional mark to exclude messages prepend_summary: Whether to prepend compressed summary **_kwargs: Additional keyword arguments (ignored) Returns: List of filtered messages """ - if not (mark is None or isinstance(mark, str)): - raise TypeError(f"The mark should be a string or None, but got {type(mark)}.") - - if not (exclude_mark is None or isinstance(exclude_mark, str)): - raise TypeError(f"The exclude_mark should be a string or None, but got {type(exclude_mark)}.") - - # Filter messages based on mark - filtered_content = [(msg, marks) for msg, marks in self.content if mark is None or mark in marks] - - # Further filter messages based on exclude_mark - if exclude_mark is not None: - filtered_content = [(msg, marks) for msg, marks in filtered_content if exclude_mark not in marks] + filtered_content = [(msg, marks) for msg, marks in self.content if _MemoryMark.COMPRESSED not in marks] if prepend_summary and self._compressed_summary: previous_summary = f""" @@ -210,7 +195,7 @@ only execute the user's new instruction. if not messages: return 0 - # Persist messages to dialog storage + # Persist messages to dialog storage instead of compressed self._append_messages_to_dialog(messages) # Remove messages from memory @@ -258,10 +243,7 @@ only execute the user's new instruction. - context_usage_ratio: Usage percentage - messages_detail: List of per-message AsMsgStat objects """ - messages = await self.get_memory( - exclude_mark=_MemoryMark.COMPRESSED, - prepend_summary=False, - ) + messages = await self.get_memory(prepend_summary=False) compressed_summary = self.get_compressed_summary() compressed_summary_tokens = await self._msg_handler.count_str_token(compressed_summary) diff --git a/reme/reme_light.py b/reme/reme_light.py index c8561550..58008120 100644 --- a/reme/reme_light.py +++ b/reme/reme_light.py @@ -61,8 +61,6 @@ class ReMeLight(Application): dialog_path (Path): Path to the dialog storage directory for raw conversation records. vector_weight (float): Weight for vector search in hybrid search (0-1). candidate_multiplier (float): Multiplier for candidate retrieval count. - tool_result_threshold (int): Character threshold for tool result compaction. - retention_days (int): Number of days to retain tool result files. summary_tasks (list[asyncio.Task]): List of active background summary tasks. """ @@ -78,8 +76,6 @@ class ReMeLight(Application): default_file_store_config: dict | None = None, vector_weight: float = 0.7, candidate_multiplier: float = 3.0, - tool_result_threshold: int = 1000, - retention_days: int = 7, enable_load_env: bool = False, ): """ @@ -111,11 +107,6 @@ class ReMeLight(Application): candidate_multiplier (float): Multiplier applied to max_results when retrieving candidates for re-ranking. Default 3.0 means 3x more candidates are retrieved than the final result count. - tool_result_threshold (int): Character count threshold for tool result - compaction. Results exceeding this length will be truncated and - saved to files. Default 1000 characters. - retention_days (int): Number of days to retain tool result files - before automatic cleanup. Default 7 days. enable_load_env (bool): Whether to load environment variables from .env file. Defaults to False. @@ -138,8 +129,6 @@ class ReMeLight(Application): self.vector_weight: float = vector_weight self.candidate_multiplier: float = candidate_multiplier - self.tool_result_threshold: int = tool_result_threshold - self.retention_days: int = retention_days # Initialize the parent Application class with comprehensive configuration super().__init__( @@ -186,19 +175,15 @@ class ReMeLight(Application): Clean up expired tool result files from the tool result directory. This method removes tool result files that have exceeded the retention - period specified during initialization. It helps manage disk space by - automatically removing old, unused tool outputs. + period. It helps manage disk space by automatically removing old, unused + tool outputs. Returns: int: The number of files that were successfully deleted """ try: - # Create a compactor instance with current configuration - compactor = ToolResultCompactor( - tool_result_dir=self.tool_result_path, - tool_result_threshold=self.tool_result_threshold, - retention_days=self.retention_days, - ) + # Create a compactor instance with default configuration + compactor = ToolResultCompactor(tool_result_dir=self.tool_result_path) # Execute cleanup and return count of deleted files return compactor.cleanup_expired_files() except Exception as e: @@ -243,7 +228,14 @@ class ReMeLight(Application): self._cleanup_tool_results() return await super().close() - async def compact_tool_result(self, messages: list[Msg]) -> list[Msg]: + async def compact_tool_result( + self, + messages: list[Msg], + recent_n: int = 1, + old_threshold: int = 500, + recent_threshold: int = 30000, + retention_days: int = 7, + ) -> list[Msg]: """ Compact tool results by truncating large outputs and saving full content to files. @@ -255,13 +247,19 @@ class ReMeLight(Application): Args: messages (list[Msg]): List of messages potentially containing tool results that may need compaction. + recent_n (int): Number of recent messages to use recent_threshold for. + Default 1. + old_threshold (int): Character threshold for old messages. Default 500. + recent_threshold (int): Character threshold for recent messages. Default 30000. + retention_days (int): Number of days to retain tool result files. + Default 7. Returns: list[Msg]: The processed list of messages with large tool results compacted. If an error occurs, returns the original unmodified messages. Note: - - Tool results shorter than tool_result_threshold are left unchanged + - Tool results are truncated based on old_threshold/recent_threshold - Full content of truncated results is saved to tool_result_path - Expired files are automatically cleaned up during this operation """ @@ -269,8 +267,10 @@ class ReMeLight(Application): # Create compactor with instance configuration compactor = ToolResultCompactor( tool_result_dir=self.tool_result_path, - tool_result_threshold=self.tool_result_threshold, - retention_days=self.retention_days, + retention_days=retention_days, + recent_n=recent_n, + old_threshold=old_threshold, + recent_threshold=recent_threshold, ) # Execute compaction and get processed messages