refactor(core): update text truncation utilities and tool result handling (#168)

* refactor(memory): remove mark filtering parameters and simplify get_memory logic

* feat(core): bump version to 0.3.1.0

* refactor(memory): update comment to clarify dialog storage persistence

* refactor(core): update text truncation utilities and tool result handling

- Add new truncate_text_head function for head-based truncation
- Introduce TRUNCATION_MARKER_START constant for truncation detection
- Replace tail-based truncation with head-based truncation in tool result compaction
- Remove tool_result_threshold and retention_days parameters from RemeLight initialization
- Update ToolResultCompactor to use configurable thresholds for recent vs old messages
- Modify compact_tool_result method to accept multiple threshold parameters
- Adjust cleanup logic to use default compactor configuration
- Simplify is_truncated function to check only start marker

* ```
feat(memory): add long line splitting in tool result compaction

- Added _split_long_lines function to break oversized lines at 10000 characters
- Implemented line splitting before saving tool results to files
- Prevents extremely long lines from breaking file-based storage
- Maintains compatibility with existing tool result format
- Preserves original content integrity through chunked processing
```

reme/__init__.py

@@ -6,7 +6,7 @@ from . import extension
 from . import memory
 from .reme import ReMe
 
-__version__ = "0.3.0.9"
+__version__ = "0.3.1.0"
 
 __all__ = [
     "config",

reme/core/utils/__init__.py

@@ -19,7 +19,7 @@ from .pydantic_utils import create_pydantic_model
 from .singleton import singleton
 from .time import timer, get_now_time
 from .hf_token_counter_utils import get_hf_token_counter
-from .truncate_text_utils import truncate_text, is_truncated
+from .truncate_text_utils import truncate_text, truncate_text_head, is_truncated, TRUNCATION_MARKER_START
 
 __all__ = [
     "convert_dashscope_to_agentscope",
@@ -52,5 +52,7 @@ __all__ = [
     "get_now_time",
     "get_hf_token_counter",
     "truncate_text",
+    "truncate_text_head",
     "is_truncated",
+    "TRUNCATION_MARKER_START",
 ]

reme/core/utils/truncate_text_utils.py

@@ -41,15 +41,42 @@ def truncate_text(text: str, max_length: int) -> str:
     )
 
 
+def truncate_text_head(text: str, max_length: int) -> str:
+    """Truncate text from the beginning, keeping only the head portion.
+
+    Args:
+        text: The text to truncate
+        max_length: Maximum allowed length
+
+    Returns:
+        Truncated text with marker indicating truncation at the end
+    """
+    text = str(text) if text else ""
+    if not text:
+        return text
+
+    if len(text) <= max_length:
+        return text
+
+    truncated_chars = len(text) - max_length
+    logger.debug(
+        "Text truncated from head: original %d chars, kept %d, removed %d chars from tail.",
+        len(text),
+        max_length,
+        truncated_chars,
+    )
+    return f"{text[:max_length]}{TRUNCATION_MARKER_START}"
+
+
 def is_truncated(text: str) -> bool:
-    """Check if the text has been truncated (contains truncation markers).
+    """Check if the text has been truncated (contains truncation marker).
 
     Args:
         text: The text to check
 
     Returns:
-        bool: True if text contains truncation markers, False otherwise
+        bool: True if text contains truncation marker, False otherwise
     """
     if not text:
         return False
-    return TRUNCATION_MARKER_START in text and TRUNCATION_MARKER_END in text
+    return TRUNCATION_MARKER_START in text

reme/memory/file_based/components/tool_result_compactor.py

@@ -8,10 +8,26 @@ from agentscope.message import Msg
 
 from ....core.op import BaseOp
 from ....core.utils import get_logger
-from ....core.utils import truncate_text, is_truncated
+from ....core.utils import truncate_text_head, TRUNCATION_MARKER_START
 
 logger = get_logger()
 
+MAX_LINE_LENGTH = 10000
+
+
+def _split_long_lines(text: str, max_len: int = MAX_LINE_LENGTH) -> str:
+    """Split lines that exceed max_len by inserting newlines."""
+    lines = text.split("\n")
+    result = []
+    for line in lines:
+        if len(line) <= max_len:
+            result.append(line)
+        else:
+            # Split line into chunks of max_len
+            for i in range(0, len(line), max_len):
+                result.append(line[i : i + max_len])
+    return "\n".join(result)
+
 
 class ToolResultCompactor(BaseOp):
     """Truncate large tool_result outputs and save full content to files."""
@@ -19,43 +35,59 @@ class ToolResultCompactor(BaseOp):
     def __init__(
         self,
         tool_result_dir: str | Path,
-        tool_result_threshold: int,
         retention_days: int = 7,
+        recent_n: int = 1,
+        old_threshold: int = 500,
+        recent_threshold: int = 30000,
         **kwargs,
     ):
         super().__init__(**kwargs)
         self.tool_result_dir = Path(tool_result_dir)
-        self.tool_result_threshold = tool_result_threshold
         self.retention_days = retention_days
+        self.recent_n = recent_n
+        self.old_threshold = old_threshold
+        self.recent_threshold = recent_threshold
 
-    def _save_and_truncate(self, content: str, tool_name: str) -> str:
+    def _save_and_truncate(self, content: str, tool_name: str, threshold: int) -> str:
         """Save full content to file and return truncated version with file reference."""
-        if not content or is_truncated(content) or len(content) <= self.tool_result_threshold:
+        if not content:
             return content
 
-        # Save full content
+        # Check if content was previously truncated
+        if TRUNCATION_MARKER_START in content:
+            parts = content.split(TRUNCATION_MARKER_START, 1)
+            if len(parts[0]) <= threshold:
+                return content
+            return f"{truncate_text_head(parts[0], threshold)}{parts[1]}"
+
+        # Not truncated before
+        if len(content) <= threshold:
+            return content
+
+        # Save full content with long lines split
         self.tool_result_dir.mkdir(parents=True, exist_ok=True)
         file_path = self.tool_result_dir / f"{uuid.uuid4().hex}.txt"
         created_at = datetime.now().isoformat()
 
+        processed_content = _split_long_lines(content)
         file_path.write_text(
-            f"# tool_name: {tool_name}\n# created_at: {created_at}\n# ---\n{content}",
+            f"# tool_name: {tool_name}\n# created_at: {created_at}\n# ---\n{processed_content}",
             encoding="utf-8",
         )
         logger.debug("Saved tool result to %s (len=%d)", file_path, len(content))
 
         # Return truncated with file reference
-        return f"{truncate_text(content, self.tool_result_threshold)}\n\n[Full content saved to: {file_path}]"
+        return f"{truncate_text_head(content, threshold)}\n\n[Full content saved to: {file_path}]"
 
-    def _process_output(self, output: str | list[dict], tool_name: str) -> str | list[dict]:
+    def _process_output(self, output: str | list[dict], tool_name: str, threshold: int) -> str | list[dict]:
         """Process tool result output, truncating if necessary."""
         if isinstance(output, str):
-            return self._save_and_truncate(output, tool_name)
+            return self._save_and_truncate(output, tool_name, threshold)
 
         if isinstance(output, list):
             return [
                 (
-                    {**b, "text": self._save_and_truncate(b.get("text", ""), tool_name)}
+                    {**b, "text": self._save_and_truncate(b.get("text", ""), tool_name, threshold)}
                     if isinstance(b, dict) and b.get("type") == "text"
                     else b
                 )
@@ -69,15 +101,21 @@ class ToolResultCompactor(BaseOp):
         if not messages:
             return messages
 
-        for msg in messages:
+        # Split messages into old and recent parts
+        split_index = max(0, len(messages) - self.recent_n)
+
+        for idx, msg in enumerate(messages):
             if not isinstance(msg.content, list):
                 continue
 
+            # Determine threshold based on message position
+            threshold = self.recent_threshold if idx >= split_index else self.old_threshold
+
             for block in msg.content:
                 if isinstance(block, dict) and block.get("type") == "tool_result":
                     output = block.get("output")
                     if output:
-                        block["output"] = self._process_output(output, block.get("name", "unknown"))
+                        block["output"] = self._process_output(output, block.get("name", "unknown"), threshold)
 
         return messages
 

reme/memory/file_based/reme_in_memory_memory.py

@@ -110,34 +110,19 @@ class ReMeInMemoryMemory(InMemoryMemory):
 
     async def get_memory(
         self,
-        mark: str | None = None,
-        exclude_mark: str | None = _MemoryMark.COMPRESSED,
         prepend_summary: bool = True,
         **_kwargs,
     ) -> list[Msg]:
         """Get the messages from the memory by mark (if provided).
 
         Args:
-            mark: Optional mark to filter messages
-            exclude_mark: Optional mark to exclude messages
             prepend_summary: Whether to prepend compressed summary
             **_kwargs: Additional keyword arguments (ignored)
 
         Returns:
             List of filtered messages
         """
-        if not (mark is None or isinstance(mark, str)):
-            raise TypeError(f"The mark should be a string or None, but got {type(mark)}.")
-
-        if not (exclude_mark is None or isinstance(exclude_mark, str)):
-            raise TypeError(f"The exclude_mark should be a string or None, but got {type(exclude_mark)}.")
-
-        # Filter messages based on mark
-        filtered_content = [(msg, marks) for msg, marks in self.content if mark is None or mark in marks]
-
-        # Further filter messages based on exclude_mark
-        if exclude_mark is not None:
-            filtered_content = [(msg, marks) for msg, marks in filtered_content if exclude_mark not in marks]
+        filtered_content = [(msg, marks) for msg, marks in self.content if _MemoryMark.COMPRESSED not in marks]
 
         if prepend_summary and self._compressed_summary:
             previous_summary = f"""
@@ -210,7 +195,7 @@ only execute the user's new instruction.
         if not messages:
             return 0
 
-        # Persist messages to dialog storage
+        # Persist messages to dialog storage instead of compressed
         self._append_messages_to_dialog(messages)
 
         # Remove messages from memory
@@ -258,10 +243,7 @@ only execute the user's new instruction.
             - context_usage_ratio: Usage percentage
             - messages_detail: List of per-message AsMsgStat objects
         """
-        messages = await self.get_memory(
-            exclude_mark=_MemoryMark.COMPRESSED,
-            prepend_summary=False,
-        )
+        messages = await self.get_memory(prepend_summary=False)
 
         compressed_summary = self.get_compressed_summary()
         compressed_summary_tokens = await self._msg_handler.count_str_token(compressed_summary)

reme/reme_light.py

@@ -61,8 +61,6 @@ class ReMeLight(Application):
         dialog_path (Path): Path to the dialog storage directory for raw conversation records.
         vector_weight (float): Weight for vector search in hybrid search (0-1).
         candidate_multiplier (float): Multiplier for candidate retrieval count.
-        tool_result_threshold (int): Character threshold for tool result compaction.
-        retention_days (int): Number of days to retain tool result files.
         summary_tasks (list[asyncio.Task]): List of active background summary tasks.
     """
 
@@ -78,8 +76,6 @@ class ReMeLight(Application):
         default_file_store_config: dict | None = None,
         vector_weight: float = 0.7,
         candidate_multiplier: float = 3.0,
-        tool_result_threshold: int = 1000,
-        retention_days: int = 7,
         enable_load_env: bool = False,
     ):
         """
@@ -111,11 +107,6 @@ class ReMeLight(Application):
             candidate_multiplier (float): Multiplier applied to max_results when
                 retrieving candidates for re-ranking. Default 3.0 means 3x more
                 candidates are retrieved than the final result count.
-            tool_result_threshold (int): Character count threshold for tool result
-                compaction. Results exceeding this length will be truncated and
-                saved to files. Default 1000 characters.
-            retention_days (int): Number of days to retain tool result files
-                before automatic cleanup. Default 7 days.
             enable_load_env (bool): Whether to load environment variables from
                 .env file. Defaults to False.
 
@@ -138,8 +129,6 @@ class ReMeLight(Application):
 
         self.vector_weight: float = vector_weight
         self.candidate_multiplier: float = candidate_multiplier
-        self.tool_result_threshold: int = tool_result_threshold
-        self.retention_days: int = retention_days
 
         # Initialize the parent Application class with comprehensive configuration
         super().__init__(
@@ -186,19 +175,15 @@ class ReMeLight(Application):
         Clean up expired tool result files from the tool result directory.
 
         This method removes tool result files that have exceeded the retention
-        period specified during initialization. It helps manage disk space by
-        automatically removing old, unused tool outputs.
+        period. It helps manage disk space by automatically removing old, unused
+        tool outputs.
 
         Returns:
             int: The number of files that were successfully deleted
         """
         try:
-            # Create a compactor instance with current configuration
-            compactor = ToolResultCompactor(
-                tool_result_dir=self.tool_result_path,
-                tool_result_threshold=self.tool_result_threshold,
-                retention_days=self.retention_days,
-            )
+            # Create a compactor instance with default configuration
+            compactor = ToolResultCompactor(tool_result_dir=self.tool_result_path)
             # Execute cleanup and return count of deleted files
             return compactor.cleanup_expired_files()
         except Exception as e:
@@ -243,7 +228,14 @@ class ReMeLight(Application):
         self._cleanup_tool_results()
         return await super().close()
 
-    async def compact_tool_result(self, messages: list[Msg]) -> list[Msg]:
+    async def compact_tool_result(
+        self,
+        messages: list[Msg],
+        recent_n: int = 1,
+        old_threshold: int = 500,
+        recent_threshold: int = 30000,
+        retention_days: int = 7,
+    ) -> list[Msg]:
         """
         Compact tool results by truncating large outputs and saving full content to files.
 
@@ -255,13 +247,19 @@ class ReMeLight(Application):
         Args:
             messages (list[Msg]): List of messages potentially containing tool results
                 that may need compaction.
+            recent_n (int): Number of recent messages to use recent_threshold for.
+                Default 1.
+            old_threshold (int): Character threshold for old messages. Default 500.
+            recent_threshold (int): Character threshold for recent messages. Default 30000.
+            retention_days (int): Number of days to retain tool result files.
+                Default 7.
 
         Returns:
             list[Msg]: The processed list of messages with large tool results compacted.
                 If an error occurs, returns the original unmodified messages.
 
         Note:
-            - Tool results shorter than tool_result_threshold are left unchanged
+            - Tool results are truncated based on old_threshold/recent_threshold
             - Full content of truncated results is saved to tool_result_path
             - Expired files are automatically cleaned up during this operation
         """
@@ -269,8 +267,10 @@ class ReMeLight(Application):
             # Create compactor with instance configuration
             compactor = ToolResultCompactor(
                 tool_result_dir=self.tool_result_path,
-                tool_result_threshold=self.tool_result_threshold,
-                retention_days=self.retention_days,
+                retention_days=retention_days,
+                recent_n=recent_n,
+                old_threshold=old_threshold,
+                recent_threshold=recent_threshold,
             )
 
             # Execute compaction and get processed messages