scriptoriumadmin/ReMe
1
0
Fork 0
mirror of https://github.com/agentscope-ai/ReMe.git synced 2026-09-09 22:31:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

refactor(core): replace text truncation utilities with new marker system (#179)

Browse source 
* refactor(core): replace text truncation utilities with new marker system

- Remove old truncate_text_utils module and its exports
- Replace TRUNCATION_MARKER_START with _TRUNCATION_NOTICE_MARKER constant
- Update as_msg_stat.py to split content using new marker format
- Modify FileIO tool to use TRUNCATION_NOTICE_MARKER for continuation hints
- Change is_truncated function checks to use marker presence detection
- Move transformers dependency from main deps to light extra dependencies
- Update tool result compactor tests to verify marker instead of is_truncated calls

* feat(file_io): enhance file operations with path resolution and append functionality

- Add expanduser() to resolve file paths with ~ symbol
- Implement proper file existence and type validation in update_file
- Add new append_file method to append content to files
- Update truncation notice format for better readability
- Fix typo in error message from "provide" to "provided"
- Update transformers dependency in pyproject.toml
- Remove duplicate transformers dependency from light extras

* refactor(file_io): disable pylint too-many-return-statements warning

* perf(file_watcher): increase default polling delay and optimize watcher configuration

- Increased default poll_delay_ms from 1000ms to 2000ms to reduce CPU usage
- Removed force_polling parameter as it's no longer needed with updated polling strategy
- Simplified async watch configuration by removing conditional force_polling logic
- Reduced overall system resource consumption during file watching operations

* refactor(memory): update conversation log documentation in memory summary

- Changed "Raw conversation logs" to "Earlier conversation logs" for clarity
- Added warning note about potentially large dialog file sizes
- Improved formatting with additional line break for better readability
- Maintained existing compressed summary integration unchanged

* feat(memory): add long-term memory support to file-based memory system

- Initialize _long_term_memory attribute as empty string
- Add memories section to content when long-term memory exists
- Consolidate summary and memories into single user message
- Format memories with markdown header # Memories
- Maintain existing compressed summary functionality
- Join multiple content parts with double newlines
This commit is contained in:
jinliyl 2026-03-26 12:10:24 +08:00 • committed by GitHub
parent f17028e1b2
commit dc8eab56a1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 136 additions and 140 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pyproject.toml
View file

@ -80,11 +80,11 @@ full = [
litellm = [
"litellm==1.80.0",
"flowllm[reme]>=0.2.0.10",
]
light = [
"agentscope==1.0.17",
"flowllm[reme]>=0.2.0.10",
]
[tool.setuptools.packages.find]

5
reme/core/file_watcher/base_file_watcher.py
View file

@ -35,7 +35,7 @@ class BaseFileWatcher:
file_store: BaseFileStore | None = None,
callback: Callable[[set[tuple[Change, str]]], None | Coroutine[Any, Any, None]] | None = None,
rebuild_index_on_start: bool = True,
poll_delay_ms: int = 1000,
poll_delay_ms: int = 2000,
**kwargs,
):
"""
@ -181,15 +181,12 @@ class BaseFileWatcher:
try:
logger.info(f"Starting watch on valid paths: {valid_paths}")
# Enable force_polling if poll_delay_ms > default 300ms to reduce CPU usage
force_polling = self.poll_delay_ms > 300
async for changes in awatch(
*valid_paths,
watch_filter=self.watch_filter,
recursive=self.recursive,
debounce=self.debounce,
poll_delay_ms=self.poll_delay_ms,
force_polling=force_polling,
stop_event=self._stop_event,
):
if self._stop_event.is_set():

5
reme/core/schema/as_msg_stat.py
View file

@ -2,6 +2,8 @@
from pydantic import BaseModel, Field
_TRUNCATION_NOTICE_MARKER = "<<<TRUNCATED>>>"
_DEFAULT_MAX_BLOCK_TEXT_PREVIEW_LENGTH = 100
_DEFAULT_MAX_FORMATTER_TEXT_LENGTH = 1000
@ -61,7 +63,8 @@ class AsBlockStat(BaseModel):
if self.block_type == "tool_result":
if not self.tool_output:
return ""
content = f"{self.tool_name} output={self._truncate(self.tool_output, max_length)}"
display_output = self.tool_output.split(_TRUNCATION_NOTICE_MARKER)[0]
content = f"{self.tool_name} output={self._truncate(display_output, max_length)}"
return f"[tool_result]: {content}"
return ""

5
reme/core/utils/__init__.py
View file

@ -19,7 +19,6 @@ from .pydantic_utils import create_pydantic_model
from .singleton import singleton
from .time import timer, get_now_time
from .hf_token_counter_utils import get_hf_token_counter
from .truncate_text_utils import truncate_text, truncate_text_head, is_truncated, TRUNCATION_MARKER_START
__all__ = [
"convert_dashscope_to_agentscope",
@ -51,8 +50,4 @@ __all__ = [
"timer",
"get_now_time",
"get_hf_token_counter",
"truncate_text",
"truncate_text_head",
"is_truncated",
"TRUNCATION_MARKER_START",
]

82
reme/core/utils/truncate_text_utils.py
View file

@ -1,82 +0,0 @@
"""Utility functions for truncating long text strings."""
from .std_logger import get_logger
logger = get_logger()
TRUNCATION_MARKER_START = "<<<TRUNCATED>>>"
TRUNCATION_MARKER_END = "<<<END_TRUNCATED>>>"
def truncate_text(text: str, max_length: int) -> str:
"""Truncate text to max length, keeping head and tail portions.
Args:
text: The text to truncate
max_length: Maximum allowed length
Returns:
Truncated text with unique markers indicating truncation
"""
text = str(text) if text else ""
if not text:
return text
if len(text) <= max_length:
return text
half_length = max_length // 2
truncated_chars = len(text) - max_length
logger.debug(
"Text truncated: original %d chars, kept head %d + tail %d, removed %d chars.",
len(text),
half_length,
half_length,
truncated_chars,
)
return (
f"{text[:half_length]}\n\n{TRUNCATION_MARKER_START} "
f"({truncated_chars} characters omitted) "
f"{TRUNCATION_MARKER_END}\n\n{text[-half_length:]}"
)
def truncate_text_head(text: str, max_length: int) -> str:
"""Truncate text from the beginning, keeping only the head portion.
Args:
text: The text to truncate
max_length: Maximum allowed length
Returns:
Truncated text with marker indicating truncation at the end
"""
text = str(text) if text else ""
if not text:
return text
if len(text) <= max_length:
return text
truncated_chars = len(text) - max_length
logger.debug(
"Text truncated from head: original %d chars, kept %d, removed %d chars from tail.",
len(text),
max_length,
truncated_chars,
)
return f"{text[:max_length]}{TRUNCATION_MARKER_START}"
def is_truncated(text: str) -> bool:
"""Check if the text has been truncated (contains truncation marker).
Args:
text: The text to check
Returns:
bool: True if text contains truncation marker, False otherwise
"""
if not text:
return False
return TRUNCATION_MARKER_START in text

26
reme/memory/file_based/reme_in_memory_memory.py
View file

@ -34,6 +34,7 @@ class ReMeInMemoryMemory(InMemoryMemory):
self._token_counter: HuggingFaceTokenCounter = token_counter
self._msg_handler: AsMsgHandler = AsMsgHandler(token_counter)
self._dialog_path: Path | None = Path(dialog_path) if dialog_path else None
self._long_term_memory: str = ""
def _append_messages_to_dialog(self, messages: list[Msg]) -> int:
"""Append messages to dialog storage file.
@ -124,21 +125,20 @@ class ReMeInMemoryMemory(InMemoryMemory):
"""
filtered_content = [(msg, marks) for msg, marks in self.content if _MemoryMark.COMPRESSED not in marks]
parts = []
if self._long_term_memory:
parts.append(f"# Memories\n\n{self._long_term_memory}")
if prepend_summary and self._compressed_summary:
previous_summary = f"""
Raw conversation logs are in dialog/YYYY-MM-DD.jsonl (or nearby date files).
Entries are chronological; read from the end for recent history.
{self._compressed_summary}
The above is a summary of previous conversation, use it as context to maintain continuity.""".strip()
parts.append(
f"# Summary of previous conversation\n\n"
f"Previous conversation logs are offloaded to dialog/YYYY-MM-DD.jsonl (or nearby date files). "
"Here is the summary:\n\n"
f"{self._compressed_summary}\n"
f"The above is a summary of previous conversation, use it as context to maintain continuity.",
)
return [
Msg(
"user",
previous_summary,
"user",
),
*[msg for msg, _ in filtered_content],
]
if parts:
return [Msg("user", "\n\n".join(parts), "user"), *[msg for msg, _ in filtered_content]]
return [msg for msg, _ in filtered_content]

115
reme/memory/file_based/tools/file_io.py
View file

@ -7,7 +7,7 @@ from typing import Optional
from agentscope.message import TextBlock
from agentscope.tool import ToolResponse
from ..utils import read_file_safe, truncate_text_output
from ..utils import read_file_safe, truncate_text_output, TRUNCATION_NOTICE_MARKER
class FileIO:
@ -32,7 +32,7 @@ class FileIO:
Returns:
The resolved absolute file path as string.
"""
path = Path(file_path)
path = Path(file_path).expanduser()
if path.is_absolute():
return str(path)
else:
@ -147,13 +147,18 @@ class FileIO:
file_path=file_path,
)
# Add continuation hint if partial read without truncation
# Add continuation hint if partial read without truncation.
# Use TRUNCATION_NOTICE_MARKER format so ToolResultCompactor can
# re-truncate with the correct start_line when compacting old messages.
if text == selected_content and e < total:
remaining = total - e
text = (
f"{file_path} (lines {s}-{e} of {total})\n{text}\n\n"
f"[{remaining} more lines. Use start_line={e + 1} to continue.]"
content_bytes = len(text.encode("utf-8"))
notice = (
TRUNCATION_NOTICE_MARKER
+ f"\nFile: {file_path}\nStarting at start_line={s}, next {content_bytes} bytes."
f"\nTotal lines: {total}"
f"\nUse start_line={e + 1} to continue."
)
text = text + notice
return ToolResponse(
content=[TextBlock(type="text", text=text)],
@ -187,7 +192,7 @@ class FileIO:
content=[
TextBlock(
type="text",
text="Error: No `file_path` provide.",
text="Error: No `file_path` provided.",
),
],
)
@ -215,6 +220,7 @@ class FileIO:
],
)
# pylint: disable=too-many-return-statements
async def edit_file(
self,
file_path: str,
@ -232,22 +238,50 @@ class FileIO:
new_text (`str`):
Replacement text.
"""
response = await self.read_file(file_path=file_path)
if response.content and len(response.content) > 0:
error_text = response.content[0].get("text", "")
if error_text.startswith("Error:"):
return response
if not response.content or len(response.content) == 0:
if not file_path:
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: Failed to read file {file_path}.",
text="Error: No `file_path` provided.",
),
],
)
resolved_path = self._resolve_file_path(file_path)
if not os.path.exists(resolved_path):
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: The file {resolved_path} does not exist.",
),
],
)
if not os.path.isfile(resolved_path):
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: The path {resolved_path} is not a file.",
),
],
)
try:
content = read_file_safe(resolved_path)
except Exception as e:
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: Read file failed due to \n{e}",
),
],
)
content = response.content[0].get("text", "")
if old_text not in content:
return ToolResponse(
content=[
@ -259,7 +293,7 @@ class FileIO:
)
new_content = content.replace(old_text, new_text)
write_response = await self.write_file(file_path=file_path, content=new_content)
write_response = await self.write_file(file_path=resolved_path, content=new_content)
if write_response.content and len(write_response.content) > 0:
write_text = write_response.content[0].get("text", "")
@ -274,3 +308,50 @@ class FileIO:
),
],
)
async def append_file(
self,
file_path: str,
content: str,
) -> ToolResponse:
"""Append content to the end of a file. Relative paths resolve from
working_dir.
Args:
file_path (`str`):
Path to the file.
content (`str`):
Content to append.
"""
if not file_path:
return ToolResponse(
content=[
TextBlock(
type="text",
text="Error: No `file_path` provided.",
),
],
)
file_path = self._resolve_file_path(file_path)
try:
with open(file_path, "a", encoding="utf-8") as file:
file.write(content)
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Appended {len(content)} bytes to {file_path}.",
),
],
)
except Exception as e:
return ToolResponse(
content=[
TextBlock(
type="text",
text=f"Error: Append file failed due to \n{e}",
),
],
)

24
reme/memory/file_based/utils/file_utils.py
View file

@ -14,12 +14,14 @@ DEFAULT_MAX_BYTES = 100 * 1024
MAX_FILE_READ_BYTES = 1024 * 1024 * 1024
# Marker prepended to every truncation notice.
# Format: <<<TRUNCATED>>>
# File: file_path
# Content from start_line=X, next N bytes.
# total_lines=Z
# Use start_line=Y to continue.
# Split on this to recover the original (un-truncated) portion:
# Format:
# <<<TRUNCATED>>>
# File: <path>
# Starting at start_line=X, next N bytes.
# Total lines: Z
# Use start_line=Y to continue.
#
# Split output on this marker to recover the original (untruncated) portion:
# original = output.split(TRUNCATION_NOTICE_MARKER)[0]
TRUNCATION_NOTICE_MARKER = "<<<TRUNCATED>>>"
@ -72,8 +74,8 @@ def truncate_text_output(
return text
# Parse start_line and total_lines from notice; return text unchanged if not found
start_match = re.search(r"start_line=(\d+),", old_notice)
total_match = re.search(r"total_lines=(\d+)", old_notice)
start_match = re.search(r"Starting at start_line=(\d+)", old_notice)
total_match = re.search(r"Total lines: (\d+)", old_notice)
if not start_match or not total_match:
return text
start_line_parsed = int(start_match.group(1))
@ -92,7 +94,7 @@ def truncate_text_output(
if has_continuation:
new_notice = re.sub(r"Use start_line=\d+", f"Use start_line={next_line}", new_notice)
elif next_line <= total_lines_parsed:
new_notice = re.sub(r"(total_lines=\d+)", f"\\1\nUse start_line={next_line} to continue.", new_notice)
new_notice = re.sub(r"(Total lines: \d+)", f"\\1\nUse start_line={next_line} to continue.", new_notice)
return result + TRUNCATION_NOTICE_MARKER + new_notice
@ -112,8 +114,8 @@ def truncate_text_output(
continuation = f"\nUse start_line={next_line} to continue." if next_line <= total_lines else ""
notice = (
TRUNCATION_NOTICE_MARKER
+ f"\n\nFile: {file_path or ''}\nContent from start_line={start_line}, next {max_bytes} bytes."
f"\ntotal_lines={total_lines}{continuation}"
+ f"\nFile: {file_path or ''}\nStarting at start_line={start_line}, next {max_bytes} bytes."
f"\nTotal lines: {total_lines}{continuation}"
)
return result + notice

12
tests/light/test_tool_result_compactor.py
View file

@ -7,8 +7,8 @@ from pathlib import Path
from agentscope.message import Msg
from reme.core.utils import is_truncated
from reme.memory.file_based.components import ToolResultCompactor
from reme.memory.file_based.utils import TRUNCATION_NOTICE_MARKER
def create_tool_result_msg(output: str | list, tool_name: str = "test_tool") -> Msg:
@ -52,7 +52,7 @@ class TestToolResultCompactor:
_ = asyncio.run(op.call(messages=messages))
output = messages[0].content[0]["output"]
assert is_truncated(output)
assert TRUNCATION_NOTICE_MARKER in output
assert "[Full content saved to:" in output
# Verify file was created
@ -87,7 +87,7 @@ class TestToolResultCompactor:
asyncio.run(op.call(messages=messages))
text_block = messages[0].content[0]["output"][0]
assert is_truncated(text_block["text"])
assert TRUNCATION_NOTICE_MARKER in text_block["text"]
assert len(list(Path(tmpdir).glob("*.txt"))) == 1
def test_list_output_no_truncation_when_short(self):
@ -116,9 +116,9 @@ class TestToolResultCompactor:
asyncio.run(op.call(messages=messages))
output = messages[0].content[0]["output"]
assert is_truncated(output[0]["text"])
assert TRUNCATION_NOTICE_MARKER in output[0]["text"]
assert output[1]["text"] == "short" # unchanged
assert is_truncated(output[2]["text"])
assert TRUNCATION_NOTICE_MARKER in output[2]["text"]
assert len(list(Path(tmpdir).glob("*.txt"))) == 2
def test_list_output_mixed_block_types(self):
@ -134,7 +134,7 @@ class TestToolResultCompactor:
asyncio.run(op.call(messages=messages))
output = messages[0].content[0]["output"]
assert is_truncated(output[0]["text"])
assert TRUNCATION_NOTICE_MARKER in output[0]["text"]
assert output[1] == {"type": "image", "source": {"type": "url", "url": "http://example.com/img.png"}}
assert len(list(Path(tmpdir).glob("*.txt"))) == 1