mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-08-28 05:25:04 +00:00
style(memory): update message formatting and improve logging (#175)
* style(memory): update message formatting and improve logging - Change default include_thinking parameter to True in as_msg_handler.py - Replace angle brackets with square brackets for block formatting in as_msg_stat.py - Add newline replacement in text truncation method in as_msg_stat.py - Add loading duration timing to embedding cache loading in base_embedding_model.py - Replace XML-style tags with markdown headers in compactor.py conversation format - Update compactor.yaml prompts to reference markdown-style headers instead of XML tags - Modify summarizer.py to use markdown-style conversation header format * refactor(file-watcher): replace scan_on_start with rebuild_index_on_start parameter - Replace scan_on_start and clear_on_start boolean parameters with single rebuild_index_on_start - Update BaseFileWatcher constructor to use rebuild_index_on_start instead of two separate flags - Modify initialization logic to clear and rescan when rebuild_index_on_start is True - Remove scan_on_start parameter from CLI and light configuration files - Update documentation to remove scan_on_start from quick start guides - Rename all test methods and classes from scan_on_start to rebuild_index_on_start - Add timezone-aware datetime helper method to summarizer component - Format log message with proper line breaks for readability * fix(core): resolve file watcher initialization issue and update version - Fixed file watcher task creation to properly handle rebuild index on start logic - Moved initialization and watch loop into async function to ensure proper execution order - Updated package version from 0.3.1.1 to 0.3.1.2 - Added missing comma in embedding model logging statement * fix(core): reduce max formatter text length limit - Changed _DEFAULT_MAX_FORMATTER_TEXT_LENGTH from 2000 to 1000 - Updated constant value in as_msg_stat.py schema module * fix(file-watcher): change default rebuild index behavior on start - Changed rebuild_index_on_start parameter default from False to True - This ensures index is rebuilt by default when file watcher starts - Maintains consistent state initialization for file watching operations * feat(compactor): add return_dict option and improve summary validation - Add _is_valid_summary function to validate summary content format - Introduce return_dict parameter to return structured results with validation - Update prompt templates with clearer task descriptions and formatting rules - Refactor update_user_message prompts to combine prefix and suffix logic - Return dictionary with user_message, history_compact, and is_valid fields when enabled - Add proper error handling for exception cases in memory compaction - Maintain backward compatibility with string return when return_dict=False * feat(memory): add thinking block configuration option - Add add_thinking_block parameter to compactor component - Pass include_thinking flag to message formatting in compactor - Add add_thinking_block parameter to reme_light compact function - Add add_thinking_block parameter to reme_light summarize function - Add add_thinking_block parameter to summarizer component - Pass include_thinking flag to message formatting in summarizer - Remove previous-summary tags from compressed summary format
This commit is contained in:
parent
0beaa035cb
commit
7b02c45218
16 changed files with 146 additions and 100 deletions
|
|
@ -176,7 +176,6 @@ Controls how context space is allocated and how memory is searched:
|
|||
| `watch_paths` | Directories/files to monitor |
|
||||
| `suffix_filters` | Which file suffixes to watch (`.md`) |
|
||||
| `recursive` | Whether to recurse into subdirectories |
|
||||
| `scan_on_start` | Whether to do a full scan on startup |
|
||||
|
||||
**token_counters — Token Counter**
|
||||
|
||||
|
|
|
|||
|
|
@ -172,7 +172,6 @@ pip install -e .
|
|||
| `watch_paths` | 要监控的目录/文件 |
|
||||
| `suffix_filters` | 只关心哪些后缀(`.md`) |
|
||||
| `recursive` | 是否递归子目录 |
|
||||
| `scan_on_start` | 启动时先全量扫一遍 |
|
||||
|
||||
**token_counters — Token 计数器**
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from . import extension
|
|||
from . import memory
|
||||
from .reme import ReMe
|
||||
|
||||
__version__ = "0.3.1.1"
|
||||
__version__ = "0.3.1.2"
|
||||
|
||||
__all__ = [
|
||||
"config",
|
||||
|
|
|
|||
|
|
@ -47,5 +47,4 @@ file_watchers:
|
|||
watch_paths: [ ".reme", ".reme/memory" ]
|
||||
suffix_filters: [ ".md" ]
|
||||
recursive: false
|
||||
scan_on_start: true
|
||||
|
||||
|
|
|
|||
|
|
@ -34,4 +34,3 @@ file_watchers:
|
|||
file_store: default
|
||||
suffix_filters: [ ".md" ]
|
||||
recursive: false
|
||||
scan_on_start: true
|
||||
|
|
|
|||
|
|
@ -159,6 +159,7 @@ class BaseEmbeddingModel(ABC):
|
|||
return
|
||||
|
||||
try:
|
||||
load_start = time.time()
|
||||
# Read all lines first (to load in reverse order)
|
||||
with open(cache_file, "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
|
|
@ -201,7 +202,9 @@ class BaseEmbeddingModel(ABC):
|
|||
logger.warning(f"Failed to parse line in cache file: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"Loaded {loaded_count} embeddings from cache file: {cache_file}")
|
||||
logger.info(
|
||||
f"Loaded {loaded_count} embeddings from cache file: {cache_file} in {time.time() - load_start:.2f}s",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load cache from {cache_file}: {e}, deleting cache file")
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -34,8 +34,7 @@ class BaseFileWatcher:
|
|||
chunk_overlap: int = 80,
|
||||
file_store: BaseFileStore | None = None,
|
||||
callback: Callable[[set[tuple[Change, str]]], None | Coroutine[Any, Any, None]] | None = None,
|
||||
scan_on_start: bool = True,
|
||||
clear_on_start: bool = True,
|
||||
rebuild_index_on_start: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
|
|
@ -50,9 +49,8 @@ class BaseFileWatcher:
|
|||
chunk_overlap: Overlap size for chunks
|
||||
file_store: File store instance
|
||||
callback: Callback function for changes
|
||||
scan_on_start: If True, scan existing files on start and trigger on_changes with Change.added
|
||||
clear_on_start: If True, clear all indexed data on start before scanning.
|
||||
Useful for full rebuild of the index.
|
||||
rebuild_index_on_start: If True, clear all indexed data on start and rescan existing files.
|
||||
If False, only monitor new changes without initialization.
|
||||
**kwargs: Additional keyword arguments
|
||||
"""
|
||||
self.watch_paths: list[str] = [watch_paths] if isinstance(watch_paths, str) else watch_paths
|
||||
|
|
@ -63,8 +61,7 @@ class BaseFileWatcher:
|
|||
self.chunk_overlap: int = chunk_overlap
|
||||
self.file_store: BaseFileStore = file_store
|
||||
self.callback = callback
|
||||
self.scan_on_start: bool = scan_on_start
|
||||
self.clear_on_start: bool = clear_on_start
|
||||
self.rebuild_index_on_start: bool = rebuild_index_on_start
|
||||
self.kwargs: dict = kwargs
|
||||
|
||||
self._stop_event = asyncio.Event()
|
||||
|
|
@ -78,16 +75,14 @@ class BaseFileWatcher:
|
|||
|
||||
self._running = True
|
||||
|
||||
# Clear all indexed data if requested
|
||||
if self.clear_on_start and self.file_store is not None:
|
||||
await self.file_store.clear_all()
|
||||
logger.info("Cleared all indexed data on start")
|
||||
async def _initialize_and_watch():
|
||||
if self.rebuild_index_on_start:
|
||||
await self.file_store.clear_all()
|
||||
logger.info("Cleared all indexed data on start")
|
||||
await self._scan_existing_files()
|
||||
await self._watch_loop()
|
||||
|
||||
# Scan existing files if requested
|
||||
if self.scan_on_start:
|
||||
await self._scan_existing_files()
|
||||
|
||||
self._watch_task = asyncio.create_task(self._watch_loop())
|
||||
self._watch_task = asyncio.create_task(_initialize_and_watch())
|
||||
logger.info(f"Started watching: {self.watch_paths}")
|
||||
|
||||
async def close(self):
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from pydantic import BaseModel, Field
|
||||
|
||||
_DEFAULT_MAX_BLOCK_TEXT_PREVIEW_LENGTH = 100
|
||||
_DEFAULT_MAX_FORMATTER_TEXT_LENGTH = 2000
|
||||
_DEFAULT_MAX_FORMATTER_TEXT_LENGTH = 1000
|
||||
|
||||
|
||||
class AsBlockStat(BaseModel):
|
||||
|
|
@ -27,7 +27,8 @@ class AsBlockStat(BaseModel):
|
|||
return self.format(_DEFAULT_MAX_BLOCK_TEXT_PREVIEW_LENGTH)
|
||||
|
||||
def _truncate(self, text: str, max_length: int) -> str:
|
||||
"""Simple truncation with ellipsis."""
|
||||
"""Truncate text with ellipsis, replacing newlines with spaces."""
|
||||
text = text.replace("\n", " ")
|
||||
if len(text) <= max_length:
|
||||
return text
|
||||
return text[:max_length] + "..."
|
||||
|
|
@ -46,22 +47,22 @@ class AsBlockStat(BaseModel):
|
|||
if self.block_type == "text":
|
||||
if not self.text:
|
||||
return ""
|
||||
return f"<text>{self._truncate(self.text, max_length)}</text>"
|
||||
return f"[text]: {self._truncate(self.text, max_length)}"
|
||||
if self.block_type == "thinking":
|
||||
if not include_thinking or not self.text:
|
||||
return ""
|
||||
return f"<thinking>{self._truncate(self.text, max_length)}</thinking>"
|
||||
return f"[think]: {self._truncate(self.text, max_length)}"
|
||||
if self.block_type in ("image", "audio", "video"):
|
||||
content = self.media_url if self.media_url else ""
|
||||
return f"<{self.block_type}>{content}</{self.block_type}>"
|
||||
return f"[{self.block_type}]: {content}"
|
||||
if self.block_type == "tool_use":
|
||||
content = f"{self.tool_name} params={self._truncate(self.tool_input, max_length)}"
|
||||
return f"<tool_use>{content}</tool_use>"
|
||||
return f"[tool_use]: {content}"
|
||||
if self.block_type == "tool_result":
|
||||
if not self.tool_output:
|
||||
return ""
|
||||
content = f"{self.tool_name} output={self._truncate(self.tool_output, max_length)}"
|
||||
return f"<tool_result>{content}</tool_result>"
|
||||
return f"[tool_result]: {content}"
|
||||
return ""
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,22 @@ from ....core.utils import get_logger
|
|||
logger = get_logger()
|
||||
|
||||
|
||||
def _is_valid_summary(content: str) -> bool:
|
||||
"""Check if the summary content is valid.
|
||||
|
||||
Args:
|
||||
content: The summary content to validate.
|
||||
|
||||
Returns:
|
||||
True if valid, False otherwise.
|
||||
"""
|
||||
if not content or not content.strip():
|
||||
return False
|
||||
if "##" not in content:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class Compactor(BaseOp):
|
||||
"""Compactor class for compacting memory messages."""
|
||||
|
||||
|
|
@ -17,17 +33,24 @@ class Compactor(BaseOp):
|
|||
self,
|
||||
memory_compact_threshold: int,
|
||||
console_enabled: bool = False,
|
||||
return_dict: bool = False,
|
||||
add_thinking_block: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.memory_compact_threshold: int = memory_compact_threshold
|
||||
self.console_enabled: bool = console_enabled
|
||||
self.return_dict: bool = return_dict
|
||||
self.add_thinking_block: bool = add_thinking_block
|
||||
|
||||
# pylint: disable=too-many-return-statements
|
||||
async def execute(self):
|
||||
messages: list[Msg] = self.context.get("messages", [])
|
||||
previous_summary: str = self.context.get("previous_summary", "")
|
||||
|
||||
if not messages:
|
||||
if self.return_dict:
|
||||
return {"user_message": "", "history_compact": "", "is_valid": False}
|
||||
return ""
|
||||
|
||||
msg_handler = AsMsgHandler(self.as_token_counter)
|
||||
|
|
@ -35,12 +58,15 @@ class Compactor(BaseOp):
|
|||
history_formatted_str: str = await msg_handler.format_msgs_to_str(
|
||||
messages=messages,
|
||||
memory_compact_threshold=self.memory_compact_threshold,
|
||||
include_thinking=self.add_thinking_block,
|
||||
)
|
||||
after_token_count = await msg_handler.count_str_token(history_formatted_str)
|
||||
logger.info(f"Compactor before_token_count={before_token_count} after_token_count={after_token_count}")
|
||||
|
||||
if not history_formatted_str:
|
||||
logger.warning(f"No history to compact. messages={messages}")
|
||||
if self.return_dict:
|
||||
return {"user_message": "", "history_compact": "", "is_valid": False}
|
||||
return ""
|
||||
|
||||
agent = ReActAgent(
|
||||
|
|
@ -52,18 +78,12 @@ class Compactor(BaseOp):
|
|||
agent.set_console_output_enabled(self.console_enabled)
|
||||
|
||||
if previous_summary:
|
||||
prefix: str = self.get_prompt("update_user_message_prefix")
|
||||
suffix: str = self.get_prompt("update_user_message_suffix")
|
||||
user_message: str = (
|
||||
f"<conversation>\n{history_formatted_str}\n</conversation>\n\n"
|
||||
f"{prefix}\n\n"
|
||||
f"<previous-summary>\n{previous_summary}\n</previous-summary>\n\n"
|
||||
f"{suffix}"
|
||||
f"# conversation\n{history_formatted_str}\n\n"
|
||||
f"# previous-summary\n{previous_summary}\n\n" + self.get_prompt("update_user_message")
|
||||
)
|
||||
else:
|
||||
user_message: str = f"<conversation>\n{history_formatted_str}\n</conversation>\n\n" + self.get_prompt(
|
||||
"initial_user_message",
|
||||
)
|
||||
user_message: str = f"# conversation\n{history_formatted_str}\n\n" + self.get_prompt("initial_user_message")
|
||||
logger.info(f"Compactor sys_prompt={agent.sys_prompt} user_message={user_message}")
|
||||
|
||||
compact_msg: Msg = await agent.reply(
|
||||
|
|
@ -75,5 +95,16 @@ class Compactor(BaseOp):
|
|||
)
|
||||
|
||||
history_compact: str = compact_msg.get_text_content()
|
||||
is_valid: bool = _is_valid_summary(history_compact)
|
||||
|
||||
if not is_valid:
|
||||
logger.warning(f"Invalid summary result: {history_compact[:200]}...")
|
||||
if self.return_dict:
|
||||
return {"user_message": user_message, "history_compact": history_compact, "is_valid": False}
|
||||
return ""
|
||||
|
||||
logger.info(f"Compactor Result:\n{history_compact}")
|
||||
|
||||
if self.return_dict:
|
||||
return {"user_message": user_message, "history_compact": history_compact, "is_valid": True}
|
||||
return history_compact
|
||||
|
|
|
|||
|
|
@ -7,10 +7,14 @@ system_prompt_zh: |
|
|||
这些摘要可以在未来会话中用于恢复上下文。专注于保留关键信息,同时减少token数量。
|
||||
|
||||
initial_user_message: |
|
||||
The messages above are a conversation to summarize. Create a structured context checkpoint summary
|
||||
that another LLM will use to continue the work.
|
||||
# Task
|
||||
Create a structured summary from the conversation above.
|
||||
|
||||
Use this EXACT format:
|
||||
# Rules:
|
||||
- Keep each section concise
|
||||
- Preserve exact file paths, function names, and error messages
|
||||
|
||||
# Output Format:
|
||||
|
||||
## Goal
|
||||
[What is the user trying to accomplish? Can be multiple items if the session covers different tasks.]
|
||||
|
|
@ -39,13 +43,17 @@ initial_user_message: |
|
|||
- [Any data, examples, or references needed to continue]
|
||||
- [Or "(none)" if not applicable]
|
||||
|
||||
Keep each section concise. Preserve exact file paths, function names, and error messages.
|
||||
Output the structured summary following the format above.
|
||||
|
||||
initial_user_message_zh: |
|
||||
上述消息是一场需要总结的对话。创建一个结构化的上下文检查点摘要,
|
||||
以便另一个LLM可以用来继续工作。
|
||||
# 任务
|
||||
根据上面的对话创建一个结构化摘要。
|
||||
|
||||
使用此确切格式:
|
||||
# 规则:
|
||||
- 保持每个部分简洁
|
||||
- 保留确切的文件路径、函数名称和错误消息
|
||||
|
||||
# 输出示例:
|
||||
|
||||
## 目标
|
||||
[用户试图完成什么?如果会话涵盖不同任务,可以有多个项目。]
|
||||
|
|
@ -74,14 +82,13 @@ initial_user_message_zh: |
|
|||
- [任何继续工作所需的数据、示例或参考资料]
|
||||
- [或者如果不适用则为"(none)"]
|
||||
|
||||
保持每个部分简洁。保留确切的文件路径、函数名称和错误消息。
|
||||
请按照上面示例的格式,输出结构化摘要。
|
||||
|
||||
update_user_message_prefix: |
|
||||
The messages above are NEW conversation messages to incorporate into the existing summary provided in
|
||||
<previous-summary> tags.
|
||||
update_user_message: |
|
||||
# Task
|
||||
Update the structured summary with new conversation messages.
|
||||
|
||||
update_user_message_suffix: |
|
||||
Update the existing structured summary with new information. RULES:
|
||||
# Rules:
|
||||
- PRESERVE all existing information from the previous summary
|
||||
- ADD new progress, decisions, and context from the new messages
|
||||
- UPDATE the Progress section: move items from "In Progress" to "Done" when completed
|
||||
|
|
@ -89,7 +96,7 @@ update_user_message_suffix: |
|
|||
- PRESERVE exact file paths, function names, and error messages
|
||||
- If something is no longer relevant, you may remove it
|
||||
|
||||
Use this EXACT format:
|
||||
# Output Format:
|
||||
|
||||
## Goal
|
||||
[Preserve existing goals, add new ones if the task expanded]
|
||||
|
|
@ -116,13 +123,13 @@ update_user_message_suffix: |
|
|||
## Critical Context
|
||||
- [Preserve important context, add new if needed]
|
||||
|
||||
Keep each section concise. Preserve exact file paths, function names, and error messages.
|
||||
Output the structured summary following the format above.
|
||||
|
||||
update_user_message_prefix_zh: |
|
||||
以上消息是需要整合到现有摘要中的新对话内容,现有摘要位于<previous-summary>标签中。
|
||||
update_user_message_zh: |
|
||||
# 任务
|
||||
使用新的对话内容来更新结构化摘要。
|
||||
|
||||
update_user_message_suffix_zh: |
|
||||
用新信息更新现有的结构化摘要。规则:
|
||||
# 规则:
|
||||
- 保留来自先前摘要的所有现有信息
|
||||
- 从新消息中添加新的进展、决策和上下文
|
||||
- 更新进度部分:当完成时将项目从"进行中"移到"已完成"
|
||||
|
|
@ -130,7 +137,7 @@ update_user_message_suffix_zh: |
|
|||
- 保留确切的文件路径、函数名称和错误消息
|
||||
- 如果某些内容不再相关,您可以删除它
|
||||
|
||||
使用此确切格式:
|
||||
# 输出示例:
|
||||
|
||||
## 目标
|
||||
[保留现有目标,如果任务扩展则添加新目标]
|
||||
|
|
@ -157,4 +164,4 @@ update_user_message_suffix_zh: |
|
|||
## 关键上下文
|
||||
- [保留重要上下文,如需要则添加新的]
|
||||
|
||||
保持每个部分简洁。保留确切的文件路径、函数名称和错误消息。
|
||||
请按照上面示例的格式,输出结构化摘要。
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ class Summarizer(BaseOp):
|
|||
toolkit: Toolkit | None = None,
|
||||
console_enabled: bool = False,
|
||||
timezone: str | None = None,
|
||||
add_thinking_block: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
|
@ -34,6 +35,16 @@ class Summarizer(BaseOp):
|
|||
self.toolkit: Toolkit | None = toolkit
|
||||
self.console_enabled: bool = console_enabled
|
||||
self.timezone: str | None = timezone
|
||||
self.add_thinking_block: bool = add_thinking_block
|
||||
|
||||
def _get_current_datetime(self) -> datetime.datetime:
|
||||
"""Get current datetime with timezone, fallback to local time if timezone is invalid."""
|
||||
if self.timezone:
|
||||
try:
|
||||
return datetime.datetime.now(zoneinfo.ZoneInfo(self.timezone))
|
||||
except Exception as e:
|
||||
logger.error(f"Invalid timezone: {self.timezone}, falling back to local time error={e}")
|
||||
return datetime.datetime.now()
|
||||
|
||||
async def execute(self):
|
||||
messages: list[Msg] = self.context.get("messages", [])
|
||||
|
|
@ -46,6 +57,7 @@ class Summarizer(BaseOp):
|
|||
history_formatted_str: str = await msg_handler.format_msgs_to_str(
|
||||
messages=messages,
|
||||
memory_compact_threshold=self.memory_compact_threshold,
|
||||
include_thinking=self.add_thinking_block,
|
||||
)
|
||||
after_token_count = await msg_handler.count_str_token(history_formatted_str)
|
||||
logger.info(f"Summarizer before_token_count={before_token_count} after_token_count={after_token_count}")
|
||||
|
|
@ -63,15 +75,9 @@ class Summarizer(BaseOp):
|
|||
)
|
||||
agent.set_console_output_enabled(self.console_enabled)
|
||||
|
||||
user_message: str = f"<conversation>\n{history_formatted_str}\n</conversation>\n" + self.prompt_format(
|
||||
user_message: str = f"# conversation\n{history_formatted_str}\n\n" + self.prompt_format(
|
||||
"user_message",
|
||||
date=(
|
||||
datetime.datetime.now(
|
||||
zoneinfo.ZoneInfo(self.timezone),
|
||||
)
|
||||
if self.timezone
|
||||
else datetime.datetime.now()
|
||||
).strftime("%Y-%m-%d"),
|
||||
date=self._get_current_datetime().strftime("%Y-%m-%d"),
|
||||
working_dir=self.working_dir,
|
||||
memory_dir=self.memory_dir,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -126,11 +126,8 @@ class ReMeInMemoryMemory(InMemoryMemory):
|
|||
|
||||
if prepend_summary and self._compressed_summary:
|
||||
previous_summary = f"""
|
||||
<previous-summary>
|
||||
{self._compressed_summary}
|
||||
</previous-summary>
|
||||
The above is a summary of our previous conversation.
|
||||
Use it as context to maintain continuity.
|
||||
The above is a summary of previous conversation, use it as context to maintain continuity.
|
||||
""".strip()
|
||||
|
||||
return [
|
||||
|
|
|
|||
|
|
@ -204,7 +204,7 @@ class AsMsgHandler:
|
|||
self,
|
||||
messages: list[Msg],
|
||||
memory_compact_threshold: int,
|
||||
include_thinking: bool = False,
|
||||
include_thinking: bool = True,
|
||||
) -> str:
|
||||
"""Format list of messages to a single formatted string.
|
||||
|
||||
|
|
|
|||
|
|
@ -347,7 +347,9 @@ class ReMeLight(Application):
|
|||
max_input_length: float = 128 * 1024,
|
||||
compact_ratio: float = 0.7,
|
||||
previous_summary: str = "",
|
||||
) -> str:
|
||||
return_dict: bool = False,
|
||||
add_thinking_block: bool = True,
|
||||
) -> str | dict:
|
||||
"""
|
||||
Compact a list of messages into a condensed summary.
|
||||
|
||||
|
|
@ -371,10 +373,13 @@ class ReMeLight(Application):
|
|||
Defaults to 0.7.
|
||||
previous_summary (str): Previous summary to incorporate into the new
|
||||
summary for continuity. Defaults to empty string.
|
||||
return_dict (bool): If True, returns a dict with user_message,
|
||||
history_compact, and is_valid. Defaults to False.
|
||||
|
||||
Returns:
|
||||
str: The condensed summary of the messages, or an empty string if
|
||||
an error occurred during compaction.
|
||||
str | dict: The condensed summary string, or a dict containing
|
||||
user_message, history_compact, and is_valid if return_dict=True.
|
||||
Returns empty string or dict with empty values if an error occurred.
|
||||
"""
|
||||
try:
|
||||
compactor = Compactor(
|
||||
|
|
@ -383,6 +388,8 @@ class ReMeLight(Application):
|
|||
as_llm_formatter=as_llm_formatter,
|
||||
as_token_counter=as_token_counter,
|
||||
language=language if language == "zh" else "",
|
||||
return_dict=return_dict,
|
||||
add_thinking_block=add_thinking_block,
|
||||
)
|
||||
|
||||
return await compactor.call(
|
||||
|
|
@ -392,8 +399,10 @@ class ReMeLight(Application):
|
|||
)
|
||||
|
||||
except Exception as e:
|
||||
# Log error and return empty string to indicate failure
|
||||
# Log error and return appropriate empty result
|
||||
logger.exception(f"Error compacting memory: {e}")
|
||||
if return_dict:
|
||||
return {"user_message": str(e), "history_compact": str(e), "is_valid": False}
|
||||
return ""
|
||||
|
||||
async def summary_memory(
|
||||
|
|
@ -407,6 +416,7 @@ class ReMeLight(Application):
|
|||
max_input_length: float = 128 * 1024,
|
||||
compact_ratio: float = 0.7,
|
||||
timezone: str | None = None,
|
||||
add_thinking_block: bool = True,
|
||||
) -> str:
|
||||
"""
|
||||
Generate a comprehensive summary of the given messages.
|
||||
|
|
@ -459,6 +469,7 @@ class ReMeLight(Application):
|
|||
as_token_counter=as_token_counter,
|
||||
language=language if language == "zh" else "",
|
||||
timezone=timezone,
|
||||
add_thinking_block=add_thinking_block,
|
||||
)
|
||||
|
||||
return await summarizer.call(messages=messages, service_context=self.service_context)
|
||||
|
|
|
|||
|
|
@ -295,7 +295,6 @@ async def test_file_watch_integration():
|
|||
"watch_paths": [TestConfig.WORKING_DIR, f"{TestConfig.WORKING_DIR}/memory"],
|
||||
"suffix_filters": [".md"],
|
||||
"recursive": False,
|
||||
"scan_on_start": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Async unit tests for BaseFileWatcher covering:
|
|||
- File suffix filtering
|
||||
- Start/stop lifecycle
|
||||
- Callback functionality
|
||||
- scan_on_start feature
|
||||
- rebuild_index_on_start feature
|
||||
|
||||
Usage:
|
||||
pytest tests/test_base_file_watcher.py -v
|
||||
|
|
@ -369,12 +369,12 @@ class TestCallbackFunctionality:
|
|||
# ==================== Test Scan on Start ====================
|
||||
|
||||
|
||||
class TestScanOnStart:
|
||||
"""Tests for scan_on_start feature."""
|
||||
class TestRebuildIndexOnStart:
|
||||
"""Tests for rebuild_index_on_start feature."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_on_start_false(self, temp_files, temp_dir: Path):
|
||||
"""Test that scan_on_start=False doesn't scan existing files."""
|
||||
async def test_rebuild_index_on_start_false(self, temp_files, temp_dir: Path):
|
||||
"""Test that rebuild_index_on_start=False doesn't scan existing files."""
|
||||
callback_called = []
|
||||
|
||||
async def callback(changes):
|
||||
|
|
@ -387,7 +387,7 @@ class TestScanOnStart:
|
|||
|
||||
watcher = BaseFileWatcher(
|
||||
watch_paths=str(temp_dir),
|
||||
scan_on_start=False,
|
||||
rebuild_index_on_start=False,
|
||||
callback=callback,
|
||||
file_store=mock_file_store,
|
||||
)
|
||||
|
|
@ -400,8 +400,8 @@ class TestScanOnStart:
|
|||
assert len(callback_called) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_on_start_true_with_files(self, temp_files, temp_dir: Path):
|
||||
"""Test that scan_on_start=True scans existing files."""
|
||||
async def test_rebuild_index_on_start_true_with_files(self, temp_files, temp_dir: Path):
|
||||
"""Test that rebuild_index_on_start=True scans existing files."""
|
||||
callback_called = []
|
||||
|
||||
async def callback(changes):
|
||||
|
|
@ -414,7 +414,7 @@ class TestScanOnStart:
|
|||
|
||||
watcher = BaseFileWatcher(
|
||||
watch_paths=str(temp_dir),
|
||||
scan_on_start=True,
|
||||
rebuild_index_on_start=True,
|
||||
callback=callback,
|
||||
file_store=mock_file_store,
|
||||
)
|
||||
|
|
@ -434,8 +434,8 @@ class TestScanOnStart:
|
|||
assert all(change == Change.added for change, _ in all_changes)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_on_start_with_suffix_filter(self, temp_files, temp_dir: Path):
|
||||
"""Test scan_on_start respects suffix filters."""
|
||||
async def test_rebuild_index_on_start_with_suffix_filter(self, temp_files, temp_dir: Path):
|
||||
"""Test rebuild_index_on_start respects suffix filters."""
|
||||
callback_called = []
|
||||
|
||||
async def callback(changes):
|
||||
|
|
@ -447,7 +447,7 @@ class TestScanOnStart:
|
|||
|
||||
watcher = BaseFileWatcher(
|
||||
watch_paths=str(temp_dir),
|
||||
scan_on_start=True,
|
||||
rebuild_index_on_start=True,
|
||||
suffix_filters=[".txt"],
|
||||
callback=callback,
|
||||
file_store=mock_file_store,
|
||||
|
|
@ -467,8 +467,8 @@ class TestScanOnStart:
|
|||
assert path.endswith(".txt"), f"Expected .txt file, got {path}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_on_start_recursive(self, temp_nested_dir: Path):
|
||||
"""Test scan_on_start with recursive=True."""
|
||||
async def test_rebuild_index_on_start_recursive(self, temp_nested_dir: Path):
|
||||
"""Test rebuild_index_on_start with recursive=True."""
|
||||
callback_called = []
|
||||
|
||||
async def callback(changes):
|
||||
|
|
@ -480,7 +480,7 @@ class TestScanOnStart:
|
|||
|
||||
watcher = BaseFileWatcher(
|
||||
watch_paths=str(temp_nested_dir),
|
||||
scan_on_start=True,
|
||||
rebuild_index_on_start=True,
|
||||
recursive=True,
|
||||
suffix_filters=[".txt"],
|
||||
callback=callback,
|
||||
|
|
@ -503,8 +503,8 @@ class TestScanOnStart:
|
|||
assert nested_found, "Should find files in nested directories"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_on_start_non_recursive(self, temp_nested_dir: Path):
|
||||
"""Test scan_on_start with recursive=False."""
|
||||
async def test_rebuild_index_on_start_non_recursive(self, temp_nested_dir: Path):
|
||||
"""Test rebuild_index_on_start with recursive=False."""
|
||||
callback_called = []
|
||||
|
||||
async def callback(changes):
|
||||
|
|
@ -516,7 +516,7 @@ class TestScanOnStart:
|
|||
|
||||
watcher = BaseFileWatcher(
|
||||
watch_paths=str(temp_nested_dir),
|
||||
scan_on_start=True,
|
||||
rebuild_index_on_start=True,
|
||||
recursive=False,
|
||||
suffix_filters=[".txt"],
|
||||
callback=callback,
|
||||
|
|
@ -538,8 +538,8 @@ class TestScanOnStart:
|
|||
assert not nested_found, "Should not find files in nested directories"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_on_start_nonexistent_path(self):
|
||||
"""Test scan_on_start with non-existent path."""
|
||||
async def test_rebuild_index_on_start_nonexistent_path(self):
|
||||
"""Test rebuild_index_on_start with non-existent path."""
|
||||
callback_called = []
|
||||
|
||||
async def callback(changes):
|
||||
|
|
@ -551,7 +551,7 @@ class TestScanOnStart:
|
|||
|
||||
watcher = BaseFileWatcher(
|
||||
watch_paths="/nonexistent/path",
|
||||
scan_on_start=True,
|
||||
rebuild_index_on_start=True,
|
||||
callback=callback,
|
||||
file_store=mock_file_store,
|
||||
)
|
||||
|
|
@ -709,7 +709,7 @@ class TestEdgeCases:
|
|||
|
||||
watcher = BaseFileWatcher(
|
||||
watch_paths=str(file_path),
|
||||
scan_on_start=True,
|
||||
rebuild_index_on_start=True,
|
||||
callback=callback,
|
||||
file_store=mock_file_store,
|
||||
)
|
||||
|
|
@ -742,7 +742,7 @@ class TestEdgeCases:
|
|||
|
||||
watcher = BaseFileWatcher(
|
||||
watch_paths=str(empty_dir),
|
||||
scan_on_start=True,
|
||||
rebuild_index_on_start=True,
|
||||
callback=callback,
|
||||
file_store=mock_file_store,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue