From 5584a5c2393369cace4ce53270b69b3fcf68d469 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Wed, 4 Mar 2026 10:55:43 +0800 Subject: [PATCH] =?UTF-8?q?feat(memory):=20add=20CoPaw=20file-based=20memo?= =?UTF-8?q?ry=20system=20with=20compaction=20and=20=E2=80=A6=20(#134)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(memory): add CoPaw file-based memory system with compaction and summarization * feat(reme): add tool result cleanup and retention management * fix(memory): resolve copaw memory processing and prompt formatting issues * docs(reme_copaw): update documentation and initialization logic * refactor(reme): remove override parameters from compact_tool_result * feat(docs): update README to reflect CoPaw memory system integration * chore(docs): update model names in documentation --- .pre-commit-config.yaml | 1 + README.md | 279 ++++--- README_ZH.md | 261 ++++--- reme/__init__.py | 4 +- reme/config/copaw.yaml | 16 + reme/config/file.yaml | 43 -- reme/memory/__init__.py | 4 +- reme/memory/{file_based => cli}/__init__.py | 0 reme/memory/{file_based => cli}/fb_cli.py | 0 reme/memory/{file_based => cli}/fb_cli.yaml | 0 .../{file_based => cli}/fb_compactor.py | 0 .../{file_based => cli}/fb_compactor.yaml | 0 .../{file_based => cli}/fb_context_checker.py | 0 .../{file_based => cli}/fb_summarizer.py | 0 .../{file_based => cli}/fb_summarizer.yaml | 0 reme/memory/file_based_copaw/__init__.py | 31 + reme/memory/file_based_copaw/compactor.py | 84 +++ reme/memory/file_based_copaw/compactor.yaml | 160 ++++ .../copaw_in_memory_memory.py | 243 ++++++ reme/memory/file_based_copaw/file_io.py | 247 ++++++ .../file_based_copaw/memory_formatter.py | 249 +++++++ reme/memory/file_based_copaw/summarizer.py | 92 +++ reme/memory/file_based_copaw/summarizer.yaml | 50 ++ .../file_based_copaw/tool_result_compactor.py | 105 +++ reme/memory/file_based_copaw/utils.py | 231 ++++++ reme/reme_cli.py | 177 ++++- reme/reme_copaw.py | 700 ++++++++++++++++++ reme/reme_fb.py | 183 ----- tests/copaw/test_compactor.py | 375 ++++++++++ tests/copaw/test_memory_formatter.py | 489 ++++++++++++ tests/copaw/test_summarizer.py | 310 ++++++++ tests/copaw/test_tool_result_compactor.py | 176 +++++ tests/copaw/test_utils.py | 89 +++ tests/test_fs_compactor.py | 6 +- tests/test_fs_context_checker.py | 8 +- tests/test_fs_file_watch_integration.py | 20 +- tests/test_fs_memory_get.py | 20 +- tests/test_fs_memory_search.py | 42 +- tests/test_fs_summary.py | 14 +- 39 files changed, 4214 insertions(+), 495 deletions(-) create mode 100644 reme/config/copaw.yaml delete mode 100644 reme/config/file.yaml rename reme/memory/{file_based => cli}/__init__.py (100%) rename reme/memory/{file_based => cli}/fb_cli.py (100%) rename reme/memory/{file_based => cli}/fb_cli.yaml (100%) rename reme/memory/{file_based => cli}/fb_compactor.py (100%) rename reme/memory/{file_based => cli}/fb_compactor.yaml (100%) rename reme/memory/{file_based => cli}/fb_context_checker.py (100%) rename reme/memory/{file_based => cli}/fb_summarizer.py (100%) rename reme/memory/{file_based => cli}/fb_summarizer.yaml (100%) create mode 100644 reme/memory/file_based_copaw/__init__.py create mode 100644 reme/memory/file_based_copaw/compactor.py create mode 100644 reme/memory/file_based_copaw/compactor.yaml create mode 100644 reme/memory/file_based_copaw/copaw_in_memory_memory.py create mode 100644 reme/memory/file_based_copaw/file_io.py create mode 100644 reme/memory/file_based_copaw/memory_formatter.py create mode 100644 reme/memory/file_based_copaw/summarizer.py create mode 100644 reme/memory/file_based_copaw/summarizer.yaml create mode 100644 reme/memory/file_based_copaw/tool_result_compactor.py create mode 100644 reme/memory/file_based_copaw/utils.py create mode 100644 reme/reme_copaw.py delete mode 100644 reme/reme_fb.py create mode 100644 tests/copaw/test_compactor.py create mode 100644 tests/copaw/test_memory_formatter.py create mode 100644 tests/copaw/test_summarizer.py create mode 100644 tests/copaw/test_tool_result_compactor.py create mode 100644 tests/copaw/test_utils.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d44e7f60..42f6ad0b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,6 +50,7 @@ repos: --disable=W0511, --disable=W0718, --disable=W0122, + --disable=W1203, --disable=C0103, --disable=R0913, --disable=R0917, diff --git a/README.md b/README.md index f6a4e67e..98c5dd8c 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@

License - English - 简体中文 + English + 简体中文 GitHub Stars

@@ -36,11 +36,14 @@ and the next conversation can recall it automatically. --- -## 📁 File-Based ReMe +## 📁 File-Based CoPaw Memory System > Memory as files, files as memory -Treat **memory as files** — readable, editable, and portable. +Treat **memory as files** — readable, editable, and portable. [CoPaw](https://github.com/agentscope-ai/CoPaw) +integrates this memory system +through [MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py), +which inherits `ReMeCopaw` and exposes memory management capabilities. | Traditional Memory Systems | File-Based ReMe | |----------------------------|--------------------| @@ -50,26 +53,31 @@ Treat **memory as files** — readable, editable, and portable. | 🚫 Hard to migrate | 📦 Copy to migrate | ``` -.reme/ -├── MEMORY.md # Long-term memory: user preferences, project config, etc. -└── memory/ - └── YYYY-MM-DD.md # Daily logs: work records for the day, written upon compact +working_dir/ +├── MEMORY.md # Long-term memory: user preferences, project config, etc. +├── memory/ +│ └── YYYY-MM-DD.md # Daily summary logs: written automatically after conversation ends +└── tool_result/ # Cache for oversized tool outputs (auto-managed, auto-cleaned when expired) + └── .txt ``` ### Core Capabilities -[ReMe File Based](reme/reme_fb.py) is the core class of the file-based memory system. It acts like an **intelligent -secretary**, managing all memory-related operations: +[ReMeCopaw](reme/reme_copaw.py) is the core class of this memory system, providing complete memory management +capabilities for AI Agents: -| Method | Function | Key Components | -|-----------------|------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `start` | 🚀 Start memory system | [BaseFileStore](reme/core/file_store/base_file_store.py) (local file storage)
[BaseFileWatcher](reme/core/file_watcher/base_file_watcher.py) (file watcher)
[BaseEmbeddingModel](reme/core/embedding/base_embedding_model.py) (embedding cache) | -| `close` | 📕 Close and save | Close file store, stop file watcher, save embedding cache | -| `context_check` | 📏 Check context limit | [ContextChecker](reme/memory/file_based/fb_context_checker.py) | -| `compact` | 📦 Compact history to summary | [Compactor](reme/memory/file_based/fb_compactor.py) | -| `summary` | 📝 Write important memory to files | [Summarizer](reme/memory/file_based/fb_summarizer.py) | -| `memory_search` | 🔍 Semantic memory search | [MemorySearch](reme/memory/tools/chunk/memory_search.py) | -| `memory_get` | 📖 Read specified memory file | [MemoryGet](reme/memory/tools/chunk/memory_get.py) | +| Method | Function | Key Components | +|--------------------------|------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `start` | 🚀 Start memory system | Initialize file store, file watcher, Embedding cache; clean up expired tool result files | +| `close` | 📕 Close and clean up | Clean tool result files, stop file watcher, save Embedding cache | +| `compact_memory` | 📦 Compact history to summary | [Compactor](reme/memory/file_based_copaw/compactor.py) — ReActAgent generates structured context checkpoint | +| `summary_memory` | 📝 Write important memory to files | [Summarizer](reme/memory/file_based_copaw/summarizer.py) — ReActAgent + file tools (read / write / edit) | +| `compact_tool_result` | ✂️ Compact oversized tool output | [ToolResultCompactor](reme/memory/file_based_copaw/tool_result_compactor.py) — Truncate and save to `tool_result/`, keep file reference in message | +| `add_async_summary_task` | ⚡ Submit background summary task | `asyncio.create_task`, summary doesn't block main conversation flow | +| `await_summary_tasks` | ⏳ Wait for background tasks | Collect results from all background summary tasks, call before closing to ensure writes complete | +| `memory_search` | 🔍 Semantic memory search | [MemorySearch](reme/memory/tools/chunk/memory_search.py) — Vector + BM25 hybrid retrieval | +| `get_in_memory_memory` | 🗂️ Create in-memory instance | [CoPawInMemoryMemory](reme/memory/file_based_copaw/copaw_in_memory_memory.py) — Token-aware memory management, supports compression summary and state serialization | +| `update_params` | ⚙️ Update runtime parameters | Adjust `max_input_length`, `memory_compact_ratio`, `language` at runtime | --- @@ -205,59 +213,81 @@ Commands starting with `/` control session state: ### Using the ReMe Package -#### File-Based ReMe +#### File-Based ReMe (CoPaw Memory System) + +`ReMeCopaw` receives AgentScope components like `ChatModelBase`, `Formatter`, `Toolkit`, and configures Embedding and +storage backend via environment variables: + +| Environment Variable | Description | Default | +|----------------------------|-----------------------------------------------|-----------------------------------------------------| +| `EMBEDDING_API_KEY` | Embedding service API Key | `""` (vector search disabled if not configured) | +| `EMBEDDING_BASE_URL` | Embedding service Base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | +| `EMBEDDING_MODEL_NAME` | Embedding model name | `""` | +| `EMBEDDING_DIMENSIONS` | Vector dimensions | `1024` | +| `EMBEDDING_CACHE_ENABLED` | Whether to enable Embedding cache | `true` | +| `EMBEDDING_MAX_CACHE_SIZE` | Maximum cache entries | `2000` | +| `FTS_ENABLED` | Whether to enable full-text search (BM25) | `true` | +| `MEMORY_STORE_BACKEND` | Storage backend (`auto` / `chroma` / `local`) | `auto` (local on Windows, chroma on others) | ```python import asyncio -from reme import ReMeFb +from agentscope.formatter import ClaudeFormatter +from agentscope.model import get_model +from agentscope.token import HuggingFaceTokenCounter +from agentscope.tool import Toolkit + +from reme.reme_copaw import ReMeCopaw async def main(): - # Initialize and start - reme = ReMeFb( - default_llm_config={ - "backend": "openai", # Backend type, OpenAI-compatible API - "model_name": "qwen3.5-plus", # Model name - }, - default_file_store_config={ - "backend": "chroma", # Store backend: sqlite/chroma/local - "fts_enabled": True, # Enable full-text search - "vector_enabled": False, # Enable vector search (set False if no embedding service) - }, - context_window_tokens=128000, # Model context window size (tokens) - reserve_tokens=36000, # Tokens reserved for output - keep_recent_tokens=20000, # Tokens to keep for recent messages - vector_weight=0.7, # Vector search weight (0–1) for hybrid search - candidate_multiplier=3.0, # Candidate multiplier for recall + # Prepare AgentScope core components + chat_model = get_model(config={"backend": "openai", "model_name": "qwen3.5-plus"}) + formatter = ClaudeFormatter() + token_counter = HuggingFaceTokenCounter() + toolkit = Toolkit() # Can register additional tools + + # Initialize ReMeCopaw + reme = ReMeCopaw( + working_dir=".reme", # Memory file storage directory + chat_model=chat_model, + formatter=formatter, + token_counter=token_counter, + toolkit=toolkit, + max_input_length=128000, # Model context window (tokens) + memory_compact_ratio=0.7, # Trigger compaction when reaching max_input_length * 0.7 + language="zh", # Summary language (zh / "") + tool_result_threshold=1000, # Auto-save tool outputs exceeding this character count + retention_days=7, # tool_result/ file retention days ) await reme.start() - messages = [ - {"role": "user", "content": "I prefer Python 3.12"}, - {"role": "assistant", "content": "Noted, you prefer Python 3.12"}, - ] + messages = [...] # list[Msg], conversation history - # Check if context exceeds limit - result = await reme.context_check(messages) - print(f"Compact result: {result}") + # 1. Compact oversized tool outputs (prevent tool results from overflowing context) + messages = await reme.compact_tool_result(messages) - # Compact conversation to summary - summary = await reme.compact(messages_to_summarize=messages) - print(f"Summary: {summary}") + # 2. Compact history to structured summary (trigger: context approaching limit) + summary = await reme.compact_memory( + messages=messages, + previous_summary="", # Can pass previous summary for incremental update + ) + print(f"Compact summary:\n{summary}") - # Write important memory to files (ReAct Agent does this automatically) - await reme.summary(messages=messages, date="2026-02-28") + # 3. Submit async summary task in background (non-blocking, writes to memory/YYYY-MM-DD.md) + reme.add_async_summary_task(messages=messages) - # Semantic search over memory - results = await reme.memory_search(query="Python version preference", max_results=5) - print(f"Search results: {results}") + # 4. Semantic memory search (Vector + BM25 hybrid retrieval) + result = await reme.memory_search(query="Python version preference", max_results=5) + print(f"Search results: {result}") - # Read specified memory file - content = await reme.memory_get(path="MEMORY.md") - print(f"Memory content: {content}") + # 5. Get in-memory instance (CoPawInMemoryMemory, manages single conversation context) + memory = reme.get_in_memory_memory() + token_stats = await memory.estimate_tokens() + print(f"Current context usage: {token_stats['context_usage_ratio']:.1f}%") - # Close (save embedding cache, stop file watcher) + # 6. Wait for background tasks before closing + await reme.await_summary_tasks() await reme.close() @@ -278,7 +308,7 @@ async def main(): working_dir=".reme", default_llm_config={ "backend": "openai", - "model_name": "qwen3-30b-a3b-thinking-2507", + "model_name": "qwen3.5-plus", }, default_embedding_model_config={ "backend": "openai", @@ -307,7 +337,8 @@ async def main(): # 2. Retrieve relevant memory memories = await reme.retrieve_memory( query="Python programming", - # user_name="alice", + user_name="alice", + # task_name="code_writing", ) print(f"Retrieve result: {memories}") @@ -358,30 +389,87 @@ if __name__ == "__main__": ## 🏛️ Technical Architecture -### File-Based ReMe Core Architecture +### File-Based CoPaw Memory System Architecture + +[CoPaw MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py) +inherits +`ReMeCopaw` and integrates memory capabilities into the Agent reasoning flow: ```mermaid graph TB - User[User / Agent] --> ReMeFb[File based ReMe] - ReMeFb --> ContextCheck[Context Check] - ReMeFb --> Compact[Context Compact] - ReMeFb --> Summary[Memory Summary] - ReMeFb --> Search[Memory Retrieval] - ContextCheck --> FbContextChecker[Check Token Limit] - Compact --> FbCompactor[Compact History to Summary] - Summary --> FbSummarizer[ReAct Agent + File Tools] - Search --> MemorySearch[Vector + BM25 Hybrid Search] - FbSummarizer --> FileTools[read / write / edit] - FileTools --> MemoryFiles[memory/*.md] + CoPaw["CoPaw MemoryManager\n(inherits ReMeCopaw)"] -->|pre_reasoning hook| Hook[MemoryCompactionHook] + CoPaw --> ReMeCopaw[ReMeCopaw] + Hook -->|exceeds threshold| ReMeCopaw + ReMeCopaw --> CompactMemory[compact_memory\nHistory compaction] + ReMeCopaw --> SummaryMemory[summary_memory\nWrite memory to files] + ReMeCopaw --> CompactToolResult[compact_tool_result\nOversized tool output compaction] + ReMeCopaw --> MemSearch[memory_search\nSemantic search] + ReMeCopaw --> InMemory[get_in_memory_memory\nCoPawInMemoryMemory] + CompactMemory --> Compactor[Compactor\nReActAgent] + SummaryMemory --> Summarizer[Summarizer\nReActAgent + file tools] + CompactToolResult --> ToolResultCompactor[ToolResultCompactor\nTruncate + save to file] + Summarizer --> FileIO[FileIO\nread / write / edit] + FileIO --> MemoryFiles[memory/YYYY-MM-DD.md] + ToolResultCompactor --> ToolResultFiles[tool_result/*.txt] MemoryFiles -.->|File change| FileWatcher[Async File Watcher] FileWatcher -->|Update index| FileStore[Local DB] - MemorySearch --> FileStore + MemSearch --> FileStore ``` +#### Auto-Compaction Trigger Flow + +`MemoryCompactionHook` checks context token usage before each reasoning step, automatically triggering compaction when +threshold is exceeded: + +```mermaid +graph LR + A[pre_reasoning] --> B{Token exceeds threshold?} + B -->|No| Z[Continue reasoning] + B -->|Yes| C[compact_tool_result\nCompact oversized tool outputs in recent messages] + C --> D[compact_memory\nGenerate structured context checkpoint] + D --> E[Mark old messages as COMPRESSED] + E --> F[add_async_summary_task\nBackground write to memory file] + F --> Z +``` + +#### Context Compaction Summary Format + +[Compactor](reme/memory/file_based_copaw/compactor.py) uses ReActAgent to compact history into structured **context +checkpoints**: + +| Field | Description | +|-----------------------|--------------------------------------------------| +| `## Goal` | 🎯 User's objectives (can be multiple) | +| `## Constraints` | ⚙️ Constraints and preferences mentioned by user | +| `## Progress` | 📈 Completed / in progress / blocked tasks | +| `## Key Decisions` | 🔑 Decisions made with brief reasons | +| `## Next Steps` | 🗺️ Next action plan (ordered list) | +| `## Critical Context` | 📌 File paths, function names, error messages | + +Supports **incremental updates**: when `previous_summary` is passed, automatically merges new conversation with old +summary, preserving historical progress. + +#### Tool Result Compaction + +[ToolResultCompactor](reme/memory/file_based_copaw/tool_result_compactor.py) solves context overflow caused by oversized +tool outputs: + +```mermaid +graph LR + A[tool_result message] --> B{Content length > threshold?} + B -->|No| C[Keep as-is] + B -->|Yes| D[Truncate to threshold characters] + D --> E[Write full content to tool_result/uuid.txt] + E --> F[Append file reference path to message] +``` + +Expired files (exceeding `retention_days`) are automatically cleaned up during `start` / `close` / +`compact_tool_result`. + #### Memory Summary: ReAct + File Tools -[Summarizer](reme/memory/file_based/fb_summarizer.py) is the core component for memory summarization. It uses the -**ReAct + file tools** pattern. +[Summarizer](reme/memory/file_based_copaw/summarizer.py) uses the **ReAct + file tools** pattern, letting AI +autonomously decide what to write and where: ```mermaid graph LR @@ -394,43 +482,28 @@ graph LR F -->|No| G[Done] ``` -#### File Tool Set +[FileIO](reme/memory/file_based_copaw/file_io.py) provides file operation tools: -Summarizer is equipped with file operation tools so the AI can work directly on memory files: +| Tool | Function | Use case | +|---------|--------------------------------|-----------------------------------------| +| `read` | Read file content (line range) | View existing memory, avoid duplicates | +| `write` | Overwrite file | Create new memory file or major rewrite | +| `edit` | Replace after exact match | Append or modify specific sections | -| Tool | Function | Use case | -|---------|-------------------|-----------------------------------------| -| `read` | Read file content | View existing memory, avoid duplicates | -| `write` | Overwrite file | Create new memory file or major rewrite | -| `edit` | Edit part of file | Append or modify specific sections | +#### In-Memory Session Management -#### Context Compaction +[CoPawInMemoryMemory](reme/memory/file_based_copaw/copaw_in_memory_memory.py) extends AgentScope's `InMemoryMemory`: -When a conversation gets too long, [Compactor](reme/memory/file_based/fb_compactor.py) compresses history into a concise -summary — like **meeting minutes**, turning long discussion into key points. - -```mermaid -graph LR - A[Messages 1..N] --> B[📦 Compact summary] -C[Recent messages] --> D[Keep as-is] -B --> E[New context] -D --> E -``` - -The compact summary includes what’s needed to continue: - -| Content | Description | -|----------------|---------------------------------------------| -| 🎯 Goals | What the user wants to accomplish | -| ⚙️ Constraints | Requirements and preferences mentioned | -| 📈 Progress | Completed / in progress / blocked tasks | -| 🔑 Decisions | Decisions made and reasons | -| 📌 Context | Key data such as file paths, function names | +| Feature | Description | +|----------------------------------|---------------------------------------------------------------------| +| `get_memory` | Filter messages by mark, auto-prepend compression summary | +| `estimate_tokens` | Precisely estimate current context token usage and ratio | +| `get_history_str` | Generate human-readable conversation summary (with token stats) | +| `state_dict` / `load_state_dict` | Support state serialization / deserialization (session persistence) | #### Memory Retrieval -[MemorySearch](reme/memory/tools/chunk/memory_search.py) provides **vector + BM25 hybrid retrieval**. The two methods -complement each other: +[MemorySearch](reme/memory/tools/chunk/memory_search.py) provides **vector + BM25 hybrid retrieval**: | Retrieval | Strength | Weakness | |---------------------|-------------------------------------------------|----------------------------------------| diff --git a/README_ZH.md b/README_ZH.md index 2dbf02f3..d5b8231b 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -33,11 +33,13 @@ ReMe 让智能体拥有**真正的记忆力**——旧对话自动浓缩，重 --- -## 📁 基于文件的 ReMe +## 📁 基于文件的 CoPaw 记忆系统 > 记忆即文件，文件即记忆 -将**记忆视为文件**——可读、可编辑、可复制。 +将**记忆视为文件**——可读、可编辑、可复制。[CoPaw](https://github.com/agentscope-ai/CoPaw) +通过 [MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py) +集成此记忆系统，继承 `ReMeCopaw` 并对外暴露记忆管理能力。 | 传统记忆系统 | File Based ReMe | |-----------|-----------------| @@ -47,25 +49,30 @@ ReMe 让智能体拥有**真正的记忆力**——旧对话自动浓缩，重 | 🚫 难迁移 | 📦 复制即迁移 | ``` -.reme/ -├── MEMORY.md # 长期记忆：用户偏好、项目配置等不常变的信息 -└── memory/ - └── YYYY-MM-DD.md # 每日日志：当天的工作记录，压缩时自动写入 +working_dir/ +├── MEMORY.md # 长期记忆：用户偏好、项目配置等持久信息 +├── memory/ +│ └── YYYY-MM-DD.md # 每日摘要日志：对话结束后自动写入 +└── tool_result/ # 超长工具输出缓存（自动管理，超期自动清理） + └── .txt ``` ### 核心能力 -[ReMe File Based](reme/reme_fb.py) 是基于文件的记忆系统的核心类，就像一个**智能秘书**，帮你管理所有记忆相关的事务： +[ReMeCopaw](reme/reme_copaw.py) 是该记忆系统的核心类，为 AI Agent 提供完整的记忆管理能力： -| 方法 | 功能 | 关键组件 | -|-----------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `start` | 🚀 启动记忆系统 | [BaseFileStore](reme/core/file_store/base_file_store.py)（本地文件store）
[BaseFileWatcher](reme/core/file_watcher/base_file_watcher.py)（文件监控）
[BaseEmbeddingModel](reme/core/embedding/base_embedding_model.py)（Embedding 缓存） | -| `close` | 📕 关闭并保存 | 关闭文件store、停止文件监控、保存 Embedding 缓存 | -| `context_check` | 📏 检查上下文是否超限 | [ContextChecker](reme/memory/file_based/fb_context_checker.py) | -| `compact` | 📦 压缩历史对话为摘要 | [Compactor](reme/memory/file_based/fb_compactor.py) | -| `summary` | 📝 将重要记忆写入文件 | [Summarizer](reme/memory/file_based/fb_summarizer.py) | -| `memory_search` | 🔍 语义搜索记忆 | [MemorySearch](reme/memory/tools/chunk/memory_search.py) | -| `memory_get` | 📖 读取指定记忆文件 | [MemoryGet](reme/memory/tools/chunk/memory_get.py) | +| 方法 | 功能 | 关键组件 | +|--------------------------|--------------|----------------------------------------------------------------------------------------------------------------| +| `start` | 🚀 启动记忆系统 | 初始化文件存储、文件监控、Embedding 缓存；清理过期工具结果文件 | +| `close` | 📕 关闭并清理 | 清理工具结果文件、停止文件监控、保存 Embedding 缓存 | +| `compact_memory` | 📦 压缩历史对话为摘要 | [Compactor](reme/memory/file_based_copaw/compactor.py) — ReActAgent 生成结构化上下文检查点 | +| `summary_memory` | 📝 将重要记忆写入文件 | [Summarizer](reme/memory/file_based_copaw/summarizer.py) — ReActAgent + 文件工具（read / write / edit） | +| `compact_tool_result` | ✂️ 压缩超长工具输出 | [ToolResultCompactor](reme/memory/file_based_copaw/tool_result_compactor.py) — 截断并转存到 `tool_result/`，消息中保留文件引用 | +| `add_async_summary_task` | ⚡ 提交后台摘要任务 | `asyncio.create_task`，摘要不阻塞主对话流程 | +| `await_summary_tasks` | ⏳ 等待后台任务完成 | 收集所有后台摘要任务的结果，关闭前调用确保写入完成 | +| `memory_search` | 🔍 语义搜索记忆 | [MemorySearch](reme/memory/tools/chunk/memory_search.py) — 向量 + BM25 混合检索 | +| `get_in_memory_memory` | 🗂️ 创建会话内存实例 | [CoPawInMemoryMemory](reme/memory/file_based_copaw/copaw_in_memory_memory.py) — Token 感知的内存管理，支持压缩摘要和状态序列化 | +| `update_params` | ⚙️ 动态更新运行时参数 | 运行时调整 `max_input_length`、`memory_compact_ratio`、`language` | ## 🗃️ 基于向量库的 ReMe @@ -196,59 +203,80 @@ remecli config=cli ### 使用 ReMe Package -#### 基于文件的 ReMe +#### 基于文件的 ReMe（CoPaw的记忆系统） + +`ReMeCopaw` 接收 AgentScope 的 `ChatModelBase`、`Formatter`、`Toolkit` 等组件，通过环境变量配置 Embedding 和存储后端： + +| 环境变量 | 说明 | 默认值 | +|----------------------------|-----------------------------------|-----------------------------------------------------| +| `EMBEDDING_API_KEY` | Embedding 服务 API Key | `""`（未配置则禁用向量搜索） | +| `EMBEDDING_BASE_URL` | Embedding 服务 Base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | +| `EMBEDDING_MODEL_NAME` | Embedding 模型名称 | `""` | +| `EMBEDDING_DIMENSIONS` | 向量维度 | `1024` | +| `EMBEDDING_CACHE_ENABLED` | 是否启用 Embedding 缓存 | `true` | +| `EMBEDDING_MAX_CACHE_SIZE` | 最大缓存条数 | `2000` | +| `FTS_ENABLED` | 是否启用全文搜索（BM25） | `true` | +| `MEMORY_STORE_BACKEND` | 存储后端（`auto` / `chroma` / `local`） | `auto`（Windows 用 local，其他用 chroma） | ```python import asyncio -from reme import ReMeFb +from agentscope.formatter import ClaudeFormatter +from agentscope.model import get_model +from agentscope.token import HuggingFaceTokenCounter +from agentscope.tool import Toolkit + +from reme.reme_copaw import ReMeCopaw async def main(): - # 初始化并启动 - reme = ReMeFb( - default_llm_config={ - "backend": "openai", # 后端类型，支持 openai 兼容接口 - "model_name": "qwen3.5-plus", # 模型名称 - }, - default_file_store_config={ - "backend": "chroma", # 存储后端，支持 sqlite/chroma/local - "fts_enabled": True, # 是否启用全文搜索 - "vector_enabled": False, # 是否启用向量搜索（无 embedding 服务可设为 False） - }, - context_window_tokens=128000, # 模型上下文窗口大小（tokens） - reserve_tokens=36000, # 预留给输出的 token 数量 - keep_recent_tokens=20000, # 保留最近消息的 token 数量 - vector_weight=0.7, # 向量搜索权重（0-1），用于混合搜索 - candidate_multiplier=3.0, # 候选结果倍数，用于召回更多候选项 + # 准备 AgentScope 核心组件 + chat_model = get_model(config={"backend": "openai", "model_name": "qwen3.5-plus"}) + formatter = ClaudeFormatter() + token_counter = HuggingFaceTokenCounter() + toolkit = Toolkit() # 可注册额外工具 + + # 初始化 ReMeCopaw + reme = ReMeCopaw( + working_dir=".reme", # 记忆文件存储目录 + chat_model=chat_model, + formatter=formatter, + token_counter=token_counter, + toolkit=toolkit, + max_input_length=128000, # 模型上下文窗口（tokens） + memory_compact_ratio=0.7, # 达到 max_input_length * 0.7 时触发压缩 + language="zh", # 摘要语言（zh / ""） + tool_result_threshold=1000, # 超过此字符数的工具输出自动转存 + retention_days=7, # tool_result/ 文件保留天数 ) await reme.start() - messages = [ - {"role": "user", "content": "我喜欢用 Python 3.12"}, - {"role": "assistant", "content": "好的，已记录你偏好 Python 3.12"}, - ] + messages = [...] # list[Msg]，对话历史 - # 检查上下文是否超限 - result = await reme.context_check(messages) - print(f"压缩结论: {result}") + # 1. 压缩超长工具输出（防止工具结果撑爆上下文） + messages = await reme.compact_tool_result(messages) - # 压缩对话为摘要 - summary = await reme.compact(messages_to_summarize=messages) - print(f"摘要: {summary}") + # 2. 将历史对话压缩为结构化摘要（触发时机：上下文接近上限） + summary = await reme.compact_memory( + messages=messages, + previous_summary="", # 可传入上轮摘要，实现增量更新 + ) + print(f"压缩摘要:\n{summary}") - # 将重要记忆写入文件（ReAct Agent 自动操作） - await reme.summary(messages=messages, date="2026-02-28") + # 3. 后台异步提交摘要任务（不阻塞对话，摘要写入 memory/YYYY-MM-DD.md） + reme.add_async_summary_task(messages=messages) - # 语义搜索记忆 - results = await reme.memory_search(query="Python 版本偏好", max_results=5) - print(f"搜索结果: {results}") + # 4. 语义搜索记忆（向量 + BM25 混合检索） + result = await reme.memory_search(query="Python 版本偏好", max_results=5) + print(f"搜索结果: {result}") - # 读取指定记忆文件 - content = await reme.memory_get(path="MEMORY.md") - print(f"记忆内容: {content}") + # 5. 获取会话内存实例（CoPawInMemoryMemory，管理单次对话的上下文） + memory = reme.get_in_memory_memory() + token_stats = await memory.estimate_tokens() + print(f"当前上下文使用率: {token_stats['context_usage_ratio']:.1f}%") - # 关闭（保存 Embedding 缓存、停止文件监控） + # 6. 关闭前等待后台任务完成 + await reme.await_summary_tasks() await reme.close() @@ -269,7 +297,7 @@ async def main(): working_dir=".reme", default_llm_config={ "backend": "openai", - "model_name": "qwen3-30b-a3b-thinking-2507", + "model_name": "qwen3.5-plus", }, default_embedding_model_config={ "backend": "openai", @@ -348,29 +376,80 @@ if __name__ == "__main__": ## 🏛️ 技术架构 -### 基于文件的 ReMe 核心架构 +### 基于文件的 CoPaw 记忆系统架构 + +[CoPaw MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py) 继承 +`ReMeCopaw`，将记忆能力集成到 Agent 推理流程中： ```mermaid graph TB - User[用户 / Agent] --> ReMeFb[File based ReMe] - ReMeFb --> ContextCheck[上下文检查] - ReMeFb --> Compact[上下文压缩] - ReMeFb --> Summary[记忆总结] - ReMeFb --> Search[记忆检索] - ContextCheck --> FbContextChecker[检查 Token 是否超限] - Compact --> FbCompactor[压缩历史对话为摘要] - Summary --> FbSummarizer[ReAct Agent + 文件工具] - Search --> MemorySearch[向量 + BM25 混合检索] - FbSummarizer --> FileTools[read / write / edit] - FileTools --> MemoryFiles[memory/*.md] + CoPaw["CoPaw MemoryManager\n(继承 ReMeCopaw)"] -->|pre_reasoning hook| Hook[MemoryCompactionHook] + CoPaw --> ReMeCopaw[ReMeCopaw] + Hook -->|超出阈值| ReMeCopaw + ReMeCopaw --> CompactMemory[compact_memory\n历史对话压缩] + ReMeCopaw --> SummaryMemory[summary_memory\n记忆写入文件] + ReMeCopaw --> CompactToolResult[compact_tool_result\n超长工具输出压缩] + ReMeCopaw --> MemSearch[memory_search\n语义搜索] + ReMeCopaw --> InMemory[get_in_memory_memory\nCoPawInMemoryMemory] + CompactMemory --> Compactor[Compactor\nReActAgent] + SummaryMemory --> Summarizer[Summarizer\nReActAgent + 文件工具] + CompactToolResult --> ToolResultCompactor[ToolResultCompactor\n截断 + 转存文件] + Summarizer --> FileIO[FileIO\nread / write / edit] + FileIO --> MemoryFiles[memory/YYYY-MM-DD.md] + ToolResultCompactor --> ToolResultFiles[tool_result/*.txt] MemoryFiles -.->|文件变更| FileWatcher[异步文件监控] FileWatcher -->|更新索引| FileStore[本地数据库] - MemorySearch --> FileStore + MemSearch --> FileStore ``` +#### 自动压缩触发流程 + +`MemoryCompactionHook` 在每次推理前检查上下文 Token 用量，超过阈值时自动触发压缩： + +```mermaid +graph LR + A[pre_reasoning] --> B{Token 超过阈值?} + B -->|否| Z[继续推理] + B -->|是| C[compact_tool_result\n压缩最近消息中的超长工具输出] + C --> D[compact_memory\n生成结构化上下文检查点] + D --> E[标记旧消息为 COMPRESSED] + E --> F[add_async_summary_task\n后台写入 memory 文件] + F --> Z +``` + +#### 上下文压缩摘要格式 + +[Compactor](reme/memory/file_based_copaw/compactor.py) 使用 ReActAgent 将历史对话压缩为结构化的**上下文检查点**： + +| 字段 | 说明 | +|-----------------------|-----------------------| +| `## Goal` | 🎯 用户要完成的目标（可多项） | +| `## Constraints` | ⚙️ 用户提到的约束和偏好 | +| `## Progress` | 📈 已完成 / 进行中 / 阻塞的任务 | +| `## Key Decisions` | 🔑 做出的决策及简短理由 | +| `## Next Steps` | 🗺️ 下一步行动计划（有序列表） | +| `## Critical Context` | 📌 文件路径、函数名、错误信息等关键数据 | + +支持**增量更新**：传入 `previous_summary` 时，自动将新对话与旧摘要合并，保留历史进展。 + +#### 工具结果压缩 + +[ToolResultCompactor](reme/memory/file_based_copaw/tool_result_compactor.py) 解决工具输出过长导致上下文膨胀的问题： + +```mermaid +graph LR + A[tool_result 消息] --> B{内容长度 > threshold?} + B -->|否| C[保留原样] + B -->|是| D[截断到 threshold 字符] + D --> E[完整内容写入 tool_result/uuid.txt] + E --> F[消息中追加文件引用路径] +``` + +过期文件（超过 `retention_days`）在 `start` / `close` / `compact_tool_result` 时自动清理。 + #### 记忆总结：ReAct + 文件工具 -[Summarizer](reme/memory/file_based/fb_summarizer.py) 是记忆总结的核心组件，它采用 **ReAct + 文件工具** 模式。 +[Summarizer](reme/memory/file_based_copaw/summarizer.py) 采用 **ReAct + 文件工具** 模式，让 AI 自主决定写什么、写到哪： ```mermaid graph LR @@ -383,49 +462,35 @@ graph LR F -->|否| G[完成] ``` -#### 文件工具集 +[FileIO](reme/memory/file_based_copaw/file_io.py) 提供文件操作工具集： -Summarizer 配备了一套文件操作工具，让 AI 能够直接操作记忆文件： +| 工具 | 功能 | 使用场景 | +|---------|---------------|---------------| +| `read` | 读取文件内容（支持行范围） | 查看现有记忆，避免重复写入 | +| `write` | 覆盖写入文件 | 创建新记忆文件或大幅重构 | +| `edit` | 精确匹配后替换 | 追加新内容或修改特定段落 | -| 工具 | 功能 | 使用场景 | -|---------|--------|--------------| -| `read` | 读取文件内容 | 查看现有记忆，避免重复 | -| `write` | 覆盖写入文件 | 创建新记忆文件或大幅重构 | -| `edit` | 编辑文件局部 | 追加新内容或修改特定部分 | +#### 会话内存管理 -#### 上下文压缩 +[CoPawInMemoryMemory](reme/memory/file_based_copaw/copaw_in_memory_memory.py) 扩展了 AgentScope 的 `InMemoryMemory`： -当对话过长时，[Compactor](reme/memory/file_based/fb_compactor.py) 负责将历史对话压缩为精华摘要——就像写**会议纪要** -，把冗长的讨论浓缩成关键要点。 - -```mermaid -graph LR - A[消息1..N] --> B[📦 压缩摘要] -C[最近消息] --> D[保留原样] -B --> E[新的上下文] -D --> E -``` - -压缩摘要包含继续工作所需的关键信息： - -| 内容 | 说明 | -|--------|---------------| -| 🎯 目标 | 用户想要完成什么 | -| ⚙️ 约束 | 用户提到的要求和偏好 | -| 📈 进展 | 已完成/进行中/阻塞的任务 | -| 🔑 决策 | 做出的决策及原因 | -| 📌 上下文 | 文件路径、函数名等关键数据 | +| 功能 | 说明 | +|----------------------------------|---------------------------| +| `get_memory` | 按标记过滤消息，自动在头部追加压缩摘要 | +| `estimate_tokens` | 精确估算当前上下文 Token 用量及使用率 | +| `get_history_str` | 生成人类可读的对话历史摘要（含 Token 统计） | +| `state_dict` / `load_state_dict` | 支持状态序列化 / 反序列化（会话持久化） | #### 记忆检索 -[MemorySearch](reme/memory/tools/chunk/memory_search.py) 提供**向量 + BM25 混合检索**能力，两种方式优势互补： +[MemorySearch](reme/memory/tools/chunk/memory_search.py) 提供**向量 + BM25 混合检索**能力： | 检索方式 | 优势 | 劣势 | |-------------|-----------------|----------------| | **向量语义** | 捕捉意义相近但措辞不同的内容 | 对精确 token 匹配较弱 | | **BM25 全文** | 精确 token 命中效果极佳 | 无法理解同义词和改写 | -**融合机制**：同时使用两路召回，按权重加权求和（向量 0.7 + BM25 0.3），确保无论是「自然语言提问」还是「精确查找」都能获得可靠结果。 +**融合机制**：两路召回后按权重加权求和（向量 0.7 + BM25 0.3），自然语言与精确查找均可命中。 ```mermaid graph LR diff --git a/reme/__init__.py b/reme/__init__.py index d5304a7f..733448bd 100644 --- a/reme/__init__.py +++ b/reme/__init__.py @@ -6,9 +6,8 @@ from . import extension from . import memory from .reme import ReMe from .reme_cli import ReMeCli -from .reme_fb import ReMeFb -__version__ = "0.3.0.2" +__version__ = "0.3.0.3" __all__ = [ "config", @@ -17,7 +16,6 @@ __all__ = [ "memory", "ReMe", "ReMeCli", - "ReMeFb", ] """ diff --git a/reme/config/copaw.yaml b/reme/config/copaw.yaml new file mode 100644 index 00000000..2a83371a --- /dev/null +++ b/reme/config/copaw.yaml @@ -0,0 +1,16 @@ +embedding_models: + default: + backend: openai + +file_stores: + default: + backend: chroma + embedding_model: default + +file_watchers: + default: + backend: full + file_store: default + suffix_filters: [ ".md" ] + recursive: false + scan_on_start: true diff --git a/reme/config/file.yaml b/reme/config/file.yaml deleted file mode 100644 index d7be65d3..00000000 --- a/reme/config/file.yaml +++ /dev/null @@ -1,43 +0,0 @@ -backend: cmd -working_dir: .reme - -llms: - default: - backend: openai - model_name: qwen3.5-plus - request_interval: 1 - -embedding_models: - default: - backend: openai - model_name: text-embedding-v4 - dimensions: 1024 - enable_cache: true - use_dimensions: false - -file_stores: - default: - backend: chroma - # backend: local - store_name: reme - embedding_model: default - fts_enabled: true - vector_enabled: false - -file_watchers: - default: - backend: full - file_store: default - watch_paths: [ ".reme", ".reme/memory" ] - suffix_filters: [ ".md" ] - recursive: false - scan_on_start: true - -token_counters: - default: - backend: base - - hf: - backend: hf - model_name: Qwen/Qwen3-Coder-30B-A3B-Instruct - use_mirror: true diff --git a/reme/memory/__init__.py b/reme/memory/__init__.py index ac825bc6..a7df138a 100644 --- a/reme/memory/__init__.py +++ b/reme/memory/__init__.py @@ -1,11 +1,11 @@ """memory""" -from . import file_based +from . import cli from . import tools from . import vector_based __all__ = [ - "file_based", + "cli", "tools", "vector_based", ] diff --git a/reme/memory/file_based/__init__.py b/reme/memory/cli/__init__.py similarity index 100% rename from reme/memory/file_based/__init__.py rename to reme/memory/cli/__init__.py diff --git a/reme/memory/file_based/fb_cli.py b/reme/memory/cli/fb_cli.py similarity index 100% rename from reme/memory/file_based/fb_cli.py rename to reme/memory/cli/fb_cli.py diff --git a/reme/memory/file_based/fb_cli.yaml b/reme/memory/cli/fb_cli.yaml similarity index 100% rename from reme/memory/file_based/fb_cli.yaml rename to reme/memory/cli/fb_cli.yaml diff --git a/reme/memory/file_based/fb_compactor.py b/reme/memory/cli/fb_compactor.py similarity index 100% rename from reme/memory/file_based/fb_compactor.py rename to reme/memory/cli/fb_compactor.py diff --git a/reme/memory/file_based/fb_compactor.yaml b/reme/memory/cli/fb_compactor.yaml similarity index 100% rename from reme/memory/file_based/fb_compactor.yaml rename to reme/memory/cli/fb_compactor.yaml diff --git a/reme/memory/file_based/fb_context_checker.py b/reme/memory/cli/fb_context_checker.py similarity index 100% rename from reme/memory/file_based/fb_context_checker.py rename to reme/memory/cli/fb_context_checker.py diff --git a/reme/memory/file_based/fb_summarizer.py b/reme/memory/cli/fb_summarizer.py similarity index 100% rename from reme/memory/file_based/fb_summarizer.py rename to reme/memory/cli/fb_summarizer.py diff --git a/reme/memory/file_based/fb_summarizer.yaml b/reme/memory/cli/fb_summarizer.yaml similarity index 100% rename from reme/memory/file_based/fb_summarizer.yaml rename to reme/memory/cli/fb_summarizer.yaml diff --git a/reme/memory/file_based_copaw/__init__.py b/reme/memory/file_based_copaw/__init__.py new file mode 100644 index 00000000..5b0405f1 --- /dev/null +++ b/reme/memory/file_based_copaw/__init__.py @@ -0,0 +1,31 @@ +"""File-based CoPaw Memory Module. + +This module provides memory management components for CoPaw (Cooperative Paw) agents, +including memory formatting, compaction, summarization, and file I/O operations. + +Components: + - MemoryFormatter: Converts message lists to formatted strings with token limiting + - CoPawInMemoryMemory: Extended InMemoryMemory with bugfixes and summary support + - Summarizer: Generates memory summaries using LLM + - Compactor: Compacts memory content to reduce token usage + - ToolResultCompactor: Truncates large tool results and saves full content to files + - FileIO: File I/O operations with configurable working directory +""" + +from . import utils +from .compactor import Compactor +from .copaw_in_memory_memory import CoPawInMemoryMemory +from .file_io import FileIO +from .memory_formatter import MemoryFormatter +from .summarizer import Summarizer +from .tool_result_compactor import ToolResultCompactor + +__all__ = [ + "MemoryFormatter", + "CoPawInMemoryMemory", + "Summarizer", + "Compactor", + "ToolResultCompactor", + "FileIO", + "utils", +] diff --git a/reme/memory/file_based_copaw/compactor.py b/reme/memory/file_based_copaw/compactor.py new file mode 100644 index 00000000..d8186d86 --- /dev/null +++ b/reme/memory/file_based_copaw/compactor.py @@ -0,0 +1,84 @@ +"""Compactor module for memory compaction operations.""" + +import logging + +from agentscope.agent import ReActAgent +from agentscope.formatter import FormatterBase +from agentscope.message import Msg +from agentscope.model import ChatModelBase +from agentscope.token import HuggingFaceTokenCounter + +from .memory_formatter import MemoryFormatter +from ...core.op import BaseOp + +logger = logging.getLogger(__name__) + + +class Compactor(BaseOp): + """Compactor class for compacting memory messages.""" + + def __init__( + self, + memory_compact_threshold: int, + chat_model: ChatModelBase, + formatter: FormatterBase, + token_counter: HuggingFaceTokenCounter, + **kwargs, + ): + super().__init__(**kwargs) + self.memory_compact_threshold: int = memory_compact_threshold + + self.chat_model: ChatModelBase = chat_model + self.formatter: FormatterBase = formatter + self.as_token_counter: HuggingFaceTokenCounter = token_counter + + async def execute(self): + messages: list[Msg] = self.context.get("messages", []) + previous_summary: str = self.context.get("previous_summary", "") + + if not messages: + return "" + + formatter = MemoryFormatter( + token_counter=self.as_token_counter, + memory_compact_threshold=self.memory_compact_threshold, + ) + history_formatted_str: str = formatter.format(messages) + + if not history_formatted_str: + logger.warning(f"No history to compact. messages={messages}") + return "" + + agent = ReActAgent( + name="reme_compactor", + model=self.chat_model, + sys_prompt=self.get_prompt("system_prompt"), + formatter=self.formatter, + ) + + if previous_summary: + prefix: str = self.get_prompt("update_user_message_prefix") + suffix: str = self.get_prompt("update_user_message_suffix") + user_message: str = ( + f"\n{history_formatted_str}\n\n\n" + f"{prefix}\n\n" + f"\n{previous_summary}\n\n\n" + f"{suffix}" + ) + else: + user_message: str = f"\n{history_formatted_str}\n\n\n" + self.get_prompt( + "initial_user_message", + ) + logger.info(f"Compactor sys_prompt={agent.sys_prompt} user_message={user_message}") + + compact_msg: Msg = await agent.reply( + Msg( + name="reme", + role="user", + content=user_message, + ), + ) + + history_compact: str = compact_msg.get_text_content() + logger.info(f"Compactor Result:\n{history_compact}") + return history_compact diff --git a/reme/memory/file_based_copaw/compactor.yaml b/reme/memory/file_based_copaw/compactor.yaml new file mode 100644 index 00000000..2e3b15a8 --- /dev/null +++ b/reme/memory/file_based_copaw/compactor.yaml @@ -0,0 +1,160 @@ +system_prompt: | + You are a context compaction assistant. Your role is to create structured summaries of conversations + that can be used to restore context in future sessions. Focus on preserving critical information while reducing token count. + +system_prompt_zh: | + 你是一个上下文压缩助手。你的角色是创建对话的结构化摘要， + 这些摘要可以在未来会话中用于恢复上下文。专注于保留关键信息，同时减少token数量。 + +initial_user_message: | + The messages above are a conversation to summarize. Create a structured context checkpoint summary + that another LLM will use to continue the work. + + Use this EXACT format: + + ## Goal + [What is the user trying to accomplish? Can be multiple items if the session covers different tasks.] + + ## Constraints & Preferences + - [Any constraints, preferences, or requirements mentioned by user] + - [Or "(none)" if none were mentioned] + + ## Progress + ### Done + - [x] [Completed tasks/changes] + + ### In Progress + - [ ] [Current work] + + ### Blocked + - [Issues preventing progress, if any] + + ## Key Decisions + - **[Decision]**: [Brief rationale] + + ## Next Steps + 1. [Ordered list of what should happen next] + + ## Critical Context + - [Any data, examples, or references needed to continue] + - [Or "(none)" if not applicable] + + Keep each section concise. Preserve exact file paths, function names, and error messages. + +initial_user_message_zh: | + 上述消息是一场需要总结的对话。创建一个结构化的上下文检查点摘要， + 以便另一个LLM可以用来继续工作。 + + 使用此确切格式： + + ## 目标 + [用户试图完成什么？如果会话涵盖不同任务，可以有多个项目。] + + ## 约束和偏好 + - [任何用户提到的约束、偏好或要求] + - [或者如果没有提到则为"(none)"] + + ## 进展 + ### 已完成 + - [x] [已完成的任务/更改] + + ### 进行中 + - [ ] [当前工作] + + ### 阻塞 + - [如果有任何阻碍进展的问题] + + ## 关键决策 + - **[决策]**: [简短理由] + + ## 下一步 + 1. [接下来应该发生的事情的有序列表] + + ## 关键上下文 + - [任何继续工作所需的数据、示例或参考资料] + - [或者如果不适用则为"(none)"] + + 保持每个部分简洁。保留确切的文件路径、函数名称和错误消息。 + +update_user_message_prefix: | + The messages above are NEW conversation messages to incorporate into the existing summary provided in + tags. + +update_user_message_suffix: | + Update the existing structured summary with new information. RULES: + - PRESERVE all existing information from the previous summary + - ADD new progress, decisions, and context from the new messages + - UPDATE the Progress section: move items from "In Progress" to "Done" when completed + - UPDATE "Next Steps" based on what was accomplished + - PRESERVE exact file paths, function names, and error messages + - If something is no longer relevant, you may remove it + + Use this EXACT format: + + ## Goal + [Preserve existing goals, add new ones if the task expanded] + + ## Constraints & Preferences + - [Preserve existing, add new ones discovered] + + ## Progress + ### Done + - [x] [Include previously done items AND newly completed items] + + ### In Progress + - [ ] [Current work - update based on progress] + + ### Blocked + - [Current blockers - remove if resolved] + + ## Key Decisions + - **[Decision]**: [Brief rationale] (preserve all previous, add new) + + ## Next Steps + 1. [Update based on current state] + + ## Critical Context + - [Preserve important context, add new if needed] + + Keep each section concise. Preserve exact file paths, function names, and error messages. + +update_user_message_prefix_zh: | + 上述消息是要整合到现有摘要中的新对话消息，这些消息在标签中提供。 + +update_user_message_suffix_zh: | + 用新信息更新现有的结构化摘要。规则： + - 保留来自先前摘要的所有现有信息 + - 从新消息中添加新的进展、决策和上下文 + - 更新进度部分：当完成时将项目从"进行中"移到"已完成" + - 根据已完成的内容更新"下一步" + - 保留确切的文件路径、函数名称和错误消息 + - 如果某些内容不再相关，您可以删除它 + + 使用此确切格式： + + ## 目标 + [保留现有目标，如果任务扩展则添加新目标] + + ## 约束和偏好 + - [保留现有内容，添加发现的新内容] + + ## 进展 + ### 已完成 + - [x] [包含以前完成的项目和新完成的项目] + + ### 进行中 + - [ ] [当前工作 - 根据进展更新] + + ### 阻塞 + - [当前阻塞问题 - 如果解决则删除] + + ## 关键决策 + - **[决策]**: [简短理由]（保留所有之前的内容，添加新的） + + ## 下一步 + 1. [根据当前状态更新] + + ## 关键上下文 + - [保留重要上下文，如需要则添加新的] + + 保持每个部分简洁。保留确切的文件路径、函数名称和错误消息。 diff --git a/reme/memory/file_based_copaw/copaw_in_memory_memory.py b/reme/memory/file_based_copaw/copaw_in_memory_memory.py new file mode 100644 index 00000000..02dc77a4 --- /dev/null +++ b/reme/memory/file_based_copaw/copaw_in_memory_memory.py @@ -0,0 +1,243 @@ +"""Custom memory implementation with bugfixes and extensions.""" + +import logging + +from agentscope.agent._react_agent import _MemoryMark +from agentscope.formatter import FormatterBase +from agentscope.memory import InMemoryMemory +from agentscope.message import Msg +from agentscope.token import HuggingFaceTokenCounter + +from .utils import safe_count_message_tokens, safe_count_str_tokens, _get_block_tokens + +logger = logging.getLogger(__name__) + + +class CoPawInMemoryMemory(InMemoryMemory): + """Extended InMemoryMemory with bugfixes and summary support.""" + + def __init__( + self, + token_counter: HuggingFaceTokenCounter, + formatter: FormatterBase, + max_input_length: int = 0, + ): + super().__init__() + self._token_counter: HuggingFaceTokenCounter = token_counter + self._formatter: FormatterBase = formatter + self._max_input_length: int = max_input_length + + async def get_memory( + self, + mark: str | None = None, + exclude_mark: str | None = _MemoryMark.COMPRESSED, + prepend_summary: bool = True, + **_kwargs, + ) -> list[Msg]: + """Get the messages from the memory by mark (if provided). + + Args: + mark: Optional mark to filter messages + exclude_mark: Optional mark to exclude messages + prepend_summary: Whether to prepend compressed summary + **_kwargs: Additional keyword arguments (ignored) + + Returns: + List of filtered messages + """ + if not (mark is None or isinstance(mark, str)): + raise TypeError(f"The mark should be a string or None, but got {type(mark)}.") + + if not (exclude_mark is None or isinstance(exclude_mark, str)): + raise TypeError(f"The exclude_mark should be a string or None, but got {type(exclude_mark)}.") + + # Filter messages based on mark + filtered_content = [(msg, marks) for msg, marks in self.content if mark is None or mark in marks] + + # Further filter messages based on exclude_mark + if exclude_mark is not None: + filtered_content = [(msg, marks) for msg, marks in filtered_content if exclude_mark not in marks] + + if prepend_summary and self._compressed_summary: + previous_summary = f""" + +{self._compressed_summary} + +The above is a summary of our previous conversation. +Use it as context to maintain continuity. + """.strip() + + return [ + Msg( + "user", + previous_summary, + "user", + ), + *[msg for msg, _ in filtered_content], + ] + + return [msg for msg, _ in filtered_content] + + def get_compressed_summary(self) -> str: + """Get the compressed summary of the memory.""" + return self._compressed_summary + + def state_dict(self) -> dict: + """Get the state dictionary for serialization.""" + return { + "content": [[msg.to_dict(), marks] for msg, marks in self.content], + "_compressed_summary": self._compressed_summary, + } + + # pylint: disable=attribute-defined-outside-init + def load_state_dict(self, state_dict: dict, strict: bool = True) -> None: + """Load the state dictionary for deserialization.""" + if strict and "content" not in state_dict: + raise KeyError("The state_dict does not contain 'content' key required for InMemoryMemory.") + + self.content = [] # pylint: disable=attribute-defined-outside-init + for item in state_dict.get("content", []): + if isinstance(item, (tuple, list)) and len(item) == 2: + msg_dict, marks = item + msg = Msg.from_dict(msg_dict) + self.content.append((msg, marks)) + + elif isinstance(item, dict): + # For compatibility with older versions + msg = Msg.from_dict(item) + self.content.append((msg, [])) + + else: + raise ValueError("Invalid item format in state_dict for InMemoryMemory.") + + self._compressed_summary = state_dict.get("_compressed_summary", "") + + async def mark_messages_compressed(self, messages: list[Msg]) -> int: + """Mark messages as compressed and return count.""" + return await self.update_messages_mark( + new_mark=_MemoryMark.COMPRESSED, + msg_ids=[msg.id for msg in messages], + ) + + def clear_compressed_summary(self): + """Clear the compressed summary.""" + self._compressed_summary = "" # pylint: disable=attribute-defined-outside-init + + def clear_content(self): + """Clear the content.""" + self.content.clear() + + async def estimate_tokens(self) -> dict: + """Estimate token usage for current memory. + + Returns: + Dict containing detailed token statistics: + - total_messages: Number of messages + - compressed_summary_tokens: Tokens in compressed summary + - messages_tokens: Tokens in messages + - estimated_tokens: Total estimated tokens + - max_input_length: Max input length from config + - context_usage_ratio: Usage percentage + - messages_detail: List of per-message token details + """ + messages = await self.get_memory( + exclude_mark=_MemoryMark.COMPRESSED, + prepend_summary=False, + ) + + compressed_summary = self.get_compressed_summary() + compressed_summary_tokens = safe_count_str_tokens(self._token_counter, compressed_summary) + + # Calculate total token count using formatter + prompt = await self._formatter.format(msgs=messages) + messages_tokens = safe_count_message_tokens(self._token_counter, prompt) + estimated_tokens = messages_tokens + compressed_summary_tokens + + # Calculate context usage ratio + max_input_length = self._max_input_length + context_usage_ratio = (estimated_tokens / max_input_length * 100) if max_input_length > 0 else 0 + + # Build per-message token details + messages_detail = [] + for i, msg in enumerate(messages, 1): + msg_detail = { + "index": i, + "role": msg.role, + "text_tokens": 0, + "blocks": [], + "preview": "", + } + try: + content = msg.content + if isinstance(content, str): + text_tokens = safe_count_str_tokens(self._token_counter, content) + msg_detail["text_tokens"] = text_tokens + msg_detail["preview"] = f"{content[:100]}..." if len(content) > 100 else content + else: + total_tokens = 0 + text_parts = [] + for block in content: + if not isinstance(block, dict): + continue + block_type = block.get("type", "unknown") + block_tokens, block_str = _get_block_tokens( + block, + block_type, + self._token_counter, + ) + total_tokens += block_tokens + text_parts.append(block_str) + msg_detail["blocks"].append( + { + "type": block_type, + "tokens": block_tokens, + }, + ) + msg_detail["text_tokens"] = total_tokens + text_preview = "".join(text_parts) + msg_detail["preview"] = f"{text_preview[:100]}..." if len(text_preview) > 100 else text_preview + except Exception as e: + msg_detail["error"] = str(e) + msg_detail["preview"] = f"" + + messages_detail.append(msg_detail) + + return { + "total_messages": len(messages), + "compressed_summary_tokens": compressed_summary_tokens, + "messages_tokens": messages_tokens, + "estimated_tokens": estimated_tokens, + "max_input_length": max_input_length, + "context_usage_ratio": context_usage_ratio, + "messages_detail": messages_detail, + } + + async def get_history_str(self) -> str: + """Get formatted history string similar to /history command output. + + Returns: + Formatted string containing conversation history details + """ + stats = await self.estimate_tokens() + + lines = [] + for msg_detail in stats["messages_detail"]: + blocks_info = "" + if msg_detail["blocks"]: + block_strs = [f"{b['type']}(tokens={b['tokens']})" for b in msg_detail["blocks"]] + blocks_info = f"\n content: [{', '.join(block_strs)}]" + + lines.append( + f"[{msg_detail['index']}] **{msg_detail['role']}** " + f"(text_tokens={msg_detail['text_tokens']})" + f"{blocks_info}\n preview: {msg_detail['preview']}", + ) + + return ( + f"**Conversation History**\n\n" + f"- Total messages: {stats['total_messages']}\n" + f"- Estimated tokens: {stats['estimated_tokens']}\n" + f"- Max input length: {stats['max_input_length']}\n" + f"- Context usage: {stats['context_usage_ratio']:.1f}%\n" + f"- Compressed summary tokens: {stats['compressed_summary_tokens']}\n\n" + "\n\n".join(lines) + ) diff --git a/reme/memory/file_based_copaw/file_io.py b/reme/memory/file_based_copaw/file_io.py new file mode 100644 index 00000000..161c3210 --- /dev/null +++ b/reme/memory/file_based_copaw/file_io.py @@ -0,0 +1,247 @@ +"""File I/O operations with a configurable working directory.""" + +import os +from pathlib import Path +from typing import Optional + +from agentscope.message import TextBlock +from agentscope.tool import ToolResponse + + +class FileIO: + """File I/O operations with a configurable working directory.""" + + def __init__(self, working_dir: str | Path): + """Initialize FileIO with a working directory. + + Args: + working_dir (`str`): + The working directory for resolving relative paths. + """ + self.working_dir = Path(working_dir) + + def _resolve_file_path(self, file_path: str) -> str: + """Resolve file path: use absolute path as-is, + resolve relative path from working_dir. + + Args: + file_path: The input file path (absolute or relative). + + Returns: + The resolved absolute file path as string. + """ + path = Path(file_path) + if path.is_absolute(): + return str(path) + else: + return str(self.working_dir / file_path) + + async def read( # pylint: disable=too-many-return-statements + self, + file_path: str, + start_line: Optional[int] = None, + end_line: Optional[int] = None, + ) -> ToolResponse: + """Read a file. Relative paths resolve from working_dir. + + Use start_line/end_line to read a specific line range (output includes + line numbers). Omit both to read the full file. + + Args: + file_path (`str`): + Path to the file. + start_line (`int`, optional): + First line to read (1-based, inclusive). + end_line (`int`, optional): + Last line to read (1-based, inclusive). + """ + file_path = self._resolve_file_path(file_path) + + if not os.path.exists(file_path): + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Error: The file {file_path} does not exist.", + ), + ], + ) + + if not os.path.isfile(file_path): + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Error: The path {file_path} is not a file.", + ), + ], + ) + + try: + with open(file_path, "r", encoding="utf-8") as f: + all_lines = f.readlines() + + range_requested = start_line is not None or end_line is not None + + if range_requested: + total = len(all_lines) + s = max(1, start_line if start_line is not None else 1) + e = min(total, end_line if end_line is not None else total) + + if s > total: + return ToolResponse( + content=[ + TextBlock( + type="text", + text=(f"Error: start_line {s} exceeds file length " f"({total} lines) in {file_path}."), + ), + ], + ) + + if s > e: + return ToolResponse( + content=[ + TextBlock( + type="text", + text=(f"Error: start_line ({s}) is greater than " f"end_line ({e}) in {file_path}."), + ), + ], + ) + + selected = all_lines[s - 1 : e] + content = "".join(selected) + header = f"{file_path} (lines {s}-{e} of {total})\n" + return ToolResponse( + content=[ + TextBlock( + type="text", + text=header + content, + ), + ], + ) + else: + content = "".join(all_lines) + return ToolResponse( + content=[ + TextBlock( + type="text", + text=content, + ), + ], + ) + + except Exception as e: + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Error: Read file failed due to \n{e}", + ), + ], + ) + + async def write( + self, + file_path: str, + content: str, + ) -> ToolResponse: + """Create or overwrite a file. Relative paths resolve from working_dir. + + Args: + file_path (`str`): + Path to the file. + content (`str`): + Content to write. + """ + if not file_path: + return ToolResponse( + content=[ + TextBlock( + type="text", + text="Error: No `file_path` provide.", + ), + ], + ) + + file_path = self._resolve_file_path(file_path) + + try: + with open(file_path, "w", encoding="utf-8") as file: + file.write(content) + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Wrote {len(content)} bytes to {file_path}.", + ), + ], + ) + except Exception as e: + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Error: Write file failed due to \n{e}", + ), + ], + ) + + async def edit( + self, + file_path: str, + old_text: str, + new_text: str, + ) -> ToolResponse: + """Find-and-replace text in a file. All occurrences of old_text are + replaced with new_text. Relative paths resolve from working_dir. + + Args: + file_path (`str`): + Path to the file. + old_text (`str`): + Exact text to find. + new_text (`str`): + Replacement text. + """ + response = await self.read(file_path=file_path) + if response.content and len(response.content) > 0: + error_text = response.content[0].get("text", "") + if error_text.startswith("Error:"): + return response + if not response.content or len(response.content) == 0: + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Error: Failed to read file {file_path}.", + ), + ], + ) + + content = response.content[0].get("text", "") + if old_text not in content: + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Error: The text to replace was not found in {file_path}.", + ), + ], + ) + + new_content = content.replace(old_text, new_text) + write_response = await self.write(file_path=file_path, content=new_content) + + if write_response.content and len(write_response.content) > 0: + write_text = write_response.content[0].get("text", "") + if write_text.startswith("Error:"): + return write_response + + return ToolResponse( + content=[ + TextBlock( + type="text", + text=f"Successfully replaced text in {file_path}.", + ), + ], + ) diff --git a/reme/memory/file_based_copaw/memory_formatter.py b/reme/memory/file_based_copaw/memory_formatter.py new file mode 100644 index 00000000..6c0e22c3 --- /dev/null +++ b/reme/memory/file_based_copaw/memory_formatter.py @@ -0,0 +1,249 @@ +"""Memory Formatter for CoPaw agents. + +Provides memory formatting capabilities including: +- Converting list of Msg to formatted string +- Memory compaction with token threshold +- Support for various content block types (text, tool_use, tool_result, etc.) +""" + +import json +import logging +import os + +from agentscope.message import Msg +from agentscope.token import HuggingFaceTokenCounter + +from .utils import safe_count_str_tokens, truncate_text + +logger = logging.getLogger(__name__) + +_DEFAULT_MAX_FORMATTER_TEXT_LENGTH = 2000 + + +class MemoryFormatter: + """Formatter that converts list of Msg to formatted string. + + Formats messages into human-readable string representation with: + - Role and timestamp information + - Text content and tool calls + - Memory compact threshold to limit total token count + """ + + def __init__( + self, + token_counter: HuggingFaceTokenCounter, + memory_compact_threshold: int, + ): + """Initialize MemoryFormatter. + + Args: + token_counter: Token counter for estimating token counts. + memory_compact_threshold: Maximum token count before skipping + older messages. + """ + self._token_counter = token_counter + self._memory_compact_threshold = memory_compact_threshold + self.max_length = int( + os.getenv("MAX_FORMATTER_TEXT_LENGTH", str(_DEFAULT_MAX_FORMATTER_TEXT_LENGTH)), + ) + + @staticmethod + def _format_tool_result_output(output: str | list[dict]) -> str: + """Convert tool result output to string. + + Args: + output: Tool result output, either string or list of content blocks. + + Returns: + Formatted string representation of the tool result. + """ + if isinstance(output, str): + return output + + textual_parts = [] + + for block in output: + try: + if not isinstance(block, dict) or "type" not in block: + logger.warning( + "Invalid block: %s, expected a dict with 'type' key, skipped.", + block, + ) + continue + + block_type = block["type"] + + if block_type == "text": + textual_parts.append(block.get("text", "")) + + elif block_type in ["image", "audio", "video"]: + source = block.get("source", {}) + url = source.get("url", "") + if url: + textual_parts.append( + f"[{block_type}] {url}", + ) + else: + textual_parts.append(f"[{block_type}]") + + elif block_type == "file": + file_path = block.get("path", "") or block.get("url", "") + file_name = block.get("name", file_path) + textual_parts.append(f"[file] {file_name}: {file_path}") + + else: + # Unknown block type: log warning and skip + logger.warning( + "Unsupported block type '%s' in tool result, skipped.", + block_type, + ) + + except Exception as e: + logger.warning( + "Failed to process block %s: %s, skipped.", + block, + e, + ) + + if not textual_parts: + return "" + if len(textual_parts) == 1: + return textual_parts[0] + return "\n".join(f"- {part}" for part in textual_parts) + + def _format_single_msg( + self, + msg: Msg, + index: int | None = None, + add_time: bool = True, + ) -> tuple[str, int]: + """Format a single Msg into string representation. + + Similar to Message.format_message style. + + Args: + msg: The Msg object to format. + index: Optional message index for round numbering. + add_time: Whether to include timestamp. + + Returns: + Tuple of (formatted_string, token_count). + """ + lines = [] + token_count = 0 + + # Build header: "round{index} [{timestamp}] {role}:" + prefix = f"round{index} " if index is not None else "" + time_str = f"[{msg.timestamp}] " if add_time and msg.timestamp else "" + role_str = msg.name or msg.role + header = f"{prefix}{time_str}{role_str}:" + lines.append(header) + token_count += safe_count_str_tokens(self._token_counter, header) + + # Process content blocks + for block in msg.get_content_blocks(): + typ = block.get("type") + + if typ == "text": + text_content = truncate_text(block.get("text", ""), self.max_length) + if text_content: + lines.append(text_content) + token_count += safe_count_str_tokens(self._token_counter, text_content) + + elif typ == "thinking": + # Skip thinking blocks to save tokens + pass + + elif typ in ["image", "audio", "video"]: + source = block.get("source", {}) + url = source.get("url", "") + if url: + lines.append(f"[{typ}] {url}") + else: + lines.append(f"[{typ}]") + # Estimate fixed token cost for media reference + token_count += 10 + + elif typ == "tool_use": + tool_name = block.get("name", "") + tool_input = block.get("input", {}) + try: + arguments_str = json.dumps(tool_input, ensure_ascii=False) + except (TypeError, ValueError): + arguments_str = str(tool_input) + truncated_args = truncate_text(arguments_str, self.max_length) + tool_line = f" - tool_call={tool_name} params={truncated_args}" + lines.append(tool_line) + token_count += safe_count_str_tokens(self._token_counter, tool_line) + + elif typ == "tool_result": + tool_name = block.get("name", "") + output = block.get("output", "") + formatted_output = self._format_tool_result_output(output) + truncated_output = truncate_text(formatted_output, self.max_length) + if truncated_output: + result_line = f" - tool_result={tool_name} output={truncated_output}" + lines.append(result_line) + token_count += safe_count_str_tokens(self._token_counter, result_line) + + else: + logger.warning( + "Unsupported block type %s in message, skipped.", + typ, + ) + + return "\n".join(lines), token_count + + def format( + self, + msgs: list[Msg], + add_time: bool = True, + add_index: bool = True, + ) -> str: + """Format list of Msg into a single formatted string. + + Messages are processed in reverse order (newest first) and older + messages are skipped when token count exceeds memory_compact_threshold. + + Args: + msgs: List of Msg objects to format. + add_time: Whether to include timestamp in each message. + add_index: Whether to include round index in each message. + + Returns: + Formatted string with all messages joined by newlines. + """ + if not msgs: + return "" + + formatted_parts: list[str] = [] + total_token_count = 0 + + # Process messages in reverse order (newest first) + for i in range(len(msgs) - 1, -1, -1): + msg = msgs[i] + index = i if add_index else None + + formatted_msg, msg_token_count = self._format_single_msg( + msg, + index=index, + add_time=add_time, + ) + + # Always include current message first, then check threshold, at least one msg + formatted_parts.append(formatted_msg) + total_token_count += msg_token_count + + # Check if we should stop adding older messages + if total_token_count >= self._memory_compact_threshold: + logger.info( + "Skipping older messages: token count %d >= %d", + total_token_count, + self._memory_compact_threshold, + ) + break + + # Reverse to restore chronological order + formatted_parts.reverse() + + return "\n\n".join(formatted_parts) diff --git a/reme/memory/file_based_copaw/summarizer.py b/reme/memory/file_based_copaw/summarizer.py new file mode 100644 index 00000000..462ea9c5 --- /dev/null +++ b/reme/memory/file_based_copaw/summarizer.py @@ -0,0 +1,92 @@ +"""Summarizer module for memory summarization operations.""" + +import datetime +import logging + +from agentscope.agent import ReActAgent +from agentscope.formatter import FormatterBase +from agentscope.message import Msg +from agentscope.model import ChatModelBase +from agentscope.token import HuggingFaceTokenCounter +from agentscope.tool import Toolkit + +from .memory_formatter import MemoryFormatter +from .file_io import FileIO +from ...core.op import BaseOp + +logger = logging.getLogger(__name__) + + +class Summarizer(BaseOp): + """Summarizer class for summarizing memory messages.""" + + def __init__( + self, + working_dir: str, + memory_dir: str, + memory_compact_threshold: int, + chat_model: ChatModelBase, + formatter: FormatterBase, + token_counter: HuggingFaceTokenCounter, + toolkit: Toolkit | None = None, + **kwargs, + ): + super().__init__(**kwargs) + self.working_dir: str = working_dir + self.memory_dir: str = memory_dir + self.memory_compact_threshold: int = memory_compact_threshold + + self.chat_model: ChatModelBase = chat_model + self.formatter: FormatterBase = formatter + self.as_token_counter: HuggingFaceTokenCounter = token_counter + if toolkit is not None: + self.toolkit: Toolkit = toolkit + else: + self.toolkit = Toolkit() + file_io = FileIO(working_dir=self.working_dir) + self.toolkit.register_tool_function(file_io.read) + self.toolkit.register_tool_function(file_io.write) + self.toolkit.register_tool_function(file_io.edit) + + async def execute(self): + messages: list[Msg] = self.context.get("messages", []) + + if not messages: + return "" + + formatter = MemoryFormatter( + token_counter=self.as_token_counter, + memory_compact_threshold=self.memory_compact_threshold, + ) + history_formatted_str: str = formatter.format(messages) + + if not history_formatted_str: + logger.warning(f"No history to summarize. messages={messages}") + return "" + + agent = ReActAgent( + name="reme_summarizer", + model=self.chat_model, + sys_prompt="You are a helpful assistant.", + formatter=self.formatter, + toolkit=self.toolkit, + ) + + user_message: str = f"\n{history_formatted_str}\n\n" + self.prompt_format( + "user_message", + date=datetime.datetime.now().strftime("%Y-%m-%d"), + working_dir=self.working_dir, + memory_dir=self.memory_dir, + ) + + summary_msg: Msg = await agent.reply( + Msg( + name="reme", + role="user", + content=user_message, + ), + ) + + history_summary: str = summary_msg.get_text_content() + logger.info(f"Summarizer Result:\n{history_summary}") + return history_summary diff --git a/reme/memory/file_based_copaw/summarizer.yaml b/reme/memory/file_based_copaw/summarizer.yaml new file mode 100644 index 00000000..9aa0892b --- /dev/null +++ b/reme/memory/file_based_copaw/summarizer.yaml @@ -0,0 +1,50 @@ +user_message: | + Memory Pre-compression Flush Cycle Initiated + The current session is about to enter the automatic compression phase. Please capture persistent memory and write it to disk. + + Current date: {date} + Working directory: {working_dir} + + Immediately store persistent memory to: {memory_dir}/YYYY-MM-DD.md + + Workflow: + 1. First, `read` {memory_dir}/YYYY-MM-DD.md (if the file doesn’t exist, an error message will be returned). + 2. Intelligently merge new information with existing content (skip merging if the file doesn’t exist): + - Avoid duplicating already recorded information + - Enrich existing entries with new details where relevant + - Maintain chronological order wherever applicable + 3. Write the updated content: + - Prefer using `edit` to update specific sections when possible + - Use `write` to overwrite the entire file only if substantial restructuring is required + + Principles: + - Always preserve timestamps and any date/time-related context + - Add only genuinely new or meaningfully enriching information + - Keep entries concise yet complete + - If there’s nothing to store, respond with [SILENT] + + +user_message_zh: | + 预压缩内存刷新轮次。 + 当前会话即将进入自动压缩阶段；请将持久化记忆捕获并写入磁盘。 + + 当前日期：{date} + 工作目录：{working_dir} + + 立即存储持久化记忆（使用路径 {memory_dir}/YYYY-MM-DD.md）。 + + 工作流程： + 1. 先 `read` {memory_dir}/YYYY-MM-DD.md（如文件不存在，会返回错误提示） + 2. 智能合并新信息与现有内容（若文件不存在则跳过合并）： + - 避免重复已记录的信息 + - 在相关时丰富现有条目的新细节 + - 在适用时保持时间顺序 + 3. 写入更新后的内容： + - 尽可能使用 `edit` 更新特定部分 + - 如需大幅重构则使用 `write` 覆盖整个文件 + + 原则： + - 始终保留时间戳、日期和时间相关上下文 + - 仅添加真正新的或有丰富价值的信息 + - 保持条目简洁但完整 + - 若无内容可存储，请回复 [SILENT] diff --git a/reme/memory/file_based_copaw/tool_result_compactor.py b/reme/memory/file_based_copaw/tool_result_compactor.py new file mode 100644 index 00000000..5b1ef0a4 --- /dev/null +++ b/reme/memory/file_based_copaw/tool_result_compactor.py @@ -0,0 +1,105 @@ +"""Tool Result Compactor: truncate large tool results and save full content to files.""" + +import logging +import uuid +from datetime import datetime, timedelta +from pathlib import Path + +from agentscope.message import Msg + +from .utils import is_truncated, truncate_text +from ...core.op import BaseOp + +logger = logging.getLogger(__name__) + + +class ToolResultCompactor(BaseOp): + """Truncate large tool_result outputs and save full content to files.""" + + def __init__( + self, + tool_result_dir: str | Path, + tool_result_threshold: int, + retention_days: int = 7, + **kwargs, + ): + super().__init__(**kwargs) + self.tool_result_dir = Path(tool_result_dir) + self.tool_result_threshold = tool_result_threshold + self.retention_days = retention_days + + def _save_and_truncate(self, content: str, tool_name: str) -> str: + """Save full content to file and return truncated version with file reference.""" + if not content or is_truncated(content) or len(content) <= self.tool_result_threshold: + return content + + # Save full content + self.tool_result_dir.mkdir(parents=True, exist_ok=True) + file_path = self.tool_result_dir / f"{uuid.uuid4().hex}.txt" + created_at = datetime.now().isoformat() + + file_path.write_text( + f"# tool_name: {tool_name}\n# created_at: {created_at}\n# ---\n{content}", + encoding="utf-8", + ) + logger.debug("Saved tool result to %s (len=%d)", file_path, len(content)) + + # Return truncated with file reference + return f"{truncate_text(content, self.tool_result_threshold)}\n\n[Full content saved to: {file_path}]" + + def _process_output(self, output: str | list[dict], tool_name: str) -> str | list[dict]: + """Process tool result output, truncating if necessary.""" + if isinstance(output, str): + return self._save_and_truncate(output, tool_name) + + if isinstance(output, list): + return [ + ( + {**b, "text": self._save_and_truncate(b.get("text", ""), tool_name)} + if isinstance(b, dict) and b.get("type") == "text" + else b + ) + for b in output + ] + return output + + async def execute(self) -> list[Msg]: + """Process all messages, truncating large tool results.""" + messages: list[Msg] = self.context.get("messages", []) + if not messages: + return messages + + for msg in messages: + if not isinstance(msg.content, list): + continue + + for block in msg.content: + if isinstance(block, dict) and block.get("type") == "tool_result": + output = block.get("output") + if output: + block["output"] = self._process_output(output, block.get("name", "unknown")) + + return messages + + def cleanup_expired_files(self) -> int: + """Clean up files older than retention_days.""" + if not self.tool_result_dir.exists(): + return 0 + + cutoff = datetime.now() - timedelta(days=self.retention_days) + deleted = 0 + + for fp in self.tool_result_dir.glob("*.txt"): + try: + for line in fp.read_text(encoding="utf-8").splitlines()[:3]: + if line.startswith("# created_at:"): + if datetime.fromisoformat(line.split(":", 1)[1].strip()) < cutoff: + fp.unlink() + deleted += 1 + break + except Exception as e: + logger.warning("Failed to process %s: %s", fp, e) + + if deleted: + logger.info("Cleaned up %d expired files", deleted) + return deleted diff --git a/reme/memory/file_based_copaw/utils.py b/reme/memory/file_based_copaw/utils.py new file mode 100644 index 00000000..a5f8f967 --- /dev/null +++ b/reme/memory/file_based_copaw/utils.py @@ -0,0 +1,231 @@ +"""Utility functions for working with text.""" + +import logging + +from agentscope.token import HuggingFaceTokenCounter + +logger = logging.getLogger(__name__) + +# Unique marker for truncated text +TRUNCATION_MARKER_START = "<<>>" +TRUNCATION_MARKER_END = "<<>>" + + +def truncate_text(text: str, max_length: int) -> str: + """Truncate text to max length, keeping head and tail portions. + + Args: + text: The text to truncate + max_length: Maximum allowed length + + Returns: + Truncated text with unique markers indicating truncation + """ + text = str(text) if text else "" + if not text: + return text + + if len(text) <= max_length: + return text + + half_length = max_length // 2 + truncated_chars = len(text) - max_length + logger.debug( + "Text truncated: original %d chars, kept head %d + tail %d, removed %d chars.", + len(text), + half_length, + half_length, + truncated_chars, + ) + return ( + f"{text[:half_length]}\n\n{TRUNCATION_MARKER_START} " + f"({truncated_chars} characters omitted) " + f"{TRUNCATION_MARKER_END}\n\n{text[-half_length:]}" + ) + + +def is_truncated(text: str) -> bool: + """Check if the text has been truncated (contains truncation markers). + + Args: + text: The text to check + + Returns: + bool: True if text contains truncation markers, False otherwise + """ + if not text: + return False + return TRUNCATION_MARKER_START in text and TRUNCATION_MARKER_END in text + + +def _extract_text_from_messages(messages: list[dict]) -> str: + """Extract text content from messages and concatenate into a string. + + Handles various message formats: + - Simple string content: {"role": "user", "content": "hello"} + - List content with text blocks: + {"role": "user", "content": [{"type": "text", "text": "hello"}]} + - List content with tool_result blocks: + {"role": "user", "content": [{"type": "tool_result", "output": "..."}]} + + Args: + messages: List of message dictionaries in chat format. + + Returns: + str: Concatenated text content from all messages. + """ + parts = [] + for msg in messages: + content = msg.get("content", "") + if isinstance(content, str): + parts.append(content) + elif isinstance(content, list): + for block in content: + if isinstance(block, dict): + block_type = block.get("type", "") + if block_type == "tool_result": + output = block.get("output", "") + if isinstance(output, str) and output: + parts.append(output) + elif isinstance(output, list): + for sub in output: + if isinstance(sub, dict): + sub_text = sub.get("text") or sub.get("content", "") + if sub_text: + parts.append(str(sub_text)) + else: + text = block.get("text") or block.get("content", "") + if text: + parts.append(str(text)) + elif isinstance(block, str): + parts.append(block) + return "\n".join(parts) + + +def safe_count_message_tokens( + token_counter: HuggingFaceTokenCounter, + messages: list[dict], +) -> int: + """Safely count tokens in messages with fallback estimation. + + This is a wrapper around count_message_tokens that catches exceptions + and falls back to a character-based estimation (len // 4) if the + tokenizer fails. + + Args: + token_counter: Token counter instance. + messages: List of message dictionaries in chat format. + + Returns: + int: The estimated number of tokens in the messages. + """ + try: + text = _extract_text_from_messages(messages) + token_ids = token_counter.tokenizer.encode(text) + token_count = len(token_ids) + return token_count + + except Exception as e: + # Fallback to character-based estimation + text = _extract_text_from_messages(messages) + estimated_tokens = len(text) // 4 + logger.warning( + "Failed to count tokens: %s, using estimated_tokens=%d", + e, + estimated_tokens, + ) + return estimated_tokens + + +def safe_count_str_tokens( + token_counter: HuggingFaceTokenCounter, + text: str, +) -> int: + """Safely count tokens in a string with fallback estimation. + + Uses the tokenizer to count tokens in the given text. If the tokenizer + fails, falls back to a character-based estimation (len // 4). + + Args: + token_counter: Token counter instance. + text: The string to count tokens for. + + Returns: + int: The estimated number of tokens in the string. + """ + try: + token_ids = token_counter.tokenizer.encode(text) + token_count = len(token_ids) + return token_count + except Exception as e: + # Fallback to character-based estimation + estimated_tokens = len(text) // 4 + logger.warning( + "Failed to count string tokens: %s, using estimated_tokens=%d", + e, + estimated_tokens, + ) + return estimated_tokens + + +def _get_block_tokens( # pylint: disable=too-many-return-statements + block: dict, + block_type: str, + token_counter: HuggingFaceTokenCounter, +) -> tuple[int, str]: + """Get token count and content string for different block types. + + Args: + block: The content block dict + block_type: The type of the block + + Returns: + Tuple of (token count, content string) + """ + if block_type == "text": + text = block.get("text", "") + return (safe_count_str_tokens(token_counter, text), text) if text else (0, "") + + if block_type == "thinking": + thinking = block.get("thinking", "") + return (safe_count_str_tokens(token_counter, thinking), thinking) if thinking else (0, "") + + if block_type == "tool_use": + # Count input dict and raw_input string + input_dict = block.get("input", {}) + raw_input = block.get("raw_input", "") + input_str = str(input_dict) if input_dict else "" + total = input_str + raw_input + return (safe_count_str_tokens(token_counter, total), total) if total else (0, "") + + if block_type == "tool_result": + output = block.get("output") + if isinstance(output, str): + return (safe_count_str_tokens(token_counter, output), output) if output else (0, "") + + if isinstance(output, list): + # Recursively count tokens in nested blocks + total_tokens = 0 + total_str = "" + for item in output: + if isinstance(item, dict): + item_type = item.get("type", "unknown") + item_tokens, item_str = _get_block_tokens(item, item_type, token_counter) + total_tokens += item_tokens + total_str += item_str + return total_tokens, total_str + return 0, "" + + if block_type in ("image", "audio", "video"): + # For media blocks, count the URL or indicate base64 size + source = block.get("source", {}) + if source.get("type") == "url": + url = source.get("url", "") + return safe_count_str_tokens(token_counter, url), url + if source.get("type") == "base64": + # Base64 data can be large, return approximate token count + data = source.get("data", "") + return (len(data) // 4, "[base64]") if data else (0, "") + return 0, "" + + return 0, "" diff --git a/reme/reme_cli.py b/reme/reme_cli.py index 9255bef1..68469425 100644 --- a/reme/reme_cli.py +++ b/reme/reme_cli.py @@ -3,13 +3,16 @@ import asyncio import os import sys +from pathlib import Path from typing import AsyncGenerator from prompt_toolkit import PromptSession +from .config import ReMeConfigParser +from .core import Application from .core.enumeration import ChunkEnum from .core.op import BaseTool -from .core.schema import StreamChunk +from .core.schema import Message, StreamChunk from .core.tools import ( BashTool, EditTool, @@ -21,17 +24,75 @@ from .core.tools import ( TavilySearch, ) from .core.utils import execute_stream_task, play_horse_easter_egg -from .memory.file_based import FbCli -from .memory.tools import MemorySearch -from .reme_fb import ReMeFb +from .memory.cli import FbCli, FbCompactor, FbContextChecker, FbSummarizer +from .memory.tools import MemoryGet, MemorySearch -class ReMeCli(ReMeFb): +class ReMeCli(Application): """ReMe Cli""" - def __init__(self, *args, config_path: str = "cli", **kwargs): + def __init__( + self, + *args, + working_dir: str = ".reme", + config_path: str = "cli", + enable_logo: bool = True, + log_to_console: bool = True, + llm_api_key: str | None = None, + llm_base_url: str | None = None, + embedding_api_key: str | None = None, + embedding_base_url: str | None = None, + default_llm_config: dict | None = None, + default_embedding_model_config: dict | None = None, + default_file_store_config: dict | None = None, + default_token_counter_config: dict | None = None, + default_file_watcher_config: dict | None = None, + context_window_tokens: int = 128000, + reserve_tokens: int = 36000, + keep_recent_tokens: int = 20000, + vector_weight: float = 0.7, + candidate_multiplier: float = 3.0, + **kwargs, + ): """Initialize ReMe with config.""" - super().__init__(*args, config_path=config_path, **kwargs) + working_path = Path(working_dir) + working_path.mkdir(parents=True, exist_ok=True) + memory_path = working_path / "memory" + memory_path.mkdir(parents=True, exist_ok=True) + self.working_dir: str = str(working_path.absolute()) + + default_file_watcher_config = default_file_watcher_config or {} + if not default_file_watcher_config.get("watch_paths", None): + default_file_watcher_config["watch_paths"] = [ + str(working_path / "MEMORY.md"), + str(working_path / "memory.md"), + str(memory_path), + ] + super().__init__( + *args, + llm_api_key=llm_api_key, + llm_base_url=llm_base_url, + embedding_api_key=embedding_api_key, + embedding_base_url=embedding_base_url, + working_dir=working_dir, + config_path=config_path, + enable_logo=enable_logo, + log_to_console=log_to_console, + parser=ReMeConfigParser, + default_llm_config=default_llm_config, + default_embedding_model_config=default_embedding_model_config, + default_file_store_config=default_file_store_config, + default_token_counter_config=default_token_counter_config, + default_file_watcher_config=default_file_watcher_config, + **kwargs, + ) + + self.service_config.metadata.setdefault("context_window_tokens", context_window_tokens) + self.service_config.metadata.setdefault("reserve_tokens", reserve_tokens) + self.service_config.metadata.setdefault("keep_recent_tokens", keep_recent_tokens) + self.service_config.metadata.setdefault("vector_weight", vector_weight) + self.service_config.metadata.setdefault("candidate_multiplier", candidate_multiplier) + self.commands = { "/new": "Create a new conversation.", "/compact": "Compact messages into a summary.", @@ -40,7 +101,6 @@ class ReMeCli(ReMeFb): "/help": "Show help.", "/horse": "A surprise.", } - self.working_dir = self.service_config.working_dir async def chat_with_remy(self, tool_result_max_size: int = 100, **kwargs): """Interactive CLI chat with Remy using simple streaming output.""" @@ -210,6 +270,107 @@ class ReMeCli(ReMeFb): print("\nGoodbye!\n") + async def context_check(self, messages: list[Message | dict]) -> dict: + """Check if messages exceed context limits.""" + checker = FbContextChecker( + context_window_tokens=self.service_config.metadata["context_window_tokens"], + reserve_tokens=self.service_config.metadata["reserve_tokens"], + keep_recent_tokens=self.service_config.metadata["keep_recent_tokens"], + ) + return await checker.call(messages=messages, service_context=self.service_context) + + async def compact( + self, + messages_to_summarize: list[Message | dict] = None, + turn_prefix_messages: list[Message | dict] = None, + previous_summary: str = "", + language: str = "zh", + **kwargs, + ) -> str | dict: + """Compact messages into a summary.""" + compactor = FbCompactor(language=language, **kwargs) + return await compactor.call( + messages_to_summarize=messages_to_summarize or [], + turn_prefix_messages=turn_prefix_messages or [], + previous_summary=previous_summary, + service_context=self.service_context, + ) + + async def summary( + self, + messages: list[Message | dict], + date: str, + version: str = "default", + language: str = "zh", + **kwargs, + ) -> str | dict: + """Generate a summary of the given messages.""" + summarizer = FbSummarizer( + tools=[ + BashTool(cwd=self.working_dir), + LsTool(cwd=self.working_dir), + ReadTool(cwd=self.working_dir), + WriteTool(cwd=self.working_dir), + EditTool(cwd=self.working_dir), + ], + working_dir=self.working_dir, + language=language, + version=version, + **kwargs, + ) + return await summarizer.call(messages=messages, date=date, service_context=self.service_context) + + async def memory_search(self, query: str, max_results: int = 5, min_score: float = 0.1) -> str: + """ + Mandatory recall step: semantically search MEMORY.md + memory/*.md (and optional session transcripts) + before answering questions about prior work, decisions, dates, people, preferences, or todos; + returns top snippets with path + lines. + + Args: + query: The semantic search query to find relevant memory snippets + max_results: Maximum number of search results to return (optional), default is 5 + min_score: Minimum similarity score threshold for results (optional), default is 0.1 + + Returns: + Search results as formatted string + """ + search_tool = MemorySearch( + vector_weight=self.service_config.metadata["vector_weight"], + candidate_multiplier=self.service_config.metadata["candidate_multiplier"], + ) + return await search_tool.call( + query=query, + max_results=max_results, + min_score=min_score, + service_context=self.service_context, + ) + + async def memory_get(self, path: str, offset: int | None = None, limit: int | None = None) -> str: + """ + Safe snippet read from MEMORY.md, memory/*.md with optional offset/limit; + use after memory_search to pull only the needed lines and keep context small. + + Args: + path: Path to the memory file to read (relative or absolute) + offset: Starting line number (1-indexed, optional) + limit: Number of lines to read from the starting line (optional) + + Returns: + Memory file content as string + """ + get_tool = MemoryGet(cwd=self.working_dir) + return await get_tool.call(path=path, offset=offset, limit=limit, service_context=self.service_context) + + async def needs_compaction(self, messages: list[Message | dict]) -> bool: + """Check if messages need compaction based on context window limits.""" + messages = [Message(**message) if isinstance(message, dict) else message for message in messages] + checker = FbContextChecker( + context_window_tokens=self.service_config.metadata["context_window_tokens"], + reserve_tokens=self.service_config.metadata["reserve_tokens"], + ) + result = await checker.call(messages=messages, service_context=self.service_context) + return result["needs_compaction"] + async def async_main(): """Main function for testing the ReMeFs CLI.""" diff --git a/reme/reme_copaw.py b/reme/reme_copaw.py new file mode 100644 index 00000000..480fda62 --- /dev/null +++ b/reme/reme_copaw.py @@ -0,0 +1,700 @@ +""" +ReMe Copaw Application Module + +This module provides the ReMeCopaw class, a specialized application built on top of +ReMe's core Application framework. It integrates memory management capabilities +including memory compaction, summarization, tool result management, and semantic +memory search functionality. + +Key Features: + - Memory compaction and summarization for long conversations + - Tool result compaction with file-based storage for large outputs + - Semantic memory search using vector and full-text search + - Configurable embedding models and vector store backends + - Async task management for background summarization +""" + +import asyncio +import logging +import os +import platform +from pathlib import Path + +from agentscope.formatter import FormatterBase +from agentscope.message import Msg, TextBlock +from agentscope.model import ChatModelBase +from agentscope.token import HuggingFaceTokenCounter +from agentscope.tool import Toolkit, ToolResponse + +from .config import ReMeConfigParser +from .core import Application +from .memory.file_based_copaw import Compactor, Summarizer, ToolResultCompactor, CoPawInMemoryMemory +from .memory.tools import MemorySearch + +# Module-level logger for tracking application events and errors +logger = logging.getLogger(__name__) + + +class ReMeCopaw(Application): + """ + ReMe Copaw Application Class + + A specialized application class that extends ReMe's core Application framework + with advanced memory management capabilities. This class is designed to handle + long-running conversations by providing intelligent memory compaction, + summarization, and semantic search features. + + Attributes: + working_path (Path): Absolute path to the working directory for storing data + memory_path (Path): Path to the memory storage directory + tool_result_path (Path): Path to store large tool result files + chat_model (ChatModelBase): Language model for generating summaries and processing + formatter (FormatterBase): Formatter for structuring model inputs/outputs + token_counter (HuggingFaceTokenCounter): Token counting utility for length management + toolkit (Toolkit): Collection of tools available to the application + max_input_length (int): Maximum allowed input length in tokens + memory_compact_threshold (int): Threshold at which memory compaction triggers + language (str): Language code for localization ("zh" for Chinese, empty for English) + vector_weight (float): Weight for vector search in hybrid search (0.0-1.0) + candidate_multiplier (float): Multiplier for candidate retrieval in search + tool_result_threshold (int): Size threshold for tool result compaction + retention_days (int): Number of days to retain tool result files + summary_tasks (list[asyncio.Task]): List of background summarization tasks + """ + + def __init__( + self, + working_dir: str, + chat_model: ChatModelBase, + formatter: FormatterBase, + token_counter: HuggingFaceTokenCounter, + toolkit: Toolkit, + max_input_length: int, + memory_compact_ratio: float, + language: str = "zh", + vector_weight: float = 0.7, + candidate_multiplier: float = 3.0, + tool_result_threshold: int = 1000, + retention_days: int = 7, + ): + # Initialize working directory structure + # All application data will be stored under this path + self.working_path = Path(working_dir).absolute() + self.working_path.mkdir(parents=True, exist_ok=True) + + # Create memory storage directory for persistent memory files + self.memory_path = self.working_path / "memory" + self.memory_path.mkdir(parents=True, exist_ok=True) + + # Create tool result directory for storing large tool outputs + self.tool_result_path = self.working_path / "tool_result" + self.tool_result_path.mkdir(parents=True, exist_ok=True) + + # Store references to core components + self.chat_model: ChatModelBase = chat_model + self.formatter: FormatterBase = formatter + self.token_counter: HuggingFaceTokenCounter = token_counter + self.toolkit: Toolkit = toolkit + + # Initialize runtime parameters (will be updated via update_params) + self.max_input_length: int = 0 + self.memory_compact_threshold: int = 0 + self.language: str = "" + + # Store configuration parameters + self.vector_weight: float = vector_weight + self.candidate_multiplier: float = candidate_multiplier + self.tool_result_threshold: int = tool_result_threshold + self.retention_days: int = retention_days + + # Apply initial parameter configuration + self.update_params( + max_input_length=max_input_length, + memory_compact_ratio=memory_compact_ratio, + language=language, + ) + + # Retrieve embedding configuration from environment variables + # These settings control the vector search capabilities + ( + embedding_api_key, + embedding_base_url, + embedding_model_name, + embedding_dimensions, + embedding_cache_enabled, + embedding_max_cache_size, + embedding_max_input_length, + embedding_max_batch_size, + ) = self.get_emb_envs() + + # Determine if vector search should be enabled based on configuration + # Vector search requires either an API key or a local model name + vector_enabled = bool(embedding_api_key) or bool(embedding_model_name) + if vector_enabled: + logger.info("Vector search enabled.") + else: + logger.warning( + "Vector search disabled. Memory search functionality will be restricted. " + "To enable, configure: EMBEDDING_API_KEY, EMBEDDING_BASE_URL, EMBEDDING_MODEL_NAME.", + ) + + # Check if full-text search (FTS) is enabled via environment variable + fts_enabled = os.environ.get("FTS_ENABLED", "true").lower() == "true" + + # Determine the memory store backend to use + # "auto" selects based on platform (local for Windows, chroma otherwise) + memory_store_backend = os.environ.get("MEMORY_STORE_BACKEND", "auto") + if memory_store_backend == "auto": + memory_backend = "local" if platform.system() == "Windows" else "chroma" + else: + memory_backend = memory_store_backend + + # Initialize the parent Application class with configuration + # Initialize the parent Application class with comprehensive configuration + super().__init__( + embedding_api_key=embedding_api_key, + embedding_base_url=embedding_base_url, + working_dir=str(self.working_path), + config_path="copaw", + enable_logo=False, + log_to_console=False, + parser=ReMeConfigParser, + default_embedding_model_config={ + "model_name": embedding_model_name, + "dimensions": embedding_dimensions, + "enable_cache": embedding_cache_enabled, + "use_dimensions": False, + "max_cache_size": embedding_max_cache_size, + "max_input_length": embedding_max_input_length, + "max_batch_size": embedding_max_batch_size, + }, + default_file_store_config={ + "backend": memory_backend, + "store_name": "copaw", + "vector_enabled": vector_enabled, + "fts_enabled": fts_enabled, + }, + default_file_watcher_config={ + "watch_paths": [ + str(self.working_path / "MEMORY.md"), + str(self.working_path / "memory.md"), + str(self.memory_path), + ], + }, + ) + + # Initialize list to track background summarization tasks + self.summary_tasks: list[asyncio.Task] = [] + + def update_params( + self, + max_input_length: int, + memory_compact_ratio: float, + language: str, + ): + """ + Update runtime parameters for memory management. + + This method allows dynamic adjustment of memory-related parameters during + runtime. It recalculates the memory compaction threshold based on the + new input length and compaction ratio. + + Args: + max_input_length (int): New maximum input length in tokens + memory_compact_ratio (float): Ratio at which to trigger compaction (0.0-1.0) + language (str): Language code for localization ("zh" or other) + + Note: + The memory_compact_threshold is calculated as: + max_input_length * memory_compact_ratio * 0.9 + The 0.9 factor provides a safety margin before reaching the absolute limit + """ + # Update the maximum allowed input length + self.max_input_length = max_input_length + + # Calculate compaction threshold with safety margin + # This ensures compaction happens before hitting the hard limit + self.memory_compact_threshold = int(max_input_length * memory_compact_ratio * 0.9) + + # Set language for localization + if language == "zh": + self.language = "zh" + else: + self.language = "" + + @staticmethod + def _safe_str(key: str, default: str) -> str: + """ + Safely retrieve a string value from an environment variable. + + Args: + key (str): The name of the environment variable to retrieve + default (str): The default value to return if the variable is not set + + Returns: + str: The value of the environment variable, or the default if not set + """ + return os.environ.get(key, default) + + @staticmethod + def _safe_int(key: str, default: int) -> int: + """ + Safely retrieve an integer value from an environment variable. + + This method handles cases where the environment variable is not set + or contains a non-integer value by returning the specified default. + + Args: + key (str): The name of the environment variable to retrieve + default (int): The default value to return on failure or if not set + + Returns: + int: The integer value of the environment variable, or the default + + Note: + Logs a warning if the value exists but cannot be parsed as an integer + """ + value = os.environ.get(key) + if value is None: + return default + + try: + return int(value) + except ValueError: + logger.warning(f"Invalid int value '{value}' for key '{key}', using default {default}") + return default + + def get_emb_envs(self): + """ + Retrieve all embedding-related configuration from environment variables. + + This method collects all settings needed for the embedding service, + including API credentials, model configuration, and caching parameters. + + Environment Variables: + EMBEDDING_API_KEY: API key for the embedding service + EMBEDDING_BASE_URL: Base URL for the embedding API (default: dashscope) + EMBEDDING_MODEL_NAME: Name of the embedding model to use + EMBEDDING_DIMENSIONS: Vector dimensions (default: 1024) + EMBEDDING_CACHE_ENABLED: Whether to enable caching (default: true) + EMBEDDING_MAX_CACHE_SIZE: Maximum cache entries (default: 2000) + EMBEDDING_MAX_INPUT_LENGTH: Max input text length (default: 8192) + EMBEDDING_MAX_BATCH_SIZE: Max batch size for requests (default: 10) + + Returns: + tuple: A tuple containing all embedding configuration values in order: + (api_key, base_url, model_name, dimensions, cache_enabled, + max_cache_size, max_input_length, max_batch_size) + """ + # API authentication and endpoint configuration + embedding_api_key = self._safe_str("EMBEDDING_API_KEY", "") + embedding_base_url = self._safe_str("EMBEDDING_BASE_URL", "https://dashscope.aliyuncs.com/compatible-mode/v1") + embedding_model_name = self._safe_str("EMBEDDING_MODEL_NAME", "") + + # Model and vector configuration + embedding_dimensions = self._safe_int("EMBEDDING_DIMENSIONS", 1024) + + # Caching configuration for performance optimization + embedding_cache_enabled = self._safe_str("EMBEDDING_CACHE_ENABLED", "true").lower() == "true" + embedding_max_cache_size = self._safe_int("EMBEDDING_MAX_CACHE_SIZE", 2000) + + # Input processing limits + embedding_max_input_length = self._safe_int("EMBEDDING_MAX_INPUT_LENGTH", 8192) + embedding_max_batch_size = self._safe_int("EMBEDDING_MAX_BATCH_SIZE", 10) + + return ( + embedding_api_key, + embedding_base_url, + embedding_model_name, + embedding_dimensions, + embedding_cache_enabled, + embedding_max_cache_size, + embedding_max_input_length, + embedding_max_batch_size, + ) + + def _cleanup_tool_results(self) -> int: + """ + Clean up expired tool result files from the tool result directory. + + This method removes tool result files that have exceeded the retention + period specified during initialization. It helps manage disk space by + automatically removing old, unused tool outputs. + + Returns: + int: The number of files that were successfully deleted + + Note: + Exceptions during cleanup are logged but do not raise errors, + ensuring the application continues to function even if cleanup fails + """ + try: + # Create a compactor instance with current configuration + compactor = ToolResultCompactor( + tool_result_dir=self.tool_result_path, + tool_result_threshold=self.tool_result_threshold, + retention_days=self.retention_days, + ) + # Execute cleanup and return count of deleted files + return compactor.cleanup_expired_files() + except Exception as e: + # Log exception details but return 0 to indicate failure gracefully + logger.exception(f"Error cleaning up tool results: {e}") + return 0 + + async def start(self): + """ + Start the application lifecycle. + + This method initializes the application by calling the parent class's + start method and performs initial cleanup of expired tool result files. + + Returns: + The result from the parent class's start method + + Note: + Tool result cleanup runs after successful startup to ensure + the application is fully initialized before performing maintenance + """ + # Initialize parent application components + result = await super().start() + # Perform initial cleanup of old tool result files + self._cleanup_tool_results() + return result + + async def close(self) -> bool: + """ + Close the application and perform cleanup. + + This method performs final cleanup of expired tool result files before + shutting down the application through the parent class's close method. + + Returns: + bool: True if shutdown was successful, False otherwise + + Note: + Cleanup is performed before calling parent close to ensure + all resources are available during the cleanup process + """ + # Clean up tool results before shutting down + self._cleanup_tool_results() + # Shutdown parent application components + return await super().close() + + async def compact_tool_result( + self, + messages: list[Msg], + ) -> list[Msg]: + """ + Compact tool results by truncating large outputs and saving full content to files. + + This method processes a list of messages and identifies tool results that exceed + the configured size threshold. Large tool outputs are truncated in the message + list while their full content is saved to files for later retrieval. + + Args: + messages (list[Msg]): List of messages to process for tool result compaction + + Returns: + list[Msg]: The processed message list with large tool results compacted + + Note: + - Tool results below the threshold remain unchanged in the messages + - Large results are replaced with truncated versions and file references + - Expired files are cleaned up as part of the compaction process + - If compaction fails, the original messages are returned unchanged + """ + try: + # Create compactor with instance configuration + compactor = ToolResultCompactor( + tool_result_dir=self.tool_result_path, + tool_result_threshold=self.tool_result_threshold, + retention_days=self.retention_days, + ) + # Set the messages context for the compactor to process + compactor.context["messages"] = messages + + # Execute compaction and get processed messages + result = await compactor.execute() + + # Clean up any expired tool result files during compaction + compactor.cleanup_expired_files() + + return result + + except Exception as e: + # Log the error and return original messages to maintain functionality + logger.exception(f"Error compacting tool results: {e}") + return messages + + async def compact_memory(self, messages: list[Msg], previous_summary: str = "") -> str: + """ + Compact a list of messages into a condensed summary. + + This method uses the Compactor to reduce the length of message history + while preserving essential information. It's useful when conversation + history approaches the maximum input length limit. + + Args: + messages (list[Msg]): The list of messages to compact + previous_summary (str): Optional previous summary to incorporate + into the compaction process for continuity + + Returns: + str: A compacted summary of the messages, or empty string on failure + + Note: + - Compaction uses the configured language model to generate summaries + - The compaction threshold determines when compaction is triggered + - If compaction fails, an empty string is returned + """ + try: + # Initialize compactor with current configuration + compactor = Compactor( + memory_compact_threshold=self.memory_compact_threshold, + chat_model=self.chat_model, + formatter=self.formatter, + token_counter=self.token_counter, + language=self.language, + ) + + # Execute compaction with optional previous summary context + return await compactor.call(messages=messages, previous_summary=previous_summary) + + except Exception as e: + # Log error and return empty string to indicate failure + logger.exception(f"Error compacting memory: {e}") + return "" + + async def summary_memory(self, messages: list[Msg]) -> str: + """ + Generate a comprehensive summary of the given messages. + + This method uses the Summarizer to create a detailed summary of the + conversation history, which can be stored as persistent memory. Unlike + compaction, summarization aims to capture key information in a format + suitable for long-term storage and retrieval. + + Args: + messages (list[Msg]): The list of messages to summarize + + Returns: + str: A generated summary of the messages, or empty string on failure + + Note: + - Summarization may use tools from the toolkit to enhance the summary + - The summary is typically stored in the memory directory + - If summarization fails, an empty string is returned + """ + try: + # Initialize summarizer with working directories and configuration + compactor = Summarizer( + working_dir=str(self.working_path), + memory_dir=str(self.memory_path), + memory_compact_threshold=self.memory_compact_threshold, + chat_model=self.chat_model, + formatter=self.formatter, + token_counter=self.token_counter, + toolkit=self.toolkit, + language=self.language, + ) + + # Execute summarization on the provided messages + return await compactor.call(messages=messages) + + except Exception as e: + # Log error and return empty string to indicate failure + logger.exception(f"Error summarizing memory: {e}") + return "" + + async def await_summary_tasks(self) -> str: + """ + Wait for all background summary tasks to complete and collect results. + + This method iterates through all pending summary tasks, waits for their + completion, and collects their results or error information. It's used + to synchronize with background summarization operations before shutdown + or when results are needed. + + Returns: + str: A concatenated string containing the status and results of + all summary tasks, with each task on a new line + + Note: + - Completed tasks are processed immediately without waiting + - Incomplete tasks are awaited with a timeout + - Cancelled tasks and exceptions are logged and included in results + - The task list is cleared after processing all tasks + """ + result = "" + for task in self.summary_tasks: + if task.done(): + # Task has already completed, check its status + if task.cancelled(): + logger.warning("Summary task was cancelled.") + result += "Summary task was cancelled.\n" + else: + # Check if the task raised an exception + exc = task.exception() + if exc is not None: + logger.exception(f"Summary task failed: {exc}") + result += f"Summary task failed: {exc}\n" + else: + # Task completed successfully, collect result + task_result = task.result() + logger.info(f"Summary task completed: {task_result}") + result += f"Summary task completed: {task_result}\n" + + else: + # Task is still running, wait for it to complete + try: + task_result = await task + logger.info(f"Summary task completed: {task_result}") + result += f"Summary task completed: {task_result}\n" + + except asyncio.CancelledError: + logger.warning("Summary task was cancelled while waiting.") + result += "Summary task was cancelled.\n" + except Exception as e: + logger.exception(f"Summary task failed: {e}") + result += f"Summary task failed: {e}\n" + + # Clear the task list after processing all tasks + self.summary_tasks.clear() + return result + + def add_async_summary_task(self, messages: list[Msg]): + """ + Add an asynchronous summary task for the given messages. + + This method creates a background task to summarize the provided messages + without blocking the main execution flow. Before adding a new task, it + cleans up any completed tasks from the task list to prevent memory leaks. + + Args: + messages (list[Msg]): The list of messages to be summarized in the + background task + + Note: + - Completed tasks are removed from the tracking list before adding + - Task status (success, failure, cancellation) is logged for monitoring + - The new task is created using asyncio.create_task for true async execution + - Failed or cancelled tasks are logged but do not prevent new tasks + """ + # Clean up completed summary tasks before adding a new one + remaining_tasks = [] + for task in self.summary_tasks: + if task.done(): + # Process completed task status + if task.cancelled(): + logger.warning("Summary task was cancelled.") + continue + exc = task.exception() + if exc is not None: + logger.exception(f"Summary task failed: {exc}") + else: + # Log successful completion with result summary + result = task.result() + logger.info(f"Summary task completed: {result}") + else: + # Keep incomplete tasks in the tracking list + remaining_tasks.append(task) + self.summary_tasks = remaining_tasks + + # Create and track the new background summarization task + task = asyncio.create_task(self.summary_memory(messages=messages)) + self.summary_tasks.append(task) + + async def memory_search(self, query: str, max_results: int = 5, min_score: float = 0.1) -> ToolResponse: + """ + Perform semantic memory search using vector and full-text search. + + This method searches the memory store for content relevant to the given query + using a hybrid approach combining vector similarity search and full-text search. + Results are ranked by relevance and filtered by the minimum score threshold. + + Args: + query (str): The search query string. Must not be empty. + max_results (int): Maximum number of results to return (1-100, default: 5) + min_score (float): Minimum relevance score threshold (0.001-0.999, default: 0.1) + + Returns: + ToolResponse: A ToolResponse containing the search results as text, + or an error message if the query is empty + + Note: + - Vector search weight is controlled by self.vector_weight + - Candidate retrieval uses self.candidate_multiplier for broader search + - Parameters are validated and clamped to valid ranges + - Requires vector search to be enabled via embedding configuration + """ + # Validate query parameter + if not query: + return ToolResponse( + content=[ + TextBlock( + type="text", + text="Error: No query provided.", + ), + ], + ) + + # Validate and clamp max_results to valid range [1, 100] + if isinstance(max_results, int): + max_results = min(max(max_results, 1), 100) + else: + max_results = 5 + + # Validate and clamp min_score to valid range [0.001, 0.999] + if isinstance(min_score, (int, float)): + min_score = min(max(min_score, 0.001), 0.999) + else: + min_score = 0.1 + + # Initialize memory search tool with configured weights + search_tool = MemorySearch( + vector_weight=self.vector_weight, + candidate_multiplier=self.candidate_multiplier, + ) + + # Execute the search with validated parameters + search_result = await search_tool.call( + query=query, + max_results=max_results, + min_score=min_score, + service_context=self.service_context, + ) + + # Return results wrapped in ToolResponse format + return ToolResponse( + content=[ + TextBlock( + type="text", + text=search_result, + ), + ], + ) + + def get_in_memory_memory(self): + """ + Create and return an in-memory memory instance. + + This method instantiates a CoPawInMemoryMemory object configured with + the current application's token counter, formatter, and input length limits. + The in-memory memory provides fast, temporary storage for conversation + context without persistence. + + Returns: + CoPawInMemoryMemory: A configured in-memory memory instance ready + for storing and retrieving conversation messages + + Note: + - In-memory memory is volatile and cleared when the instance is destroyed + - Useful for managing conversation context within a single session + - Shares the same token counter and formatter as the main application + """ + return CoPawInMemoryMemory( + token_counter=self.token_counter, + formatter=self.formatter, + max_input_length=self.max_input_length, + ) diff --git a/reme/reme_fb.py b/reme/reme_fb.py deleted file mode 100644 index 914d237b..00000000 --- a/reme/reme_fb.py +++ /dev/null @@ -1,183 +0,0 @@ -"""ReMe File Based""" - -from pathlib import Path - -from .config import ReMeConfigParser -from .core import Application -from .core.schema import Message -from .core.tools import ( - BashTool, - EditTool, - LsTool, - ReadTool, - WriteTool, -) -from .memory.file_based import FbCompactor, FbContextChecker, FbSummarizer -from .memory.tools import MemoryGet, MemorySearch - - -class ReMeFb(Application): - """ReMe File Based""" - - def __init__( - self, - *args, - working_dir: str = ".reme", - config_path: str = "file", - enable_logo: bool = True, - log_to_console: bool = True, - llm_api_key: str | None = None, - llm_base_url: str | None = None, - embedding_api_key: str | None = None, - embedding_base_url: str | None = None, - default_llm_config: dict | None = None, - default_embedding_model_config: dict | None = None, - default_file_store_config: dict | None = None, - default_token_counter_config: dict | None = None, - default_file_watcher_config: dict | None = None, - context_window_tokens: int = 128000, - reserve_tokens: int = 36000, - keep_recent_tokens: int = 20000, - vector_weight: float = 0.7, - candidate_multiplier: float = 3.0, - **kwargs, - ): - """Initialize ReMe with config.""" - working_path = Path(working_dir) - working_path.mkdir(parents=True, exist_ok=True) - memory_path = working_path / "memory" - memory_path.mkdir(parents=True, exist_ok=True) - self.working_dir: str = str(working_path.absolute()) - - default_file_watcher_config = default_file_watcher_config or {} - if not default_file_watcher_config.get("watch_paths", None): - default_file_watcher_config["watch_paths"] = [ - str(working_path / "MEMORY.md"), - str(working_path / "memory.md"), - str(memory_path), - ] - super().__init__( - *args, - llm_api_key=llm_api_key, - llm_base_url=llm_base_url, - embedding_api_key=embedding_api_key, - embedding_base_url=embedding_base_url, - working_dir=working_dir, - config_path=config_path, - enable_logo=enable_logo, - log_to_console=log_to_console, - parser=ReMeConfigParser, - default_llm_config=default_llm_config, - default_embedding_model_config=default_embedding_model_config, - default_file_store_config=default_file_store_config, - default_token_counter_config=default_token_counter_config, - default_file_watcher_config=default_file_watcher_config, - **kwargs, - ) - - self.service_config.metadata.setdefault("context_window_tokens", context_window_tokens) - self.service_config.metadata.setdefault("reserve_tokens", reserve_tokens) - self.service_config.metadata.setdefault("keep_recent_tokens", keep_recent_tokens) - self.service_config.metadata.setdefault("vector_weight", vector_weight) - self.service_config.metadata.setdefault("candidate_multiplier", candidate_multiplier) - - async def context_check(self, messages: list[Message | dict]) -> dict: - """Check if messages exceed context limits.""" - checker = FbContextChecker( - context_window_tokens=self.service_config.metadata["context_window_tokens"], - reserve_tokens=self.service_config.metadata["reserve_tokens"], - keep_recent_tokens=self.service_config.metadata["keep_recent_tokens"], - ) - return await checker.call(messages=messages, service_context=self.service_context) - - async def compact( - self, - messages_to_summarize: list[Message | dict] = None, - turn_prefix_messages: list[Message | dict] = None, - previous_summary: str = "", - language: str = "zh", - **kwargs, - ) -> str | dict: - """Compact messages into a summary.""" - compactor = FbCompactor(language=language, **kwargs) - return await compactor.call( - messages_to_summarize=messages_to_summarize or [], - turn_prefix_messages=turn_prefix_messages or [], - previous_summary=previous_summary, - service_context=self.service_context, - ) - - async def summary( - self, - messages: list[Message | dict], - date: str, - version: str = "default", - language: str = "zh", - **kwargs, - ) -> str | dict: - """Generate a summary of the given messages.""" - summarizer = FbSummarizer( - tools=[ - BashTool(cwd=self.working_dir), - LsTool(cwd=self.working_dir), - ReadTool(cwd=self.working_dir), - WriteTool(cwd=self.working_dir), - EditTool(cwd=self.working_dir), - ], - working_dir=self.working_dir, - language=language, - version=version, - **kwargs, - ) - return await summarizer.call(messages=messages, date=date, service_context=self.service_context) - - async def memory_search(self, query: str, max_results: int = 5, min_score: float = 0.1) -> str: - """ - Mandatory recall step: semantically search MEMORY.md + memory/*.md (and optional session transcripts) - before answering questions about prior work, decisions, dates, people, preferences, or todos; - returns top snippets with path + lines. - - Args: - query: The semantic search query to find relevant memory snippets - max_results: Maximum number of search results to return (optional), default is 5 - min_score: Minimum similarity score threshold for results (optional), default is 0.1 - - Returns: - Search results as formatted string - """ - search_tool = MemorySearch( - vector_weight=self.service_config.metadata["vector_weight"], - candidate_multiplier=self.service_config.metadata["candidate_multiplier"], - ) - return await search_tool.call( - query=query, - max_results=max_results, - min_score=min_score, - service_context=self.service_context, - ) - - async def memory_get(self, path: str, offset: int | None = None, limit: int | None = None) -> str: - """ - Safe snippet read from MEMORY.md, memory/*.md with optional offset/limit; - use after memory_search to pull only the needed lines and keep context small. - - Args: - path: Path to the memory file to read (relative or absolute) - offset: Starting line number (1-indexed, optional) - limit: Number of lines to read from the starting line (optional) - - Returns: - Memory file content as string - """ - get_tool = MemoryGet(cwd=self.working_dir) - return await get_tool.call(path=path, offset=offset, limit=limit, service_context=self.service_context) - - async def needs_compaction(self, messages: list[Message | dict]) -> bool: - """Check if messages need compaction based on context window limits.""" - messages = [Message(**message) if isinstance(message, dict) else message for message in messages] - checker = FbContextChecker( - context_window_tokens=self.service_config.metadata["context_window_tokens"], - reserve_tokens=self.service_config.metadata["reserve_tokens"], - ) - result = await checker.call(messages=messages, service_context=self.service_context) - return result["needs_compaction"] diff --git a/tests/copaw/test_compactor.py b/tests/copaw/test_compactor.py new file mode 100644 index 00000000..d044f376 --- /dev/null +++ b/tests/copaw/test_compactor.py @@ -0,0 +1,375 @@ +"""Tests for Compactor.""" + +import asyncio +import logging + +from agentscope.message import Msg + +from test_utils import ( + get_dash_chat_model, + get_formatter, + get_token_counter, +) +from reme.memory.file_based_copaw import Compactor + +# 配置日志输出到控制台 +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + + +# ANSI 颜色码 +class Colors: + """ANSI color codes for terminal output.""" + + GREEN = "\033[92m" + RED = "\033[91m" + YELLOW = "\033[93m" + BLUE = "\033[94m" + CYAN = "\033[96m" + BOLD = "\033[1m" + RESET = "\033[0m" + + +def print_pass(test_name: str): + """打印测试通过信息""" + print(f"{Colors.GREEN}{Colors.BOLD}✓ {test_name} PASSED{Colors.RESET}") + + +def print_fail(test_name: str, error: str): + """打印测试失败信息""" + print(f"{Colors.RED}{Colors.BOLD}✗ {test_name} FAILED: {error}{Colors.RESET}") + + +def print_error(test_name: str, error: str): + """打印测试错误信息""" + print(f"{Colors.YELLOW}{Colors.BOLD}⚠ {test_name} ERROR: {error}{Colors.RESET}") + + +def print_test_header(test_name: str): + """打印测试标题""" + print(f"\n{Colors.CYAN}{'=' * 60}{Colors.RESET}") + print(f"{Colors.BLUE}{Colors.BOLD}Running: {test_name}{Colors.RESET}") + print(f"{Colors.CYAN}{'=' * 60}{Colors.RESET}") + + +def create_user_msg(content: str) -> Msg: + """Create a user message.""" + return Msg(name="user", role="user", content=content) + + +def create_assistant_msg(content: str) -> Msg: + """Create an assistant message.""" + return Msg(name="assistant", role="assistant", content=content) + + +def create_tool_use_msg(tool_name: str, tool_input: dict) -> Msg: + """Create a message with tool_use content block.""" + return Msg( + name="assistant", + role="assistant", + content=[ + { + "type": "tool_use", + "id": "call_123", + "name": tool_name, + "input": tool_input, + }, + ], + ) + + +def create_tool_result_msg(tool_name: str, output: str) -> Msg: + """Create a message with tool_result content block.""" + return Msg( + name="tool", + role="user", + content=[ + { + "type": "tool_result", + "id": "call_123", + "name": tool_name, + "output": output, + }, + ], + ) + + +def create_compactor(): + """Create a Compactor instance for testing.""" + return Compactor( + memory_compact_threshold=4000, + chat_model=get_dash_chat_model(), + formatter=get_formatter(), + token_counter=get_token_counter(), + ) + + +def test_empty_messages(): + """Test that empty messages return empty string.""" + compactor = create_compactor() + result = asyncio.run(compactor.call(messages=[])) + assert result == "", f"Expected empty string, got: {result}" + print("test_empty_messages PASSED") + + +def test_short_conversation(): + """Test compaction of a short conversation.""" + compactor = create_compactor() + messages = [ + create_user_msg("Hello, I need help with Python."), + create_assistant_msg("Sure, I'd be happy to help. What do you need?"), + create_user_msg("How do I read a file?"), + create_assistant_msg("You can use open() function: with open('file.txt', 'r') as f: content = f.read()"), + ] + + logger.info(f"Input messages count: {len(messages)}") + for i, msg in enumerate(messages): + logger.debug( + f"Message {i}: role={msg.role}, content=" + f"{msg.content[:50] if isinstance(msg.content, str) else msg.content}...", + ) + + result = asyncio.run(compactor.call(messages=messages)) + + logger.info(f"Result type: {type(result)}") + logger.info(f"Result: {result}") + + assert result, "Result should not be empty" + assert isinstance(result, str), f"Result should be string, got: {type(result)}" + assert "##" in result, "Result should have markdown headers" + print_pass("test_short_conversation") + + +def test_medium_conversation(): + """Test compaction of a medium-length conversation with tool calls.""" + compactor = create_compactor() + messages = [ + create_user_msg("Help me create a Python script to process data."), + create_assistant_msg("I'll help you create a data processing script. Let me first check the data format."), + create_tool_use_msg("read_file", {"path": "/data/input.csv"}), + create_tool_result_msg("read_file", "id,name,value\n1,Alice,100\n2,Bob,200\n3,Charlie,300"), + create_assistant_msg( + "I see the data is in CSV format. Here's a script to process it:\n" + "```python\nimport csv\n\ndef process_data(filepath):\n" + " with open(filepath, 'r') as f:\n reader = csv.DictReader(f)\n" + " return [row for row in reader]\n```", + ), + create_user_msg("Can you add a filter function?"), + create_assistant_msg( + "Sure, here's the updated script with filtering:\n" + "```python\ndef filter_by_value(data, min_value):\n" + " return [row for row in data if int(row['value']) >= min_value]\n```", + ), + ] + + result = asyncio.run(compactor.call(messages=messages)) + + assert result, "Result should not be empty" + assert isinstance(result, str), f"Result should be string, got: {type(result)}" + assert "##" in result, "Result should have markdown headers" + print_pass("test_medium_conversation") + + +def test_long_conversation(): + """Test compaction of a long conversation that exceeds threshold.""" + compactor = create_compactor() + messages = [] + for i in range(100000): + messages.append(create_user_msg(f"Question {i}: How do I implement feature {i}?")) + messages.append( + create_assistant_msg( + f"Answer {i}: Here's how to implement feature {i}. " * 20, + ), + ) + + result = asyncio.run(compactor.call(messages=messages)) + + assert result, "Result should not be empty" + assert isinstance(result, str), f"Result should be string, got: {type(result)}" + assert "##" in result, "Result should have markdown headers" + print_pass("test_long_conversation") + + +def test_with_previous_summary(): + """Test compaction with an existing previous summary.""" + compactor = create_compactor() + previous_summary = """## Goal +User wants to build a REST API with FastAPI. + +## Constraints & Preferences +- Use Python 3.10+ +- Follow RESTful best practices + +## Progress +### Done +- [x] Set up project structure +- [x] Created main.py with basic FastAPI app + +### In Progress +- [ ] Add user authentication + +### Blocked +- (none) + +## Key Decisions +- **Framework**: FastAPI for performance and type hints + +## Next Steps +1. Implement JWT authentication +2. Add user endpoints + +## Critical Context +- Using SQLAlchemy for database +- PostgreSQL as database backend +""" + + messages = [ + create_user_msg("Let's implement the JWT authentication now."), + create_assistant_msg("I'll implement JWT authentication. First, let me install the required packages."), + create_tool_use_msg("run_command", {"command": "pip install python-jose[cryptography] passlib[bcrypt]"}), + create_tool_result_msg("run_command", "Successfully installed python-jose-3.3.0 passlib-1.7.4"), + create_assistant_msg("Dependencies installed. Now let's create the auth module with JWT token generation."), + ] + + result = asyncio.run( + compactor.call( + messages=messages, + previous_summary=previous_summary, + ), + ) + + assert result, "Result should not be empty" + assert isinstance(result, str), f"Result should be string, got: {type(result)}" + assert "##" in result, "Result should have markdown headers" + print_pass("test_with_previous_summary") + + +def test_conversation_with_multiple_tool_calls(): + """Test compaction of conversation with multiple sequential tool calls.""" + compactor = create_compactor() + messages = [ + create_user_msg("Help me debug this Python script that's failing."), + create_assistant_msg("Let me check the script first."), + create_tool_use_msg("read_file", {"path": "/app/main.py"}), + create_tool_result_msg( + "read_file", + "def process():\n data = load_data()\n result = analyze(data)\n return result", + ), + create_tool_use_msg("read_file", {"path": "/app/utils.py"}), + create_tool_result_msg("read_file", "def load_data():\n return open('data.json').read()"), + create_tool_use_msg("run_command", {"command": "python /app/main.py"}), + create_tool_result_msg("run_command", "FileNotFoundError: [Errno 2] No such file or directory: 'data.json'"), + create_assistant_msg( + "I found the issue! The script is looking for 'data.json' " + "in the current directory instead of an absolute path.", + ), + create_user_msg("How should I fix it?"), + create_assistant_msg( + "Update load_data() to use an absolute path:\n" + "```python\nimport os\ndef load_data():\n" + " script_dir = os.path.dirname(__file__)\n" + " return open(os.path.join(script_dir, 'data.json')).read()\n```", + ), + ] + + result = asyncio.run(compactor.call(messages=messages)) + + assert result, "Result should not be empty" + assert isinstance(result, str), f"Result should be string, got: {type(result)}" + print_pass("test_conversation_with_multiple_tool_calls") + + +def test_low_threshold(): + """Test compaction with low memory threshold.""" + compactor = Compactor( + memory_compact_threshold=500, + chat_model=get_dash_chat_model(), + formatter=get_formatter(), + token_counter=get_token_counter(), + ) + + messages = [ + create_user_msg("Build a web scraper."), + create_assistant_msg("I'll create a web scraper using BeautifulSoup and requests."), + create_user_msg("Make it handle pagination."), + create_assistant_msg("Here's the paginated scraper implementation with error handling."), + ] + + result = asyncio.run(compactor.call(messages=messages)) + + assert result, "Result should not be empty" + assert isinstance(result, str), f"Result should be string, got: {type(result)}" + print("test_low_threshold PASSED") + + +def test_high_threshold(): + """Test compaction with high memory threshold.""" + compactor = Compactor( + memory_compact_threshold=10000, + chat_model=get_dash_chat_model(), + formatter=get_formatter(), + token_counter=get_token_counter(), + ) + + messages = [ + create_user_msg("Create a calculator class."), + create_assistant_msg("Here's a Calculator class with basic operations: add, subtract, multiply, divide."), + ] + + result = asyncio.run(compactor.call(messages=messages)) + + assert result, "Result should not be empty" + assert isinstance(result, str), f"Result should be string, got: {type(result)}" + print("test_high_threshold PASSED") + + +def run_all_tests(): + """Run all tests.""" + tests = [ + test_empty_messages, + test_short_conversation, + test_medium_conversation, + test_long_conversation, + test_with_previous_summary, + test_conversation_with_multiple_tool_calls, + test_low_threshold, + test_high_threshold, + ] + + passed = 0 + failed = 0 + + for test in tests: + try: + print_test_header(test.__name__) + test() + passed += 1 + except AssertionError as e: + print_fail(test.__name__, str(e)) + failed += 1 + except Exception as e: + print_error(test.__name__, str(e)) + failed += 1 + + # 打印最终统计结果 + print(f"\n{Colors.CYAN}{'=' * 60}{Colors.RESET}") + print(f"{Colors.BOLD}Test Results Summary{Colors.RESET}") + print(f"{Colors.CYAN}{'=' * 60}{Colors.RESET}") + print(f"{Colors.GREEN}{Colors.BOLD}✓ Passed: {passed}{Colors.RESET}") + if failed > 0: + print(f"{Colors.RED}{Colors.BOLD}✗ Failed: {failed}{Colors.RESET}") + else: + print(f"{Colors.GREEN}✗ Failed: {failed}{Colors.RESET}") + print(f"{Colors.CYAN}{'=' * 60}{Colors.RESET}") + + if failed == 0: + print(f"\n{Colors.GREEN}{Colors.BOLD}🎉 All tests passed!{Colors.RESET}") + else: + print(f"\n{Colors.RED}{Colors.BOLD}💥 Some tests failed!{Colors.RESET}") + + +if __name__ == "__main__": + run_all_tests() diff --git a/tests/copaw/test_memory_formatter.py b/tests/copaw/test_memory_formatter.py new file mode 100644 index 00000000..7da3db2d --- /dev/null +++ b/tests/copaw/test_memory_formatter.py @@ -0,0 +1,489 @@ +"""Tests for MemoryFormatter.""" + +# pylint: disable=W0212 + +import logging + +from agentscope.message import Msg + +from test_utils import get_token_counter +from reme.memory.file_based_copaw import MemoryFormatter + +# 配置日志输出到控制台 +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + + +# ANSI 颜色码 +class Colors: + """ANSI color codes for terminal output.""" + + GREEN = "\033[92m" + RED = "\033[91m" + YELLOW = "\033[93m" + BLUE = "\033[94m" + CYAN = "\033[96m" + BOLD = "\033[1m" + RESET = "\033[0m" + + +def print_pass(test_name: str): + """打印测试通过信息""" + print(f"{Colors.GREEN}{Colors.BOLD}✓ {test_name} PASSED{Colors.RESET}") + + +def print_fail(test_name: str, error: str): + """打印测试失败信息""" + print(f"{Colors.RED}{Colors.BOLD}✗ {test_name} FAILED: {error}{Colors.RESET}") + + +def print_error(test_name: str, error: str): + """打印测试错误信息""" + print(f"{Colors.YELLOW}{Colors.BOLD}⚠ {test_name} ERROR: {error}{Colors.RESET}") + + +def print_test_header(test_name: str): + """打印测试标题""" + print(f"\n{Colors.CYAN}{'=' * 60}{Colors.RESET}") + print(f"{Colors.BLUE}{Colors.BOLD}Running: {test_name}{Colors.RESET}") + print(f"{Colors.CYAN}{'=' * 60}{Colors.RESET}") + + +def create_user_msg(content: str) -> Msg: + """Create a user message.""" + return Msg(name="user", role="user", content=content) + + +def create_assistant_msg(content: str) -> Msg: + """Create an assistant message.""" + return Msg(name="assistant", role="assistant", content=content) + + +def create_tool_use_msg(tool_name: str, tool_input: dict) -> Msg: + """Create a message with tool_use content block.""" + return Msg( + name="assistant", + role="assistant", + content=[ + { + "type": "tool_use", + "id": "call_123", + "name": tool_name, + "input": tool_input, + }, + ], + ) + + +def create_tool_result_msg(tool_name: str, output: str | list[dict]) -> Msg: + """Create a message with tool_result content block.""" + return Msg( + name="tool", + role="user", + content=[ + { + "type": "tool_result", + "id": "call_123", + "name": tool_name, + "output": output, + }, + ], + ) + + +def create_thinking_msg(thinking_content: str) -> Msg: + """Create a message with thinking content block.""" + return Msg( + name="assistant", + role="assistant", + content=[ + { + "type": "thinking", + "text": thinking_content, + }, + ], + ) + + +def create_image_msg(url: str = "") -> Msg: + """Create a message with image content block.""" + content = [ + { + "type": "image", + "source": {"url": url} if url else {}, + }, + ] + return Msg(name="assistant", role="assistant", content=content) + + +def create_formatter(memory_compact_threshold: int = 4000) -> MemoryFormatter: + """Create a MemoryFormatter instance for testing.""" + return MemoryFormatter( + token_counter=get_token_counter(), + memory_compact_threshold=memory_compact_threshold, + ) + + +# ==================== _format_tool_result_output Tests ==================== + + +def test_format_tool_result_output_string(): + """Test _format_tool_result_output with string input.""" + result = MemoryFormatter._format_tool_result_output("Hello, world!") + assert result == "Hello, world!", f"Expected 'Hello, world!', got: {result}" + print_pass("test_format_tool_result_output_string") + + +def test_format_tool_result_output_text_block(): + """Test _format_tool_result_output with text block.""" + output = [{"type": "text", "text": "This is text content"}] + result = MemoryFormatter._format_tool_result_output(output) + assert result == "This is text content", f"Expected 'This is text content', got: {result}" + print_pass("test_format_tool_result_output_text_block") + + +def test_format_tool_result_output_image_block(): + """Test _format_tool_result_output with image block.""" + output = [{"type": "image", "source": {"url": "https://example.com/image.png"}}] + result = MemoryFormatter._format_tool_result_output(output) + assert "[image]" in result, f"Expected '[image]' in result, got: {result}" + assert "https://example.com/image.png" in result, f"Expected URL in result, got: {result}" + print_pass("test_format_tool_result_output_image_block") + + +def test_format_tool_result_output_file_block(): + """Test _format_tool_result_output with file block.""" + output = [{"type": "file", "path": "/path/to/file.txt", "name": "file.txt"}] + result = MemoryFormatter._format_tool_result_output(output) + assert "[file]" in result, f"Expected '[file]' in result, got: {result}" + assert "file.txt" in result, f"Expected 'file.txt' in result, got: {result}" + print_pass("test_format_tool_result_output_file_block") + + +def test_format_tool_result_output_multiple_blocks(): + """Test _format_tool_result_output with multiple blocks.""" + output = [ + {"type": "text", "text": "First part"}, + {"type": "text", "text": "Second part"}, + ] + result = MemoryFormatter._format_tool_result_output(output) + assert "First part" in result, f"Expected 'First part' in result, got: {result}" + assert "Second part" in result, f"Expected 'Second part' in result, got: {result}" + # Multiple parts should be joined with newlines and bullets + assert "- " in result, f"Expected bullet format in result, got: {result}" + print_pass("test_format_tool_result_output_multiple_blocks") + + +def test_format_tool_result_output_empty_list(): + """Test _format_tool_result_output with empty list.""" + result = MemoryFormatter._format_tool_result_output([]) + assert result == "", f"Expected empty string, got: {result}" + print_pass("test_format_tool_result_output_empty_list") + + +def test_format_tool_result_output_invalid_block(): + """Test _format_tool_result_output with invalid block (missing type).""" + output = [{"text": "No type key"}] + result = MemoryFormatter._format_tool_result_output(output) + assert result == "", f"Expected empty string for invalid block, got: {result}" + print_pass("test_format_tool_result_output_invalid_block") + + +def test_format_tool_result_output_unknown_type(): + """Test _format_tool_result_output with unknown block type.""" + output = [{"type": "unknown_type", "data": "some data"}] + result = MemoryFormatter._format_tool_result_output(output) + assert result == "", f"Expected empty string for unknown type, got: {result}" + print_pass("test_format_tool_result_output_unknown_type") + + +# ==================== format (single message) Tests ==================== + + +def test_format_empty_messages(): + """Test format with empty message list.""" + formatter = create_formatter() + result = formatter.format([]) + assert result == "", f"Expected empty string, got: {result}" + print_pass("test_format_empty_messages") + + +def test_format_single_user_message(): + """Test format with a single user message.""" + formatter = create_formatter() + msgs = [create_user_msg("Hello, how are you?")] + result = formatter.format(msgs) + + assert "user:" in result, f"Expected 'user:' in result, got: {result}" + assert "Hello, how are you?" in result, f"Expected content in result, got: {result}" + print_pass("test_format_single_user_message") + + +def test_format_single_assistant_message(): + """Test format with a single assistant message.""" + formatter = create_formatter() + msgs = [create_assistant_msg("I am fine, thank you!")] + result = formatter.format(msgs) + + assert "assistant:" in result, f"Expected 'assistant:' in result, got: {result}" + assert "I am fine, thank you!" in result, f"Expected content in result, got: {result}" + print_pass("test_format_single_assistant_message") + + +def test_format_with_tool_use(): + """Test format with tool_use message.""" + formatter = create_formatter() + msgs = [create_tool_use_msg("read_file", {"path": "/test.txt"})] + result = formatter.format(msgs) + + assert "tool_call=read_file" in result, f"Expected 'tool_call=read_file' in result, got: {result}" + assert "params=" in result, f"Expected 'params=' in result, got: {result}" + print_pass("test_format_with_tool_use") + + +def test_format_with_tool_result(): + """Test format with tool_result message.""" + formatter = create_formatter() + msgs = [create_tool_result_msg("read_file", "file content here")] + result = formatter.format(msgs) + + assert "tool_result=read_file" in result, f"Expected 'tool_result=read_file' in result, got: {result}" + assert "output=" in result, f"Expected 'output=' in result, got: {result}" + print_pass("test_format_with_tool_result") + + +def test_format_with_thinking_block(): + """Test that thinking blocks are skipped.""" + formatter = create_formatter() + msgs = [create_thinking_msg("Let me think about this...")] + result = formatter.format(msgs) + + # Thinking content should NOT appear in the result + assert "Let me think about this" not in result, f"Thinking content should be skipped, got: {result}" + print_pass("test_format_with_thinking_block") + + +def test_format_with_image(): + """Test format with image content block.""" + formatter = create_formatter() + msgs = [create_image_msg("https://example.com/image.png")] + result = formatter.format(msgs) + + assert "[image]" in result, f"Expected '[image]' in result, got: {result}" + print_pass("test_format_with_image") + + +# ==================== format (multiple messages) Tests ==================== + + +def test_format_conversation(): + """Test format with a conversation.""" + formatter = create_formatter() + msgs = [ + create_user_msg("What is Python?"), + create_assistant_msg("Python is a programming language."), + create_user_msg("Tell me more."), + create_assistant_msg("Python is known for its readability and simplicity."), + ] + result = formatter.format(msgs) + + assert "round0" in result, f"Expected 'round0' in result, got: {result}" + assert "round1" in result, f"Expected 'round1' in result, got: {result}" + assert "round2" in result, f"Expected 'round2' in result, got: {result}" + assert "round3" in result, f"Expected 'round3' in result, got: {result}" + print_pass("test_format_conversation") + + +def test_format_without_index(): + """Test format without round index.""" + formatter = create_formatter() + msgs = [ + create_user_msg("Hello"), + create_assistant_msg("Hi there!"), + ] + result = formatter.format(msgs, add_index=False) + + assert "round" not in result, f"Expected no 'round' prefix, got: {result}" + print_pass("test_format_without_index") + + +def test_format_without_time(): + """Test format without timestamp.""" + formatter = create_formatter() + msgs = [create_user_msg("Test message")] + result = formatter.format(msgs, add_time=False) + + # The result should not have timestamp brackets at the beginning + # Note: this test may need adjustment based on actual timestamp format + assert "user:" in result, f"Expected 'user:' in result, got: {result}" + print_pass("test_format_without_time") + + +def test_format_with_tool_conversation(): + """Test format with tool use and result in conversation.""" + formatter = create_formatter() + msgs = [ + create_user_msg("Read the file."), + create_tool_use_msg("read_file", {"path": "/data.txt"}), + create_tool_result_msg("read_file", "File content here"), + create_assistant_msg("The file contains: File content here"), + ] + result = formatter.format(msgs) + + assert "user:" in result + assert "tool_call=read_file" in result + assert "tool_result=read_file" in result + assert "assistant:" in result + print_pass("test_format_with_tool_conversation") + + +# ==================== Token Threshold Tests ==================== + + +def test_format_low_threshold(): + """Test that older messages are skipped with low threshold.""" + formatter = create_formatter(memory_compact_threshold=100) + msgs = [] + for i in range(20): + msgs.append(create_user_msg(f"Question {i}: " + "x" * 50)) + msgs.append(create_assistant_msg(f"Answer {i}: " + "y" * 50)) + + result = formatter.format(msgs) + + # With low threshold, not all messages should be included + # The newest messages should be present + assert "round39" in result or "round38" in result, f"Expected recent round in result, got: {result}" + # Older messages might be truncated + logger.info(f"Result length: {len(result)}") + print_pass("test_format_low_threshold") + + +def test_format_high_threshold(): + """Test that all messages are included with high threshold.""" + formatter = create_formatter(memory_compact_threshold=100000) + msgs = [ + create_user_msg("Message 1"), + create_assistant_msg("Response 1"), + create_user_msg("Message 2"), + create_assistant_msg("Response 2"), + ] + result = formatter.format(msgs) + + # All messages should be included + assert "round0" in result + assert "round1" in result + assert "round2" in result + assert "round3" in result + print_pass("test_format_high_threshold") + + +# ==================== Edge Cases Tests ==================== + + +def test_format_long_text_truncation(): + """Test that long text is truncated.""" + formatter = create_formatter() + long_text = "x" * 5000 # Much longer than default max length + msgs = [create_user_msg(long_text)] + result = formatter.format(msgs) + + # The result should be shorter due to truncation + assert len(result) < len(long_text), f"Expected truncated result, got length: {len(result)}" + print_pass("test_format_long_text_truncation") + + +def test_format_special_characters(): + """Test format with special characters in content.""" + formatter = create_formatter() + msgs = [create_user_msg("Test with 中文, 日本語, émojis 🎉")] + result = formatter.format(msgs) + + assert "中文" in result, f"Expected Chinese characters in result, got: {result}" + print_pass("test_format_special_characters") + + +def test_format_tool_result_with_complex_output(): + """Test format with complex tool result output.""" + formatter = create_formatter() + complex_output = [ + {"type": "text", "text": "Operation completed"}, + {"type": "image", "source": {"url": "https://example.com/result.png"}}, + ] + msgs = [create_tool_result_msg("process_data", complex_output)] + result = formatter.format(msgs) + + assert "tool_result=process_data" in result, f"Expected tool result in result, got: {result}" + print_pass("test_format_tool_result_with_complex_output") + + +def run_all_tests(): + """Run all tests.""" + tests = [ + # _format_tool_result_output tests + test_format_tool_result_output_string, + test_format_tool_result_output_text_block, + test_format_tool_result_output_image_block, + test_format_tool_result_output_file_block, + test_format_tool_result_output_multiple_blocks, + test_format_tool_result_output_empty_list, + test_format_tool_result_output_invalid_block, + test_format_tool_result_output_unknown_type, + # format tests (single message) + test_format_empty_messages, + test_format_single_user_message, + test_format_single_assistant_message, + test_format_with_tool_use, + test_format_with_tool_result, + test_format_with_thinking_block, + test_format_with_image, + # format tests (multiple messages) + test_format_conversation, + test_format_without_index, + test_format_without_time, + test_format_with_tool_conversation, + # threshold tests + test_format_low_threshold, + test_format_high_threshold, + # edge cases + test_format_long_text_truncation, + test_format_special_characters, + test_format_tool_result_with_complex_output, + ] + + passed = 0 + failed = 0 + + for test in tests: + try: + print_test_header(test.__name__) + test() + passed += 1 + except AssertionError as e: + print_fail(test.__name__, str(e)) + failed += 1 + except Exception as e: + print_error(test.__name__, str(e)) + failed += 1 + + # 打印最终统计结果 + print(f"\n{Colors.CYAN}{'=' * 60}{Colors.RESET}") + print(f"{Colors.BOLD}Test Results Summary{Colors.RESET}") + print(f"{Colors.CYAN}{'=' * 60}{Colors.RESET}") + print(f"{Colors.GREEN}{Colors.BOLD}✓ Passed: {passed}{Colors.RESET}") + if failed > 0: + print(f"{Colors.RED}{Colors.BOLD}✗ Failed: {failed}{Colors.RESET}") + else: + print(f"{Colors.GREEN}✗ Failed: {failed}{Colors.RESET}") + print(f"{Colors.CYAN}{'=' * 60}{Colors.RESET}") + + if failed == 0: + print(f"\n{Colors.GREEN}{Colors.BOLD}🎉 All tests passed!{Colors.RESET}") + else: + print(f"\n{Colors.RED}{Colors.BOLD}💥 Some tests failed!{Colors.RESET}") + + +if __name__ == "__main__": + run_all_tests() diff --git a/tests/copaw/test_summarizer.py b/tests/copaw/test_summarizer.py new file mode 100644 index 00000000..d0f7eb5f --- /dev/null +++ b/tests/copaw/test_summarizer.py @@ -0,0 +1,310 @@ +"""Tests for Summarizer.""" + +import asyncio +import datetime +import logging +import tempfile +from pathlib import Path + +from agentscope.message import Msg + +from test_utils import ( + get_dash_chat_model, + get_formatter, + get_token_counter, +) +from reme.memory.file_based_copaw import Summarizer + +# 配置日志输出到控制台 +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + + +# ANSI 颜色码 +class Colors: + """ANSI color codes for terminal output.""" + + GREEN = "\033[92m" + RED = "\033[91m" + YELLOW = "\033[93m" + BLUE = "\033[94m" + CYAN = "\033[96m" + BOLD = "\033[1m" + RESET = "\033[0m" + + +def print_pass(test_name: str): + """打印测试通过信息""" + print(f"{Colors.GREEN}{Colors.BOLD}✓ {test_name} PASSED{Colors.RESET}") + + +def print_fail(test_name: str, error: str): + """打印测试失败信息""" + print(f"{Colors.RED}{Colors.BOLD}✗ {test_name} FAILED: {error}{Colors.RESET}") + + +def print_error(test_name: str, error: str): + """打印测试错误信息""" + print(f"{Colors.YELLOW}{Colors.BOLD}⚠ {test_name} ERROR: {error}{Colors.RESET}") + + +def print_test_header(test_name: str): + """打印测试标题""" + print(f"\n{Colors.CYAN}{'=' * 60}{Colors.RESET}") + print(f"{Colors.BLUE}{Colors.BOLD}Running: {test_name}{Colors.RESET}") + print(f"{Colors.CYAN}{'=' * 60}{Colors.RESET}") + + +def create_user_msg(content: str) -> Msg: + """Create a user message.""" + return Msg(name="user", role="user", content=content) + + +def create_assistant_msg(content: str) -> Msg: + """Create an assistant message.""" + return Msg(name="assistant", role="assistant", content=content) + + +def create_tool_use_msg(tool_name: str, tool_input: dict) -> Msg: + """Create a message with tool_use content block.""" + return Msg( + name="assistant", + role="assistant", + content=[ + { + "type": "tool_use", + "id": "call_123", + "name": tool_name, + "input": tool_input, + }, + ], + ) + + +def create_tool_result_msg(tool_name: str, output: str) -> Msg: + """Create a message with tool_result content block.""" + return Msg( + name="tool", + role="user", + content=[ + { + "type": "tool_result", + "id": "call_123", + "name": tool_name, + "output": output, + }, + ], + ) + + +def create_summarizer(working_dir: str = None, memory_dir: str = "memory"): + """Create a Summarizer instance for testing.""" + if working_dir is None: + working_dir = tempfile.mkdtemp() + + # 确保 memory_dir 存在 + memory_path = Path(working_dir) / memory_dir + memory_path.mkdir(parents=True, exist_ok=True) + + return ( + Summarizer( + working_dir=working_dir, + memory_dir=memory_dir, + memory_compact_threshold=4000, + chat_model=get_dash_chat_model(), + formatter=get_formatter(), + token_counter=get_token_counter(), + ), + working_dir, + ) + + +def test_empty_messages(): + """Test that empty messages return empty string.""" + summarizer, _ = create_summarizer() + result = asyncio.run(summarizer.call(messages=[])) + assert result == "", f"Expected empty string, got: {result}" + print_pass("test_empty_messages") + + +def test_short_conversation(): + """Test summarization of a short conversation.""" + summarizer, working_dir = create_summarizer() + messages = [ + create_user_msg("Hello, I need help with Python."), + create_assistant_msg("Sure, I'd be happy to help. What do you need?"), + create_user_msg("How do I read a file?"), + create_assistant_msg("You can use open() function: with open('file.txt', 'r') as f: content = f.read()"), + ] + + logger.info(f"Input messages count: {len(messages)}") + logger.info(f"Working directory: {working_dir}") + + result = asyncio.run(summarizer.call(messages=messages)) + + logger.info(f"Result type: {type(result)}") + logger.info(f"Result: {result}") + + assert result, "Result should not be empty" + assert isinstance(result, str), f"Result should be string, got: {type(result)}" + print_pass("test_short_conversation") + + +def test_conversation_with_tool_calls(): + """Test summarization of conversation with tool calls.""" + summarizer, working_dir = create_summarizer() + messages = [ + create_user_msg("Help me debug this Python script."), + create_assistant_msg("Let me check the script first."), + create_tool_use_msg("read_file", {"path": "/app/main.py"}), + create_tool_result_msg("read_file", "def process():\n data = load_data()\n return data"), + create_assistant_msg("I found the issue! The script needs error handling."), + create_user_msg("How should I fix it?"), + create_assistant_msg("Add try-except block around the load_data() call."), + ] + + logger.info(f"Working directory: {working_dir}") + result = asyncio.run(summarizer.call(messages=messages)) + + assert result, "Result should not be empty" + assert isinstance(result, str), f"Result should be string, got: {type(result)}" + print_pass("test_conversation_with_tool_calls") + + +def test_consecutive_summaries(): + """Test consecutive summaries in the same directory. + + This test verifies that: + 1. First summary creates the memory file + 2. Second summary reads and updates the existing file + """ + # 使用固定的临时目录 + working_dir = tempfile.mkdtemp() + memory_dir = "memory" + memory_path = Path(working_dir) / memory_dir + memory_path.mkdir(parents=True, exist_ok=True) + + logger.info(f"Working directory: {working_dir}") + logger.info(f"Memory path: {memory_path}") + + # 创建 Summarizer 实例 + summarizer = Summarizer( + working_dir=working_dir, + memory_dir=memory_dir, + memory_compact_threshold=4000, + chat_model=get_dash_chat_model(), + formatter=get_formatter(), + token_counter=get_token_counter(), + ) + + # 第一轮对话 + messages_round1 = [ + create_user_msg("My name is Alice and I'm learning Python."), + create_assistant_msg("Nice to meet you, Alice! Python is a great language to learn."), + create_user_msg("I prefer using VS Code as my editor."), + create_assistant_msg("VS Code is excellent for Python development with great extensions."), + ] + + logger.info("=" * 40) + logger.info("Round 1: First summary (creating new file)") + logger.info("=" * 40) + result1 = asyncio.run(summarizer.call(messages=messages_round1)) + logger.info(f"Round 1 Result:\n{result1}") + + # 检查文件是否被创建 + today = datetime.datetime.now().strftime("%Y-%m-%d") + expected_file = memory_path / f"{today}.md" + logger.info(f"Expected file: {expected_file}") + + # 列出目录内容 + files_after_round1 = list(memory_path.iterdir()) + logger.info(f"Files after round 1: {files_after_round1}") + + assert expected_file.exists(), f"Memory file should be created at {expected_file}" + + # 读取第一轮写入的内容 + content_after_round1 = expected_file.read_text() + logger.info(f"Content after round 1:\n{content_after_round1}") + + # 第二轮对话 + messages_round2 = [ + create_user_msg("I also like using Docker for my projects."), + create_assistant_msg("Docker is great for containerization and deployment."), + create_user_msg("My favorite framework is FastAPI."), + create_assistant_msg("FastAPI is excellent for building modern APIs with Python."), + ] + + logger.info("=" * 40) + logger.info("Round 2: Second summary (reading and updating existing file)") + logger.info("=" * 40) + result2 = asyncio.run(summarizer.call(messages=messages_round2)) + logger.info(f"Round 2 Result:\n{result2}") + + # 读取第二轮写入后的内容 + content_after_round2 = expected_file.read_text() + logger.info(f"Content after round 2:\n{content_after_round2}") + + # 验证 + assert result1, "Round 1 result should not be empty" + assert result2, "Round 2 result should not be empty" + + # 验证第二轮内容包含新信息（Docker 或 FastAPI） + # 注意：具体内容取决于 LLM 的响应 + assert len(content_after_round2) > 0, "Content after round 2 should not be empty" + + logger.info("=" * 40) + logger.info("Consecutive summaries test completed successfully!") + logger.info("=" * 40) + + print_pass("test_consecutive_summaries") + + +def run_all_tests(): + """Run all tests.""" + tests = [ + test_consecutive_summaries, + test_empty_messages, + test_short_conversation, + test_conversation_with_tool_calls, + ] + + passed = 0 + failed = 0 + + for test in tests: + try: + print_test_header(test.__name__) + test() + passed += 1 + except AssertionError as e: + print_fail(test.__name__, str(e)) + failed += 1 + except Exception as e: + import traceback + + print_error(test.__name__, str(e)) + traceback.print_exc() + failed += 1 + + # 打印最终统计结果 + print(f"\n{Colors.CYAN}{'=' * 60}{Colors.RESET}") + print(f"{Colors.BOLD}Test Results Summary{Colors.RESET}") + print(f"{Colors.CYAN}{'=' * 60}{Colors.RESET}") + print(f"{Colors.GREEN}{Colors.BOLD}✓ Passed: {passed}{Colors.RESET}") + if failed > 0: + print(f"{Colors.RED}{Colors.BOLD}✗ Failed: {failed}{Colors.RESET}") + else: + print(f"{Colors.GREEN}✗ Failed: {failed}{Colors.RESET}") + print(f"{Colors.CYAN}{'=' * 60}{Colors.RESET}") + + if failed == 0: + print(f"\n{Colors.GREEN}{Colors.BOLD}🎉 All tests passed!{Colors.RESET}") + else: + print(f"\n{Colors.RED}{Colors.BOLD}💥 Some tests failed!{Colors.RESET}") + + +if __name__ == "__main__": + run_all_tests() diff --git a/tests/copaw/test_tool_result_compactor.py b/tests/copaw/test_tool_result_compactor.py new file mode 100644 index 00000000..ba225255 --- /dev/null +++ b/tests/copaw/test_tool_result_compactor.py @@ -0,0 +1,176 @@ +"""Tests for ToolResultCompactor.""" + +import asyncio +import tempfile +from datetime import datetime, timedelta +from pathlib import Path + +from agentscope.message import Msg + +from reme.memory.file_based_copaw.tool_result_compactor import ToolResultCompactor +from reme.memory.file_based_copaw.utils import TRUNCATION_MARKER_START + + +def create_tool_result_msg(output: str | list, tool_name: str = "test_tool") -> Msg: + """Create a Msg with tool_result content block.""" + return Msg( + name="tool", + role="user", + content=[ + { + "type": "tool_result", + "id": "call_123", + "name": tool_name, + "output": output, + }, + ], + ) + + +class TestToolResultCompactor: + """Tests for ToolResultCompactor.""" + + def test_no_truncation_when_under_threshold(self): + """Test that short content is not truncated.""" + with tempfile.TemporaryDirectory() as tmpdir: + op = ToolResultCompactor(tool_result_dir=tmpdir, tool_result_threshold=1000) + messages = [create_tool_result_msg("short content")] + + result = asyncio.run(op.call(messages=messages)) + + assert result == messages + assert messages[0].content[0]["output"] == "short content" + assert len(list(Path(tmpdir).glob("*.txt"))) == 0 + + def test_truncation_when_over_threshold(self): + """Test that long content is truncated and saved to file.""" + with tempfile.TemporaryDirectory() as tmpdir: + op = ToolResultCompactor(tool_result_dir=tmpdir, tool_result_threshold=100) + long_content = "x" * 500 + messages = [create_tool_result_msg(long_content)] + + _ = asyncio.run(op.call(messages=messages)) + + output = messages[0].content[0]["output"] + assert TRUNCATION_MARKER_START in output + assert "[Full content saved to:" in output + + # Verify file was created + files = list(Path(tmpdir).glob("*.txt")) + assert len(files) == 1 + + # Verify file content + content = files[0].read_text() + assert "# tool_name: test_tool" in content + assert "# created_at:" in content + assert long_content in content + + def test_skip_already_truncated(self): + """Test that already truncated content is not re-truncated.""" + with tempfile.TemporaryDirectory() as tmpdir: + op = ToolResultCompactor(tool_result_dir=tmpdir, tool_result_threshold=100) + truncated_content = f"head{TRUNCATION_MARKER_START}(100 chars omitted)<<>>tail" + messages = [create_tool_result_msg(truncated_content)] + + asyncio.run(op.call(messages=messages)) + + assert messages[0].content[0]["output"] == truncated_content + assert len(list(Path(tmpdir).glob("*.txt"))) == 0 + + def test_truncation_list_output(self): + """Test truncation of list output with text blocks.""" + with tempfile.TemporaryDirectory() as tmpdir: + op = ToolResultCompactor(tool_result_dir=tmpdir, tool_result_threshold=100) + list_output = [{"type": "text", "text": "y" * 500}] + messages = [create_tool_result_msg(list_output)] + + asyncio.run(op.call(messages=messages)) + + text_block = messages[0].content[0]["output"][0] + assert TRUNCATION_MARKER_START in text_block["text"] + assert len(list(Path(tmpdir).glob("*.txt"))) == 1 + + def test_list_output_no_truncation_when_short(self): + """Test that short list output is not truncated.""" + with tempfile.TemporaryDirectory() as tmpdir: + op = ToolResultCompactor(tool_result_dir=tmpdir, tool_result_threshold=1000) + list_output = [{"type": "text", "text": "short"}] + messages = [create_tool_result_msg(list_output)] + + asyncio.run(op.call(messages=messages)) + + assert messages[0].content[0]["output"][0]["text"] == "short" + assert len(list(Path(tmpdir).glob("*.txt"))) == 0 + + def test_list_output_multiple_text_blocks(self): + """Test truncation of multiple text blocks in list output.""" + with tempfile.TemporaryDirectory() as tmpdir: + op = ToolResultCompactor(tool_result_dir=tmpdir, tool_result_threshold=100) + list_output = [ + {"type": "text", "text": "a" * 500}, + {"type": "text", "text": "short"}, + {"type": "text", "text": "b" * 500}, + ] + messages = [create_tool_result_msg(list_output)] + + asyncio.run(op.call(messages=messages)) + + output = messages[0].content[0]["output"] + assert TRUNCATION_MARKER_START in output[0]["text"] + assert output[1]["text"] == "short" # unchanged + assert TRUNCATION_MARKER_START in output[2]["text"] + assert len(list(Path(tmpdir).glob("*.txt"))) == 2 + + def test_list_output_mixed_block_types(self): + """Test that non-text blocks in list output are unchanged.""" + with tempfile.TemporaryDirectory() as tmpdir: + op = ToolResultCompactor(tool_result_dir=tmpdir, tool_result_threshold=100) + list_output = [ + {"type": "text", "text": "c" * 500}, + {"type": "image", "source": {"type": "url", "url": "http://example.com/img.png"}}, + ] + messages = [create_tool_result_msg(list_output)] + + asyncio.run(op.call(messages=messages)) + + output = messages[0].content[0]["output"] + assert TRUNCATION_MARKER_START in output[0]["text"] + assert output[1] == {"type": "image", "source": {"type": "url", "url": "http://example.com/img.png"}} + assert len(list(Path(tmpdir).glob("*.txt"))) == 1 + + def test_cleanup_expired_files(self): + """Test cleanup of expired files.""" + with tempfile.TemporaryDirectory() as tmpdir: + op = ToolResultCompactor(tool_result_dir=tmpdir, tool_result_threshold=100, retention_days=1) + + # Create an old file + old_time = (datetime.now() - timedelta(days=2)).isoformat() + old_file = Path(tmpdir) / "old_file.txt" + old_file.write_text(f"# tool_name: test\n# created_at: {old_time}\n# ---\ncontent") + + # Create a new file + new_time = datetime.now().isoformat() + new_file = Path(tmpdir) / "new_file.txt" + new_file.write_text(f"# tool_name: test\n# created_at: {new_time}\n# ---\ncontent") + + deleted = op.cleanup_expired_files() + + assert deleted == 1 + assert not old_file.exists() + assert new_file.exists() + + def test_string_content_msg_unchanged(self): + """Test that messages with string content are unchanged.""" + with tempfile.TemporaryDirectory() as tmpdir: + op = ToolResultCompactor(tool_result_dir=tmpdir, tool_result_threshold=100) + messages = [Msg(name="user", role="user", content="hello world")] + + asyncio.run(op.call(messages=messages)) + + assert messages[0].content == "hello world" + + +if __name__ == "__main__": + import pytest + + pytest.main([__file__, "-v"]) diff --git a/tests/copaw/test_utils.py b/tests/copaw/test_utils.py new file mode 100644 index 00000000..f5da46e2 --- /dev/null +++ b/tests/copaw/test_utils.py @@ -0,0 +1,89 @@ +"""Test utilities for copaw tests.""" + +import os +from pathlib import Path +from typing import Any + +from loguru import logger + +_token_counter = None + + +def get_token_counter(): + """Get or initialize the global token counter instance. + + Returns: + TokenCounterBase: The token counter instance for Qwen models. + + Raises: + RuntimeError: If token counter initialization fails. + """ + global _token_counter + if _token_counter is None: + from agentscope.token import HuggingFaceTokenCounter + + # Use Qwen tokenizer for DashScope models + # Qwen3 series uses the same tokenizer as Qwen2.5 + + # Try local tokenizer first, fall back to online if not found + local_tokenizer_path = Path(__file__).parent.parent.parent / "tokenizer" + + if local_tokenizer_path.exists() and (local_tokenizer_path / "tokenizer.json").exists(): + tokenizer_path = str(local_tokenizer_path) + logger.info(f"Using local Qwen tokenizer from {tokenizer_path}") + else: + tokenizer_path = "Qwen/Qwen2.5-7B-Instruct" + logger.info( + "Local tokenizer not found, downloading from HuggingFace", + ) + + _token_counter = HuggingFaceTokenCounter( + pretrained_model_name_or_path=tokenizer_path, + use_mirror=True, # Use HF mirror for users in China + use_fast=True, + trust_remote_code=True, + ) + logger.debug("Token counter initialized with Qwen tokenizer") + return _token_counter + + +def get_dash_chat_model(model_name: str = "qwen3.5-plus"): + """Get DashScope chat model instance.""" + from agentscope.model import OpenAIChatModel + from reme.core.utils import load_env + + load_env() + return OpenAIChatModel( + api_key=os.environ["REME_LLM_API_KEY"], + client_kwargs={"base_url": os.environ["REME_LLM_BASE_URL"]}, + model_name=model_name, + ) + + +def get_formatter(): + """Get formatter instance.""" + from agentscope.formatter import OpenAIChatFormatter + from agentscope.token import HuggingFaceTokenCounter + from reme.memory.file_based_copaw.utils import _extract_text_from_messages + + class ReMeChatFormatter(OpenAIChatFormatter): + """ReMe chat formatter class.""" + + async def _count(self, msgs: list[dict[str, Any]]) -> int | None: + """Count the number of tokens in the input messages. If token counter + is not provided, `None` will be returned. + + Args: + msgs (`list[Msg]`): + The input messages to count tokens for. + """ + if self.token_counter is None: + return None + + assert isinstance(self.token_counter, HuggingFaceTokenCounter) + text = _extract_text_from_messages(msgs) + token_ids = self.token_counter.tokenizer.encode(text) + token_count = len(token_ids) + return token_count + + return ReMeChatFormatter(token_counter=get_token_counter()) diff --git a/tests/test_fs_compactor.py b/tests/test_fs_compactor.py index 00be5320..1204b17a 100644 --- a/tests/test_fs_compactor.py +++ b/tests/test_fs_compactor.py @@ -6,7 +6,7 @@ which creates compact summaries of conversation history using LLM. import asyncio -from reme import ReMeFb +from reme import ReMeCli from reme.core.enumeration import Role from reme.core.schema import Message @@ -560,7 +560,7 @@ async def test_full_compact_with_summary(): print("TEST: Full Compaction with LLM Summary Generation") print("=" * 80) - reme_fs = ReMeFb( + reme_fs = ReMeCli( enable_logo=False, vector_store=None, compact_params={ @@ -607,7 +607,7 @@ async def test_realistic_personal_conversation_compact(): print("TEST: Realistic Personal Conversation Compaction") print("=" * 80) - reme_fs = ReMeFb( + reme_fs = ReMeCli( enable_logo=False, vector_store=None, compact_params={ diff --git a/tests/test_fs_context_checker.py b/tests/test_fs_context_checker.py index 14a7fe9f..8a3ed618 100644 --- a/tests/test_fs_context_checker.py +++ b/tests/test_fs_context_checker.py @@ -6,7 +6,7 @@ which determines where to split conversation history when token limits are excee import asyncio -from reme import ReMeFb +from reme import ReMeCli from reme.core.enumeration import Role from reme.core.schema import Message @@ -67,7 +67,7 @@ async def test_no_compaction_needed(): print("TEST 1: Below Threshold - No Cut Point Needed") print("=" * 80) - reme_fs = ReMeFb( + reme_fs = ReMeCli( "vector_stores={}", # Override config to disable vector stores enable_logo=False, context_window_tokens=5000, @@ -112,7 +112,7 @@ async def test_compaction_needed_above_threshold(): print("TEST 2: Compaction Needed Above Threshold") print("=" * 80) - reme_fs = ReMeFb( + reme_fs = ReMeCli( "vector_stores={}", # Override config to disable vector stores enable_logo=False, context_window_tokens=1500, @@ -181,7 +181,7 @@ async def test_split_turn_scenario(): print("TEST 3: Split Turn - Cut in Middle of Assistant Response") print("=" * 80) - reme_fs = ReMeFb( + reme_fs = ReMeCli( "vector_stores={}", # Override config to disable vector stores enable_logo=False, context_window_tokens=2000, diff --git a/tests/test_fs_file_watch_integration.py b/tests/test_fs_file_watch_integration.py index 8f47b472..9f679d38 100644 --- a/tests/test_fs_file_watch_integration.py +++ b/tests/test_fs_file_watch_integration.py @@ -1,8 +1,8 @@ -"""Integration test for ReMeFb file watching with memory_search and memory_get. +"""Integration test for ReMeCli file watching with memory_search and memory_get. This test demonstrates the complete workflow: 1. Create markdown files with personal information in test_reme folder -2. Initialize ReMeFb with file watching enabled +2. Initialize ReMeCli with file watching enabled 3. Start file watching to automatically index files into the database 4. Use memory_search and memory_get to retrieve the indexed content 5. Modify the markdown files @@ -19,7 +19,7 @@ import json import shutil from pathlib import Path -from reme import ReMeFb +from reme import ReMeCli # ==================== Test Configuration ==================== @@ -278,10 +278,10 @@ async def test_file_watch_integration(): test_files = create_test_markdown_files(TestConfig.WORKING_DIR) print(f"\n✓ Created {len(test_files)} markdown files in {TestConfig.WORKING_DIR}") - # ==================== STEP 2: Initialize ReMeFb ==================== - print_separator("STEP 2: Initializing ReMeFb with File Watching") + # ==================== STEP 2: Initialize ReMeCli ==================== + print_separator("STEP 2: Initializing ReMeCli with File Watching") - reme_fs = ReMeFb( + reme_fs = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, default_file_store_config={ @@ -299,7 +299,7 @@ async def test_file_watch_integration(): }, ) - print("✓ ReMeFb instance created") + print("✓ ReMeCli instance created") print(f" Working directory: {TestConfig.WORKING_DIR}") print(f" Watch paths: {TestConfig.WORKING_DIR}, {TestConfig.WORKING_DIR}/memory") print(" File filters: .md files") @@ -469,7 +469,7 @@ async def test_file_watch_integration(): print_separator("STEP 10: Cleanup") await reme_fs.close() - print("✓ ReMeFb closed") + print("✓ ReMeCli closed") # Clean up test directory if test_dir.exists(): @@ -489,11 +489,11 @@ async def test_file_watch_integration(): async def main(): """Run the file watch integration test.""" print("\n" + "=" * 80) - print(" ReMeFb File Watch Integration Test") + print(" ReMeCli File Watch Integration Test") print("=" * 80) print("\nThis test validates the complete file watching workflow:") print(" 1. Create markdown files with personal information") - print(" 2. Initialize ReMeFb and start file watching") + print(" 2. Initialize ReMeCli and start file watching") print(" 3. Verify automatic indexing into database") print(" 4. Search and retrieve initial content") print(" 5. Modify files and verify re-indexing") diff --git a/tests/test_fs_memory_get.py b/tests/test_fs_memory_get.py index d353751d..d6d8f5e3 100644 --- a/tests/test_fs_memory_get.py +++ b/tests/test_fs_memory_get.py @@ -1,6 +1,6 @@ -"""Tests for ReMeFb memory_get interface. +"""Tests for ReMeCli memory_get interface. -This module tests the memory_get() method of ReMeFb class which provides +This module tests the memory_get() method of ReMeCli class which provides a high-level interface for reading specific snippets from memory files. The memory_get function should enable the LLM to: @@ -13,7 +13,7 @@ import asyncio import os from pathlib import Path -from reme import ReMeFb +from reme import ReMeCli def print_result(content: str, title: str = "RESULT", max_len: int = 300): @@ -105,7 +105,7 @@ async def test_memory_get_full_file(): print("=" * 80) workspace_dir = ".reme_test_get" - reme_fs = ReMeFb(enable_logo=False, working_dir=workspace_dir) + reme_fs = ReMeCli(enable_logo=False, working_dir=workspace_dir) await reme_fs.start() # Create test file @@ -144,7 +144,7 @@ async def test_memory_get_with_offset(): print("=" * 80) workspace_dir = ".reme_test_get" - reme_fs = ReMeFb(enable_logo=False, working_dir=workspace_dir) + reme_fs = ReMeCli(enable_logo=False, working_dir=workspace_dir) await reme_fs.start() test_file_path = "memory/test_profile.md" @@ -180,7 +180,7 @@ async def test_memory_get_with_offset_and_limit(): print("=" * 80) workspace_dir = ".reme_test_get" - reme_fs = ReMeFb(enable_logo=False, working_dir=workspace_dir) + reme_fs = ReMeCli(enable_logo=False, working_dir=workspace_dir) await reme_fs.start() test_file_path = "memory/test_profile.md" @@ -219,7 +219,7 @@ async def test_memory_get_beginning_lines(): print("=" * 80) workspace_dir = ".reme_test_get" - reme_fs = ReMeFb(enable_logo=False, working_dir=workspace_dir) + reme_fs = ReMeCli(enable_logo=False, working_dir=workspace_dir) await reme_fs.start() test_file_path = "memory/test_profile.md" @@ -257,7 +257,7 @@ async def test_memory_get_single_line(): print("=" * 80) workspace_dir = ".reme_test_get" - reme_fs = ReMeFb(enable_logo=False, working_dir=workspace_dir) + reme_fs = ReMeCli(enable_logo=False, working_dir=workspace_dir) await reme_fs.start() test_file_path = "memory/test_profile.md" @@ -294,7 +294,7 @@ async def test_memory_get_with_absolute_path(): print("=" * 80) workspace_dir = ".reme_test_get" - reme_fs = ReMeFb(enable_logo=False, working_dir=workspace_dir) + reme_fs = ReMeCli(enable_logo=False, working_dir=workspace_dir) await reme_fs.start() # Get absolute path @@ -324,7 +324,7 @@ async def test_memory_get_with_absolute_path(): async def main(): """Run core memory_get interface tests.""" print("\n" + "=" * 80) - print("ReMeFb Memory Get Interface - Tests") + print("ReMeCli Memory Get Interface - Tests") print("=" * 80) print("\nThis test suite validates that the memory_get() function:") print(" 1. Reads entire memory files without parameters") diff --git a/tests/test_fs_memory_search.py b/tests/test_fs_memory_search.py index 2a65fd5a..d546f27d 100644 --- a/tests/test_fs_memory_search.py +++ b/tests/test_fs_memory_search.py @@ -1,6 +1,6 @@ -"""Tests for ReMeFb memory_search interface. +"""Tests for ReMeCli memory_search interface. -This module tests the memory_search() method of ReMeFb class which provides +This module tests the memory_search() method of ReMeCli class which provides a high-level interface for searching personal information stored in memory files. The memory_search function should enable: @@ -16,7 +16,7 @@ import hashlib import shutil from pathlib import Path -from reme import ReMeFb +from reme import ReMeCli from reme.core.enumeration import MemorySource from reme.core.schema import FileMetadata, MemoryChunk @@ -207,8 +207,8 @@ async def test_memory_search_basic(): print("TEST 1: Basic Memory Search") print("=" * 80) - # Initialize ReMeFb with unique store name - reme_fs = ReMeFb( + # Initialize ReMeCli with unique store name + reme_fs = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, default_file_store_config={ @@ -268,8 +268,8 @@ async def test_memory_search_technical_content(): print("TEST 2: Technical Content Search") print("=" * 80) - # Initialize ReMeFb with unique store name - reme_fs = ReMeFb( + # Initialize ReMeCli with unique store name + reme_fs = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, default_file_store_config={ @@ -334,8 +334,8 @@ async def test_memory_search_with_source_filter(): print("TEST 3: Memory Search with Source Filter") print("=" * 80) - # Initialize ReMeFb with unique store name - reme_fs = ReMeFb( + # Initialize ReMeCli with unique store name + reme_fs = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, ) @@ -374,7 +374,7 @@ async def test_memory_search_with_source_filter(): # Search only MEMORY source print(f"\n--- Searching MEMORY source for: '{query}' ---") # Create a new instance with MEMORY source filter - reme_fs_memory = ReMeFb( + reme_fs_memory = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, search_params={"sources": [MemorySource.MEMORY]}, @@ -396,7 +396,7 @@ async def test_memory_search_with_source_filter(): # Search only SESSIONS source print(f"\n--- Searching SESSIONS source for: '{query}' ---") # Create a new instance with SESSIONS source filter - reme_fs_sessions = ReMeFb( + reme_fs_sessions = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, search_params={"sources": [MemorySource.SESSIONS]}, @@ -437,8 +437,8 @@ async def test_memory_search_score_filtering(): print("TEST 4: Memory Search with Score Filtering") print("=" * 80) - # Initialize ReMeFb with unique store name - reme_fs = ReMeFb( + # Initialize ReMeCli with unique store name + reme_fs = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, default_file_store_config={ @@ -503,8 +503,8 @@ async def test_memory_search_max_results(): print("TEST 5: Memory Search with Result Limiting") print("=" * 80) - # Initialize ReMeFb with unique store name - reme_fs = ReMeFb( + # Initialize ReMeCli with unique store name + reme_fs = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, default_file_store_config={ @@ -569,8 +569,8 @@ async def test_memory_search_hybrid_mode(): print("TEST 6: Memory Search with Hybrid Mode") print("=" * 80) - # Initialize ReMeFb with unique store name - reme_fs = ReMeFb( + # Initialize ReMeCli with unique store name + reme_fs = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, default_file_store_config={ @@ -601,7 +601,7 @@ async def test_memory_search_hybrid_mode(): # Test with hybrid enabled print(f"\n--- Hybrid search (enabled) for: '{query}' ---") # Create instance with hybrid enabled - reme_fs_hybrid = ReMeFb( + reme_fs_hybrid = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, default_file_store_config={ @@ -630,7 +630,7 @@ async def test_memory_search_hybrid_mode(): # Test with hybrid disabled (vector only) print(f"\n--- Vector-only search for: '{query}' ---") # Create instance with hybrid disabled - reme_fs_vector = ReMeFb( + reme_fs_vector = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, default_file_store_config={ @@ -662,7 +662,7 @@ async def test_memory_search_hybrid_mode(): for vec_weight, text_weight in weight_configs: # Create instance with specific weights - reme_fs_weights = ReMeFb( + reme_fs_weights = ReMeCli( enable_logo=False, working_dir=TestConfig.WORKING_DIR, default_file_store_config={ @@ -708,7 +708,7 @@ async def cleanup_test_data(): async def main(): """Run all memory search tests.""" print("\n" + "=" * 80) - print("ReMeFb Memory Search Interface Tests") + print("ReMeCli Memory Search Interface Tests") print("=" * 80) print("\nThis test suite validates the memory_search() function:") print(" 1. Basic semantic search functionality") diff --git a/tests/test_fs_summary.py b/tests/test_fs_summary.py index b2701344..8a692500 100644 --- a/tests/test_fs_summary.py +++ b/tests/test_fs_summary.py @@ -1,6 +1,6 @@ -"""ReMeFb summary接口测试。 +"""ReMeCli summary接口测试。 -本模块测试ReMeFb类的summary()方法，该方法提供了 +本模块测试ReMeCli类的summary()方法，该方法提供了 将用户个人信息存储到记忆文件的高级接口。 summary函数应该能够让LLM： @@ -13,7 +13,7 @@ import asyncio import shutil from pathlib import Path -from reme import ReMeFb +from reme import ReMeCli from reme.core.enumeration import Role from reme.core.schema import Message @@ -114,7 +114,7 @@ async def test_summary_first_write(): if Path(working_dir).exists(): shutil.rmtree(working_dir) - reme_fs = ReMeFb(enable_logo=False, working_dir=working_dir, vector_store=None) + reme_fs = ReMeCli(enable_logo=False, working_dir=working_dir, vector_store=None) await reme_fs.start() # 确保记忆文件已删除 @@ -177,7 +177,7 @@ async def test_summary_complementary_info(): if Path(working_dir).exists(): shutil.rmtree(working_dir) - reme_fs = ReMeFb(enable_logo=False, working_dir=working_dir, vector_store=None) + reme_fs = ReMeCli(enable_logo=False, working_dir=working_dir, vector_store=None) await reme_fs.start() # 确保记忆文件已删除 @@ -271,7 +271,7 @@ async def test_summary_conflicting_info(): if Path(working_dir).exists(): shutil.rmtree(working_dir) - reme_fs = ReMeFb(enable_logo=False, working_dir=working_dir, vector_store=None) + reme_fs = ReMeCli(enable_logo=False, working_dir=working_dir, vector_store=None) await reme_fs.start() # 确保记忆文件已删除 @@ -353,7 +353,7 @@ async def test_summary_conflicting_info(): async def main(): """运行时间对齐的summary接口测试。""" print("\n" + "=" * 80) - print("ReMeFb Summary接口 - 时间对齐的记忆存储测试") + print("ReMeCli Summary接口 - 时间对齐的记忆存储测试") print("=" * 80) print("\n本测试套件验证summary()函数：") print(" 1. 正确处理消息中的time_created字段（%Y-%m-%d %H:%M:%S）")