From 9e2e98ef40a232c519a524de8bda1ca86e85466a Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Wed, 4 Mar 2026 18:43:21 +0800 Subject: [PATCH] Dev/readme (#137) * refactor(memory): rename copaw to reme and update module structure * chore(release): bump version to 0.3.0.6b1 * refactor(docs): update README and ReMeLight implementation * refactor(reme): rename ReMeCopaw to ReMeLight and remove CLI module * docs(readme): update Chinese documentation with enhanced structure and content * docs(readme): update Chinese documentation for context compression * docs(readme): update context compression section header * docs(readme): update documentation with installation and usage guide * docs(readme): update Chinese documentation table * chore(deps): update dependency extras configuration * chore(test): remove deprecated test files for message operations * docs(readme): update documentation with ReMeLight implementation changes --- README.md | 476 ++++++---------- README_ZH.md | 539 +++++++----------- example.env | 14 +- pyproject.toml | 6 +- reme/__init__.py | 4 +- reme/config/{copaw.yaml => light.yaml} | 0 reme/core/llm/base_llm.py | 4 +- reme/core/service_context.py | 8 +- reme/{memory => extension}/cli/__init__.py | 0 reme/{memory => extension}/cli/fb_cli.py | 0 reme/{memory => extension}/cli/fb_cli.yaml | 0 .../{memory => extension}/cli/fb_compactor.py | 0 .../cli/fb_compactor.yaml | 0 .../cli/fb_context_checker.py | 0 .../cli/fb_summarizer.py | 0 .../cli/fb_summarizer.yaml | 0 reme/{ => extension}/reme_cli.py | 0 reme/memory/__init__.py | 4 +- .../__init__.py | 10 +- .../compactor.py | 0 .../compactor.yaml | 0 .../file_io.py | 0 .../memory_formatter.py | 0 reme/memory/file_based/reme_chat_formatter.py | 29 + .../reme_in_memory_memory.py} | 2 +- .../summarizer.py | 0 .../summarizer.yaml | 0 .../tool_result_compactor.py | 0 .../{file_based_copaw => file_based}/utils.py | 40 ++ reme/memory/skills/__init__.py | 0 reme/{reme_copaw.py => reme_light.py} | 153 ++--- test/test_agentic_retrieve_op.py | 113 ---- {tests => test}/test_fs_compactor.py | 0 {tests => test}/test_fs_context_checker.py | 0 .../test_fs_file_watch_integration.py | 0 {tests => test}/test_fs_memory_get.py | 0 {tests => test}/test_fs_memory_search.py | 0 {tests => test}/test_fs_summary.py | 0 test/test_message_compact_op.py | 81 --- test/test_message_compress_op.py | 285 --------- test/test_message_offload_op.py | 247 -------- tests/{copaw => light}/test_compactor.py | 2 +- .../{copaw => light}/test_memory_formatter.py | 2 +- tests/light/test_reme_light.py | 198 +++++++ tests/{copaw => light}/test_summarizer.py | 2 +- .../test_tool_result_compactor.py | 4 +- tests/{copaw => light}/test_utils.py | 2 +- 47 files changed, 745 insertions(+), 1480 deletions(-) rename reme/config/{copaw.yaml => light.yaml} (100%) rename reme/{memory => extension}/cli/__init__.py (100%) rename reme/{memory => extension}/cli/fb_cli.py (100%) rename reme/{memory => extension}/cli/fb_cli.yaml (100%) rename reme/{memory => extension}/cli/fb_compactor.py (100%) rename reme/{memory => extension}/cli/fb_compactor.yaml (100%) rename reme/{memory => extension}/cli/fb_context_checker.py (100%) rename reme/{memory => extension}/cli/fb_summarizer.py (100%) rename reme/{memory => extension}/cli/fb_summarizer.yaml (100%) rename reme/{ => extension}/reme_cli.py (100%) rename reme/memory/{file_based_copaw => file_based}/__init__.py (76%) rename reme/memory/{file_based_copaw => file_based}/compactor.py (100%) rename reme/memory/{file_based_copaw => file_based}/compactor.yaml (100%) rename reme/memory/{file_based_copaw => file_based}/file_io.py (100%) rename reme/memory/{file_based_copaw => file_based}/memory_formatter.py (100%) create mode 100644 reme/memory/file_based/reme_chat_formatter.py rename reme/memory/{file_based_copaw/copaw_in_memory_memory.py => file_based/reme_in_memory_memory.py} (99%) rename reme/memory/{file_based_copaw => file_based}/summarizer.py (100%) rename reme/memory/{file_based_copaw => file_based}/summarizer.yaml (100%) rename reme/memory/{file_based_copaw => file_based}/tool_result_compactor.py (100%) rename reme/memory/{file_based_copaw => file_based}/utils.py (84%) delete mode 100644 reme/memory/skills/__init__.py rename reme/{reme_copaw.py => reme_light.py} (88%) delete mode 100644 test/test_agentic_retrieve_op.py rename {tests => test}/test_fs_compactor.py (100%) rename {tests => test}/test_fs_context_checker.py (100%) rename {tests => test}/test_fs_file_watch_integration.py (100%) rename {tests => test}/test_fs_memory_get.py (100%) rename {tests => test}/test_fs_memory_search.py (100%) rename {tests => test}/test_fs_summary.py (100%) delete mode 100644 test/test_message_compact_op.py delete mode 100644 test/test_message_compress_op.py delete mode 100644 test/test_message_offload_op.py rename tests/{copaw => light}/test_compactor.py (99%) rename tests/{copaw => light}/test_memory_formatter.py (99%) create mode 100644 tests/light/test_reme_light.py rename tests/{copaw => light}/test_summarizer.py (99%) rename tests/{copaw => light}/test_tool_result_compactor.py (97%) rename tests/{copaw => light}/test_utils.py (97%) diff --git a/README.md b/README.md index 98c5dd8c..c2c17069 100644 --- a/README.md +++ b/README.md @@ -17,144 +17,59 @@

- A memory management toolkit for AI agents — Remember Me, Refine Me.
+ Memory Management Toolkit for AI Agents, Remember Me, Refine Me.

-> For legacy versions, see [0.2.x Documentation](docs/README_0_2_x.md) +> For legacy versions, please refer to [0.2.x Documentation](docs/README_0_2_x.md) --- -🧠 ReMe is a **memory management framework** built for **AI agents**, offering both **file-based** and **vector-based** -memory systems. +🧠 ReMe is a memory management framework built specifically for **AI Agents**, offering both file-based and vector-based memory systems. -It addresses two core problems of agent memory: **limited context windows** (early information gets truncated or lost -during -long conversations) and **stateless sessions** (new conversations cannot inherit history and always start from scratch). +It addresses two core memory challenges for agents: **Limited context window** (early information gets truncated or lost in long conversations), and **Stateless sessions** (new conversations cannot inherit history, starting from scratch every time). -ReMe gives agents **real memory** — old conversations are automatically condensed, important information is persisted, -and the next conversation can recall it automatically. +ReMe gives agents **true memory capability** — old conversations are automatically condensed, important information is persistently stored, and relevant context is automatically recalled in future conversations. --- -## 📁 File-Based CoPaw Memory System +## 📁 File-Based Memory System -> Memory as files, files as memory +> Memory as Files, Files as Memory -Treat **memory as files** — readable, editable, and portable. [CoPaw](https://github.com/agentscope-ai/CoPaw) -integrates this memory system -through [MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py), -which inherits `ReMeCopaw` and exposes memory management capabilities. +Treat **memory as files** — readable, editable, and copyable. [CoPaw](https://github.com/agentscope-ai/CoPaw)'s memory system inherits from `ReMeLight`, implementing memory management capabilities. -| Traditional Memory Systems | File-Based ReMe | -|----------------------------|--------------------| -| 🗄️ Database storage | 📝 Markdown files | -| 🔒 Opaque | 👀 Read anytime | -| ❌ Hard to modify | ✏️ Edit directly | +| Traditional Memory System | File Based ReMe | +|---------------------------|-------------------| +| 🗄️ Database storage | 📝 Markdown files | +| 🔒 Invisible | 👀 Always readable | +| ❌ Hard to modify | ✏️ Direct editing | | 🚫 Hard to migrate | 📦 Copy to migrate | ``` working_dir/ -├── MEMORY.md # Long-term memory: user preferences, project config, etc. +├── MEMORY.md # Long-term memory: user preferences, project configs, etc. ├── memory/ -│ └── YYYY-MM-DD.md # Daily summary logs: written automatically after conversation ends -└── tool_result/ # Cache for oversized tool outputs (auto-managed, auto-cleaned when expired) +│ └── YYYY-MM-DD.md # Daily summary logs: auto-written after conversations +└── tool_result/ # Long tool output cache (auto-managed, auto-cleanup on expiry) └── .txt ``` ### Core Capabilities -[ReMeCopaw](reme/reme_copaw.py) is the core class of this memory system, providing complete memory management -capabilities for AI Agents: +[ReMeLight](reme/reme_light.py) is the core class of this memory system, providing complete memory management capabilities for AI Agents: -| Method | Function | Key Components | -|--------------------------|------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `start` | 🚀 Start memory system | Initialize file store, file watcher, Embedding cache; clean up expired tool result files | -| `close` | 📕 Close and clean up | Clean tool result files, stop file watcher, save Embedding cache | -| `compact_memory` | 📦 Compact history to summary | [Compactor](reme/memory/file_based_copaw/compactor.py) — ReActAgent generates structured context checkpoint | -| `summary_memory` | 📝 Write important memory to files | [Summarizer](reme/memory/file_based_copaw/summarizer.py) — ReActAgent + file tools (read / write / edit) | -| `compact_tool_result` | ✂️ Compact oversized tool output | [ToolResultCompactor](reme/memory/file_based_copaw/tool_result_compactor.py) — Truncate and save to `tool_result/`, keep file reference in message | -| `add_async_summary_task` | ⚡ Submit background summary task | `asyncio.create_task`, summary doesn't block main conversation flow | -| `await_summary_tasks` | ⏳ Wait for background tasks | Collect results from all background summary tasks, call before closing to ensure writes complete | -| `memory_search` | 🔍 Semantic memory search | [MemorySearch](reme/memory/tools/chunk/memory_search.py) — Vector + BM25 hybrid retrieval | -| `get_in_memory_memory` | 🗂️ Create in-memory instance | [CoPawInMemoryMemory](reme/memory/file_based_copaw/copaw_in_memory_memory.py) — Token-aware memory management, supports compression summary and state serialization | -| `update_params` | ⚙️ Update runtime parameters | Adjust `max_input_length`, `memory_compact_ratio`, `language` at runtime | - ---- - -## 🗃️ Vector-Based ReMe - -[ReMe Vector Based](reme/reme.py) is the core class for the vector-based memory system, supporting unified management of -three memory types: - -| Memory Type | Purpose | Usage Context | -|------------------------------|-----------------------------------------------------|---------------| -| **Personal memory** | User preferences, habits | `user_name` | -| **Task / procedural memory** | Task execution experience, success/failure patterns | `task_name` | -| **Tool memory** | Tool usage experience, parameter tuning | `tool_name` | - -### Core Capabilities - -| Method | Function | Description | -|--------------------|---------------------|-----------------------------------------------------------| -| `summarize_memory` | 🧠 Summarize memory | Automatically extract and store memory from conversations | -| `retrieve_memory` | 🔍 Retrieve memory | Retrieve relevant memory by query | -| `add_memory` | ➕ Add memory | Manually add memory to vector store | -| `get_memory` | 📖 Get memory | Fetch a single memory by ID | -| `update_memory` | ✏️ Update memory | Update content or metadata of existing memory | -| `delete_memory` | 🗑️ Delete memory | Delete specified memory | -| `list_memory` | 📋 List memory | List memories with filtering and sorting | - ---- - -## 💻 ReMeCli: Terminal Assistant with File-Based Memory - - - - - - - -
- 马
上
有
钱 - 		- - - 马
到
成
功 -
- -### When Is Memory Written? - -| Scenario | Written to | Trigger | -|---------------------------------------------|------------------------|------------------------------------| -| Auto-compact when context is too long | `memory/YYYY-MM-DD.md` | Automatic in background | -| User runs `/compact` | `memory/YYYY-MM-DD.md` | Manual compact + background save | -| User runs `/new` | `memory/YYYY-MM-DD.md` | New conversation + background save | -| User says "remember this" | `MEMORY.md` or log | Agent writes via `write` tool | -| Agent finds important decisions/preferences | `MEMORY.md` | Agent writes proactively | - -### Memory Retrieval Tools - -| Method | Tool | When to use | Example | -|-----------------|-----------------|----------------------------------|---------------------------------------| -| Semantic search | `memory_search` | Unsure where it is, fuzzy lookup | "Earlier discussion about deployment" | -| Direct read | `read` | Know the date or file | Read `memory/2025-02-13.md` | - -Search uses **vector + BM25 hybrid retrieval** (vector weight 0.7, BM25 weight 0.3), so queries using both natural -language and exact -keywords can match. - -### Built-in Tools - -| Tool | Function | Details | -|-----------------|----------------|------------------------------------------------------------| -| `memory_search` | Search memory | Vector + BM25 hybrid search over MEMORY.md and memory/*.md | -| `bash` | Run commands | Execute bash commands with timeout and output truncation | -| `ls` | List directory | Show directory structure | -| `read` | Read file | Text and images supported, with segmented reading | -| `edit` | Edit file | Replace after exact text match | -| `write` | Write file | Create or overwrite, auto-create directories | -| `execute_code` | Run Python | Execute code snippets | -| `web_search` | Web search | Search via Tavily | +| Method | Function | Key Components | +|---------------------------|-----------------------------|---------------------------------------------------------------------------------------------------------------------| +| `start` | 🚀 Start memory system | Initialize file store, file watcher, embedding cache; cleanup expired tool result files | +| `close` | 📕 Close and cleanup | Cleanup tool result files, stop file watcher, save embedding cache | +| `compact_memory` | 📦 Compress history to summary | [Compactor](reme/memory/file_based/compactor.py) — ReActAgent generates structured context checkpoints | +| `summary_memory` | 📝 Write important memories to files | [Summarizer](reme/memory/file_based/summarizer.py) — ReActAgent + file tools (read / write / edit) | +| `compact_tool_result` | ✂️ Compress long tool outputs | [ToolResultCompactor](reme/memory/file_based/tool_result_compactor.py) — Truncate and save to `tool_result/`, keep file reference in message | +| `add_async_summary_task` | ⚡ Submit background summary task | `asyncio.create_task`, summary doesn't block main conversation flow | +| `await_summary_tasks` | ⏳ Wait for background tasks | Collect results from all background summary tasks, call before closing to ensure writes complete | +| `memory_search` | 🔍 Semantic memory search | [MemorySearch](reme/memory/tools/chunk/memory_search.py) — Vector + BM25 hybrid retrieval | +| `get_in_memory_memory` | 🗂️ Create in-memory instance | [ReMeInMemoryMemory](reme/memory/file_based/reme_in_memory_memory.py) — Token-aware memory management, supports compression summaries and state serialization | +| `update_params` | ⚙️ Dynamically update runtime params | Runtime adjustment of `max_input_length`, `memory_compact_ratio`, `language` | --- @@ -163,131 +78,66 @@ keywords can match. ### Installation ```bash -pip install -U reme-ai +pip install -U reme-ai[as] ``` ### Environment Variables -API keys are set via environment variables; you can put them in a `.env` file in the project root: +`ReMeLight` environment variables configure embedding and storage backends -| Variable | Description | Example | -|---------------------------|----------------------------------|-----------------------------------------------------| -| `REME_LLM_API_KEY` | LLM API key | `sk-xxx` | -| `REME_LLM_BASE_URL` | LLM base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | -| `REME_EMBEDDING_API_KEY` | Embedding API key | `sk-xxx` | -| `REME_EMBEDDING_BASE_URL` | Embedding base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | -| `TAVILY_API_KEY` | Tavily search API key (optional) | `tvly-xxx` | - -### Using ReMeCli - -#### Start ReMeCli - -```bash -remecli config=cli -``` - -#### ReMeCli System Commands - -> Year of the Horse easter egg: `/horse` — fireworks, galloping animation, and random horse-year blessings. - -Commands starting with `/` control session state: - -| Command | Description | Waits for response | -|------------|--------------------------------------------------------------------|--------------------| -| `/compact` | Manually compact current conversation and save to long-term memory | Yes | -| `/new` | Start new conversation; history saved to long-term memory | No | -| `/clear` | Clear everything, **without saving** | No | -| `/history` | View uncompressed messages in current conversation | No | -| `/help` | Show command list | No | -| `/exit` | Exit | No | - -**Difference between the three commands** - -| Command | Compact summary | Long-term memory | Message history | -|------------|-----------------|------------------|-----------------| -| `/compact` | New summary | Saved | Keep recent | -| `/new` | Cleared | Saved | Cleared | -| `/clear` | Cleared | Not saved | Cleared | - -> `/clear` permanently deletes; nothing is persisted anywhere. - -### Using the ReMe Package - -#### File-Based ReMe (CoPaw Memory System) - -`ReMeCopaw` receives AgentScope components like `ChatModelBase`, `Formatter`, `Toolkit`, and configures Embedding and -storage backend via environment variables: - -| Environment Variable | Description | Default | -|----------------------------|-----------------------------------------------|-----------------------------------------------------| -| `EMBEDDING_API_KEY` | Embedding service API Key | `""` (vector search disabled if not configured) | -| `EMBEDDING_BASE_URL` | Embedding service Base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | -| `EMBEDDING_MODEL_NAME` | Embedding model name | `""` | -| `EMBEDDING_DIMENSIONS` | Vector dimensions | `1024` | -| `EMBEDDING_CACHE_ENABLED` | Whether to enable Embedding cache | `true` | -| `EMBEDDING_MAX_CACHE_SIZE` | Maximum cache entries | `2000` | -| `FTS_ENABLED` | Whether to enable full-text search (BM25) | `true` | -| `MEMORY_STORE_BACKEND` | Storage backend (`auto` / `chroma` / `local`) | `auto` (local on Windows, chroma on others) | +| Environment Variable | Description | Default | +|-----------------------------|--------------------------------------|-----------------------------------------------------| +| `EMBEDDING_API_KEY` | Embedding service API Key | `""` | +| `EMBEDDING_BASE_URL` | Embedding service Base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | +| `EMBEDDING_MODEL_NAME` | Embedding model name | `""` | +| `EMBEDDING_DIMENSIONS` | Vector dimensions | `1024` | +| `EMBEDDING_CACHE_ENABLED` | Enable embedding cache | `true` | +| `EMBEDDING_MAX_CACHE_SIZE` | Maximum cache entries | `2000` | +| `FTS_ENABLED` | Enable full-text search (BM25) | `true` | +| `MEMORY_STORE_BACKEND` | Storage backend (`auto` / `chroma` / `local`) | `auto` (local on Windows, chroma otherwise) | ```python import asyncio -from agentscope.formatter import ClaudeFormatter -from agentscope.model import get_model -from agentscope.token import HuggingFaceTokenCounter -from agentscope.tool import Toolkit - -from reme.reme_copaw import ReMeCopaw - +from agentscope.message import Msg +from reme.reme_light import ReMeLight async def main(): - # Prepare AgentScope core components - chat_model = get_model(config={"backend": "openai", "model_name": "qwen3.5-plus"}) - formatter = ClaudeFormatter() - token_counter = HuggingFaceTokenCounter() - toolkit = Toolkit() # Can register additional tools - - # Initialize ReMeCopaw - reme = ReMeCopaw( + reme = ReMeLight( working_dir=".reme", # Memory file storage directory - chat_model=chat_model, - formatter=formatter, - token_counter=token_counter, - toolkit=toolkit, max_input_length=128000, # Model context window (tokens) - memory_compact_ratio=0.7, # Trigger compaction when reaching max_input_length * 0.7 + memory_compact_ratio=0.7, # Trigger compression at max_input_length * 0.7 language="zh", # Summary language (zh / "") tool_result_threshold=1000, # Auto-save tool outputs exceeding this character count retention_days=7, # tool_result/ file retention days ) await reme.start() - messages = [...] # list[Msg], conversation history + messages = [...] - # 1. Compact oversized tool outputs (prevent tool results from overflowing context) + # 1. Compress long tool outputs (prevent tool results from bloating context) messages = await reme.compact_tool_result(messages) - # 2. Compact history to structured summary (trigger: context approaching limit) - summary = await reme.compact_memory( - messages=messages, - previous_summary="", # Can pass previous summary for incremental update - ) - print(f"Compact summary:\n{summary}") + # 2. Compress conversation history to structured summary (triggered when context approaches limit), pass previous summary for incremental updates + summary = await reme.compact_memory(messages=messages, previous_summary="") # 3. Submit async summary task in background (non-blocking, writes to memory/YYYY-MM-DD.md) reme.add_async_summary_task(messages=messages) # 4. Semantic memory search (Vector + BM25 hybrid retrieval) result = await reme.memory_search(query="Python version preference", max_results=5) - print(f"Search results: {result}") - # 5. Get in-memory instance (CoPawInMemoryMemory, manages single conversation context) + # 5. Get in-memory instance (ReMeInMemoryMemory, manages single conversation context) AgentScope InMemoryMemory memory = reme.get_in_memory_memory() token_stats = await memory.estimate_tokens() print(f"Current context usage: {token_stats['context_usage_ratio']:.1f}%") + print(f"Message tokens: {token_stats['messages_tokens']}") + print(f"Estimated total tokens: {token_stats['estimated_tokens']}") # 6. Wait for background tasks before closing - await reme.await_summary_tasks() + summary_result = await reme.await_summary_tasks() + + # Close ReMeLight await reme.close() @@ -297,6 +147,28 @@ if __name__ == "__main__": #### Vector-Based ReMe +## 🗃️ Vector-Based ReMe + +[ReMe Vector Based](reme/reme.py) is the core class of the vector-based memory system, supporting unified management of three memory types: + +| Memory Type | Purpose | Use Case | +|----------------------|--------------------------------------|--------------| +| **Personal Memory** | Record user preferences, habits | `user_name` | +| **Task/Procedural Memory** | Record task execution experience, success/failure patterns | `task_name` | +| **Tool Memory** | Record tool usage experience, parameter optimization | `tool_name` | + +### Core Capabilities + +| Method | Function | Description | +|---------------------|------------------|------------------------------------| +| `summarize_memory` | 🧠 Memory Summary | Auto-extract and store memories from conversations | +| `retrieve_memory` | 🔍 Memory Retrieval | Retrieve relevant memories based on query | +| `add_memory` | ➕ Add Memory | Manually add memory to vector store | +| `get_memory` | 📖 Get Memory | Get single memory by ID | +| `update_memory` | ✏️ Update Memory | Update existing memory content or metadata | +| `delete_memory` | 🗑️ Delete Memory | Delete specified memory | +| `list_memory` | 📋 List Memories | List memories by type, supports filtering and sorting | + ```python import asyncio from reme import ReMe @@ -323,24 +195,24 @@ async def main(): messages = [ {"role": "user", "content": "Help me write a Python script", "time_created": "2026-02-28 10:00:00"}, - {"role": "assistant", "content": "Sure, I'll help you write it", "time_created": "2026-02-28 10:00:05"}, + {"role": "assistant", "content": "OK, let me help you", "time_created": "2026-02-28 10:00:05"}, ] - # 1. Summarize memory from conversation (auto-extract user preferences, task experience, etc.) + # 1. Summarize memories from conversation (auto-extract user preferences, task experience, etc.) result = await reme.summarize_memory( messages=messages, user_name="alice", # Personal memory # task_name="code_writing", # Task memory ) - print(f"Summarize result: {result}") + print(f"Summary result: {result}") - # 2. Retrieve relevant memory + # 2. Retrieve relevant memories memories = await reme.retrieve_memory( query="Python programming", user_name="alice", # task_name="code_writing", ) - print(f"Retrieve result: {memories}") + print(f"Retrieval result: {memories}") # 3. Manually add memory memory_node = await reme.add_memory( @@ -358,11 +230,11 @@ async def main(): updated_memory = await reme.update_memory( memory_id=memory_id, user_name="alice", - memory_content="User prefers concise, well-commented code style", + memory_content="User prefers concise code style with comments", ) print(f"Updated memory: {updated_memory}") - # 6. List all memories for user (with filtering and sorting) + # 6. List all user memories (supports filtering and sorting) all_memories = await reme.list_memory( user_name="alice", limit=10, @@ -385,91 +257,81 @@ if __name__ == "__main__": asyncio.run(main()) ``` ---- - ## 🏛️ Technical Architecture -### File-Based CoPaw Memory System Architecture +### File-Based ReMeLight Memory System Architecture -[CoPaw MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py) -inherits -`ReMeCopaw` and integrates memory capabilities into the Agent reasoning flow: +[CoPaw MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py) inherits from `ReMeLight`, integrating memory capabilities into the Agent reasoning flow: ```mermaid graph TB - CoPaw["CoPaw MemoryManager\n(inherits ReMeCopaw)"] -->|pre_reasoning hook| Hook[MemoryCompactionHook] - CoPaw --> ReMeCopaw[ReMeCopaw] - Hook -->|exceeds threshold| ReMeCopaw - ReMeCopaw --> CompactMemory[compact_memory\nHistory compaction] - ReMeCopaw --> SummaryMemory[summary_memory\nWrite memory to files] - ReMeCopaw --> CompactToolResult[compact_tool_result\nOversized tool output compaction] - ReMeCopaw --> MemSearch[memory_search\nSemantic search] - ReMeCopaw --> InMemory[get_in_memory_memory\nCoPawInMemoryMemory] + CoPaw["CoPaw MemoryManager\n(inherits ReMeLight)"] -->|pre_reasoning hook| Hook[MemoryCompactionHook] + CoPaw --> ReMeLight[ReMeLight] + Hook -->|exceeds threshold| ReMeLight + ReMeLight --> CompactMemory[compact_memory\nHistory Compression] + ReMeLight --> SummaryMemory[summary_memory\nWrite Memory to Files] + ReMeLight --> CompactToolResult[compact_tool_result\nLong Tool Output Compression] + ReMeLight --> MemSearch[memory_search\nSemantic Search] + ReMeLight --> InMemory[get_in_memory_memory\nReMeInMemoryMemory] CompactMemory --> Compactor[Compactor\nReActAgent] - SummaryMemory --> Summarizer[Summarizer\nReActAgent + file tools] - CompactToolResult --> ToolResultCompactor[ToolResultCompactor\nTruncate + save to file] + SummaryMemory --> Summarizer[Summarizer\nReActAgent + File Tools] + CompactToolResult --> ToolResultCompactor[ToolResultCompactor\nTruncate + Save to File] Summarizer --> FileIO[FileIO\nread / write / edit] FileIO --> MemoryFiles[memory/YYYY-MM-DD.md] ToolResultCompactor --> ToolResultFiles[tool_result/*.txt] - MemoryFiles -.->|File change| FileWatcher[Async File Watcher] - FileWatcher -->|Update index| FileStore[Local DB] + MemoryFiles -.->|file changes| FileWatcher[Async File Watcher] + FileWatcher -->|update index| FileStore[Local Database] MemSearch --> FileStore ``` -#### Auto-Compaction Trigger Flow +#### Auto-Compression Trigger Flow -`MemoryCompactionHook` checks context token usage before each reasoning step, automatically triggering compaction when -threshold is exceeded: +`MemoryCompactionHook` checks context token usage before each reasoning step, automatically triggering compression when threshold is exceeded: ```mermaid graph LR A[pre_reasoning] --> B{Token exceeds threshold?} B -->|No| Z[Continue reasoning] - B -->|Yes| C[compact_tool_result\nCompact oversized tool outputs in recent messages] + B -->|Yes| C[compact_tool_result\nCompress long tool outputs in recent messages] C --> D[compact_memory\nGenerate structured context checkpoint] D --> E[Mark old messages as COMPRESSED] - E --> F[add_async_summary_task\nBackground write to memory file] + E --> F[add_async_summary_task\nBackground write to memory files] F --> Z ``` -#### Context Compaction Summary Format +#### Context Compression Summary Format -[Compactor](reme/memory/file_based_copaw/compactor.py) uses ReActAgent to compact history into structured **context -checkpoints**: +[Compactor](reme/memory/file_based/compactor.py) uses ReActAgent to compress conversation history into structured **context checkpoints**: -| Field | Description | -|-----------------------|--------------------------------------------------| -| `## Goal` | 🎯 User's objectives (can be multiple) | -| `## Constraints` | ⚙️ Constraints and preferences mentioned by user | -| `## Progress` | 📈 Completed / in progress / blocked tasks | -| `## Key Decisions` | 🔑 Decisions made with brief reasons | -| `## Next Steps` | 🗺️ Next action plan (ordered list) | -| `## Critical Context` | 📌 File paths, function names, error messages | +| Field | Description | +|------------------------|----------------------------------------------| +| `## Goal` | 🎯 Goals the user wants to accomplish (can be multiple) | +| `## Constraints` | ⚙️ Constraints and preferences mentioned by user | +| `## Progress` | 📈 Completed / In-progress / Blocked tasks | +| `## Key Decisions` | 🔑 Decisions made with brief rationale | +| `## Next Steps` | 🗺️ Next action plan (ordered list) | +| `## Critical Context` | 📌 Key data like file paths, function names, error messages | -Supports **incremental updates**: when `previous_summary` is passed, automatically merges new conversation with old -summary, preserving historical progress. +Supports **incremental updates**: When `previous_summary` is provided, new conversation is automatically merged with old summary, preserving historical progress. -#### Tool Result Compaction +#### Tool Result Compression -[ToolResultCompactor](reme/memory/file_based_copaw/tool_result_compactor.py) solves context overflow caused by oversized -tool outputs: +[ToolResultCompactor](reme/memory/file_based/tool_result_compactor.py) solves the problem of context bloat caused by overly long tool outputs: ```mermaid graph LR A[tool_result message] --> B{Content length > threshold?} - B -->|No| C[Keep as-is] + B -->|No| C[Keep as is] B -->|Yes| D[Truncate to threshold characters] D --> E[Write full content to tool_result/uuid.txt] E --> F[Append file reference path to message] ``` -Expired files (exceeding `retention_days`) are automatically cleaned up during `start` / `close` / -`compact_tool_result`. +Expired files (exceeding `retention_days`) are automatically cleaned up during `start` / `close` / `compact_tool_result`. -#### Memory Summary: ReAct + File Tools +#### Memory Summarization: ReAct + File Tools -[Summarizer](reme/memory/file_based_copaw/summarizer.py) uses the **ReAct + file tools** pattern, letting AI -autonomously decide what to write and where: +[Summarizer](reme/memory/file_based/summarizer.py) uses the **ReAct + File Tools** pattern, letting AI autonomously decide what to write and where: ```mermaid graph LR @@ -477,68 +339,86 @@ graph LR B --> C[Act: read memory/YYYY-MM-DD.md] C --> D{Think: How to merge with existing content?} D --> E[Act: edit to update file] - E --> F{Think: Anything missing?} + E --> F{Think: Anything missed?} F -->|Yes| B - F -->|No| G[Done] + F -->|No| G[Complete] ``` -[FileIO](reme/memory/file_based_copaw/file_io.py) provides file operation tools: +[FileIO](reme/memory/file_based/file_io.py) provides file operation tools: -| Tool | Function | Use case | -|---------|--------------------------------|-----------------------------------------| -| `read` | Read file content (line range) | View existing memory, avoid duplicates | -| `write` | Overwrite file | Create new memory file or major rewrite | -| `edit` | Replace after exact match | Append or modify specific sections | +| Tool | Function | Use Case | +|---------|-------------------------------|-----------------------------------| +| `read` | Read file content (supports line ranges) | View existing memories, avoid duplicate writes | +| `write` | Overwrite file | Create new memory files or major restructuring | +| `edit` | Replace after exact match | Append new content or modify specific sections | -#### In-Memory Session Management +#### In-Memory Management -[CoPawInMemoryMemory](reme/memory/file_based_copaw/copaw_in_memory_memory.py) extends AgentScope's `InMemoryMemory`: +[ReMeInMemoryMemory](reme/memory/file_based/reme_in_memory_memory.py) extends AgentScope's `InMemoryMemory`: -| Feature | Description | -|----------------------------------|---------------------------------------------------------------------| -| `get_memory` | Filter messages by mark, auto-prepend compression summary | -| `estimate_tokens` | Precisely estimate current context token usage and ratio | -| `get_history_str` | Generate human-readable conversation summary (with token stats) | -| `state_dict` / `load_state_dict` | Support state serialization / deserialization (session persistence) | +| Feature | Description | +|-----------------------------------|------------------------------------------| +| `get_memory` | Filter messages by tag, auto-prepend compression summary at head | +| `estimate_tokens` | Precisely estimate current context token usage and utilization | +| `get_history_str` | Generate human-readable conversation history summary (with token stats) | +| `state_dict` / `load_state_dict` | Support state serialization / deserialization (session persistence) | +| `mark_messages_compressed` | Mark messages as compressed state | +| `get_compressed_summary` | Get compressed summary content | #### Memory Retrieval -[MemorySearch](reme/memory/tools/chunk/memory_search.py) provides **vector + BM25 hybrid retrieval**: +[MemorySearch](reme/memory/tools/chunk/memory_search.py) provides **Vector + BM25 hybrid retrieval** capabilities: -| Retrieval | Strength | Weakness | -|---------------------|-------------------------------------------------|----------------------------------------| -| **Vector semantic** | Captures similar meaning with different wording | Weaker on exact token match | -| **BM25 full-text** | Strong exact token match | No synonym or paraphrase understanding | +| Retrieval Method | Advantage | Disadvantage | +|------------------|----------------------------------------|----------------------------------| +| **Vector Semantic** | Captures semantically similar but differently worded content | Weak on exact token matching | +| **BM25 Full-text** | Excellent for exact token hits | Cannot understand synonyms and paraphrases | -**Fusion**: Both retrieval paths are used; results are combined by weighted sum (vector 0.7 + BM25 0.3), so both -natural-language queries and exact lookups get reliable results. +**Fusion Mechanism**: After dual-path recall, weighted sum is applied (Vector 0.7 + BM25 0.3), enabling both natural language and exact lookups to hit. ```mermaid graph LR - Q[Search query] --> V[Vector search × 0.7] -Q --> B[BM25 × 0.3] -V --> M[Dedupe + weighted merge] -B --> M -M --> R[Top-N results] + Q[Search Query] --> V[Vector Search × 0.7] + Q --> B[BM25 × 0.3] + V --> M[Dedupe + Weighted Fusion] + B --> M + M --> R[Top-N Results] ``` --- ### Vector-Based ReMe Core Architecture +### Installation + +```bash +pip install -U reme-ai +``` + +### Environment Variables + +API keys are set via environment variables, can be written in `.env` file in project root: + +| Environment Variable | Description | Example | +|----------------------------|--------------------------|-----------------------------------------------------| +| `REME_LLM_API_KEY` | LLM API Key | `sk-xxx` | +| `REME_LLM_BASE_URL` | LLM Base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | +| `REME_EMBEDDING_API_KEY` | Embedding API Key | `sk-xxx` | +| `REME_EMBEDDING_BASE_URL` | Embedding Base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | + ```mermaid graph TB User[User / Agent] --> ReMe[Vector Based ReMe] - ReMe --> Summarize[Memory Summarize] - ReMe --> Retrieve[Memory Retrieve] - ReMe --> CRUD[CRUD] + ReMe --> Summarize[Memory Summarization] + ReMe --> Retrieve[Memory Retrieval] + ReMe --> CRUD[CRUD Operations] Summarize --> PersonalSum[PersonalSummarizer] Summarize --> ProceduralSum[ProceduralSummarizer] Summarize --> ToolSum[ToolSummarizer] Retrieve --> PersonalRet[PersonalRetriever] Retrieve --> ProceduralRet[ProceduralRetriever] Retrieve --> ToolRet[ToolRetriever] - PersonalSum --> VectorStore[Vector DB] + PersonalSum --> VectorStore[Vector Database] ProceduralSum --> VectorStore ToolSum --> VectorStore PersonalRet --> VectorStore @@ -546,19 +426,13 @@ graph TB ToolRet --> VectorStore ``` ---- - ## ⭐ Community & Support -- **Star & Watch**: Star helps more agent developers discover ReMe; Watch keeps you updated on new releases and - features. -- **Share your work**: In Issues or Discussions, share what ReMe unlocks for your agents — we’re happy to highlight - great community examples. -- **Need a new feature?** Open a Feature Request; we’ll iterate with the community. -- **Code contributions**: All forms of code contribution are welcome. See - the [Contribution Guide](docs/contribution.md). -- **Acknowledgments**: Thanks to OpenClaw, Mem0, MemU, CoPaw, and other open-source projects for inspiration and - support. +- **Star & Watch**: Star helps more agent developers discover ReMe; Watch keeps you informed about new releases and features. +- **Share Your Work**: Share what ReMe unlocked for your agent in Issues or Discussions — we'd love to showcase community achievements. +- **Need a Feature?** Submit a Feature Request, and we'll work with the community to improve. +- **Code Contributions**: All forms of code contributions are welcome, please see the [Contribution Guide](docs/contribution.md). +- **Acknowledgments**: Thanks to OpenClaw, Mem0, MemU, CoPaw, and other excellent open-source projects for their inspiration and help. --- @@ -577,7 +451,7 @@ graph TB ## ⚖️ License -This project is open source under the Apache License 2.0. See the [LICENSE](./LICENSE) file for details. +This project is open-sourced under the Apache License 2.0, see the [LICENSE](./LICENSE) file for details. --- diff --git a/README_ZH.md b/README_ZH.md index d5b8231b..99e6b1e1 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -33,13 +33,12 @@ ReMe 让智能体拥有**真正的记忆力**——旧对话自动浓缩，重 --- -## 📁 基于文件的 CoPaw 记忆系统 +## 📁 基于文件的记忆系统 > 记忆即文件，文件即记忆 -将**记忆视为文件**——可读、可编辑、可复制。[CoPaw](https://github.com/agentscope-ai/CoPaw) -通过 [MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py) -集成此记忆系统，继承 `ReMeCopaw` 并对外暴露记忆管理能力。 +将**记忆视为文件**——可读、可编辑、可复制。 +[CoPaw](https://github.com/agentscope-ai/CoPaw)通过继承 `ReMeLight` 实现了长期记忆和上下文的管理。 | 传统记忆系统 | File Based ReMe | |-----------|-----------------| @@ -59,22 +58,209 @@ working_dir/ ### 核心能力 -[ReMeCopaw](reme/reme_copaw.py) 是该记忆系统的核心类，为 AI Agent 提供完整的记忆管理能力： +[ReMeLight](reme/reme_light.py) 是该记忆系统的核心类，为 AI Agent 提供完整的记忆管理能力： -| 方法 | 功能 | 关键组件 | -|--------------------------|--------------|----------------------------------------------------------------------------------------------------------------| -| `start` | 🚀 启动记忆系统 | 初始化文件存储、文件监控、Embedding 缓存；清理过期工具结果文件 | -| `close` | 📕 关闭并清理 | 清理工具结果文件、停止文件监控、保存 Embedding 缓存 | -| `compact_memory` | 📦 压缩历史对话为摘要 | [Compactor](reme/memory/file_based_copaw/compactor.py) — ReActAgent 生成结构化上下文检查点 | -| `summary_memory` | 📝 将重要记忆写入文件 | [Summarizer](reme/memory/file_based_copaw/summarizer.py) — ReActAgent + 文件工具（read / write / edit） | -| `compact_tool_result` | ✂️ 压缩超长工具输出 | [ToolResultCompactor](reme/memory/file_based_copaw/tool_result_compactor.py) — 截断并转存到 `tool_result/`，消息中保留文件引用 | -| `add_async_summary_task` | ⚡ 提交后台摘要任务 | `asyncio.create_task`，摘要不阻塞主对话流程 | -| `await_summary_tasks` | ⏳ 等待后台任务完成 | 收集所有后台摘要任务的结果，关闭前调用确保写入完成 | -| `memory_search` | 🔍 语义搜索记忆 | [MemorySearch](reme/memory/tools/chunk/memory_search.py) — 向量 + BM25 混合检索 | -| `get_in_memory_memory` | 🗂️ 创建会话内存实例 | [CoPawInMemoryMemory](reme/memory/file_based_copaw/copaw_in_memory_memory.py) — Token 感知的内存管理，支持压缩摘要和状态序列化 | -| `update_params` | ⚙️ 动态更新运行时参数 | 运行时调整 `max_input_length`、`memory_compact_ratio`、`language` | +| 方法 | 功能 | 关键组件 | +|--------------------------|--------------|----------------------------------------------------------------------------------------------------------| +| `start` | 🚀 启动记忆系统 | 初始化文件存储、文件监控、Embedding 缓存；清理过期工具结果文件 | +| `close` | 📕 关闭并清理 | 清理工具结果文件、停止文件监控、保存 Embedding 缓存 | +| `compact_memory` | 📦 压缩历史对话为摘要 | [Compactor](reme/memory/file_based/compactor.py) — ReActAgent 生成结构化上下文检查点 | +| `summary_memory` | 📝 将重要记忆写入文件 | [Summarizer](reme/memory/file_based/summarizer.py) — ReActAgent + 文件工具（read / write / edit） | +| `compact_tool_result` | ✂️ 压缩超长工具输出 | [ToolResultCompactor](reme/memory/file_based/tool_result_compactor.py) — 截断并转存到 `tool_result/`，消息中保留文件引用 | | +| `memory_search` | 🔍 语义搜索记忆 | [MemorySearch](reme/memory/tools/chunk/memory_search.py) — 向量 + BM25 混合检索 | +| `get_in_memory_memory` | 🗂️ 创建会话内存实例 | [ReMeInMemoryMemory](reme/memory/file_based/reme_in_memory_memory.py) — Token 感知的内存管理，支持压缩摘要和状态序列化 | -## 🗃️ 基于向量库的 ReMe +--- + +### 🚀 快速开始 + +#### 安装 + +```bash +pip install -U reme-ai[light] +``` + +#### 环境变量 + +`ReMeLight` 环境变量配置 Embedding 和存储后端 + +| Variable | Description | Example | +|----------------------|--------------------|-----------------------------------------------------| +| `LLM_API_KEY` | LLM API key | `sk-xxx` | +| `LLM_BASE_URL` | LLM base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | +| `EMBEDDING_API_KEY` | Embedding API key | `sk-xxx` | +| `EMBEDDING_BASE_URL` | Embedding base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | +| `LLM_MODEL_NAME` | LLM model name | `qwen3.5-plus` | + +#### Python使用 + +```python +import asyncio + +from agentscope.message import Msg +from reme.reme_light import ReMeLight + + +async def main(): + reme = ReMeLight( + working_dir=".reme", # 记忆文件存储目录 + max_input_length=128000, # 模型上下文窗口（tokens） + memory_compact_ratio=0.7, # 达到 max_input_length * 0.7 时触发压缩 + language="zh", # 摘要语言（zh / ""） + tool_result_threshold=1000, # 超过此字符数的工具输出自动转存 + retention_days=7, # tool_result/ 文件保留天数 + ) + await reme.start() + + messages = [...] + + # 1. 压缩超长工具输出（防止工具结果撑爆上下文） + messages = await reme.compact_tool_result(messages) + + # 2. 将历史对话压缩为结构化摘要（触发时机：上下文接近上限），可传入上轮摘要，实现增量更新 + summary = await reme.compact_memory(messages=messages, previous_summary="") + + # 3. 后台异步提交摘要任务（不阻塞对话，摘要写入 memory/YYYY-MM-DD.md） + reme.add_async_summary_task(messages=messages) + + # 4. 语义搜索记忆（向量 + BM25 混合检索） + result = await reme.memory_search(query="Python 版本偏好", max_results=5) + + # 5. 获取会话内存实例（ReMeInMemoryMemory，管理单次对话的上下文）AgentScope InMemoryMemory + memory = reme.get_in_memory_memory() + token_stats = await memory.estimate_tokens() + print(f"当前上下文使用率: {token_stats['context_usage_ratio']:.1f}%") + print(f"消息 Token 数: {token_stats['messages_tokens']}") + print(f"预估总 Token 数: {token_stats['estimated_tokens']}") + + # 6. 关闭前等待后台任务完成 + summary_result = await reme.await_summary_tasks() + + # 关闭 ReMeLight + await reme.close() + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### 基于文件的 ReMeLight 记忆系统架构 + +[CoPaw MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py) 继承 +`ReMeLight`，将记忆能力集成到 Agent 推理流程中： + +```mermaid +graph TB + CoPaw["CoPaw MemoryManager\n(继承 ReMeLight)"] -->|pre_reasoning hook| Hook[MemoryCompactionHook] + CoPaw --> ReMeLight[ReMeLight] + Hook -->|超出阈值| ReMeLight + ReMeLight --> CompactMemory[compact_memory\n历史对话压缩] + ReMeLight --> SummaryMemory[summary_memory\n记忆写入文件] + ReMeLight --> CompactToolResult[compact_tool_result\n超长工具输出压缩] + ReMeLight --> MemSearch[memory_search\n语义搜索] + ReMeLight --> InMemory[get_in_memory_memory\nReMeInMemoryMemory] + CompactMemory --> Compactor[Compactor\nReActAgent] + SummaryMemory --> Summarizer[Summarizer\nReActAgent + 文件工具] + CompactToolResult --> ToolResultCompactor[ToolResultCompactor\n截断 + 转存文件] + Summarizer --> FileIO[FileIO\nread / write / edit] + FileIO --> MemoryFiles[memory/YYYY-MM-DD.md] + ToolResultCompactor --> ToolResultFiles[tool_result/*.txt] + MemoryFiles -.->|文件变更| FileWatcher[异步文件监控] + FileWatcher -->|更新索引| FileStore[本地数据库] + MemSearch --> FileStore +``` + +### 上下文压缩机制 + +#### 上下文压缩 + +[Compactor](reme/memory/file_based/compactor.py) 使用 ReActAgent 将历史对话压缩为结构化的**上下文检查点**： + +| 字段 | 说明 | +|-----------------------|-----------------------| +| `## Goal` | 🎯 用户要完成的目标（可多项） | +| `## Constraints` | ⚙️ 用户提到的约束和偏好 | +| `## Progress` | 📈 已完成 / 进行中 / 阻塞的任务 | +| `## Key Decisions` | 🔑 做出的决策及简短理由 | +| `## Next Steps` | 🗺️ 下一步行动计划（有序列表） | +| `## Critical Context` | 📌 文件路径、函数名、错误信息等关键数据 | + +支持**增量更新**：传入 `previous_summary` 时，自动将新对话与旧摘要合并，保留历史进展。 + +#### 工具结果压缩 + +[ToolResultCompactor](reme/memory/file_based/tool_result_compactor.py) 解决工具输出过长（比如 browser use）导致上下文膨胀的问题： + +```mermaid +graph LR + A[tool_result 消息] --> B{内容长度 > threshold?} + B -->|否| C[保留原样] + B -->|是| D[截断到 threshold 字符] + D --> E[完整内容写入 tool_result/uuid.txt] + E --> F[消息中追加文件引用路径] +``` + +过期文件（超过 `retention_days`）在 `start` / `close` / `compact_tool_result` 时自动清理。 + +### 记忆总结：ReAct + 文件工具 + +[Summarizer](reme/memory/file_based/summarizer.py) 采用 **ReAct + 文件工具** 模式，让 AI 自主决定写什么、写到哪： + +```mermaid +graph LR + A[接收对话] --> B{思考: 有什么值得记录?} + B --> C[行动: read memory/YYYY-MM-DD.md] + C --> D{思考: 如何与现有内容合并?} + D --> E[行动: edit 更新文件] + E --> F{思考: 还有遗漏吗?} + F -->|是| B + F -->|否| G[完成] +``` + +[FileIO](reme/memory/file_based/file_io.py) 提供文件操作工具集： + +| 工具 | 功能 | 使用场景 | +|---------|---------------|---------------| +| `read` | 读取文件内容（支持行范围） | 查看现有记忆，避免重复写入 | +| `write` | 覆盖写入文件 | 创建新记忆文件或大幅重构 | +| `edit` | 精确匹配后替换 | 追加新内容或修改特定段落 | + +### 会话内存管理 + +[ReMeInMemoryMemory](reme/memory/file_based/reme_in_memory_memory.py) 扩展了 AgentScope 的 `InMemoryMemory`： + +| 功能 | 说明 | +|----------------------------------|---------------------------| +| `get_memory` | 按标记过滤消息，自动在头部追加压缩摘要 | +| `estimate_tokens` | 精确估算当前上下文 Token 用量及使用率 | +| `get_history_str` | 生成人类可读的对话历史摘要（含 Token 统计） | +| `state_dict` / `load_state_dict` | 支持状态序列化 / 反序列化（会话持久化） | +| `mark_messages_compressed` | 标记消息为已压缩状态 | +| `get_compressed_summary` | 获取已压缩的摘要内容 | + +### 记忆检索 + +[MemorySearch](reme/memory/tools/chunk/memory_search.py) 提供**向量 + BM25 混合检索**能力： + +| 检索方式 | 优势 | 劣势 | +|-------------|-----------------|----------------| +| **向量语义** | 捕捉意义相近但措辞不同的内容 | 对精确 token 匹配较弱 | +| **BM25 全文** | 精确 token 命中效果极佳 | 无法理解同义词和改写 | + +**融合机制**：两路召回后按权重加权求和（向量 0.7 + BM25 0.3），自然语言与精确查找均可命中。 + +```mermaid +graph LR + Q[搜索查询] --> V[向量搜索 × 0.7] +Q --> B[BM25 × 0.3] +V --> M[去重 + 加权融合] +B --> M +M --> R[Top-N 结果] +``` + +--- + +## 🗃️ 基于向量库的记忆系统 [ReMe Vector Based](reme/reme.py) 是基于向量库的记忆系统核心类，支持三种记忆类型的统一管理： @@ -96,60 +282,6 @@ working_dir/ | `delete_memory` | 🗑️ 删除记忆 | 删除指定记忆 | | `list_memory` | 📋 列出记忆 | 列出某类记忆，支持过滤和排序 | ---- - -## 💻 ReMeCli：基于文件记忆的终端助手 - - - - - - - -
- 马
上
有
钱 - 		- - - 马
到
成
功 -
- -### 什么时候会写记忆？ - -| 场景 | 写到哪 | 怎么触发 | -|------------------|------------------------|----------------------| -| 上下文超长自动压缩 | `memory/YYYY-MM-DD.md` | 后台自动 | -| 用户执行 `/compact` | `memory/YYYY-MM-DD.md` | 手动压缩 + 后台保存 | -| 用户执行 `/new` | `memory/YYYY-MM-DD.md` | 新对话 + 后台保存 | -| 用户说"记住这个" | `MEMORY.md` 或日志 | Agent 用 `write` 工具写入 | -| Agent 发现了重要决策/偏好 | `MEMORY.md` | Agent 主动写 | - -### 记忆检索工具 - -| 方式 | 工具 | 什么时候用 | 举例 | -|------|-----------------|------------|--------------------------| -| 语义搜索 | `memory_search` | 不确定记在哪，模糊找 | "之前关于部署的讨论" | -| 直接读 | `read` | 知道是哪天、哪个文件 | 读 `memory/2025-02-13.md` | - -搜索用的是**向量 + BM25 混合检索**（向量权重 0.7，BM25 权重 0.3），无论自然语言还是精确关键词都能命中。 - -### 内置工具 - -| 工具 | 功能 | 细节 | -|-----------------|----------|----------------------------------------| -| `memory_search` | 搜记忆 | MEMORY.md 和 memory/*.md 里做向量+BM25 混合检索 | -| `bash` | 跑命令 | 执行 bash 命令，有超时和输出截断 | -| `ls` | 看目录 | 列目录结构 | -| `read` | 读文件 | 文本和图片都行，支持分段读 | -| `edit` | 改文件 | 精确匹配文本后替换 | -| `write` | 写文件 | 创建或覆盖，自动建目录 | -| `execute_code` | 跑 Python | 运行代码片段 | -| `web_search` | 联网搜索 | 通过 Tavily | - ---- - -## 🚀 快速开始 - ### 安装 ```bash @@ -160,134 +292,17 @@ pip install -U reme-ai API 密钥通过环境变量设置，可写在项目根目录的 `.env` 文件中： -| 环境变量 | 说明 | 示例 | -|---------------------------|-----------------------|-----------------------------------------------------| -| `REME_LLM_API_KEY` | LLM 的 API Key | `sk-xxx` | -| `REME_LLM_BASE_URL` | LLM 的 Base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | -| `REME_EMBEDDING_API_KEY` | Embedding 的 API Key | `sk-xxx` | -| `REME_EMBEDDING_BASE_URL` | Embedding 的 Base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | -| `TAVILY_API_KEY` | Tavily 搜索 API Key（可选） | `tvly-xxx` | - -### 使用 ReMeCli - -#### 启动 ReMeCli - -```bash -remecli config=cli -``` - -#### ReMeCli 系统命令 - -> 马年彩蛋：`/horse` 触发——烟花、奔马动画和随机马年祝福。 - -对话里输入 `/` 开头的命令控制状态： - -| 命令 | 说明 | 需等待响应 | -|------------|---------------------|-------| -| `/compact` | 手动压缩当前对话，同时后台存到长期记忆 | 是 | -| `/new` | 开始新对话，历史后台保存到长期记忆 | 否 | -| `/clear` | 清空一切，**不保存** | 否 | -| `/history` | 看当前对话里未压缩的消息 | 否 | -| `/help` | 看命令列表 | 否 | -| `/exit` | 退出 | 否 | - -**三个命令的区别** - -| 命令 | 压缩摘要 | 长期记忆 | 消息历史 | -|------------|-------|------|-------| -| `/compact` | 生成新摘要 | 保存 | 保留最近的 | -| `/new` | 清空 | 保存 | 清空 | -| `/clear` | 清空 | 不保存 | 清空 | - -> `/clear` 是真删，删了就没了，不会存到任何地方。 - -### 使用 ReMe Package - -#### 基于文件的 ReMe（CoPaw的记忆系统） - -`ReMeCopaw` 接收 AgentScope 的 `ChatModelBase`、`Formatter`、`Toolkit` 等组件，通过环境变量配置 Embedding 和存储后端： - -| 环境变量 | 说明 | 默认值 | -|----------------------------|-----------------------------------|-----------------------------------------------------| -| `EMBEDDING_API_KEY` | Embedding 服务 API Key | `""`（未配置则禁用向量搜索） | -| `EMBEDDING_BASE_URL` | Embedding 服务 Base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | -| `EMBEDDING_MODEL_NAME` | Embedding 模型名称 | `""` | -| `EMBEDDING_DIMENSIONS` | 向量维度 | `1024` | -| `EMBEDDING_CACHE_ENABLED` | 是否启用 Embedding 缓存 | `true` | -| `EMBEDDING_MAX_CACHE_SIZE` | 最大缓存条数 | `2000` | -| `FTS_ENABLED` | 是否启用全文搜索（BM25） | `true` | -| `MEMORY_STORE_BACKEND` | 存储后端（`auto` / `chroma` / `local`） | `auto`（Windows 用 local，其他用 chroma） | +| 环境变量 | 说明 | 示例 | +|-----------------|----------------------|-----------------------------------------------------| +| `LLM_API_KEY` | LLM 的 API Key | `sk-xxx` | +| `LLM_BASE_URL` | LLM 的 Base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | +| `EMBEDDING_API_KEY` | Embedding 的 API Key | `sk-xxx` | +| `EMBEDDING_BASE_URL` | Embedding 的 Base URL | `https://dashscope.aliyuncs.com/compatible-mode/v1` | +### Python使用 ```python import asyncio -from agentscope.formatter import ClaudeFormatter -from agentscope.model import get_model -from agentscope.token import HuggingFaceTokenCounter -from agentscope.tool import Toolkit - -from reme.reme_copaw import ReMeCopaw - - -async def main(): - # 准备 AgentScope 核心组件 - chat_model = get_model(config={"backend": "openai", "model_name": "qwen3.5-plus"}) - formatter = ClaudeFormatter() - token_counter = HuggingFaceTokenCounter() - toolkit = Toolkit() # 可注册额外工具 - - # 初始化 ReMeCopaw - reme = ReMeCopaw( - working_dir=".reme", # 记忆文件存储目录 - chat_model=chat_model, - formatter=formatter, - token_counter=token_counter, - toolkit=toolkit, - max_input_length=128000, # 模型上下文窗口（tokens） - memory_compact_ratio=0.7, # 达到 max_input_length * 0.7 时触发压缩 - language="zh", # 摘要语言（zh / ""） - tool_result_threshold=1000, # 超过此字符数的工具输出自动转存 - retention_days=7, # tool_result/ 文件保留天数 - ) - await reme.start() - - messages = [...] # list[Msg]，对话历史 - - # 1. 压缩超长工具输出（防止工具结果撑爆上下文） - messages = await reme.compact_tool_result(messages) - - # 2. 将历史对话压缩为结构化摘要（触发时机：上下文接近上限） - summary = await reme.compact_memory( - messages=messages, - previous_summary="", # 可传入上轮摘要，实现增量更新 - ) - print(f"压缩摘要:\n{summary}") - - # 3. 后台异步提交摘要任务（不阻塞对话，摘要写入 memory/YYYY-MM-DD.md） - reme.add_async_summary_task(messages=messages) - - # 4. 语义搜索记忆（向量 + BM25 混合检索） - result = await reme.memory_search(query="Python 版本偏好", max_results=5) - print(f"搜索结果: {result}") - - # 5. 获取会话内存实例（CoPawInMemoryMemory，管理单次对话的上下文） - memory = reme.get_in_memory_memory() - token_stats = await memory.estimate_tokens() - print(f"当前上下文使用率: {token_stats['context_usage_ratio']:.1f}%") - - # 6. 关闭前等待后台任务完成 - await reme.await_summary_tasks() - await reme.close() - - -if __name__ == "__main__": - asyncio.run(main()) -``` - -#### 基于向量库的 ReMe - -```python -import asyncio from reme import ReMe @@ -374,137 +389,7 @@ if __name__ == "__main__": asyncio.run(main()) ``` -## 🏛️ 技术架构 - -### 基于文件的 CoPaw 记忆系统架构 - -[CoPaw MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py) 继承 -`ReMeCopaw`，将记忆能力集成到 Agent 推理流程中： - -```mermaid -graph TB - CoPaw["CoPaw MemoryManager\n(继承 ReMeCopaw)"] -->|pre_reasoning hook| Hook[MemoryCompactionHook] - CoPaw --> ReMeCopaw[ReMeCopaw] - Hook -->|超出阈值| ReMeCopaw - ReMeCopaw --> CompactMemory[compact_memory\n历史对话压缩] - ReMeCopaw --> SummaryMemory[summary_memory\n记忆写入文件] - ReMeCopaw --> CompactToolResult[compact_tool_result\n超长工具输出压缩] - ReMeCopaw --> MemSearch[memory_search\n语义搜索] - ReMeCopaw --> InMemory[get_in_memory_memory\nCoPawInMemoryMemory] - CompactMemory --> Compactor[Compactor\nReActAgent] - SummaryMemory --> Summarizer[Summarizer\nReActAgent + 文件工具] - CompactToolResult --> ToolResultCompactor[ToolResultCompactor\n截断 + 转存文件] - Summarizer --> FileIO[FileIO\nread / write / edit] - FileIO --> MemoryFiles[memory/YYYY-MM-DD.md] - ToolResultCompactor --> ToolResultFiles[tool_result/*.txt] - MemoryFiles -.->|文件变更| FileWatcher[异步文件监控] - FileWatcher -->|更新索引| FileStore[本地数据库] - MemSearch --> FileStore -``` - -#### 自动压缩触发流程 - -`MemoryCompactionHook` 在每次推理前检查上下文 Token 用量，超过阈值时自动触发压缩： - -```mermaid -graph LR - A[pre_reasoning] --> B{Token 超过阈值?} - B -->|否| Z[继续推理] - B -->|是| C[compact_tool_result\n压缩最近消息中的超长工具输出] - C --> D[compact_memory\n生成结构化上下文检查点] - D --> E[标记旧消息为 COMPRESSED] - E --> F[add_async_summary_task\n后台写入 memory 文件] - F --> Z -``` - -#### 上下文压缩摘要格式 - -[Compactor](reme/memory/file_based_copaw/compactor.py) 使用 ReActAgent 将历史对话压缩为结构化的**上下文检查点**： - -| 字段 | 说明 | -|-----------------------|-----------------------| -| `## Goal` | 🎯 用户要完成的目标（可多项） | -| `## Constraints` | ⚙️ 用户提到的约束和偏好 | -| `## Progress` | 📈 已完成 / 进行中 / 阻塞的任务 | -| `## Key Decisions` | 🔑 做出的决策及简短理由 | -| `## Next Steps` | 🗺️ 下一步行动计划（有序列表） | -| `## Critical Context` | 📌 文件路径、函数名、错误信息等关键数据 | - -支持**增量更新**：传入 `previous_summary` 时，自动将新对话与旧摘要合并，保留历史进展。 - -#### 工具结果压缩 - -[ToolResultCompactor](reme/memory/file_based_copaw/tool_result_compactor.py) 解决工具输出过长导致上下文膨胀的问题： - -```mermaid -graph LR - A[tool_result 消息] --> B{内容长度 > threshold?} - B -->|否| C[保留原样] - B -->|是| D[截断到 threshold 字符] - D --> E[完整内容写入 tool_result/uuid.txt] - E --> F[消息中追加文件引用路径] -``` - -过期文件（超过 `retention_days`）在 `start` / `close` / `compact_tool_result` 时自动清理。 - -#### 记忆总结：ReAct + 文件工具 - -[Summarizer](reme/memory/file_based_copaw/summarizer.py) 采用 **ReAct + 文件工具** 模式，让 AI 自主决定写什么、写到哪： - -```mermaid -graph LR - A[接收对话] --> B{思考: 有什么值得记录?} - B --> C[行动: read memory/YYYY-MM-DD.md] - C --> D{思考: 如何与现有内容合并?} - D --> E[行动: edit 更新文件] - E --> F{思考: 还有遗漏吗?} - F -->|是| B - F -->|否| G[完成] -``` - -[FileIO](reme/memory/file_based_copaw/file_io.py) 提供文件操作工具集： - -| 工具 | 功能 | 使用场景 | -|---------|---------------|---------------| -| `read` | 读取文件内容（支持行范围） | 查看现有记忆，避免重复写入 | -| `write` | 覆盖写入文件 | 创建新记忆文件或大幅重构 | -| `edit` | 精确匹配后替换 | 追加新内容或修改特定段落 | - -#### 会话内存管理 - -[CoPawInMemoryMemory](reme/memory/file_based_copaw/copaw_in_memory_memory.py) 扩展了 AgentScope 的 `InMemoryMemory`： - -| 功能 | 说明 | -|----------------------------------|---------------------------| -| `get_memory` | 按标记过滤消息，自动在头部追加压缩摘要 | -| `estimate_tokens` | 精确估算当前上下文 Token 用量及使用率 | -| `get_history_str` | 生成人类可读的对话历史摘要（含 Token 统计） | -| `state_dict` / `load_state_dict` | 支持状态序列化 / 反序列化（会话持久化） | - -#### 记忆检索 - -[MemorySearch](reme/memory/tools/chunk/memory_search.py) 提供**向量 + BM25 混合检索**能力： - -| 检索方式 | 优势 | 劣势 | -|-------------|-----------------|----------------| -| **向量语义** | 捕捉意义相近但措辞不同的内容 | 对精确 token 匹配较弱 | -| **BM25 全文** | 精确 token 命中效果极佳 | 无法理解同义词和改写 | - -**融合机制**：两路召回后按权重加权求和（向量 0.7 + BM25 0.3），自然语言与精确查找均可命中。 - -```mermaid -graph LR - Q[搜索查询] --> V[向量搜索 × 0.7] -Q --> B[BM25 × 0.3] -V --> M[去重 + 加权融合] -B --> M -M --> R[Top-N 结果] -``` - ---- - -### 基于向量库的 ReMe 核心架构 - +### 技术架构 ```mermaid graph TB User[用户 / Agent] --> ReMe[Vector Based ReMe] diff --git a/example.env b/example.env index b988f2b2..7bcdbf6a 100644 --- a/example.env +++ b/example.env @@ -1,11 +1,7 @@ -FLOW_EMBEDDING_API_KEY=sk-xxxx -FLOW_EMBEDDING_BASE_URL=https://xxxx/v1 -FLOW_LLM_API_KEY=sk-xxxx -FLOW_LLM_BASE_URL=https://xxxx/v1 - -REME_LLM_API_KEY=sk-xxxx -REME_LLM_BASE_URL=https://xxxx/v1 -REME_EMBEDDING_API_KEY=sk-xxxx -REME_EMBEDDING_BASE_URL=https://xxxx/v1 +LLM_API_KEY=sk-xxxx +LLM_BASE_URL=https://xxxx/v1 +EMBEDDING_API_KEY=sk-xxxx +EMBEDDING_BASE_URL=https://xxxx/v1 +LLM_MODEL_NAME=qwen3.5-plus TAVILY_API_KEY=xxxx diff --git a/pyproject.toml b/pyproject.toml index a3f99fbc..43da64dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,11 +77,11 @@ dev = [ ] full = [ - "reme_ai[dev,ray]", + "reme_ai[dev,ray,light]", ] -as = [ - "agentscope", +light = [ + "agentscope==1.0.16.dev0", ] [tool.setuptools.packages.find] diff --git a/reme/__init__.py b/reme/__init__.py index add74af3..9842ce1a 100644 --- a/reme/__init__.py +++ b/reme/__init__.py @@ -5,9 +5,8 @@ from . import core from . import extension from . import memory from .reme import ReMe -from .reme_cli import ReMeCli -__version__ = "0.3.0.5" +__version__ = "0.3.0.6b1" __all__ = [ "config", @@ -15,7 +14,6 @@ __all__ = [ "extension", "memory", "ReMe", - "ReMeCli", ] """ diff --git a/reme/config/copaw.yaml b/reme/config/light.yaml similarity index 100% rename from reme/config/copaw.yaml rename to reme/config/light.yaml diff --git a/reme/core/llm/base_llm.py b/reme/core/llm/base_llm.py index 856c7f3b..ba13ff10 100644 --- a/reme/core/llm/base_llm.py +++ b/reme/core/llm/base_llm.py @@ -50,12 +50,12 @@ class BaseLLM(ABC): @property def api_key(self) -> str | None: """Get API key from environment variable.""" - return os.getenv("REME_LLM_API_KEY") or self._api_key + return os.getenv("LLM_API_KEY") or self._api_key @property def base_url(self) -> str | None: """Get base URL from environment variable.""" - return os.getenv("REME_LLM_BASE_URL") or self._base_url + return os.getenv("LLM_BASE_URL") or self._base_url @staticmethod def _accumulate_tool_call_chunk(tool_call, ret_tools: list[ToolCall]): diff --git a/reme/core/service_context.py b/reme/core/service_context.py index 0b9c5a1b..ecb6c3a3 100644 --- a/reme/core/service_context.py +++ b/reme/core/service_context.py @@ -50,10 +50,10 @@ class ServiceContext(BaseDict): load_env() # Update common environment variables for LLM and embedding services. - self.update_env("REME_LLM_API_KEY", llm_api_key) - self.update_env("REME_LLM_BASE_URL", llm_base_url) - self.update_env("REME_EMBEDDING_API_KEY", embedding_api_key) - self.update_env("REME_EMBEDDING_BASE_URL", embedding_base_url) + self.update_env("LLM_API_KEY", llm_api_key) + self.update_env("LLM_BASE_URL", llm_base_url) + self.update_env("EMBEDDING_API_KEY", embedding_api_key) + self.update_env("EMBEDDING_BASE_URL", embedding_base_url) if service_config is None: parser_class = parser if parser is not None else PydanticConfigParser diff --git a/reme/memory/cli/__init__.py b/reme/extension/cli/__init__.py similarity index 100% rename from reme/memory/cli/__init__.py rename to reme/extension/cli/__init__.py diff --git a/reme/memory/cli/fb_cli.py b/reme/extension/cli/fb_cli.py similarity index 100% rename from reme/memory/cli/fb_cli.py rename to reme/extension/cli/fb_cli.py diff --git a/reme/memory/cli/fb_cli.yaml b/reme/extension/cli/fb_cli.yaml similarity index 100% rename from reme/memory/cli/fb_cli.yaml rename to reme/extension/cli/fb_cli.yaml diff --git a/reme/memory/cli/fb_compactor.py b/reme/extension/cli/fb_compactor.py similarity index 100% rename from reme/memory/cli/fb_compactor.py rename to reme/extension/cli/fb_compactor.py diff --git a/reme/memory/cli/fb_compactor.yaml b/reme/extension/cli/fb_compactor.yaml similarity index 100% rename from reme/memory/cli/fb_compactor.yaml rename to reme/extension/cli/fb_compactor.yaml diff --git a/reme/memory/cli/fb_context_checker.py b/reme/extension/cli/fb_context_checker.py similarity index 100% rename from reme/memory/cli/fb_context_checker.py rename to reme/extension/cli/fb_context_checker.py diff --git a/reme/memory/cli/fb_summarizer.py b/reme/extension/cli/fb_summarizer.py similarity index 100% rename from reme/memory/cli/fb_summarizer.py rename to reme/extension/cli/fb_summarizer.py diff --git a/reme/memory/cli/fb_summarizer.yaml b/reme/extension/cli/fb_summarizer.yaml similarity index 100% rename from reme/memory/cli/fb_summarizer.yaml rename to reme/extension/cli/fb_summarizer.yaml diff --git a/reme/reme_cli.py b/reme/extension/reme_cli.py similarity index 100% rename from reme/reme_cli.py rename to reme/extension/reme_cli.py diff --git a/reme/memory/__init__.py b/reme/memory/__init__.py index a7df138a..ac825bc6 100644 --- a/reme/memory/__init__.py +++ b/reme/memory/__init__.py @@ -1,11 +1,11 @@ """memory""" -from . import cli +from . import file_based from . import tools from . import vector_based __all__ = [ - "cli", + "file_based", "tools", "vector_based", ] diff --git a/reme/memory/file_based_copaw/__init__.py b/reme/memory/file_based/__init__.py similarity index 76% rename from reme/memory/file_based_copaw/__init__.py rename to reme/memory/file_based/__init__.py index 5b0405f1..29f33749 100644 --- a/reme/memory/file_based_copaw/__init__.py +++ b/reme/memory/file_based/__init__.py @@ -1,11 +1,11 @@ -"""File-based CoPaw Memory Module. +"""File-based Memory Module. This module provides memory management components for CoPaw (Cooperative Paw) agents, including memory formatting, compaction, summarization, and file I/O operations. Components: - MemoryFormatter: Converts message lists to formatted strings with token limiting - - CoPawInMemoryMemory: Extended InMemoryMemory with bugfixes and summary support + - ReMeInMemoryMemory: Extended InMemoryMemory with bugfixes and summary support - Summarizer: Generates memory summaries using LLM - Compactor: Compacts memory content to reduce token usage - ToolResultCompactor: Truncates large tool results and saves full content to files @@ -14,18 +14,20 @@ Components: from . import utils from .compactor import Compactor -from .copaw_in_memory_memory import CoPawInMemoryMemory from .file_io import FileIO from .memory_formatter import MemoryFormatter +from .reme_chat_formatter import ReMeChatFormatter +from .reme_in_memory_memory import ReMeInMemoryMemory from .summarizer import Summarizer from .tool_result_compactor import ToolResultCompactor __all__ = [ "MemoryFormatter", - "CoPawInMemoryMemory", + "ReMeInMemoryMemory", "Summarizer", "Compactor", "ToolResultCompactor", "FileIO", "utils", + "ReMeChatFormatter", ] diff --git a/reme/memory/file_based_copaw/compactor.py b/reme/memory/file_based/compactor.py similarity index 100% rename from reme/memory/file_based_copaw/compactor.py rename to reme/memory/file_based/compactor.py diff --git a/reme/memory/file_based_copaw/compactor.yaml b/reme/memory/file_based/compactor.yaml similarity index 100% rename from reme/memory/file_based_copaw/compactor.yaml rename to reme/memory/file_based/compactor.yaml diff --git a/reme/memory/file_based_copaw/file_io.py b/reme/memory/file_based/file_io.py similarity index 100% rename from reme/memory/file_based_copaw/file_io.py rename to reme/memory/file_based/file_io.py diff --git a/reme/memory/file_based_copaw/memory_formatter.py b/reme/memory/file_based/memory_formatter.py similarity index 100% rename from reme/memory/file_based_copaw/memory_formatter.py rename to reme/memory/file_based/memory_formatter.py diff --git a/reme/memory/file_based/reme_chat_formatter.py b/reme/memory/file_based/reme_chat_formatter.py new file mode 100644 index 00000000..b70e700c --- /dev/null +++ b/reme/memory/file_based/reme_chat_formatter.py @@ -0,0 +1,29 @@ +"""ReMe chat formatter.""" + +from typing import Any + +from agentscope.formatter import OpenAIChatFormatter +from agentscope.token import HuggingFaceTokenCounter + +from .utils import _extract_text_from_messages + + +class ReMeChatFormatter(OpenAIChatFormatter): + """ReMe chat formatter class.""" + + async def _count(self, msgs: list[dict[str, Any]]) -> int | None: + """Count the number of tokens in the input messages. If token counter + is not provided, `None` will be returned. + + Args: + msgs (`list[Msg]`): + The input messages to count tokens for. + """ + if self.token_counter is None: + return None + + assert isinstance(self.token_counter, HuggingFaceTokenCounter) + text = _extract_text_from_messages(msgs) + token_ids = self.token_counter.tokenizer.encode(text) + token_count = len(token_ids) + return token_count diff --git a/reme/memory/file_based_copaw/copaw_in_memory_memory.py b/reme/memory/file_based/reme_in_memory_memory.py similarity index 99% rename from reme/memory/file_based_copaw/copaw_in_memory_memory.py rename to reme/memory/file_based/reme_in_memory_memory.py index 02dc77a4..0e915382 100644 --- a/reme/memory/file_based_copaw/copaw_in_memory_memory.py +++ b/reme/memory/file_based/reme_in_memory_memory.py @@ -13,7 +13,7 @@ from .utils import safe_count_message_tokens, safe_count_str_tokens, _get_block_ logger = logging.getLogger(__name__) -class CoPawInMemoryMemory(InMemoryMemory): +class ReMeInMemoryMemory(InMemoryMemory): """Extended InMemoryMemory with bugfixes and summary support.""" def __init__( diff --git a/reme/memory/file_based_copaw/summarizer.py b/reme/memory/file_based/summarizer.py similarity index 100% rename from reme/memory/file_based_copaw/summarizer.py rename to reme/memory/file_based/summarizer.py diff --git a/reme/memory/file_based_copaw/summarizer.yaml b/reme/memory/file_based/summarizer.yaml similarity index 100% rename from reme/memory/file_based_copaw/summarizer.yaml rename to reme/memory/file_based/summarizer.yaml diff --git a/reme/memory/file_based_copaw/tool_result_compactor.py b/reme/memory/file_based/tool_result_compactor.py similarity index 100% rename from reme/memory/file_based_copaw/tool_result_compactor.py rename to reme/memory/file_based/tool_result_compactor.py diff --git a/reme/memory/file_based_copaw/utils.py b/reme/memory/file_based/utils.py similarity index 84% rename from reme/memory/file_based_copaw/utils.py rename to reme/memory/file_based/utils.py index a5f8f967..f460f96f 100644 --- a/reme/memory/file_based_copaw/utils.py +++ b/reme/memory/file_based/utils.py @@ -1,6 +1,7 @@ """Utility functions for working with text.""" import logging +from pathlib import Path from agentscope.token import HuggingFaceTokenCounter @@ -229,3 +230,42 @@ def _get_block_tokens( # pylint: disable=too-many-return-statements return 0, "" return 0, "" + + +_token_counter = None + + +def get_token_counter(): + """Get or initialize the global token counter instance. + + Returns: + TokenCounterBase: The token counter instance for Qwen models. + + Raises: + RuntimeError: If token counter initialization fails. + """ + global _token_counter + if _token_counter is None: + # Use Qwen tokenizer for DashScope models + # Qwen3 series uses the same tokenizer as Qwen2.5 + + # Try local tokenizer first, fall back to online if not found + local_tokenizer_path = Path(__file__).parent.parent.parent / "tokenizer" + + if local_tokenizer_path.exists() and (local_tokenizer_path / "tokenizer.json").exists(): + tokenizer_path = str(local_tokenizer_path) + logger.info(f"Using local Qwen tokenizer from {tokenizer_path}") + else: + tokenizer_path = "Qwen/Qwen2.5-7B-Instruct" + logger.info( + "Local tokenizer not found, downloading from HuggingFace", + ) + + _token_counter = HuggingFaceTokenCounter( + pretrained_model_name_or_path=tokenizer_path, + use_mirror=True, # Use HF mirror for users in China + use_fast=True, + trust_remote_code=True, + ) + logger.debug("Token counter initialized with Qwen tokenizer") + return _token_counter diff --git a/reme/memory/skills/__init__.py b/reme/memory/skills/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/reme/reme_copaw.py b/reme/reme_light.py similarity index 88% rename from reme/reme_copaw.py rename to reme/reme_light.py index 2c76bf5e..4396c16f 100644 --- a/reme/reme_copaw.py +++ b/reme/reme_light.py @@ -1,7 +1,7 @@ """ -ReMe Copaw Application Module +ReMe Light Application Module -This module provides the ReMeCopaw class, a specialized application built on top of +This module provides the ReMeLight class, a specialized application built on top of ReMe's core Application framework. It integrates memory management capabilities including memory compaction, summarization, tool result management, and semantic memory search functionality. @@ -22,22 +22,23 @@ from pathlib import Path from agentscope.formatter import FormatterBase from agentscope.message import Msg, TextBlock -from agentscope.model import ChatModelBase +from agentscope.model import ChatModelBase, OpenAIChatModel from agentscope.token import HuggingFaceTokenCounter from agentscope.tool import Toolkit, ToolResponse from .config import ReMeConfigParser from .core import Application -from .memory.file_based_copaw import Compactor, Summarizer, ToolResultCompactor, CoPawInMemoryMemory +from .memory.file_based import Compactor, Summarizer, ToolResultCompactor, ReMeInMemoryMemory, ReMeChatFormatter +from .memory.file_based.utils import get_token_counter from .memory.tools import MemorySearch +from .core.utils import load_env -# Module-level logger for tracking application events and errors logger = logging.getLogger(__name__) -class ReMeCopaw(Application): +class ReMeLight(Application): """ - ReMe Copaw Application Class + ReMe Light Application Class A specialized application class that extends ReMe's core Application framework with advanced memory management capabilities. This class is designed to handle @@ -64,13 +65,17 @@ class ReMeCopaw(Application): def __init__( self, - working_dir: str, - chat_model: ChatModelBase, - formatter: FormatterBase, - token_counter: HuggingFaceTokenCounter, - toolkit: Toolkit, - max_input_length: int, - memory_compact_ratio: float, + working_dir: str = ".reme", + llm_api_key: str | None = None, + llm_base_url: str | None = None, + embedding_api_key: str | None = None, + embedding_base_url: str | None = None, + chat_model: ChatModelBase | None = None, + formatter: FormatterBase | None = None, + token_counter: HuggingFaceTokenCounter | None = None, + toolkit: Toolkit | None = None, + max_input_length: int = 128000, + memory_compact_ratio: float = 0.7, language: str = "zh", vector_weight: float = 0.7, candidate_multiplier: float = 3.0, @@ -90,23 +95,11 @@ class ReMeCopaw(Application): self.tool_result_path = self.working_path / "tool_result" self.tool_result_path.mkdir(parents=True, exist_ok=True) - # Store references to core components - self.chat_model: ChatModelBase = chat_model - self.formatter: FormatterBase = formatter - self.token_counter: HuggingFaceTokenCounter = token_counter - self.toolkit: Toolkit = toolkit - # Initialize runtime parameters (will be updated via update_params) self.max_input_length: int = 0 self.memory_compact_threshold: int = 0 self.language: str = "" - # Store configuration parameters - self.vector_weight: float = vector_weight - self.candidate_multiplier: float = candidate_multiplier - self.tool_result_threshold: int = tool_result_threshold - self.retention_days: int = retention_days - # Apply initial parameter configuration self.update_params( max_input_length=max_input_length, @@ -114,18 +107,21 @@ class ReMeCopaw(Application): language=language, ) - # Retrieve embedding configuration from environment variables - # These settings control the vector search capabilities - ( - embedding_api_key, - embedding_base_url, - embedding_model_name, - embedding_dimensions, - embedding_cache_enabled, - embedding_max_cache_size, - embedding_max_input_length, - embedding_max_batch_size, - ) = self.get_emb_envs() + # Store configuration parameters + self.vector_weight: float = vector_weight + self.candidate_multiplier: float = candidate_multiplier + self.tool_result_threshold: int = tool_result_threshold + self.retention_days: int = retention_days + + load_env() + + llm_model_name = self._safe_str("LLM_MODEL_NAME", "") + embedding_model_name = self._safe_str("EMBEDDING_MODEL_NAME", "") + embedding_dimensions = self._safe_int("EMBEDDING_DIMENSIONS", 1024) + embedding_cache_enabled = self._safe_str("EMBEDDING_CACHE_ENABLED", "true").lower() == "true" + embedding_max_cache_size = self._safe_int("EMBEDDING_MAX_CACHE_SIZE", 2000) + embedding_max_input_length = self._safe_int("EMBEDDING_MAX_INPUT_LENGTH", 8192) + embedding_max_batch_size = self._safe_int("EMBEDDING_MAX_BATCH_SIZE", 10) # Determine if vector search should be enabled based on configuration # Vector search requires either an API key or a local model name @@ -149,13 +145,14 @@ class ReMeCopaw(Application): else: memory_backend = memory_store_backend - # Initialize the parent Application class with configuration # Initialize the parent Application class with comprehensive configuration super().__init__( + llm_api_key=llm_api_key, + llm_base_url=llm_base_url, embedding_api_key=embedding_api_key, embedding_base_url=embedding_base_url, working_dir=str(self.working_path), - config_path="copaw", + config_path="light", enable_logo=False, log_to_console=False, parser=ReMeConfigParser, @@ -183,6 +180,27 @@ class ReMeCopaw(Application): }, ) + if chat_model is not None: + self.chat_model: ChatModelBase = chat_model + else: + # add more params later + self.chat_model = OpenAIChatModel( + api_key=os.environ["LLM_API_KEY"], + client_kwargs={"base_url": os.environ["LLM_BASE_URL"]}, + model_name=llm_model_name, + ) + + if token_counter is not None: + self.token_counter: HuggingFaceTokenCounter = token_counter + else: + self.token_counter = get_token_counter() + + if formatter is not None: + self.formatter: FormatterBase = formatter + else: + self.formatter = ReMeChatFormatter(token_counter=self.token_counter) + self.toolkit: Toolkit | None = toolkit + # Initialize list to track background summarization tasks self.summary_tasks: list[asyncio.Task] = [] @@ -264,55 +282,6 @@ class ReMeCopaw(Application): logger.warning(f"Invalid int value '{value}' for key '{key}', using default {default}") return default - def get_emb_envs(self): - """ - Retrieve all embedding-related configuration from environment variables. - - This method collects all settings needed for the embedding service, - including API credentials, model configuration, and caching parameters. - - Environment Variables: - EMBEDDING_API_KEY: API key for the embedding service - EMBEDDING_BASE_URL: Base URL for the embedding API (default: dashscope) - EMBEDDING_MODEL_NAME: Name of the embedding model to use - EMBEDDING_DIMENSIONS: Vector dimensions (default: 1024) - EMBEDDING_CACHE_ENABLED: Whether to enable caching (default: true) - EMBEDDING_MAX_CACHE_SIZE: Maximum cache entries (default: 2000) - EMBEDDING_MAX_INPUT_LENGTH: Max input text length (default: 8192) - EMBEDDING_MAX_BATCH_SIZE: Max batch size for requests (default: 10) - - Returns: - tuple: A tuple containing all embedding configuration values in order: - (api_key, base_url, model_name, dimensions, cache_enabled, - max_cache_size, max_input_length, max_batch_size) - """ - # API authentication and endpoint configuration - embedding_api_key = self._safe_str("EMBEDDING_API_KEY", "") - embedding_base_url = self._safe_str("EMBEDDING_BASE_URL", "https://dashscope.aliyuncs.com/compatible-mode/v1") - embedding_model_name = self._safe_str("EMBEDDING_MODEL_NAME", "") - - # Model and vector configuration - embedding_dimensions = self._safe_int("EMBEDDING_DIMENSIONS", 1024) - - # Caching configuration for performance optimization - embedding_cache_enabled = self._safe_str("EMBEDDING_CACHE_ENABLED", "true").lower() == "true" - embedding_max_cache_size = self._safe_int("EMBEDDING_MAX_CACHE_SIZE", 2000) - - # Input processing limits - embedding_max_input_length = self._safe_int("EMBEDDING_MAX_INPUT_LENGTH", 8192) - embedding_max_batch_size = self._safe_int("EMBEDDING_MAX_BATCH_SIZE", 10) - - return ( - embedding_api_key, - embedding_base_url, - embedding_model_name, - embedding_dimensions, - embedding_cache_enabled, - embedding_max_cache_size, - embedding_max_input_length, - embedding_max_batch_size, - ) - def _cleanup_tool_results(self) -> int: """ Clean up expired tool result files from the tool result directory. @@ -681,13 +650,13 @@ class ReMeCopaw(Application): """ Create and return an in-memory memory instance. - This method instantiates a CoPawInMemoryMemory object configured with + This method instantiates a ReMeInMemoryMemory object configured with the current application's token counter, formatter, and input length limits. The in-memory memory provides fast, temporary storage for conversation context without persistence. Returns: - CoPawInMemoryMemory: A configured in-memory memory instance ready + ReMeInMemoryMemory: A configured in-memory memory instance ready for storing and retrieving conversation messages Note: @@ -695,7 +664,7 @@ class ReMeCopaw(Application): - Useful for managing conversation context within a single session - Shares the same token counter and formatter as the main application """ - return CoPawInMemoryMemory( + return ReMeInMemoryMemory( token_counter=self.token_counter, formatter=self.formatter, max_input_length=self.max_input_length, diff --git a/test/test_agentic_retrieve_op.py b/test/test_agentic_retrieve_op.py deleted file mode 100644 index 3cb9d851..00000000 --- a/test/test_agentic_retrieve_op.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Test script for AgenticRetrieveOp. - -This script provides a simple end-to-end test case for AgenticRetrieveOp. -It can be run directly with: python test_agentic_retrieve_op.py -""" - -import asyncio -import json - -from flowllm.core.enumeration import Role -from flowllm.core.schema import Message, ToolCall -from loguru import logger - -from reme_ai.agent.react.agentic_retrieve_op import AgenticRetrieveOp -from reme_ai.main import ReMeApp - - -async def test_agentic_retrieve_basic(): - """Basic test for AgenticRetrieveOp with a short conversation history.""" - logger.info("\n" + "=" * 60) - logger.info("Test: AgenticRetrieveOp basic behavior") - logger.info("=" * 60) - - tool_call_id = "call_6596dafa2a6a46f7a217da" - f = open("README.md", encoding="utf-8") - readme_content = f.read() - f.close() - - messages = [ - Message( - role=Role.SYSTEM, - content=( - "You are a helpful assistant. " - "请先使用`Grep`匹配关键词或者正则表达式所在行数，然后通过`ReadFile`读取位置附近的代码。" - "如果没有找到匹配项，永远不要放弃尝试，尝试其他的参数，比如只搜索部分关键词。" - "`Grep`之后通过 `ReadFile` 命令，你可以从指定偏移位置`offset`+长度`limit`开始查看内容，不要超过100行。" - "如果当前内容不足，`ReadFile` 命令也可以不断尝试不同的`offset`和`limit`参数" - ), - ), - Message( - role=Role.USER, - content="搜索下reme项目的的README内容", - ), - Message( - role=Role.ASSISTANT, - content="", - tool_calls=[ - ToolCall( - **{ - "index": 0, - "id": tool_call_id, - "function": { - "arguments": '{"query": "readme"}', - "name": "web_search", - }, - "type": "function", - }, - ), - ], - ), - Message( - role=Role.TOOL, - content=readme_content * 4, - tool_call_id=tool_call_id, - ), - Message( - role=Role.USER, - content="根据readme回答task memory在appworld的效果是多少，需要具体的数值", - ), - ] - - # llm = "qwen3_coder_plus" - llm = "qwen3_30b_instruct" - # llm = "qwen3_30b_thinking" - # llm = "qwen3_coder_30b_instruct" - # llm = "qwen3_max_instruct" - op = AgenticRetrieveOp(llm=llm) - - await op.async_call( - messages=[m.model_dump() for m in messages], - working_summary_mode="auto", - compact_ratio_threshold=0.75, - max_total_tokens=20000, - max_tool_message_tokens=2000, - group_token_threshold=None, - keep_recent_count=1, - store_dir="./test_working_memory", - chat_id="c123", - ) - - answer = op.context.response.answer - messages = op.context.response.metadata["messages"] - logger.info(f"✓ AgenticRetrieveOp result answer: {answer}") - logger.info(f"✓ AgenticRetrieveOp result messages: {json.dumps(messages, ensure_ascii=False, indent=2)}") - logger.info(f" Success: {op.context.response.success}") - - -async def async_main(): - """Entry point for running AgenticRetrieveOp test.""" - async with ReMeApp(): - logger.info("=" * 80) - logger.info("Testing AgenticRetrieveOp - ReAct Retrieval Workflow") - logger.info("=" * 80) - - await test_agentic_retrieve_basic() - - logger.info("\n" + "=" * 80) - logger.info("All AgenticRetrieveOp tests completed!") - logger.info("=" * 80) - - -if __name__ == "__main__": - asyncio.run(async_main()) diff --git a/tests/test_fs_compactor.py b/test/test_fs_compactor.py similarity index 100% rename from tests/test_fs_compactor.py rename to test/test_fs_compactor.py diff --git a/tests/test_fs_context_checker.py b/test/test_fs_context_checker.py similarity index 100% rename from tests/test_fs_context_checker.py rename to test/test_fs_context_checker.py diff --git a/tests/test_fs_file_watch_integration.py b/test/test_fs_file_watch_integration.py similarity index 100% rename from tests/test_fs_file_watch_integration.py rename to test/test_fs_file_watch_integration.py diff --git a/tests/test_fs_memory_get.py b/test/test_fs_memory_get.py similarity index 100% rename from tests/test_fs_memory_get.py rename to test/test_fs_memory_get.py diff --git a/tests/test_fs_memory_search.py b/test/test_fs_memory_search.py similarity index 100% rename from tests/test_fs_memory_search.py rename to test/test_fs_memory_search.py diff --git a/tests/test_fs_summary.py b/test/test_fs_summary.py similarity index 100% rename from tests/test_fs_summary.py rename to test/test_fs_summary.py diff --git a/test/test_message_compact_op.py b/test/test_message_compact_op.py deleted file mode 100644 index cd8e4edc..00000000 --- a/test/test_message_compact_op.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Test script for MessageCompactOp. - -This script provides test cases for MessageCompactOp class. -It can be run directly with: python test_context_compact_op.py -""" - -import asyncio - -from flowllm.core.enumeration import Role -from flowllm.core.schema import Message - -from reme_ai.main import ReMeApp -from reme_ai.retrieve.working import BatchWriteFileOp -from reme_ai.summary.working import MessageCompactOp - - -async def async_main(): - """Test function for MessageCompactOp.""" - async with ReMeApp(): - # Create test messages with system, user, assistant, tool sequence - messages = [ - Message(role=Role.SYSTEM, content="You are a helpful assistant."), - Message(role=Role.USER, content="What is the weather today?"), - Message( - role=Role.ASSISTANT, - content="I'll check the weather for you.", - ), - Message( - role=Role.TOOL, - content="A" * 5000, # Large tool message that should be compacted - tool_call_id="call_001", - ), - Message( - role=Role.ASSISTANT, - content="Let me also check the forecast.", - ), - Message( - role=Role.TOOL, - content="B" * 5000, # Another large tool message - tool_call_id="call_002", - ), - Message( - role=Role.USER, - content="What about tomorrow?", - ), - Message( - role=Role.ASSISTANT, - content="I'll check tomorrow's weather.", - ), - Message( - role=Role.TOOL, - content="C" * 5000, # Third large tool message - tool_call_id="call_003", - ), - Message( - role=Role.TOOL, - content="Recent result", # Recent tool message (should be kept) - tool_call_id="call_004", - ), - ] - - # Create op with lower thresholds for testing - op = MessageCompactOp() >> BatchWriteFileOp() - - # Execute the compaction - await op.async_call( - messages=[m.model_dump() for m in messages], - max_total_tokens=1000, # Low threshold to trigger compaction - max_tool_message_tokens=100, # Low threshold to compact tool messages - preview_char_length=50, # Keep 50 chars in preview - keep_recent_count=1, # Keep 1 recent tool message - store_dir="./test_compact_storage", - ) - - # Print results - result = op.context.response.answer - print(f"Context compaction result: {result}") - - -if __name__ == "__main__": - asyncio.run(async_main()) diff --git a/test/test_message_compress_op.py b/test/test_message_compress_op.py deleted file mode 100644 index 51423c3a..00000000 --- a/test/test_message_compress_op.py +++ /dev/null @@ -1,285 +0,0 @@ -""" -Test script for MessageCompressOp. - -This script demonstrates how to use the message compression operation to reduce -token usage in conversation histories using language models. -""" - -import asyncio - -from loguru import logger - -from reme_ai.main import ReMeApp -from reme_ai.summary.working import MessageCompressOp - - -async def main(): - """Main function to test MessageCompressOp.""" - - async with ReMeApp(): - logger.info("=" * 80) - logger.info("Testing MessageCompressOp - LLM-based Context Compression") - logger.info("=" * 80) - - # Create a mock conversation with multiple messages - messages = [ - { - "role": "system", - "content": "You are a helpful AI assistant specialized in software development.", - }, - { - "role": "user", - "content": "I need help building a REST API in Python. I want to use FastAPI.", - }, - { - "role": "assistant", - "content": "Great choice! FastAPI is an excellent framework for building REST APIs. " - "It's fast, modern, and has automatic API documentation. To get started, you'll need " - "to install FastAPI and uvicorn. Would you like me to guide you through setting up " - "your first endpoint?", - }, - { - "role": "user", - "content": "Yes please. I want to create a user management API with CRUD operations.", - }, - { - "role": "assistant", - "content": "Perfect! For a user management API, I recommend this structure:\n" - "1. Define a User model using Pydantic\n" - "2. Create POST /users endpoint for creating users\n" - "3. Create GET /users and GET /users/{id} for reading\n" - "4. Create PUT /users/{id} for updates\n" - "5. Create DELETE /users/{id} for deletion\n" - "We'll also need a database. Would you prefer SQLite, PostgreSQL, or MongoDB?", - }, - { - "role": "user", - "content": "Let's use PostgreSQL. Also, I need JWT authentication.", - }, - { - "role": "assistant", - "content": "Excellent. PostgreSQL is a robust choice. For JWT authentication, we'll use " - "python-jose library. Here's what we'll implement:\n" - "1. User registration endpoint\n" - "2. Login endpoint that returns JWT token\n" - "3. Protected endpoints that require valid JWT\n" - "4. Password hashing using bcrypt\n" - "Let me show you the code for the User model first.", - }, - { - "role": "user", - "content": "Before we proceed, I also need rate limiting and input validation.", - }, - { - "role": "assistant", - "content": "Good thinking! For rate limiting, we can use slowapi library which integrates " - "well with FastAPI. For input validation, Pydantic (which FastAPI uses) handles most of it, " - "but we can add custom validators. I'll also add request validation middleware. " - "Let's start implementing all of this step by step.", - }, - { - "role": "user", - "content": "I need to build a distributed task queue system in Python that can handle millions of tasks" - " per day. It needs to be horizontally scalable and fault-tolerant.", - }, - { - "role": "assistant", - "content": "That's a challenging but exciting project! For a high-throughput distributed task queue, " - "I recommend a architecture with:\n\n1. **Message Broker**: Redis or RabbitMQ for task " - "distribution\n2. **Task Workers**: Multiple worker processes across multiple machines\n3." - " **Result Backend**: Redis or PostgreSQL for storing task results\n4. **Monitoring**:" - " Prometheus + Grafana for metrics\n5. **API Layer**: FastAPI for task submission and " - "status queries\n\nFor the core library, we can build on top of Celery or create a custo" - "m solution. Would you like me to design the system architecture first, or do you have pr" - "eferences for specific technologies?", - }, - { - "role": "user", - "content": "I want to build this from scratch without Celery. I need custom features like task" - " priorities, retry policies with exponential backoff, and task dependencies. Also," - " I need it to support both synchronous and asynchronous task execution patterns.", - }, - { - "role": "assistant", - "content": "Excellent! Building from scratch gives us full control. Let me design a comprehensive " - "system architecture:\n\n**Core Components:**\n1. **Task Router**: Distributes tasks based" - " on priority queues\n2. **Worker Manager**: Handles worker lifecycle and load balancing\n3." - " **Retry Engine**: Implements exponential backoff with jitter\n4. **Dependency Graph**:" - " Manages task dependencies using topological sorting\n5. **State Manager**: Tracks task " - "states (pending, running, completed, failed)\n6. **Metrics Collector**: Real-time " - "performance metrics\n\n**Data Structures:**\n- Priority queues using Redis Sorted Sets\n-" - " Task metadata in PostgreSQL with JSONB columns\n- Distributed locks with Redis Redlock" - " algorithm\n\n**Features:**\n- At-least-once delivery guarantee\n- Dead letter queue for " - "failed tasks\n- Rate limiting per task type\n- Circuit breaker pattern for failing " - "services\n\nWould you like me to start with the core task model and priority queue " - "implementation?", - }, - { - "role": "user", - "content": "That sounds good. But I also need WebSocket support for real-time task status updates, a " - "web dashboard for monitoring, and the ability to dynamically scale workers based on queue" - " depth. Plus, I need task deduplication and exactly-once processing semantics.", - }, - { - "role": "assistant", - "content": "Perfect! Let me enhance the architecture with these advanced requirements:\n\n**Real-time " - "Features:**\n1. **WebSocket Manager**: Using Python's websocket-server with Redis pub/sub " - "for multi-instance communication\n2. **Event Stream**: Task state changes published to Re" - "dis streams\n3. **Client SDK**: JavaScript/TypeScript library for real-time subscriptio" - "ns\n\n**Auto-scaling Architecture:**\n1. **Metrics Aggregator**: Collects queue depth, " - "processing latency, error rates\n2. **Scaler Service**: Kubernetes HPA integration or cu" - "stom Docker Swarm scaling\n3. **Predictive Scaling**: ML model to predict load based on hi" - "storical patterns\n\n**Exactly-once Processing:**\n1. **Idempotency Keys**: Each task has " - "UUID-based idempotency key\n2. **Deduplication Store**: Redis with TTL for processed task " - "IDs\n3. **Transactional Outbox**: PostgreSQL outbox pattern for reliable event publishing" - "\n4. **Two-phase Commit**: For distributed task processing\n\n**Dashboard Features:**\n- " - "Real-time queue metrics with D3.js visualizations\n- Task timeline view with dependency g" - "raphs\n- Worker health monitoring with automatic restart\n- Alert system for SLA violati" - "ons\n\n**Additional Components:**\n- Task scheduler for delayed tasks using Redis keys w" - "ith TTL\n- Batch processing support for bulk operations\n- Multi-tenant isolation with na" - "mespace support\n\nThis is getting quite comprehensive! Should I start with the core tas" - "k model and database schema, or would you prefer to begin with the WebSocket real-time up" - "date system?", - }, - { - "role": "user", - "content": "This is getting complex. I also need multi-region support with eventual consistency, the " - "ability to pause/resume task processing by task type, and A/B testing capabilities for " - "different worker implementations. Also, I need comprehensive audit logging and GDPR comp" - "liance features.", - }, - { - "role": "assistant", - "content": "Absolutely! This is evolving into an enterprise-grade system. Let me architect the compl" - "ete solution:\n\n**Multi-region Architecture:**\n1. **Global Load Balancer**: GeoDNS with" - " health checks\n2. **Cross-region Replication**: Redis Cluster with active-active setup " - "using CRDTs\n3. **Conflict Resolution**: Vector clocks for task ordering, last-writer-win" - "s for metadata\n4. **Region-aware Routing**: Route tasks to workers in same region when p" - "ossible\n5. **Failover Mechanism**: Automatic region failover with 30-second RTO\n\n**Adv" - "anced Control Features:**\n1. **Task Type Governance**: \n - Pause/resume via Redis fe" - "ature flags with immediate propagation\n - Rate limits per task type with burst capaci" - "ty\n - Resource quotas (CPU/memory) per task category\n2. **A/B Testing Framework**:\n" - " - Task routing based on consistent hashing of task ID\n - Variant assignment with s" - "tickiness\n - Statistical significance tracking for performance metrics\n - Automati" - "c winner selection based on success rate and latency\n\n**Compliance & Audit:**\n1. **A" - "udit Trail**:\n - Immutable task history in PostgreSQL with row-level security\n -" - " Change data capture (CDC) using Debezium\n - Cryptographic signing of audit logs\n " - " - 7-year retention policy with automated archival to S3\n2. **GDPR Compliance**:\n " - " - Right to be forgotten: Cascade delete with verification\n - Data portability: JSO" - "N export of all user tasks\n - Consent management: Task processing consent tracking" - "\n - Data anonymization: PII encryption with rotating keys\n\n**Enhanced Monitoring:*" - "*\n1. **Distributed Tracing**: OpenTelemetry integration with Jaeger\n2. **Performance " - "Profiling**: Py-spy continuous profiling\n3. **Cost Analytics**: Per-task-type resource" - " consumption tracking\n4. **SLA Monitoring**: 99.9% task processing latency targets\n\n" - "**Security Features:**\n1. **Zero-trust Architecture**: mTLS between all services\n2. **" - "Secrets Management**: HashiCorp Vault integration\n3. **Network Segmentation**: VPC is" - "olation with private subnets\n4. **Encryption**: AES-256 at rest, TLS 1.3 in transit\n" - "\n**Operational Excellence:**\n1. **Blue-green Deployments**: Zero-downtime updates\n2." - " **Canary Releases**: Gradual rollout of worker changes\n3. **Chaos Engineering**: Rand" - "om worker termination testing\n4. **Capacity Planning**: Automated infrastructure provi" - "sioning\n\nThis is becoming a mission-critical system! Would you like me to start with " - "the database schema and core task lifecycle management, or should we begin with the mul" - "ti-region replication strategy? I can also create a detailed implementation roadmap wi" - "th phases if you prefer.", - }, - { - "role": "user", - "content": "How do I reverse a string in Python?", - }, - { - "role": "assistant", - "content": "You can reverse a string in Python using slicing: `reversed_string = original_string[::" - "-1]`. The `[::-1]` slice notation means start at the end of the string and end at posit" - "ion 0, moving with the step -1.", - }, - { - "role": "user", - "content": "What's the difference between a list and a tuple?", - }, - { - "role": "assistant", - "content": "Lists are mutable (you can change their content after creation) and use square bracket" - "s `[]`. Tuples are immutable and use parentheses `()`. Lists are better for collectio" - "ns that need to change, while tuples are faster and can be used as dictionary keys.", - }, - { - "role": "user", - "content": "How do I handle exceptions in Python?", - }, - { - "role": "assistant", - "content": "Use try-except blocks: `try:` followed by your risky code, then `except SpecificError" - ":` to catch particular exceptions. Always catch specific exceptions rather than the g" - "eneric `Exception` when possible. You can also use `finally:` for cleanup code that ru" - "ns regardless.", - }, - ] - - # Test 1: Messages below threshold (should not compress) - logger.info("\n" + "=" * 60) - logger.info("Test 1: Messages below threshold (should skip compression)") - logger.info("=" * 60) - - compress_op1 = MessageCompressOp() - - await compress_op1.async_call( - messages=messages, - max_total_tokens=50000, # High threshold, won't trigger - keep_recent_count=2, - ) - - result_messages1 = compress_op1.context.response.answer - logger.info(f"✓ Result: {len(result_messages1)} messages (unchanged)") - - # Test 2: Messages above threshold (should compress) - logger.info("\n" + "=" * 60) - logger.info("Test 2: Messages above threshold (should compress)") - logger.info("=" * 60) - - compress_op2 = MessageCompressOp() - - await compress_op2.async_call( - messages=messages, - max_total_tokens=2000, # Low threshold, will trigger - keep_recent_count=2, - compress_system_message=False, - ) - - result_messages2 = compress_op2.context.response.answer - logger.info(f"✓ Result: {len(result_messages2)} messages (compressed)") - - # Display compression results - logger.info("\n" + "=" * 60) - logger.info("Compression Result Details:") - logger.info("=" * 60) - logger.info(f"Original messages: {len(messages)}") - logger.info(f"Compressed messages: {len(result_messages2)}") - - # Test 3: Messages above threshold (should compress) - logger.info("\n" + "=!" * 30) - logger.info("Test 3: Messages above micro threshold (should compress)") - logger.info("=!" * 30) - - compress_op2 = MessageCompressOp() - - await compress_op2.async_call( - messages=messages, - max_total_tokens=2000, # Low threshold, will trigger - keep_recent_count=2, - compress_system_message=False, # Don't compress system messages - group_token_threshold=1500, - ) - - result_messages2 = compress_op2.context.response.answer - logger.info(f"✓ Result: {len(result_messages2)} messages (compressed)") - - # Display compression results - logger.info("\n" + "=" * 60) - logger.info("Compression Result Details:") - logger.info("=" * 60) - logger.info(f"Original messages: {len(messages)}") - logger.info(f"Compressed messages: {len(result_messages2)}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/test/test_message_offload_op.py b/test/test_message_offload_op.py deleted file mode 100644 index 403e89fd..00000000 --- a/test/test_message_offload_op.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Test script for MessageOffloadOp. - -This script provides test cases for MessageOffloadOp class. -It can be run directly with: python test_context_offload_op.py -""" - -import asyncio - -from flowllm.core.enumeration import Role -from flowllm.core.schema import Message -from loguru import logger - -from reme_ai.enumeration import WorkingSummaryMode -from reme_ai.main import ReMeApp -from reme_ai.retrieve.working import BatchWriteFileOp -from reme_ai.summary.working import MessageOffloadOp - - -async def test_compact_mode(): - """Test COMPACT mode - Only apply compaction with MessageOffloadOp.""" - logger.info("\n" + "=" * 60) - logger.info("Test: COMPACT mode - Only apply compaction") - logger.info("=" * 60) - - # Create test messages with system, user, assistant, tool sequence - messages = [ - Message(role=Role.SYSTEM, content="You are a helpful assistant."), - Message(role=Role.USER, content="What is the weather today?"), - Message( - role=Role.ASSISTANT, - content="I'll check the weather for you.", - ), - Message( - role=Role.TOOL, - content="A" * 5000, # Large tool message that should be compacted - tool_call_id="call_001", - ), - Message( - role=Role.ASSISTANT, - content="Let me also check the forecast.", - ), - Message( - role=Role.TOOL, - content="B" * 5000, # Another large tool message - tool_call_id="call_002", - ), - Message( - role=Role.USER, - content="What about tomorrow?", - ), - Message( - role=Role.ASSISTANT, - content="I'll check tomorrow's weather.", - ), - Message( - role=Role.TOOL, - content="C" * 5000, # Third large tool message - tool_call_id="call_003", - ), - Message( - role=Role.TOOL, - content="Recent result", # Recent tool message (should be kept) - tool_call_id="call_004", - ), - ] - - op = MessageOffloadOp() >> BatchWriteFileOp() - - await op.async_call( - messages=[m.model_dump() for m in messages], - context_manage_mode=WorkingSummaryMode.COMPACT, - max_total_tokens=1000, # Low threshold to trigger compaction - max_tool_message_tokens=100, # Low threshold to compact tool messages - preview_char_length=50, # Keep 50 chars in preview - keep_recent_count=1, # Keep 1 recent tool message - store_dir="./test_compact_storage", - ) - - result = op.context.response.answer - logger.info(f"✓ COMPACT mode result: {len(result)} messages") - logger.info(f" Success: {op.context.response.success}") - - -async def test_compress_mode(): - """Test COMPRESS mode - Only apply compression with MessageOffloadOp.""" - logger.info("\n" + "=" * 60) - logger.info("Test: COMPRESS mode - Only apply compression") - logger.info("=" * 60) - - # Create test messages with system, user, assistant, tool sequence - messages = [ - Message(role=Role.SYSTEM, content="You are a helpful assistant."), - Message(role=Role.USER, content="What is the weather today?"), - Message( - role=Role.ASSISTANT, - content="I'll check the weather for you.", - ), - Message( - role=Role.TOOL, - content="A" * 5000, # Large tool message that should be compacted - tool_call_id="call_001", - ), - Message( - role=Role.ASSISTANT, - content="Let me also check the forecast.", - ), - Message( - role=Role.TOOL, - content="B" * 5000, # Another large tool message - tool_call_id="call_002", - ), - Message( - role=Role.USER, - content="What about tomorrow?", - ), - Message( - role=Role.ASSISTANT, - content="I'll check tomorrow's weather.", - ), - Message( - role=Role.TOOL, - content="C" * 5000, # Third large tool message - tool_call_id="call_003", - ), - Message( - role=Role.TOOL, - content="Recent result", # Recent tool message (should be kept) - tool_call_id="call_004", - ), - ] - - op = MessageOffloadOp() >> BatchWriteFileOp() - - await op.async_call( - messages=[m.model_dump() for m in messages], - context_manage_mode=WorkingSummaryMode.COMPRESS, - max_total_tokens=2000, # Low threshold to trigger compression - keep_recent_count=2, - store_dir="./test_compact_storage", - ) - - result = op.context.response.answer - logger.info(f"✓ COMPRESS mode result: {len(result)} messages") - logger.info(f" Success: {op.context.response.success}") - - -async def test_auto_mode(): - """Test AUTO mode - Apply compaction first, then compression if needed using MessageOffloadOp.""" - logger.info("\n" + "=" * 60) - logger.info("Test: AUTO mode - Apply compaction first, then compression if needed") - logger.info("=" * 60) - - # Create messages with extensive user content to ensure compact ratio exceeds threshold - auto_messages = [ - Message(role=Role.SYSTEM, content="You are a helpful assistant."), - Message(role=Role.USER, content="What is the weather today?"), - Message( - role=Role.ASSISTANT, - content="I'll check the weather for you.", - ), - Message( - role=Role.TOOL, - content="A" * 5000, # Large tool message that should be compacted - tool_call_id="call_001", - ), - Message( - role=Role.USER, - content="I need detailed information about the weather forecast for the next week. " - "Please provide temperature, humidity, wind speed, and precipitation chances for each day. " - "Also, I want to know about any weather warnings or advisories. " - "This is very important for my travel planning." * 50, # Long user message - ), - Message( - role=Role.ASSISTANT, - content="I'll gather comprehensive weather information for you. Let me check multiple sources." * 3, - ), - Message( - role=Role.TOOL, - content="B" * 5000, # Another large tool message - tool_call_id="call_002", - ), - Message( - role=Role.USER, - content="Can you also provide information about air quality, UV index, and sunrise/sunset times? " - "I'm planning outdoor activities and need to know the best times to be outside. " - "Also, please include historical weather data for comparison." * 4, # More long user content - ), - Message( - role=Role.ASSISTANT, - content="Absolutely! I'll get all that information for you including air quality metrics and UV data." * 2, - ), - Message( - role=Role.TOOL, - content="C" * 5000, # Third large tool message - tool_call_id="call_003", - ), - Message( - role=Role.USER, - content="What about tomorrow?", - ), - Message( - role=Role.ASSISTANT, - content="I'll check tomorrow's weather.", - ), - Message( - role=Role.TOOL, - content="Recent result", # Recent tool message (should be kept) - tool_call_id="call_004", - ), - ] - - op = MessageOffloadOp() >> BatchWriteFileOp() - - await op.async_call( - messages=[m.model_dump() for m in auto_messages], - context_manage_mode=WorkingSummaryMode.AUTO, - compact_ratio_threshold=0.2, # Low threshold, should trigger compression after compact - max_total_tokens=1000, - max_tool_message_tokens=100, - preview_char_length=50, - keep_recent_count=1, - store_dir="./test_compact_storage", - ) - - result = op.context.response.answer - logger.info(f"✓ AUTO mode result: {len(result)} messages") - logger.info(f" Success: {op.context.response.success}") - - -async def async_main(): - """Test function for MessageOffloadOp.""" - async with ReMeApp(): - logger.info("=" * 80) - logger.info("Testing MessageOffloadOp - Context Management Orchestration") - logger.info("=" * 80) - - await test_compact_mode() - await test_compress_mode() - await test_auto_mode() - - logger.info("\n" + "=" * 80) - logger.info("All tests completed!") - logger.info("=" * 80) - - -if __name__ == "__main__": - asyncio.run(async_main()) diff --git a/tests/copaw/test_compactor.py b/tests/light/test_compactor.py similarity index 99% rename from tests/copaw/test_compactor.py rename to tests/light/test_compactor.py index d044f376..32dfd9d3 100644 --- a/tests/copaw/test_compactor.py +++ b/tests/light/test_compactor.py @@ -10,7 +10,7 @@ from test_utils import ( get_formatter, get_token_counter, ) -from reme.memory.file_based_copaw import Compactor +from reme.memory.file_based import Compactor # 配置日志输出到控制台 logging.basicConfig( diff --git a/tests/copaw/test_memory_formatter.py b/tests/light/test_memory_formatter.py similarity index 99% rename from tests/copaw/test_memory_formatter.py rename to tests/light/test_memory_formatter.py index 7da3db2d..00f45bb1 100644 --- a/tests/copaw/test_memory_formatter.py +++ b/tests/light/test_memory_formatter.py @@ -7,7 +7,7 @@ import logging from agentscope.message import Msg from test_utils import get_token_counter -from reme.memory.file_based_copaw import MemoryFormatter +from reme.memory.file_based import MemoryFormatter # 配置日志输出到控制台 logging.basicConfig( diff --git a/tests/light/test_reme_light.py b/tests/light/test_reme_light.py new file mode 100644 index 00000000..9e2ac70a --- /dev/null +++ b/tests/light/test_reme_light.py @@ -0,0 +1,198 @@ +"""测试 ReMeLight""" + +import asyncio + +from agentscope.message import Msg +from reme.reme_light import ReMeLight + + +# ==================== 消息创建辅助函数 ==================== +def create_user_msg(content: str) -> Msg: + """创建用户消息""" + return Msg(name="user", role="user", content=content) + + +def create_assistant_msg(content: str) -> Msg: + """创建助手消息""" + return Msg(name="assistant", role="assistant", content=content) + + +def create_tool_use_msg(tool_id: str, tool_name: str, tool_input: dict) -> Msg: + """创建工具调用消息""" + return Msg( + name="assistant", + role="assistant", + content=[ + { + "type": "tool_use", + "id": tool_id, + "name": tool_name, + "input": tool_input, + }, + ], + ) + + +def create_tool_result_msg(tool_id: str, tool_name: str, output: str) -> Msg: + """创建工具结果消息""" + return Msg( + name="tool", + role="user", + content=[ + { + "type": "tool_result", + "id": tool_id, + "name": tool_name, + "output": output, + }, + ], + ) + + +def create_thinking_msg(thinking_content: str) -> Msg: + """创建思考消息""" + return Msg( + name="assistant", + role="assistant", + content=[ + { + "type": "thinking", + "text": thinking_content, + }, + ], + ) + + +# ==================== 构建模拟对话历史 ==================== +def build_sample_messages() -> list[Msg]: + """构建一段包含多种消息类型的模拟对话""" + messages = [ + # 用户询问 Python 版本 + create_user_msg("我想设置一个 Python 开发环境，你有什么建议？"), + # 助手思考 + create_thinking_msg("用户想要搭建 Python 开发环境，我需要了解他的需求和偏好..."), + # 助手回复 + create_assistant_msg( + "好的！我建议使用 Python 3.11 或 3.12 版本，它们性能更好且功能丰富。" + "你希望用于什么类型的开发？Web、数据科学还是其他？", + ), + # 用户提供更多信息 + create_user_msg("主要是做 Web 开发，使用 FastAPI 框架。另外我喜欢用 pyenv 管理版本。"), + # 助手调用工具查询 + create_tool_use_msg( + tool_id="call_001", + tool_name="search_web", + tool_input={"query": "FastAPI Python version compatibility 2024"}, + ), + # 工具返回结果（模拟较长的输出） + create_tool_result_msg( + tool_id="call_001", + tool_name="search_web", + output=( + "FastAPI 官方推荐使用 Python 3.8+ 版本，但 3.11/3.12 性能最佳。\n" + "主要依赖：\n" + "- Starlette: ASGI 框架\n" + "- Pydantic v2: 数据验证\n" + "- Uvicorn: ASGI 服务器\n" + "最新版本 FastAPI 0.109+ 完全支持 Python 3.12。\n" + "建议搭配 uv 或 pip-tools 进行依赖管理。" + ), + ), + # 助手总结建议 + create_assistant_msg( + "根据查询结果，我的建议是：\n" + "1. **Python 版本**: 使用 Python 3.11 或 3.12（通过 pyenv 安装）\n" + "2. **框架**: FastAPI 0.109+ 完全兼容这些版本\n" + "3. **依赖管理**: 推荐使用 uv（更快）或 pip-tools\n" + "4. **ASGI 服务器**: Uvicorn 配合 gunicorn 用于生产环境\n\n" + "需要我帮你生成一个项目模板吗？", + ), + # 用户确认偏好 + create_user_msg("好的，我决定用 Python 3.12 + FastAPI + uv。请记住我的这些偏好。"), + # 助手确认 + create_assistant_msg( + "已记录你的开发偏好：\n" + "- Python 版本: 3.12 (通过 pyenv 管理)\n" + "- Web 框架: FastAPI\n" + "- 包管理器: uv\n" + "以后有相关问题我会参考这些偏好给你建议！", + ), + ] + return messages + + +# ==================== 主测试流程 ==================== +async def main(): + """ReMeLight 主测试流程，演示完整的记忆管理功能。""" + # 初始化 ReMeLight + reme = ReMeLight( + working_dir=".reme", # 记忆文件存储目录 + max_input_length=128000, # 模型上下文窗口（tokens） + memory_compact_ratio=0.7, # 达到 max_input_length * 0.7 时触发压缩 + language="zh", # 摘要语言（zh / ""） + tool_result_threshold=1000, # 超过此字符数的工具输出自动转存 + retention_days=7, # tool_result/ 文件保留天数 + ) + await reme.start() + print("=" * 60) + print("ReMeLight 已启动") + print("=" * 60) + + # 构建模拟对话历史 + messages = build_sample_messages() + print(f"\n[原始消息数量]: {len(messages)} 条") + + # 1. 压缩超长工具输出（防止工具结果撑爆上下文） + print("\n" + "-" * 40) + print("[步骤 1] 压缩超长工具输出...") + messages = await reme.compact_tool_result(messages) + print(f"处理后消息数量: {len(messages)} 条") + + # 2. 将历史对话压缩为结构化摘要（触发时机：上下文接近上限） + print("\n" + "-" * 40) + print("[步骤 2] 生成结构化压缩摘要...") + summary = await reme.compact_memory( + messages=messages, + previous_summary="", # 可传入上轮摘要，实现增量更新 + ) + print(f"压缩摘要:\n{summary[:500]}..." if len(summary) > 500 else f"压缩摘要:\n{summary}") + + # 3. 后台异步提交摘要任务（不阻塞对话，摘要写入 memory/YYYY-MM-DD.md） + print("\n" + "-" * 40) + print("[步骤 3] 提交后台异步摘要任务...") + reme.add_async_summary_task(messages=messages) + print("异步任务已提交") + + # 4. 语义搜索记忆（向量 + BM25 混合检索） + print("\n" + "-" * 40) + print("[步骤 4] 语义搜索记忆...") + result = await reme.memory_search(query="Python 版本偏好", max_results=5) + print(f"搜索结果: {result}") + + # 5. 获取会话内存实例（ReMeInMemoryMemory，管理单次对话的上下文） + print("\n" + "-" * 40) + print("[步骤 5] 获取会话内存实例并估算 Token 使用...") + memory = reme.get_in_memory_memory() + # 将消息添加到内存中以便估算 + for msg in messages: + await memory.add(msg) + token_stats = await memory.estimate_tokens() + print(f"当前上下文使用率: {token_stats['context_usage_ratio']:.1f}%") + print(f"消息 Token 数: {token_stats['messages_tokens']}") + print(f"预估总 Token 数: {token_stats['estimated_tokens']}") + + # 6. 关闭前等待后台任务完成 + print("\n" + "-" * 40) + print("[步骤 6] 等待后台任务完成...") + summary_result = await reme.await_summary_tasks() + print(f"后台摘要任务完成，结果长度: {len(summary_result)} 字符") + + # 关闭 ReMeLight + await reme.close() + print("\n" + "=" * 60) + print("ReMeLight 已关闭") + print("=" * 60) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/copaw/test_summarizer.py b/tests/light/test_summarizer.py similarity index 99% rename from tests/copaw/test_summarizer.py rename to tests/light/test_summarizer.py index d0f7eb5f..560efabd 100644 --- a/tests/copaw/test_summarizer.py +++ b/tests/light/test_summarizer.py @@ -13,7 +13,7 @@ from test_utils import ( get_formatter, get_token_counter, ) -from reme.memory.file_based_copaw import Summarizer +from reme.memory.file_based import Summarizer # 配置日志输出到控制台 logging.basicConfig( diff --git a/tests/copaw/test_tool_result_compactor.py b/tests/light/test_tool_result_compactor.py similarity index 97% rename from tests/copaw/test_tool_result_compactor.py rename to tests/light/test_tool_result_compactor.py index ba225255..059e626e 100644 --- a/tests/copaw/test_tool_result_compactor.py +++ b/tests/light/test_tool_result_compactor.py @@ -7,8 +7,8 @@ from pathlib import Path from agentscope.message import Msg -from reme.memory.file_based_copaw.tool_result_compactor import ToolResultCompactor -from reme.memory.file_based_copaw.utils import TRUNCATION_MARKER_START +from reme.memory.file_based.tool_result_compactor import ToolResultCompactor +from reme.memory.file_based.utils import TRUNCATION_MARKER_START def create_tool_result_msg(output: str | list, tool_name: str = "test_tool") -> Msg: diff --git a/tests/copaw/test_utils.py b/tests/light/test_utils.py similarity index 97% rename from tests/copaw/test_utils.py rename to tests/light/test_utils.py index f5da46e2..f5cae021 100644 --- a/tests/copaw/test_utils.py +++ b/tests/light/test_utils.py @@ -64,7 +64,7 @@ def get_formatter(): """Get formatter instance.""" from agentscope.formatter import OpenAIChatFormatter from agentscope.token import HuggingFaceTokenCounter - from reme.memory.file_based_copaw.utils import _extract_text_from_messages + from reme.memory.file_based.utils import _extract_text_from_messages class ReMeChatFormatter(OpenAIChatFormatter): """ReMe chat formatter class."""