From 92f76566db31aad0dfe3a10293f9e11c43582609 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Thu, 25 Sep 2025 10:57:27 +0800 Subject: [PATCH 01/13] ``` docs(readme): update latest updates section with new ReMe releases and directions - Add entry for2025-09-25 about tool memory and Personal Memory Application/Agent - Update ReMe v0.1.9 release information with agentscope-runtime integration - Maintain existing content about asynchronous operations support ``` --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9c9add8f..cca3f146 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ Personal memory helps "**understand user preferences**", while task memory helps ## 📰 Latest Updates -- **[2025-09]** 🎉 ReMe v0.1.8 has been officially released, adding support for asynchronous operations. It has also been +- **[2025-09-25]** 🎉 ReMe is exploring the directions of tool memory and Personal Memory Application/Agent. +- **[2025-09]** 🎉 ReMe v0.1.9 has been officially released, adding support for asynchronous operations. It has also been integrated into the memory service of agentscope-runtime. - **[2025-09]** 🎉 ReMe v0.1 officially released, integrating task memory and personal memory. If you want to use the original memoryscope project, you can find it From b9da8d2a7fc6d10e27826e5f4da97c3cdbd2dc80 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Thu, 16 Oct 2025 10:50:27 +0800 Subject: [PATCH 02/13] feat(memory): add tool call result handling and memory management - Add new ToolCallResult and ToolMemory schemas for tracking tool executions - Implement tool memory retrieval and summarization flows in default config - Register new parse_tool_call_result_op for processing tool call results - Extend memory conversion logic to support tool memory type - Add test cases for tool memory serialization and deserialization - Include token counting utilities for text processing tasks --- reme_ai/config/default.yaml | 32 ++- reme_ai/retrieve/__init__.py | 1 + reme_ai/retrieve/tool/__init__.py | 0 reme_ai/schema/memory.py | 197 +++++++++++++++++- reme_ai/summary/__init__.py | 1 + reme_ai/summary/tool/__init__.py | 1 + .../summary/tool/parse_tool_call_result_op.py | 13 ++ test/http_client_test.py | 31 ++- test/test5.py | 6 + test/test6.py | 13 ++ 10 files changed, 280 insertions(+), 15 deletions(-) create mode 100644 reme_ai/retrieve/tool/__init__.py create mode 100644 reme_ai/summary/tool/__init__.py create mode 100644 reme_ai/summary/tool/parse_tool_call_result_op.py create mode 100644 test/test5.py create mode 100644 test/test6.py diff --git a/reme_ai/config/default.yaml b/reme_ai/config/default.yaml index 7984694a..4cefb810 100644 --- a/reme_ai/config/default.yaml +++ b/reme_ai/config/default.yaml @@ -49,13 +49,31 @@ flow: description: "A list of conversation trajectory information, including message content and score. This field does not need to be filled in, the system will complete it automatically." required: false - retrieve_task_memory_simple: - flow_content: build_query_op >> recall_vector_store_op >> merge_memory_op + retrieve_tool_memory: + flow_content: parse_tool_call_result_op + description: "Retrieve tool related memories from the memory database based on queries to improve tool recall" + input_schema: + tool_names: + type: string + description: "tool_name1,tool_name2" + required: true + + add_tool_call_result: + flow_content: parse_tool_call_result_op description: "Retrieves the most relevant top-k memory experiences from historical data based on the current query with simplified processing" input_schema: - query: + tool_call_results: + type: array + description: "tool_call_result" + required: true + + summary_tool_call_result: + flow_content: parse_tool_call_result_op + description: "Retrieves the most relevant top-k memory experiences from historical data based on the current query with simplified processing" + input_schema: + tool_names: type: string - description: "current query" + description: "tool_name1,tool_name2" required: true summary_task_memory_simple: @@ -143,6 +161,12 @@ llm: params: temperature: 0.6 + wk1: + backend: openai_compatible + model_name: qwen3-30b-a3b-instruct-2507 + params: + temperature: 0.6 + qwen3_30b_instruct: backend: openai_compatible model_name: qwen3-30b-a3b-instruct-2507 diff --git a/reme_ai/retrieve/__init__.py b/reme_ai/retrieve/__init__.py index 8922c1ab..4f26e94d 100644 --- a/reme_ai/retrieve/__init__.py +++ b/reme_ai/retrieve/__init__.py @@ -1,2 +1,3 @@ from . import personal from . import task +from . import tool diff --git a/reme_ai/retrieve/tool/__init__.py b/reme_ai/retrieve/tool/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reme_ai/schema/memory.py b/reme_ai/schema/memory.py index 8ed090cd..63a4c004 100644 --- a/reme_ai/schema/memory.py +++ b/reme_ai/schema/memory.py @@ -1,8 +1,10 @@ import datetime from abc import ABC +from typing import List from uuid import uuid4 from flowllm.schema.vector_node import VectorNode +from mcp.types import CallToolResult, TextContent from pydantic import BaseModel, Field @@ -105,6 +107,118 @@ class PersonalMemory(BaseMemory): metadata=metadata.pop("metadata", {})) +class ToolCallResult(BaseModel): + create_time: str = Field(default="", description="Time of tool invocation") + tool_name: str = Field(default=..., description="Name of the tool") + input: dict | str = Field(default="", description="Tool input") + output: str = Field(default="", description="Tool output") + token_cost: int = Field(default=-1, description="Token consumption of the tool") + success: bool = Field(default=True, description="Whether the tool invocation was successful") + time_cost: float = Field(default=0, description="Time consumed by the tool invocation, in seconds") + evaluation: str = Field(default="", description="Evaluation for the tool invocation") + score: float = Field(default=0, description="Score of the Evaluation") + # updated: bool = Field(default=False, description="Whether tool memory has been updated by `ToolCallMetadata`") + + metadata: dict = Field(default_factory=dict) + + def from_mcp_tool_result(self, tool_result: CallToolResult, max_char_len: int = None): + text_list = [] + for content in tool_result.content: + if isinstance(content, TextContent): + text_list.append(content.text) + + else: + raise NotImplementedError(f"content.type={type(content)} not supported") + content = "\n".join(text_list) + + if max_char_len: + content = content[:max_char_len] + self.output = content + + self.success = tool_result.is_error + self.metadata.update(tool_result.meta) + + +class ToolMemory(BaseMemory): + memory_type: str = Field(default="tool") + tool_call_results: List[ToolCallResult] = Field(default_factory=list) + + def to_vector_node(self) -> VectorNode: + return VectorNode(unique_id=self.memory_id, + workspace_id=self.workspace_id, + content=self.when_to_use, + metadata={ + "memory_type": self.memory_type, + "content": self.content, + "score": self.score, + "time_created": self.time_created, + "time_modified": self.time_modified, + "author": self.author, + "tool_call_results": [x.model_dump() for x in self.tool_call_results], + "metadata": self.metadata, + }) + + def statistic(self, recent_frequency: int = 20) -> dict: + """ + Calculate statistical information for the most recent 20 tool calls. + Returns avg token_cost, success rate, avg time_cost, and avg score. + """ + if not self.tool_call_results: + return { + "total_calls": 0, + "recent_calls_analyzed": 0, + "avg_token_cost": 0.0, + "success_rate": 0.0, + "avg_time_cost": 0.0, + "avg_score": 0.0 + } + + # Get the most recent 20 tool calls (or all if less than 20) + recent_calls = self.tool_call_results[-20:] + total_calls = len(self.tool_call_results) + recent_calls_count = len(recent_calls) + + # Calculate statistics + total_token_cost = sum(call.token_cost for call in recent_calls if call.token_cost >= 0) + valid_token_calls = [call for call in recent_calls if call.token_cost >= 0] + avg_token_cost = total_token_cost / len(valid_token_calls) if valid_token_calls else 0.0 + + successful_calls = sum(1 for call in recent_calls if call.success) + success_rate = successful_calls / recent_calls_count if recent_calls_count > 0 else 0.0 + + total_time_cost = sum(call.time_cost for call in recent_calls) + avg_time_cost = total_time_cost / recent_calls_count if recent_calls_count > 0 else 0.0 + + total_score = sum(call.score for call in recent_calls) + avg_score = total_score / recent_calls_count if recent_calls_count > 0 else 0.0 + + return { + "total_calls": total_calls, + "recent_calls_analyzed": recent_calls_count, + "avg_token_cost": round(avg_token_cost, 2), + "success_rate": round(success_rate, 4), + "avg_time_cost": round(avg_time_cost, 3), + "avg_score": round(avg_score, 3) + } + + @classmethod + def from_vector_node(cls, node: VectorNode) -> "ToolMemory": + metadata = node.metadata.copy() + tool_call_results = [ToolCallResult(**result) for result in metadata.pop("tool_call_results", [])] + return cls(workspace_id=node.workspace_id, + memory_id=node.unique_id, + when_to_use=node.content, + memory_type=metadata.pop("memory_type"), + content=metadata.pop("content"), + score=metadata.pop("score"), + time_created=metadata.pop("time_created"), + time_modified=metadata.pop("time_modified"), + author=metadata.pop("author"), + tool_call_results=tool_call_results, + metadata=metadata.pop("metadata", {})) + + + def vector_node_to_memory(node: VectorNode) -> BaseMemory: memory_type = node.metadata.get("memory_type") if memory_type == "task": @@ -113,6 +227,9 @@ def vector_node_to_memory(node: VectorNode) -> BaseMemory: elif memory_type == "personal": return PersonalMemory.from_vector_node(node) + elif memory_type == "tool": + return ToolMemory.from_vector_node(node) + else: raise RuntimeError(f"memory_type={memory_type} not supported!") @@ -125,11 +242,14 @@ def dict_to_memory(memory_dict: dict): elif memory_type == "personal": return PersonalMemory(**memory_dict) + elif memory_type == "tool": + return ToolMemory(**memory_dict) + else: raise RuntimeError(f"memory_type={memory_type} not supported!") -if __name__ == "__main__": +def task_main(): e1 = TaskMemory( workspace_id="w_1024", memory_id="123", @@ -142,3 +262,78 @@ if __name__ == "__main__": print(v1.model_dump_json(indent=2)) e2 = vector_node_to_memory(v1) print(e2.model_dump_json(indent=2)) + + +def personal_main(): + p1 = PersonalMemory( + workspace_id="w_2048", + memory_id="456", + when_to_use="personal memory test case", + content="personal test content", + target="user_preferences", + reflection_subject="learning_style", + score=0.85, + metadata={"category": "user_profile"}) + print("PersonalMemory test:") + print(p1.model_dump_json(indent=2)) + v1 = p1.to_vector_node() + print("VectorNode:") + print(v1.model_dump_json(indent=2)) + p2 = vector_node_to_memory(v1) + print("Reconstructed PersonalMemory:") + print(p2.model_dump_json(indent=2)) + + +def tool_main(): + # Create sample tool call results + tool_result1 = ToolCallResult( + create_time="2025-10-15 10:30:00", + tool_name="file_reader", + input={"file_path": "/test/file.txt"}, + output="File content successfully read", + token_cost=50, + success=True, + time_cost=0.5, + evaluation="Successfully executed", + score=0.95 + ) + + tool_result2 = ToolCallResult( + create_time="2025-10-15 10:31:00", + tool_name="data_processor", + input={"data": "sample_data", "format": "json"}, + output="Data processed successfully", + token_cost=75, + success=True, + time_cost=1.2, + evaluation="Good performance", + score=0.88 + ) + + t1 = ToolMemory( + workspace_id="w_4096", + memory_id="789", + memory_type="tool", + when_to_use="tool execution memory test", + content="tool execution test content", + score=0.92, + tool_call_results=[tool_result1, tool_result2], + metadata={"execution_context": "test_environment"}) + + print("ToolMemory test:") + print(t1.model_dump_json(indent=2)) + v1 = t1.to_vector_node() + print("VectorNode:") + print(v1.model_dump_json(indent=2)) + t2 = ToolMemory.from_vector_node(v1) + print("Reconstructed ToolMemory:") + print(t2.model_dump_json(indent=2)) + + +if __name__ == "__main__": + print("=== Task Memory Test ===") + # task_main() + print("\n=== Personal Memory Test ===") + # personal_main() + print("\n=== Tool Memory Test ===") + tool_main() diff --git a/reme_ai/summary/__init__.py b/reme_ai/summary/__init__.py index 8922c1ab..4f26e94d 100644 --- a/reme_ai/summary/__init__.py +++ b/reme_ai/summary/__init__.py @@ -1,2 +1,3 @@ from . import personal from . import task +from . import tool diff --git a/reme_ai/summary/tool/__init__.py b/reme_ai/summary/tool/__init__.py new file mode 100644 index 00000000..384b8389 --- /dev/null +++ b/reme_ai/summary/tool/__init__.py @@ -0,0 +1 @@ +from .parse_tool_call_result_op import ParseToolCallResultOp diff --git a/reme_ai/summary/tool/parse_tool_call_result_op.py b/reme_ai/summary/tool/parse_tool_call_result_op.py new file mode 100644 index 00000000..b5198279 --- /dev/null +++ b/reme_ai/summary/tool/parse_tool_call_result_op.py @@ -0,0 +1,13 @@ +from flowllm import C, BaseAsyncOp +from loguru import logger + +@C.register_op() +class ParseToolCallResultOp(BaseAsyncOp): + file_path: str = __file__ + + async def async_execute(self): + tool_call_results: list = self.context.get("tool_call_results", []) + logger.info(f"tool_call_results={tool_call_results}") + + + self.context.response.answer = "haha" \ No newline at end of file diff --git a/test/http_client_test.py b/test/http_client_test.py index 0e276ff3..064d9c9f 100644 --- a/test/http_client_test.py +++ b/test/http_client_test.py @@ -110,23 +110,34 @@ async def run2(session): result = await response.json() print(json.dumps(result, ensure_ascii=False)) + +async def run3(session): + workspace_id = "default2" + + async with session.post( + f"{base_url}/add_tool_call_result", + json={ + "tool_call_results": [ + {"a": 1}, + {"a": 2}, + ], + "workspace_id": workspace_id, + }, + headers={"Content-Type": "application/json"} + ) as response: + result = await response.json() + print(json.dumps(result, ensure_ascii=False)) + + async def main(): async with aiohttp.ClientSession() as session: # 获取工具列表 print("获取工具列表...") - async with session.get(f"{base_url}/list") as response: - if response.status == 200: - tools = await response.json() - print("可用工具:") - for tool in tools: - print(json.dumps(tool, ensure_ascii=False)) - else: - print(f"获取工具列表失败: {response.status}") - return # await run1(session) - await run2(session) + # await run2(session) + await run3(session) if __name__ == "__main__": asyncio.run(main()) diff --git a/test/test5.py b/test/test5.py new file mode 100644 index 00000000..d0845c55 --- /dev/null +++ b/test/test5.py @@ -0,0 +1,6 @@ +import tiktoken +enc = tiktoken.get_encoding("o200k_base") + +# r = enc.encode("我爱吃西瓜，你说啥") +r = enc.encode("hello world aaaaaaaaaaaa") +print(len(r)) \ No newline at end of file diff --git a/test/test6.py b/test/test6.py new file mode 100644 index 00000000..66ceae98 --- /dev/null +++ b/test/test6.py @@ -0,0 +1,13 @@ +import tiktoken + +def count_tokens(text: str) -> int: + """计算给定文本在指定模型下的 token 数量""" + encoding = tiktoken.get_encoding("o200k_base") + tokens = encoding.encode(text) + return len(tokens) + +# 示例使用 +text = "你好，世界！Hello, world!" +token_count = count_tokens(text) +print(f"Token 数量: {token_count}") +print(len(text) / 4) From b07213791fda6a6c1364c2878e107ea60a751b8c Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Thu, 16 Oct 2025 15:49:49 +0800 Subject: [PATCH 03/13] feat(tool): implement tool memory management and summarization - Added SummaryToolMemoryOp for analyzing tool usage patterns- Implemented RetrieveToolMemoryOp for fetching tool memories - Enhanced ParseToolCallResultOp with evaluation and scoring logic - Updated ToolMemory schema with summary, evaluation, and score fields - Added new prompt templates for tool evaluation and summarization - Configured tool memory operations in default.yaml - Fixed success flag logic in ToolCallResult processing - Improved statistical analysis for tool call history - Added concurrent processing for tool call evaluations - Integrated tool memory updates with vector store operations --- reme_ai/config/default.yaml | 39 +- .../retrieve/tool/retrieve_tool_memory_op.py | 71 ++++ reme_ai/schema/memory.py | 16 +- reme_ai/summary/tool/__init__.py | 1 + .../summary/tool/parse_tool_call_result_op.py | 168 ++++++++- .../tool/parse_tool_call_result_prompt.yaml | 46 +++ .../summary/tool/summary_tool_memory_op.py | 351 ++++++++++++++++++ .../tool/summary_tool_memory_prompt.yaml | 27 ++ 8 files changed, 699 insertions(+), 20 deletions(-) create mode 100644 reme_ai/retrieve/tool/retrieve_tool_memory_op.py create mode 100644 reme_ai/summary/tool/parse_tool_call_result_prompt.yaml create mode 100644 reme_ai/summary/tool/summary_tool_memory_op.py create mode 100644 reme_ai/summary/tool/summary_tool_memory_prompt.yaml diff --git a/reme_ai/config/default.yaml b/reme_ai/config/default.yaml index 4cefb810..e5b91518 100644 --- a/reme_ai/config/default.yaml +++ b/reme_ai/config/default.yaml @@ -50,30 +50,34 @@ flow: required: false retrieve_tool_memory: - flow_content: parse_tool_call_result_op - description: "Retrieve tool related memories from the memory database based on queries to improve tool recall" + flow_content: retrieve_tool_memory_op + description: "Retrieves tool memories from the vector database based on tool names to provide tool usage patterns and best practices" input_schema: tool_names: type: string - description: "tool_name1,tool_name2" + description: "Comma-separated tool names (e.g., 'tool_name1,tool_name2')" required: true add_tool_call_result: - flow_content: parse_tool_call_result_op - description: "Retrieves the most relevant top-k memory experiences from historical data based on the current query with simplified processing" + flow_content: parse_tool_call_result_op >> update_vector_store_op + description: "Evaluates and adds tool call results to the tool memory database, creating new memory or updating existing memory for the specified tool" input_schema: + tool_name: + type: string + description: "The name of the tool to add call results for" + required: true tool_call_results: type: array - description: "tool_call_result" + description: "List of tool call result objects, each containing: tool_name, input, output, success, time_cost, token_cost, create_time" required: true - summary_tool_call_result: - flow_content: parse_tool_call_result_op - description: "Retrieves the most relevant top-k memory experiences from historical data based on the current query with simplified processing" + summary_tool_memory: + flow_content: summary_tool_memory_op >> update_vector_store_op + description: "Analyzes tool call history and generates comprehensive usage patterns, best practices, and recommendations for the specified tools" input_schema: tool_names: type: string - description: "tool_name1,tool_name2" + description: "Comma-separated tool names to summarize (e.g., 'tool_name1,tool_name2')" required: true summary_task_memory_simple: @@ -154,6 +158,21 @@ op: params: enable_llm_rewrite: true + # tool memory ops + parse_tool_call_result_op: + backend: parse_tool_call_result_op + llm: default + params: + max_history_tool_call_cnt: 100 + evaluation_sleep_interval: 1.0 + + summary_tool_memory_op: + backend: summary_tool_memory_op + llm: default + params: + recent_call_count: 20 + summary_sleep_interval: 1.0 + llm: default: backend: openai_compatible diff --git a/reme_ai/retrieve/tool/retrieve_tool_memory_op.py b/reme_ai/retrieve/tool/retrieve_tool_memory_op.py new file mode 100644 index 00000000..1b909a01 --- /dev/null +++ b/reme_ai/retrieve/tool/retrieve_tool_memory_op.py @@ -0,0 +1,71 @@ +from typing import List + +from flowllm import C, BaseAsyncOp +from flowllm.schema.vector_node import VectorNode +from loguru import logger + +from reme_ai.schema.memory import ToolMemory, vector_node_to_memory + + +@C.register_op() +class RetrieveToolMemoryOp(BaseAsyncOp): + file_path: str = __file__ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + async def async_execute(self): + tool_names: str = self.context.get("tool_names", "") + workspace_id: str = self.context.workspace_id + + if not tool_names: + logger.warning("tool_names is empty, skipping processing") + self.context.response.answer = "tool_names is required" + self.context.response.success = False + return + + # Split tool names by comma + tool_name_list = [name.strip() for name in tool_names.split(",") if name.strip()] + logger.info(f"workspace_id={workspace_id} retrieving {len(tool_name_list)} tools: {tool_name_list}") + + # Search for each tool in the vector store + matched_tool_memories: List[ToolMemory] = [] + + for tool_name in tool_name_list: + nodes: List[VectorNode] = await self.vector_store.async_search( + query=tool_name, + workspace_id=workspace_id, + top_k=1 + ) + + if nodes: + top_node = nodes[0] + memory = vector_node_to_memory(top_node) + + # Ensure it's a ToolMemory and when_to_use matches + if isinstance(memory, ToolMemory) and memory.when_to_use == tool_name: + matched_tool_memories.append(memory) + logger.info(f"Found tool_memory for tool_name={tool_name}, " + f"memory_id={memory.memory_id}, " + f"total_calls={len(memory.tool_call_results)}") + else: + logger.warning(f"No exact match found for tool_name={tool_name}") + else: + logger.warning(f"No memory found for tool_name={tool_name}") + + if not matched_tool_memories: + logger.info("No matching tool memories found") + self.context.response.answer = "No matching tool memories found" + self.context.response.success = False + return + + # Set response + self.context.response.answer = f"Successfully retrieved {len(matched_tool_memories)} tool memories" + self.context.response.success = True + self.context.response.metadata["memory_list"] = matched_tool_memories + + # Log retrieval results + for memory in matched_tool_memories: + logger.info(f"Retrieved tool: {memory.when_to_use}, " + f"total_calls={len(memory.tool_call_results)}, " + f"content_length={len(memory.content)}") diff --git a/reme_ai/schema/memory.py b/reme_ai/schema/memory.py index 63a4c004..a1dd6a31 100644 --- a/reme_ai/schema/memory.py +++ b/reme_ai/schema/memory.py @@ -115,9 +115,9 @@ class ToolCallResult(BaseModel): token_cost: int = Field(default=-1, description="Token consumption of the tool") success: bool = Field(default=True, description="Whether the tool invocation was successful") time_cost: float = Field(default=0, description="Time consumed by the tool invocation, in seconds") - evaluation: str = Field(default="", description="Evaluation for the tool invocation") - score: float = Field(default=0, description="Score of the Evaluation") - # updated: bool = Field(default=False, description="Whether tool memory has been updated by `ToolCallMetadata`") + summary: str = Field(default="", description="Brief summary of the tool call result") + evaluation: str = Field(default="", description="Detailed evaluation for the tool invocation") + score: float = Field(default=0, description="Score of the Evaluation (0.0, 0.5, or 1.0)") metadata: dict = Field(default_factory=dict) @@ -135,7 +135,7 @@ class ToolCallResult(BaseModel): content = content[:max_char_len] self.output = content - self.success = tool_result.is_error + self.success = not tool_result.is_error self.metadata.update(tool_result.meta) @@ -160,7 +160,7 @@ class ToolMemory(BaseMemory): def statistic(self, recent_frequency: int = 20) -> dict: """ - Calculate statistical information for the most recent 20 tool calls. + Calculate statistical information for the most recent N tool calls. Returns avg token_cost, success rate, avg time_cost, and avg score. """ if not self.tool_call_results: @@ -173,8 +173,8 @@ class ToolMemory(BaseMemory): "avg_score": 0.0 } - # Get the most recent 20 tool calls (or all if less than 20) - recent_calls = self.tool_call_results[-20:] + # Get the most recent N tool calls (or all if less than N) + recent_calls = self.tool_call_results[-recent_frequency:] total_calls = len(self.tool_call_results) recent_calls_count = len(recent_calls) @@ -219,7 +219,7 @@ class ToolMemory(BaseMemory): -def vector_node_to_memory(node: VectorNode) -> BaseMemory: +def vector_node_to_memory(node: VectorNode): memory_type = node.metadata.get("memory_type") if memory_type == "task": return TaskMemory.from_vector_node(node) diff --git a/reme_ai/summary/tool/__init__.py b/reme_ai/summary/tool/__init__.py index 384b8389..e0b06b7c 100644 --- a/reme_ai/summary/tool/__init__.py +++ b/reme_ai/summary/tool/__init__.py @@ -1 +1,2 @@ from .parse_tool_call_result_op import ParseToolCallResultOp +from .summary_tool_memory_op import SummaryToolMemoryOp diff --git a/reme_ai/summary/tool/parse_tool_call_result_op.py b/reme_ai/summary/tool/parse_tool_call_result_op.py index b5198279..1a28fdbc 100644 --- a/reme_ai/summary/tool/parse_tool_call_result_op.py +++ b/reme_ai/summary/tool/parse_tool_call_result_op.py @@ -1,13 +1,177 @@ +import asyncio +from typing import List + from flowllm import C, BaseAsyncOp +from flowllm.enumeration.role import Role +from flowllm.schema.message import Message +from flowllm.schema.vector_node import VectorNode +from flowllm.utils.common_utils import extract_content from loguru import logger +from reme_ai.schema.memory import ToolMemory, ToolCallResult, vector_node_to_memory + + @C.register_op() class ParseToolCallResultOp(BaseAsyncOp): file_path: str = __file__ + def __init__(self, + max_history_tool_call_cnt: int = 100, + evaluation_sleep_interval: float = 1.0, + **kwargs): + super().__init__(**kwargs) + self.max_history_tool_call_cnt: int = max_history_tool_call_cnt + self.evaluation_sleep_interval: float = evaluation_sleep_interval + + async def _evaluate_single_tool_call(self, tool_call_result: ToolCallResult, index: int) -> ToolCallResult: + await asyncio.sleep(self.evaluation_sleep_interval * index) + + prompt = self.prompt_format( + prompt_name="evaluate_tool_call_prompt", + tool_name=tool_call_result.tool_name, + input_params=str(tool_call_result.input), + output=tool_call_result.output, + success_flag=str(tool_call_result.success), + time_cost=tool_call_result.time_cost, + token_cost=tool_call_result.token_cost) + + def parse_evaluation(message: Message) -> ToolCallResult: + content = message.content.strip() + eval_data = extract_content(content, "json") + + # 更新 tool_call_result - 包含 summary, evaluation 和 score + tool_call_result.summary = eval_data.get("summary", "") + tool_call_result.evaluation = eval_data.get("evaluation", "") + tool_call_result.score = float(eval_data.get("score", 0.0)) + + # 验证 score 是否符合 3 档要求 (0.0, 0.5, 1.0) + if tool_call_result.score not in [0.0, 0.5, 1.0]: + logger.warning(f"Score {tool_call_result.score} not in [0.0, 0.5, 1.0], rounding to nearest") + if tool_call_result.score < 0.25: + tool_call_result.score = 0.0 + elif tool_call_result.score < 0.75: + tool_call_result.score = 0.5 + else: + tool_call_result.score = 1.0 + + logger.info(f"Evaluated tool call {index}: tool_name={tool_call_result.tool_name}, " + f"score={tool_call_result.score}, summary={tool_call_result.summary[:50]}...") + return tool_call_result + + # 调用 LLM 进行评估 + result = await self.llm.achat(messages=[Message(role=Role.USER, content=prompt)], callback_fn=parse_evaluation) + + return result + async def async_execute(self): tool_call_results: list = self.context.get("tool_call_results", []) - logger.info(f"tool_call_results={tool_call_results}") + tool_call_results = [ToolCallResult(**x) if isinstance(x, dict) else x for x in tool_call_results] + tool_name: str = self.context.get("tool_name", "") + workspace_id: str = self.context.workspace_id + logger.info(f"workspace_id={workspace_id} count={len(tool_call_results)} tool_name={tool_name}") + + if not tool_name: + logger.warning("tool_name is empty, skipping processing") + self.context.response.answer = "tool_name is required" + self.context.response.success = False + return + + if not tool_call_results: + logger.info("No valid tool_call_results to process") + self.context.response.answer = "No valid tool_call_results" + self.context.response.success = False + return + + # 并发评估所有 tool_call_results + logger.info(f"Starting concurrent evaluation of {len(tool_call_results)} tool call results") + + # 使用基类的 submit_async_task 提交所有评估任务 + for index, tool_call_result in enumerate(tool_call_results): + self.submit_async_task(self._evaluate_single_tool_call, tool_call_result, index) + + # 使用基类的 join_async_task 等待所有任务完成 + # 注意: 基类已经过滤掉异常,返回的只包含成功的结果 + tool_call_results = await self.join_async_task(return_exceptions=True) + logger.info(f"Completed evaluation of {len(tool_call_results)} tool call results") + + nodes: List[VectorNode] = await self.vector_store.async_search(query=tool_name, + workspace_id=workspace_id, + top_k=1) + + tool_memory: ToolMemory | None = None + exist_node: bool = False + + if nodes: + top_node = nodes[0] + memory: ToolMemory = vector_node_to_memory(top_node) + + # 确保是 ToolMemory 类型且 when_to_use 与 tool_name 匹配 + if isinstance(memory, ToolMemory) and memory.when_to_use == tool_name: + tool_memory = memory + exist_node = True + logger.info(f"Found existing tool_memory for tool_name={tool_name}, memory_id={tool_memory.memory_id}") + else: + logger.info(f"Top result does not match tool_name={tool_name}, will create new memory") + + # 如果没有找到匹配的 memory，创建新的 + if tool_memory is None: + tool_memory = ToolMemory(workspace_id=workspace_id, when_to_use=tool_name) + logger.info(f"Created new tool_memory for tool_name={tool_name}, memory_id={tool_memory.memory_id}") + + tool_memory.tool_call_results.extend(tool_call_results) + + # 保留最近的 n 个 + if len(tool_memory.tool_call_results) > self.max_history_tool_call_cnt: + tool_memory.tool_call_results = tool_memory.tool_call_results[-self.max_history_tool_call_cnt:] + logger.info(f"Trimmed tool_call_results to {self.max_history_tool_call_cnt} most recent entries") + + # 更新修改时间 + tool_memory.update_modified_time() + + # 4. 将更新后的 memory 保存到向量数据库 + # 如果是更新现有的 memory，需要先删除旧的 + deleted_memory_ids = [] + if exist_node: + deleted_memory_ids = [tool_memory.memory_id] + + # 设置返回结果 + self.context.response.metadata["deleted_memory_ids"] = deleted_memory_ids + self.context.response.metadata["memory_list"] = [tool_memory] + + logger.info(f"Updated tool_memory: tool_name={tool_name}, total_results={len(tool_memory.tool_call_results)}") - self.context.response.answer = "haha" \ No newline at end of file +async def main(): + """Simple test for ParseToolCallResultOp""" + from flowllm.app import FlowLLMApp + from datetime import datetime + + async with FlowLLMApp(load_default_config=True): + op = ParseToolCallResultOp() + + # Create simple test data + tool_call_results = [ + { + "create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": "test_tool", + "input": { + "query": "search for python asyncio documentation", + "max_results": 10, + "filter_type": "official_docs", + "language": "en" + }, + "output": "Found 10 relevant documentation pages for Python asyncio. Top results include: 1) Official Python docs for asyncio module, 2) Real Python asyncio tutorial, 3) Stack Overflow asyncio examples. All results are from official sources as requested.", + "token_cost": 150, + "success": True, + "time_cost": 2.3 + } + ] + tool_name = "test_tool" + workspace_id = "test_workspace1" + + await op.async_call(tool_call_results=tool_call_results, tool_name=tool_name, workspace_id=workspace_id) + logger.info(f"Response: {op.context.response.model_dump_json()}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/reme_ai/summary/tool/parse_tool_call_result_prompt.yaml b/reme_ai/summary/tool/parse_tool_call_result_prompt.yaml new file mode 100644 index 00000000..acc530ce --- /dev/null +++ b/reme_ai/summary/tool/parse_tool_call_result_prompt.yaml @@ -0,0 +1,46 @@ +evaluate_tool_call_prompt: | + You are an expert in evaluating tool invocation results. Please evaluate the following tool call based on two criteria: + + 1. **Success**: Whether the tool invocation was successful + 2. **Parameter Alignment**: Whether the result aligns with the expected behavior given the input parameters + + ## Tool Call Information: + - Tool Name: {tool_name} + - Success Flag: {success_flag} + - Time Cost: {time_cost}s + - Token Cost: {token_cost} tokens + - Input Parameters: {input_params} + - Output: {output} + + ## Evaluation Criteria: + + ### 1. Success Evaluation + - Check if the success flag indicates successful execution + - Verify if the output contains error messages or indicates failure + - Consider if time and token costs are reasonable + - Assess if there are any error indicators in the output + + ### 2. Parameter Alignment Evaluation + - Evaluate if the output matches what would be expected given the input parameters + - Consider if the input parameters are appropriate for the tool + - Check for any parameter mismatches or unexpected behaviors + - Verify the output is consistent with the tool's intended purpose + + ## Response Format: + Please provide your evaluation in the following JSON format: + + ```json + {{ + "summary": "A brief one-sentence summary of the tool call result", + "evaluation": "A brief evaluation (2-3 sentences) explaining the success status and parameter alignment.", + "score": 1.0 + }} + ``` + + ## Scoring Guidelines: + - **1.0**: Success - Tool executed successfully with good parameter alignment + - **0.5**: Partial Success - Tool executed but with parameter alignment issues or minor errors + - **0.0**: Failure - Tool execution failed or severe parameter misalignment + + Please evaluate carefully and provide a score of exactly 0.0, 0.5, or 1.0. + diff --git a/reme_ai/summary/tool/summary_tool_memory_op.py b/reme_ai/summary/tool/summary_tool_memory_op.py new file mode 100644 index 00000000..fab8ec14 --- /dev/null +++ b/reme_ai/summary/tool/summary_tool_memory_op.py @@ -0,0 +1,351 @@ +import asyncio +from typing import List + +from flowllm import C, BaseAsyncOp +from flowllm.enumeration.role import Role +from flowllm.schema.message import Message +from flowllm.schema.vector_node import VectorNode +from flowllm.utils.common_utils import extract_content +from loguru import logger + +from reme_ai.schema.memory import ToolMemory, vector_node_to_memory + + +@C.register_op() +class SummaryToolMemoryOp(BaseAsyncOp): + file_path: str = __file__ + + def __init__(self, + recent_call_count: int = 20, + summary_sleep_interval: float = 1.0, + **kwargs): + super().__init__(**kwargs) + self.recent_call_count: int = recent_call_count + self.summary_sleep_interval: float = summary_sleep_interval + + @staticmethod + def _format_call_summaries_markdown(recent_calls: List) -> str: + """Format tool call summaries as markdown.""" + if not recent_calls: + return "No recent calls available." + + lines = [] + for i, call in enumerate(recent_calls, 1): + lines.append(f"### Call #{i}") + lines.append(f"- **Summary**: {call.summary}") + lines.append(f"- **Evaluation**: {call.evaluation}") + lines.append(f"- **Score**: {call.score}") + lines.append(f"- **Success**: {call.success}") + lines.append(f"- **Time Cost**: {call.time_cost}s") + lines.append(f"- **Token Cost**: {call.token_cost}") + lines.append("") + + return "\n".join(lines) + + @staticmethod + def _format_statistics_markdown(statistics: dict) -> str: + """Format statistics as markdown.""" + lines = [f"- **Total Calls**: {statistics.get('total_calls', 0)}", + f"- **Recent Calls**: {statistics.get('recent_calls', 0)}", + f"- **Success Rate**: {statistics.get('success_rate', 0):.2%}", + f"- **Recent Success Rate**: {statistics.get('recent_success_rate', 0):.2%}", + f"- **Average Score**: {statistics.get('avg_score', 0):.3f}", + f"- **Recent Average Score**: {statistics.get('recent_avg_score', 0):.3f}", + f"- **Average Time Cost**: {statistics.get('avg_time_cost', 0):.3f}s", + f"- **Average Token Cost**: {statistics.get('avg_token_cost', 0):.1f}"] + + return "\n".join(lines) + + async def _summarize_single_tool(self, tool_memory: ToolMemory, index: int) -> ToolMemory: + await asyncio.sleep(self.summary_sleep_interval * index) + + # Get the most recent N tool calls + recent_calls = tool_memory.tool_call_results[-self.recent_call_count:] + + if not recent_calls: + logger.warning(f"No tool call results found for tool: {tool_memory.when_to_use}") + return tool_memory + + # Get statistics + statistics = tool_memory.statistic(recent_frequency=self.recent_call_count) + + # Format data as markdown + call_summaries_md = self._format_call_summaries_markdown(recent_calls) + statistics_md = self._format_statistics_markdown(statistics) + + prompt = self.prompt_format(prompt_name="summarize_tool_usage_prompt", + tool_name=tool_memory.when_to_use, + call_summaries=call_summaries_md, + statistics=statistics_md) + + def parse_summary(message: Message) -> ToolMemory: + content = message.content.strip() + # Extract content from txt code block + tool_memory.content = extract_content(content, "txt") + + # Update modified time + tool_memory.update_modified_time() + + logger.info(f"Summarized tool {index}: tool_name={tool_memory.when_to_use}, " + f"content_length={len(tool_memory.content)}") + return tool_memory + + # Call LLM to generate summary + result = await self.llm.achat(messages=[Message(role=Role.USER, content=prompt)], callback_fn=parse_summary) + + return result + + async def async_execute(self): + tool_names: str = self.context.get("tool_names", "") + workspace_id: str = self.context.workspace_id + + if not tool_names: + logger.warning("tool_names is empty, skipping processing") + self.context.response.answer = "tool_names is required" + self.context.response.success = False + return + + # Split tool names by comma + tool_name_list = [name.strip() for name in tool_names.split(",") if name.strip()] + logger.info(f"workspace_id={workspace_id} processing {len(tool_name_list)} tools: {tool_name_list}") + + # Search for each tool in the vector store + matched_tool_memories: List[ToolMemory] = [] + + for tool_name in tool_name_list: + nodes: List[VectorNode] = await self.vector_store.async_search(query=tool_name, + workspace_id=workspace_id, + top_k=1) + + if nodes: + top_node = nodes[0] + memory = vector_node_to_memory(top_node) + + # Ensure it's a ToolMemory and when_to_use matches + if isinstance(memory, ToolMemory) and memory.when_to_use == tool_name: + matched_tool_memories.append(memory) + logger.info(f"Found tool_memory for tool_name={tool_name}, " + f"memory_id={memory.memory_id}, " + f"total_calls={len(memory.tool_call_results)}") + else: + logger.warning(f"No exact match found for tool_name={tool_name}") + else: + logger.warning(f"No memory found for tool_name={tool_name}") + + if not matched_tool_memories: + logger.info("No matching tool memories found") + self.context.response.answer = "No matching tool memories found" + self.context.response.success = False + return + + # Concurrently summarize all tool memories + logger.info(f"Starting concurrent summarization of {len(matched_tool_memories)} tool memories") + + # 使用基类的 submit_async_task 提交所有总结任务 + for index, tool_memory in enumerate(matched_tool_memories): + self.submit_async_task(self._summarize_single_tool, tool_memory, index) + + # 使用基类的 join_async_task 等待所有任务完成 + # 注意: 基类已经过滤掉异常,返回的只包含成功的结果 + valid_summarized_memories = await self.join_async_task(return_exceptions=True) + logger.info(f"Completed summarization of {len(valid_summarized_memories)} tool memories") + + # Set response + self.context.response.answer = f"Successfully summarized {len(valid_summarized_memories)} tool memories" + self.context.response.success = True + self.context.response.metadata["memory_list"] = valid_summarized_memories + self.context.response.metadata["deleted_memory_ids"] = [m.memory_id for m in valid_summarized_memories] + + # Log summary for each tool + for memory in valid_summarized_memories: + logger.info(f"Tool: {memory.when_to_use}, " + f"Content: {memory.content[:100]}...") + + +async def main(): + from flowllm.app import FlowLLMApp + from reme_ai.summary.tool.parse_tool_call_result_op import ParseToolCallResultOp + from reme_ai.vector_store.update_vector_store_op import UpdateVectorStoreOp + from datetime import datetime, timedelta + import random + + async with FlowLLMApp(load_default_config=True): + workspace_id = "test_workspace_complex" + tool_name = "web_search_tool" + + # ===== 第一步: 准备 30 条模拟的工具调用记录 ===== + # 模拟不同场景的调用记录:成功、参数错误、超时、返回空结果等 + logger.info("=" * 80) + logger.info("步骤1: 准备 30 条工具调用记录,模拟真实使用场景") + logger.info("=" * 80) + + base_time = datetime.now() - timedelta(days=7) + tool_call_results = [] + + # 场景1: 成功的搜索 (15条) + success_queries = [ + "Python asyncio tutorial", "machine learning basics", "React hooks guide", + "Docker best practices", "SQL optimization tips", "Git workflow strategies", + "RESTful API design", "microservices architecture", "Redis caching patterns", + "Kubernetes deployment", "GraphQL advantages", "MongoDB schema design", + "JWT authentication", "OAuth2 flow", "WebSocket real-time" + ] + + for i, query in enumerate(success_queries): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=i * 2)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "query": query, + "max_results": random.randint(5, 20), + "language": "en", + "filter_type": "technical_docs" + }, + "output": f"Found {random.randint(8, 20)} relevant results for '{query}'. Top results include official documentation, tutorials, and best practice guides.", + "token_cost": random.randint(100, 300), + "success": True, + "time_cost": round(random.uniform(1.5, 3.5), 2) + }) + + # 场景2: 参数不合理导致的部分成功 (8条) + for i in range(8): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=30 + i * 3)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "query": f"test query {i}", # 查询词过于简单 + "max_results": 100, # 请求过多结果 + "language": "unknown", # 语言参数错误 + }, + "output": f"Warning: language 'unknown' not supported, using default. Query too generic, returning limited results. Found {random.randint(2, 5)} results.", + "token_cost": random.randint(50, 150), + "success": True, + "time_cost": round(random.uniform(2.0, 4.0), 2) + }) + + # 场景3: 超时或失败 (5条) + for i in range(5): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=54 + i * 4)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "query": f"extremely complex query with many conditions {i}", + "max_results": 50, + "language": "en", + "filter_type": "all" + }, + "output": "Error: Request timeout after 10 seconds. Try simplifying the query or reducing max_results.", + "token_cost": 20, + "success": False, + "time_cost": 10.0 + }) + + # 场景4: 空结果 (2条) + for i in range(2): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=74 + i * 5)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "query": f"xyzabc123nonexistent{i}", # 不存在的内容 + "max_results": 10, + "language": "en", + }, + "output": "No results found for the given query. Please try different keywords.", + "token_cost": 30, + "success": True, + "time_cost": 1.2 + }) + + logger.info(f"准备了 {len(tool_call_results)} 条工具调用记录") + logger.info(f"- 成功调用: 15 条") + logger.info(f"- 参数不合理: 8 条") + logger.info(f"- 超时失败: 5 条") + logger.info(f"- 空结果: 2 条") + + # ===== 第二步: 使用 ParseToolCallResultOp >> UpdateVectorStoreOp 串联运行 ===== + logger.info("\n" + "=" * 80) + logger.info("步骤2: ParseToolCallResultOp >> UpdateVectorStoreOp 串联评估并保存") + logger.info("=" * 80) + + # 使用 >> 操作符串联两个Op,自动传递context和metadata + pipeline = ParseToolCallResultOp(evaluation_sleep_interval=0.1) >> UpdateVectorStoreOp() + + await pipeline.async_call( + tool_call_results=tool_call_results, + tool_name=tool_name, + workspace_id=workspace_id + ) + + if not pipeline.context.response.success: + logger.error(f"Pipeline failed: {pipeline.context.response.answer}") + return + + logger.info(f"✓ Pipeline 完成") + logger.info(f" 评估了 {len(tool_call_results)} 条记录") + logger.info(f" 每条记录包含: summary, evaluation, score (0.0/0.5/1.0)") + + # 显示一些评估结果示例 + memory_list = pipeline.context.response.metadata.get("memory_list", []) + if memory_list: + tool_memory = memory_list[0] + logger.info(f"\n评估结果示例 (前3条):") + for i, result in enumerate(tool_memory.tool_call_results[:3], 1): + logger.info(f" 调用 #{i}:") + logger.info(f" 查询: {result.input.get('query', 'N/A')}") + logger.info(f" 评分: {result.score}") + logger.info(f" 总结: {result.summary[:80]}...") + logger.info(f" 评价: {result.evaluation[:80]}...") + + # 显示向量数据库更新结果 + update_result = pipeline.context.response.metadata.get("update_result", {}) + logger.info(f"\n✓ 向量数据库更新完成:") + logger.info(f" 删除记录数: {update_result.get('deleted_count', 0)}") + logger.info(f" 插入记录数: {update_result.get('inserted_count', 0)}") + + # ===== 第三步: 使用 SummaryToolMemoryOp 总结工具使用模式 ===== + logger.info("\n" + "=" * 80) + logger.info("步骤3: 使用 SummaryToolMemoryOp 从 30 条记录中提取使用模式和建议") + logger.info("=" * 80) + + summary_op = SummaryToolMemoryOp( + recent_call_count=30, # 分析最近30条记录 + summary_sleep_interval=0.5 + ) + await summary_op.async_call( + tool_names=tool_name, + workspace_id=workspace_id + ) + + if not summary_op.context.response.success: + logger.error(f"SummaryToolMemoryOp failed: {summary_op.context.response.answer}") + return + + logger.info(f"✓ SummaryToolMemoryOp 完成") + + # ===== 第四步: 展示 summary 的价值 ===== + logger.info("\n" + "=" * 80) + logger.info("步骤4: 展示 Summary 如何将分散的调用记录转化为有价值的使用指南") + logger.info("=" * 80) + + summarized_memories = summary_op.context.response.metadata.get("memory_list", []) + if summarized_memories: + summarized_memory = summarized_memories[0] + + logger.info(f"\n工具名称: {summarized_memory.when_to_use}") + logger.info(f"\n统计信息:") + stats = summarized_memory.statistic(recent_frequency=30) + logger.info(f" 总调用次数: {stats['total_calls']}") + logger.info(f" 成功率: {stats['success_rate']:.1%}") + logger.info(f" 平均评分: {stats['avg_score']:.2f}") + logger.info(f" 平均耗时: {stats['avg_time_cost']:.2f}s") + logger.info(f" 平均Token消耗: {stats['avg_token_cost']:.1f}") + + logger.info(f"\n" + "=" * 60) + logger.info("Summary 生成的使用指南 (从30条分散记录中提取):") + logger.info("=" * 60) + logger.info(summarized_memory.content) + logger.info("=" * 60) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/reme_ai/summary/tool/summary_tool_memory_prompt.yaml b/reme_ai/summary/tool/summary_tool_memory_prompt.yaml new file mode 100644 index 00000000..9fd1e075 --- /dev/null +++ b/reme_ai/summary/tool/summary_tool_memory_prompt.yaml @@ -0,0 +1,27 @@ +summarize_tool_usage_prompt: | + You are an expert in analyzing tool usage patterns and generating practical usage guidelines. + + ## Tool Information: + - Tool Name: {tool_name} + + ## Recent Tool Call History: + {call_summaries} + + ## Statistical Summary: + {statistics} + + ## Your Task: + Based on the tool call history, generate a concise and logical tool usage description following this structure: + + 1. **Core Function**: What this tool does and when to use it + 2. **Success Patterns**: Parameter patterns and usage scenarios that work well (with key metrics) + 3. **Common Issues**: Main pitfalls to avoid and why they fail + 4. **Best Practices**: 2-3 actionable recommendations + + ## Response Format: + Provide a structured, concise description (max 200 words). Focus on actionable insights derived from actual usage data. Avoid generic advice. + + ```txt + [Your concise, data-driven tool usage description] + ``` + From 82405c4a7c92241e02447a22f664c8b31ca08272 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Thu, 16 Oct 2025 16:36:40 +0800 Subject: [PATCH 04/13] docs(task_memory): enhance documentation with data structures and usage examples - Add TaskMemory and Trajectory data structure definitions - Include detailed processing flows for record and delete operations - Add comparison table between Task Memory and Tool Memory - Provide best practices for trajectory scoring and maintenance - Enhance build_query_op documentation with processing flow - Add format examples for rewrite_memory_op with and without LLM - Include usage examples for simple_summary_op and comparative summary - Add comparison table between simple and full summary pipelines - Improve parameter descriptions and when-to-use guidance - Add Chinese translations for key concepts and examples --- docs/task_memory/task_memory.md | 148 +++++++++++++++++++++++++- docs/task_memory/task_retrieve_ops.md | 117 +++++++++++++++++++- docs/task_memory/task_summary_ops.md | 55 +++++++++- 3 files changed, 313 insertions(+), 7 deletions(-) diff --git a/docs/task_memory/task_memory.md b/docs/task_memory/task_memory.md index 5e232225..a3e07295 100644 --- a/docs/task_memory/task_memory.md +++ b/docs/task_memory/task_memory.md @@ -11,9 +11,39 @@ Task Memory represents knowledge extracted from previous task executions, includ Each task memory contains: - `when_to_use`: Conditions that indicate when this memory is relevant -- `content`: The actual knowledge or memory to be applied +- `content`: The actual knowledge or experience to be applied +- `score`: Quality score of the memory (assigned during validation) - Metadata about the memory's source and utility +## Task Memory的数据结构 + +### TaskMemory + +```python +class TaskMemory(BaseMemory): + memory_type: str = "task" + workspace_id: str # 工作空间ID + memory_id: str # 记忆的唯一ID + when_to_use: str # 何时使用此记忆的条件 + content: str # 具体的经验内容 + score: float # 质量评分（validation阶段设置） + time_created: str # 创建时间 + time_modified: str # 最后修改时间 + author: str # 创建者（通常是LLM模型名） + metadata: dict # 其他元数据 +``` + +### Trajectory + +```python +class Trajectory(BaseModel): + messages: List[Message] # 对话消息列表 + score: float # 轨迹评分（用于判断成功/失败） + metadata: dict # 可选的元数据（如segments、query等） +``` + +Task Memory通过分析Trajectory来提取经验。轨迹的score决定了它被分类为成功还是失败案例。 + ## Configuration Logic Task Memory in ReMe is configured through two main flows: @@ -196,7 +226,119 @@ response = requests.post( ReMe also provides additional task memory operations: -- `record_task_memory`: Update frequency and utility attributes of retrieved memories -- `delete_task_memory`: Delete memories based on utility/frequency thresholds +### Record Task Memory + +`record_task_memory` flow用于更新检索到的任务记忆的使用频率和效用属性： + +```yaml +record_task_memory: + flow_content: update_memory_freq_op >> update_memory_utility_op >> update_vector_store_op + description: "Update the freq & utility attributes of retrieved task memories" + input_schema: + workspace_id: + type: string + required: true + memory_dicts: + type: array + description: "A list of retrieved task memory" + required: true + update_utility: + type: boolean + description: "Whether to update the utility attribute" + required: true +``` + +使用示例： + +```python +# 记录检索到的记忆被使用 +response = requests.post( + url=f"{BASE_URL}record_task_memory", + json={ + "workspace_id": WORKSPACE_ID, + "memory_dicts": retrieved_memories, # 从retrieve_task_memory获取的记忆列表 + "update_utility": True # 是否更新效用值 + } +) +``` + +### Delete Task Memory + +`delete_task_memory` flow用于删除低效用的任务记忆： + +```yaml +delete_task_memory: + flow_content: delete_memory_op >> update_vector_store_op + description: "Delete task memories when utility/freq < utility_threshold and freq >= freq_threshold" + input_schema: + workspace_id: + type: string + required: true + freq_threshold: + type: integer + description: "The retrieved frequency threshold" + required: true + utility_threshold: + type: number + description: "The utility/freq threshold" + required: true +``` + +使用示例： + +```python +# 删除使用频率高但效用低的记忆 +response = requests.post( + url=f"{BASE_URL}delete_task_memory", + json={ + "workspace_id": WORKSPACE_ID, + "freq_threshold": 10, # 至少被检索过10次 + "utility_threshold": 0.3 # 但效用/频率比 < 0.3 + } +) +``` + +这个机制确保记忆库的质量：频繁被检索但实际没什么帮助的记忆会被清理。 + +## 与Tool Memory的对比 + +| 特性 | Task Memory | Tool Memory | +|------|-------------|-------------| +| **目的** | 记录任务解决的经验 | 记录工具调用的使用模式 | +| **输入** | 对话轨迹（Trajectory） | 工具调用记录（ToolCallResult） | +| **索引方式** | when_to_use条件（语义搜索） | 工具名称（精确匹配） | +| **提取方式** | 从完整轨迹中分析提取 | 从单次调用评估和统计 | +| **内容** | 任务解决经验和策略 | 工具使用指南和最佳实践 | +| **更新方式** | 从成功/失败/对比中提取 | 每次调用后追加记录 | +| **检索流程** | 复杂：构建查询→召回→重排→重写 | 简单：精确匹配→返回 | +| **去重机制** | 语义去重（deduplication） | 按工具名唯一，限制历史条数 | +| **验证机制** | LLM验证质量（validation） | 评估单次调用质量 | + +## 最佳实践 + +1. **轨迹评分标准**： + - 明确定义成功和失败的评分标准 + - `success_threshold`通常设为1.0（完美成功） + - 部分成功的轨迹（如0.5-0.9）可以提供对比经验 + +2. **轨迹质量**： + - 确保轨迹包含完整的问题解决过程 + - 包含足够的上下文信息在metadata中 + - 避免过短或过长的轨迹 + +3. **定期维护**： + - 使用`record_task_memory`跟踪记忆使用情况 + - 定期使用`delete_task_memory`清理低效记忆 + - 保持记忆库的质量和相关性 + +4. **合理使用简化版本**： + - 简单场景使用`summary_task_memory_simple` + - 复杂场景使用完整的`summary_task_memory` flow + - 根据实际需求选择是否启用轨迹分割 + +5. **检索优化**： + - 调整`top_k`参数（默认5）控制返回的记忆数量 + - 启用`enable_llm_rerank`提升相关性 + - 启用`enable_llm_rewrite`使记忆更适配当前任务 For more detailed examples, see the `use_task_memory_demo.py` file in the cookbook directory of the ReMe project. diff --git a/docs/task_memory/task_retrieve_ops.md b/docs/task_memory/task_retrieve_ops.md index a12e8698..6c61c88f 100644 --- a/docs/task_memory/task_retrieve_ops.md +++ b/docs/task_memory/task_retrieve_ops.md @@ -8,16 +8,61 @@ Constructs a query for memory retrieval either from a direct query input or by a ### Functionality -- If a direct `query` is provided in the context, it uses that query +- If a direct `query` is provided in the context, it uses that query directly - If `messages` are provided in the context, it can: - - Use an LLM to generate a query based on the conversation context - - Or create a simple query from recent messages without using an LLM + - **LLM-based**: Use an LLM to generate a focused query based on the conversation context + - **Simple**: Create a simple query from the last 3 messages without using an LLM +- Raises an error if neither `query` nor `messages` is provided + +### Processing Flow + +1. **Check for Direct Query**: + - If `context.query` exists, use it directly + - This is the most common case when retrieving for a specific task + +2. **Build from Messages** (if no direct query): + - Extract last 3 messages from `context.messages` + - If `enable_llm_build=true`: + - Merge all messages into execution_process text + - Use LLM to analyze and generate a focused query + - If `enable_llm_build=false`: + - Truncate each message content to 200 chars + - Format as "- role: content" for each message + - Concatenate into a simple query + +3. **Set Context**: Store the built query in `context.query` for downstream ops ### Parameters - `op.build_query_op.params.enable_llm_build` (boolean, default: `true`): - When `true`, uses an LLM to generate a query from conversation messages - When `false`, creates a simple query by concatenating recent messages + - LLM-based queries are more focused but require an extra LLM call + - Simple queries are faster but may be less targeted + +### Usage Example + +```python +import requests + +# Method 1: Direct query (recommended) +response = requests.post( + url="http://0.0.0.0:8002/retrieve_task_memory", + json={ + "workspace_id": "my_workspace", + "query": "How to handle file upload errors in Python?" + } +) + +# Method 2: From messages (when integrated in conversation) +# This is handled internally by the flow when messages are provided +``` + +### When to Use Which Mode + +- **Direct Query**: Most common, use when you know what to search for +- **LLM-based Build**: When you have a conversation and want the LLM to extract the key question +- **Simple Build**: When you want fast retrieval without extra LLM cost ## RerankMemoryOp @@ -53,12 +98,78 @@ Rewrites and formats the retrieved memories to make them more relevant and actio - Formats retrieved memories into a structured format - Can use an LLM to rewrite memories to better fit the current context (optional) - Generates a cohesive context message from multiple memories +- Extracts current conversation context to inform the rewriting process + +### Processing Flow + +1. **Get Inputs**: + - `memory_list`: Retrieved and reranked memories from previous ops + - `query`: The search query + - `messages`: Current conversation messages (optional) + +2. **Format Memories**: + - Create numbered memory entries with "When to use" and "Content" + - Example format: + ``` + Memory 1: + When to use: [condition] + Content: [experience] + ``` + +3. **LLM Rewrite** (if `enable_llm_rewrite=true`): + - Extract current context from last 3 messages + - Provide original formatted memories and current context to LLM + - LLM adapts the memories to be more relevant to the current query + - Extracts `rewritten_context` from LLM response (JSON format) + +4. **Return**: Set `context.response.answer` to the final formatted/rewritten memories ### Parameters - `op.rewrite_memory_op.params.enable_llm_rewrite` (boolean, default: `true`): - When `true`, uses an LLM to rewrite the memories to make them more relevant and actionable - When `false`, simply formats the memories without LLM-based rewriting + - LLM rewriting makes memories more contextual but adds latency + +### Format Examples + +**Without LLM Rewrite** (enable_llm_rewrite=false): +``` +Memory 1 : + When to use: When handling file upload errors in web applications + Content: Use try-except blocks to catch specific exceptions like PermissionError... + +Memory 2 : + When to use: When validating file types before processing + Content: Check file extensions and MIME types to ensure security... +``` + +**With LLM Rewrite** (enable_llm_rewrite=true): +``` +Based on your question about file upload errors in Python web applications: + +1. Error Handling Strategy: + - Wrap file operations in try-except blocks + - Catch specific exceptions: PermissionError, IOError, OSError + - [Adapted from Memory 1] + +2. Input Validation: + - Always validate file types before processing + - Check both extension and MIME type + - [Adapted from Memory 2] +``` + +### When to Enable LLM Rewrite + +- **Enable** when: + - Memories need to be adapted to specific context + - User query is complex or nuanced + - Better integration with conversation flow is needed + +- **Disable** when: + - Fast retrieval is prioritized + - Memories are already clear and relevant + - Reducing LLM calls and costs is important ## MergeMemoryOp diff --git a/docs/task_memory/task_summary_ops.md b/docs/task_memory/task_summary_ops.md index 138f5812..f1364baf 100644 --- a/docs/task_memory/task_summary_ops.md +++ b/docs/task_memory/task_summary_ops.md @@ -138,13 +138,66 @@ A simplified version of memory extraction that processes entire trajectories in ### Functionality - Classifies trajectories as success or failure based on score threshold -- Extracts memories directly from complete trajectories +- Extracts memories directly from complete trajectories without segmentation +- Uses a single LLM prompt to extract multiple memories from each trajectory +- Parses JSON response containing `when_to_use` and `memory` pairs - Useful for simpler use cases where detailed segmentation is not required +### Processing Flow + +1. **Receive Trajectories**: Get a list of Trajectory objects with messages and scores +2. **For Each Trajectory**: + - Merge all messages into execution_process text + - Determine execution_result ("success" or "fail") based on score threshold + - Format prompt with execution_process and execution_result + - Call LLM to extract memories in JSON format +3. **Parse Response**: Extract JSON array of {when_to_use, memory} objects +4. **Create TaskMemory**: Create TaskMemory objects for each extracted memory +5. **Return**: Set memory_list in context.response.metadata + ### Parameters - `op.simple_summary_op.params.success_score_threshold` (float, default: `0.9`): - The threshold score that determines if a trajectory is considered successful + - Trajectories with `score >= success_score_threshold` are marked as "success" + - Trajectories with `score < success_score_threshold` are marked as "fail" + +### Usage Example + +```python +import requests + +# Prepare trajectory with messages +trajectory = { + "messages": [ + {"role": "user", "content": "How do I sort a list in Python?"}, + {"role": "assistant", "content": "You can use the sorted() function..."}, + {"role": "user", "content": "Thanks! What about reverse sorting?"}, + {"role": "assistant", "content": "Use sorted(list, reverse=True)..."} + ], + "score": 1.0 # Successful trajectory +} + +# Summarize using simple summary +response = requests.post( + url="http://0.0.0.0:8002/summary_task_memory_simple", + json={ + "workspace_id": "my_workspace", + "trajectories": [trajectory] + } +) +``` + +### Comparison with Full Summary Pipeline + +| Feature | SimpleSummaryOp | Full Summary Pipeline | +|---------|-----------------|----------------------| +| **Complexity** | Single operation | Multi-stage pipeline | +| **Segmentation** | No segmentation | Optional segmentation | +| **Extraction** | Single prompt per trajectory | Separate prompts for success/failure/comparative | +| **Validation** | No validation | LLM validation step | +| **Deduplication** | No deduplication | Optional deduplication | +| **Use Case** | Quick prototyping | Production environments | ## SimpleComparativeSummaryOp From 40538f974a63cbb65f7a30b97edbf4439c544374 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Thu, 16 Oct 2025 16:39:40 +0800 Subject: [PATCH 05/13] docs(task_memory): simplify documentation and remove redundant content - Removed detailed data structure definitions for TaskMemory and Trajectory - Eliminated extensive examples and usage instructions for record_task_memory and delete_task_memory - Removed comparison table with Tool Memory and best practices section - Simplified build_query_op and rewrite_memory_op documentation by removing processing flows - Removed detailed examples and parameter descriptions for simple_summary_op- Added new tool_memory.md documentation with complete tool memory implementation - Added new tool_retrieve_ops.md documentation with detailed retrieval operations- Created comprehensive tool memory data structures and API documentation - Added usage examples and integration workflows for tool memory operations - Documented configuration parameters and best practices for tool memory management --- docs/task_memory/task_memory.md | 150 +------ docs/task_memory/task_retrieve_ops.md | 119 +----- docs/task_memory/task_summary_ops.md | 57 +-- docs/tool_memory/tool_memory.md | 557 ++++++++++++++++++++++++++ docs/tool_memory/tool_retrieve_ops.md | 548 +++++++++++++++++++++++++ docs/tool_memory/tool_summary_ops.md | 465 +++++++++++++++++++++ 6 files changed, 1580 insertions(+), 316 deletions(-) create mode 100644 docs/tool_memory/tool_memory.md create mode 100644 docs/tool_memory/tool_retrieve_ops.md create mode 100644 docs/tool_memory/tool_summary_ops.md diff --git a/docs/task_memory/task_memory.md b/docs/task_memory/task_memory.md index a3e07295..ae66d9f3 100644 --- a/docs/task_memory/task_memory.md +++ b/docs/task_memory/task_memory.md @@ -11,39 +11,9 @@ Task Memory represents knowledge extracted from previous task executions, includ Each task memory contains: - `when_to_use`: Conditions that indicate when this memory is relevant -- `content`: The actual knowledge or experience to be applied -- `score`: Quality score of the memory (assigned during validation) +- `content`: The actual knowledge or memory to be applied - Metadata about the memory's source and utility -## Task Memory的数据结构 - -### TaskMemory - -```python -class TaskMemory(BaseMemory): - memory_type: str = "task" - workspace_id: str # 工作空间ID - memory_id: str # 记忆的唯一ID - when_to_use: str # 何时使用此记忆的条件 - content: str # 具体的经验内容 - score: float # 质量评分（validation阶段设置） - time_created: str # 创建时间 - time_modified: str # 最后修改时间 - author: str # 创建者（通常是LLM模型名） - metadata: dict # 其他元数据 -``` - -### Trajectory - -```python -class Trajectory(BaseModel): - messages: List[Message] # 对话消息列表 - score: float # 轨迹评分（用于判断成功/失败） - metadata: dict # 可选的元数据（如segments、query等） -``` - -Task Memory通过分析Trajectory来提取经验。轨迹的score决定了它被分类为成功还是失败案例。 - ## Configuration Logic Task Memory in ReMe is configured through two main flows: @@ -226,119 +196,7 @@ response = requests.post( ReMe also provides additional task memory operations: -### Record Task Memory +- `record_task_memory`: Update frequency and utility attributes of retrieved memories +- `delete_task_memory`: Delete memories based on utility/frequency thresholds -`record_task_memory` flow用于更新检索到的任务记忆的使用频率和效用属性： - -```yaml -record_task_memory: - flow_content: update_memory_freq_op >> update_memory_utility_op >> update_vector_store_op - description: "Update the freq & utility attributes of retrieved task memories" - input_schema: - workspace_id: - type: string - required: true - memory_dicts: - type: array - description: "A list of retrieved task memory" - required: true - update_utility: - type: boolean - description: "Whether to update the utility attribute" - required: true -``` - -使用示例： - -```python -# 记录检索到的记忆被使用 -response = requests.post( - url=f"{BASE_URL}record_task_memory", - json={ - "workspace_id": WORKSPACE_ID, - "memory_dicts": retrieved_memories, # 从retrieve_task_memory获取的记忆列表 - "update_utility": True # 是否更新效用值 - } -) -``` - -### Delete Task Memory - -`delete_task_memory` flow用于删除低效用的任务记忆： - -```yaml -delete_task_memory: - flow_content: delete_memory_op >> update_vector_store_op - description: "Delete task memories when utility/freq < utility_threshold and freq >= freq_threshold" - input_schema: - workspace_id: - type: string - required: true - freq_threshold: - type: integer - description: "The retrieved frequency threshold" - required: true - utility_threshold: - type: number - description: "The utility/freq threshold" - required: true -``` - -使用示例： - -```python -# 删除使用频率高但效用低的记忆 -response = requests.post( - url=f"{BASE_URL}delete_task_memory", - json={ - "workspace_id": WORKSPACE_ID, - "freq_threshold": 10, # 至少被检索过10次 - "utility_threshold": 0.3 # 但效用/频率比 < 0.3 - } -) -``` - -这个机制确保记忆库的质量：频繁被检索但实际没什么帮助的记忆会被清理。 - -## 与Tool Memory的对比 - -| 特性 | Task Memory | Tool Memory | -|------|-------------|-------------| -| **目的** | 记录任务解决的经验 | 记录工具调用的使用模式 | -| **输入** | 对话轨迹（Trajectory） | 工具调用记录（ToolCallResult） | -| **索引方式** | when_to_use条件（语义搜索） | 工具名称（精确匹配） | -| **提取方式** | 从完整轨迹中分析提取 | 从单次调用评估和统计 | -| **内容** | 任务解决经验和策略 | 工具使用指南和最佳实践 | -| **更新方式** | 从成功/失败/对比中提取 | 每次调用后追加记录 | -| **检索流程** | 复杂：构建查询→召回→重排→重写 | 简单：精确匹配→返回 | -| **去重机制** | 语义去重（deduplication） | 按工具名唯一，限制历史条数 | -| **验证机制** | LLM验证质量（validation） | 评估单次调用质量 | - -## 最佳实践 - -1. **轨迹评分标准**： - - 明确定义成功和失败的评分标准 - - `success_threshold`通常设为1.0（完美成功） - - 部分成功的轨迹（如0.5-0.9）可以提供对比经验 - -2. **轨迹质量**： - - 确保轨迹包含完整的问题解决过程 - - 包含足够的上下文信息在metadata中 - - 避免过短或过长的轨迹 - -3. **定期维护**： - - 使用`record_task_memory`跟踪记忆使用情况 - - 定期使用`delete_task_memory`清理低效记忆 - - 保持记忆库的质量和相关性 - -4. **合理使用简化版本**： - - 简单场景使用`summary_task_memory_simple` - - 复杂场景使用完整的`summary_task_memory` flow - - 根据实际需求选择是否启用轨迹分割 - -5. **检索优化**： - - 调整`top_k`参数（默认5）控制返回的记忆数量 - - 启用`enable_llm_rerank`提升相关性 - - 启用`enable_llm_rewrite`使记忆更适配当前任务 - -For more detailed examples, see the `use_task_memory_demo.py` file in the cookbook directory of the ReMe project. +For more detailed examples, see the `use_task_memory_demo.py` file in the cookbook directory of the ReMe project. \ No newline at end of file diff --git a/docs/task_memory/task_retrieve_ops.md b/docs/task_memory/task_retrieve_ops.md index 6c61c88f..f4ec6eb5 100644 --- a/docs/task_memory/task_retrieve_ops.md +++ b/docs/task_memory/task_retrieve_ops.md @@ -8,61 +8,16 @@ Constructs a query for memory retrieval either from a direct query input or by a ### Functionality -- If a direct `query` is provided in the context, it uses that query directly +- If a direct `query` is provided in the context, it uses that query - If `messages` are provided in the context, it can: - - **LLM-based**: Use an LLM to generate a focused query based on the conversation context - - **Simple**: Create a simple query from the last 3 messages without using an LLM -- Raises an error if neither `query` nor `messages` is provided - -### Processing Flow - -1. **Check for Direct Query**: - - If `context.query` exists, use it directly - - This is the most common case when retrieving for a specific task - -2. **Build from Messages** (if no direct query): - - Extract last 3 messages from `context.messages` - - If `enable_llm_build=true`: - - Merge all messages into execution_process text - - Use LLM to analyze and generate a focused query - - If `enable_llm_build=false`: - - Truncate each message content to 200 chars - - Format as "- role: content" for each message - - Concatenate into a simple query - -3. **Set Context**: Store the built query in `context.query` for downstream ops + - Use an LLM to generate a query based on the conversation context + - Or create a simple query from recent messages without using an LLM ### Parameters - `op.build_query_op.params.enable_llm_build` (boolean, default: `true`): - When `true`, uses an LLM to generate a query from conversation messages - When `false`, creates a simple query by concatenating recent messages - - LLM-based queries are more focused but require an extra LLM call - - Simple queries are faster but may be less targeted - -### Usage Example - -```python -import requests - -# Method 1: Direct query (recommended) -response = requests.post( - url="http://0.0.0.0:8002/retrieve_task_memory", - json={ - "workspace_id": "my_workspace", - "query": "How to handle file upload errors in Python?" - } -) - -# Method 2: From messages (when integrated in conversation) -# This is handled internally by the flow when messages are provided -``` - -### When to Use Which Mode - -- **Direct Query**: Most common, use when you know what to search for -- **LLM-based Build**: When you have a conversation and want the LLM to extract the key question -- **Simple Build**: When you want fast retrieval without extra LLM cost ## RerankMemoryOp @@ -98,78 +53,12 @@ Rewrites and formats the retrieved memories to make them more relevant and actio - Formats retrieved memories into a structured format - Can use an LLM to rewrite memories to better fit the current context (optional) - Generates a cohesive context message from multiple memories -- Extracts current conversation context to inform the rewriting process - -### Processing Flow - -1. **Get Inputs**: - - `memory_list`: Retrieved and reranked memories from previous ops - - `query`: The search query - - `messages`: Current conversation messages (optional) - -2. **Format Memories**: - - Create numbered memory entries with "When to use" and "Content" - - Example format: - ``` - Memory 1: - When to use: [condition] - Content: [experience] - ``` - -3. **LLM Rewrite** (if `enable_llm_rewrite=true`): - - Extract current context from last 3 messages - - Provide original formatted memories and current context to LLM - - LLM adapts the memories to be more relevant to the current query - - Extracts `rewritten_context` from LLM response (JSON format) - -4. **Return**: Set `context.response.answer` to the final formatted/rewritten memories ### Parameters - `op.rewrite_memory_op.params.enable_llm_rewrite` (boolean, default: `true`): - When `true`, uses an LLM to rewrite the memories to make them more relevant and actionable - When `false`, simply formats the memories without LLM-based rewriting - - LLM rewriting makes memories more contextual but adds latency - -### Format Examples - -**Without LLM Rewrite** (enable_llm_rewrite=false): -``` -Memory 1 : - When to use: When handling file upload errors in web applications - Content: Use try-except blocks to catch specific exceptions like PermissionError... - -Memory 2 : - When to use: When validating file types before processing - Content: Check file extensions and MIME types to ensure security... -``` - -**With LLM Rewrite** (enable_llm_rewrite=true): -``` -Based on your question about file upload errors in Python web applications: - -1. Error Handling Strategy: - - Wrap file operations in try-except blocks - - Catch specific exceptions: PermissionError, IOError, OSError - - [Adapted from Memory 1] - -2. Input Validation: - - Always validate file types before processing - - Check both extension and MIME type - - [Adapted from Memory 2] -``` - -### When to Enable LLM Rewrite - -- **Enable** when: - - Memories need to be adapted to specific context - - User query is complex or nuanced - - Better integration with conversation flow is needed - -- **Disable** when: - - Fast retrieval is prioritized - - Memories are already clear and relevant - - Reducing LLM calls and costs is important ## MergeMemoryOp @@ -181,4 +70,4 @@ An alternative to RewriteMemoryOp that merges multiple memories into a single re - Collects the content from all memories in the memory list - Formats them into a single response with a standard structure -- Adds a prompt to consider the helpful parts when answering the question +- Adds a prompt to consider the helpful parts when answering the question \ No newline at end of file diff --git a/docs/task_memory/task_summary_ops.md b/docs/task_memory/task_summary_ops.md index f1364baf..db6dd1af 100644 --- a/docs/task_memory/task_summary_ops.md +++ b/docs/task_memory/task_summary_ops.md @@ -138,66 +138,13 @@ A simplified version of memory extraction that processes entire trajectories in ### Functionality - Classifies trajectories as success or failure based on score threshold -- Extracts memories directly from complete trajectories without segmentation -- Uses a single LLM prompt to extract multiple memories from each trajectory -- Parses JSON response containing `when_to_use` and `memory` pairs +- Extracts memories directly from complete trajectories - Useful for simpler use cases where detailed segmentation is not required -### Processing Flow - -1. **Receive Trajectories**: Get a list of Trajectory objects with messages and scores -2. **For Each Trajectory**: - - Merge all messages into execution_process text - - Determine execution_result ("success" or "fail") based on score threshold - - Format prompt with execution_process and execution_result - - Call LLM to extract memories in JSON format -3. **Parse Response**: Extract JSON array of {when_to_use, memory} objects -4. **Create TaskMemory**: Create TaskMemory objects for each extracted memory -5. **Return**: Set memory_list in context.response.metadata - ### Parameters - `op.simple_summary_op.params.success_score_threshold` (float, default: `0.9`): - The threshold score that determines if a trajectory is considered successful - - Trajectories with `score >= success_score_threshold` are marked as "success" - - Trajectories with `score < success_score_threshold` are marked as "fail" - -### Usage Example - -```python -import requests - -# Prepare trajectory with messages -trajectory = { - "messages": [ - {"role": "user", "content": "How do I sort a list in Python?"}, - {"role": "assistant", "content": "You can use the sorted() function..."}, - {"role": "user", "content": "Thanks! What about reverse sorting?"}, - {"role": "assistant", "content": "Use sorted(list, reverse=True)..."} - ], - "score": 1.0 # Successful trajectory -} - -# Summarize using simple summary -response = requests.post( - url="http://0.0.0.0:8002/summary_task_memory_simple", - json={ - "workspace_id": "my_workspace", - "trajectories": [trajectory] - } -) -``` - -### Comparison with Full Summary Pipeline - -| Feature | SimpleSummaryOp | Full Summary Pipeline | -|---------|-----------------|----------------------| -| **Complexity** | Single operation | Multi-stage pipeline | -| **Segmentation** | No segmentation | Optional segmentation | -| **Extraction** | Single prompt per trajectory | Separate prompts for success/failure/comparative | -| **Validation** | No validation | LLM validation step | -| **Deduplication** | No deduplication | Optional deduplication | -| **Use Case** | Quick prototyping | Production environments | ## SimpleComparativeSummaryOp @@ -213,4 +160,4 @@ A simplified version of comparative memory extraction. ### Parameters -No specific parameters beyond the LLM configuration. +No specific parameters beyond the LLM configuration. \ No newline at end of file diff --git a/docs/tool_memory/tool_memory.md b/docs/tool_memory/tool_memory.md new file mode 100644 index 00000000..024a6409 --- /dev/null +++ b/docs/tool_memory/tool_memory.md @@ -0,0 +1,557 @@ +# Tool Memory in ReMe + +Tool Memory is a specialized component of ReMe that captures and learns from tool usage patterns, enabling AI agents to improve their tool invocation strategies over time. This document explains how tool memory works and how to use it in your applications. + +## What is Tool Memory? + +Tool Memory represents knowledge extracted from historical tool invocations, including: +- Usage patterns and best practices for specific tools +- Common parameter configurations that lead to success or failure +- Performance characteristics (time cost, token cost, success rate) +- Actionable recommendations based on real usage data + +Each tool memory contains: +- `when_to_use`: The tool name (used as the unique identifier) +- `content`: Synthesized usage guidelines and best practices +- `tool_call_results`: Historical invocation records with evaluations +- Statistical metrics about tool performance + +## Tool Memory Data Structure + +### ToolMemory + +```python +class ToolMemory(BaseMemory): + memory_type: str = "tool" + workspace_id: str # Workspace identifier + memory_id: str # Unique memory ID + when_to_use: str # Tool name (serves as identifier) + content: str # Synthesized usage guidelines + score: float # Overall quality score + time_created: str # Creation timestamp + time_modified: str # Last modification timestamp + author: str # Creator (typically LLM model name) + tool_call_results: List[ToolCallResult] # Historical invocation records + metadata: dict # Additional metadata +``` + +### ToolCallResult + +```python +class ToolCallResult(BaseModel): + create_time: str # Invocation timestamp + tool_name: str # Name of the tool + input: dict | str # Input parameters + output: str # Tool output + token_cost: int # Token consumption + success: bool # Whether invocation succeeded + time_cost: float # Time consumed (seconds) + summary: str # Brief summary of the result + evaluation: str # Detailed evaluation + score: float # Evaluation score (0.0, 0.5, or 1.0) + metadata: dict # Additional metadata +``` + +Tool Memory learns from each tool invocation by evaluating the result and accumulating insights over time. + +## Configuration Logic + +Tool Memory in ReMe is configured through three main flows: + +### 1. Add Tool Call Result + +The `add_tool_call_result` flow processes individual tool invocations and adds them to memory: + +```yaml +add_tool_call_result: + flow_content: parse_tool_call_result_op >> update_vector_store_op + description: "Evaluates and adds tool call results to the tool memory database" +``` + +```mermaid +graph LR + A[Tool Call Results] --> B[parse_tool_call_result_op] + B --> C[Evaluate Each Call] + C --> D[Generate Summary & Score] + D --> E[Update/Create ToolMemory] + E --> F[update_vector_store_op] + F --> G[Store in Vector DB] +``` + +This flow: +1. Receives tool call results with input, output, and metadata +2. Evaluates each call using LLM (generates summary, evaluation, score) +3. Appends evaluated results to the tool's memory +4. Updates the vector store with the modified memory + +### 2. Retrieve Tool Memory + +The `retrieve_tool_memory` flow fetches usage guidelines for specific tools: + +```yaml +retrieve_tool_memory: + flow_content: retrieve_tool_memory_op + description: "Retrieves tool memories from the vector database based on tool names" +``` + +```mermaid +graph LR + A[Tool Names] --> B[retrieve_tool_memory_op] + B --> C[Search by Tool Name] + C --> D[Exact Match Check] + D --> E[Return ToolMemory] + E --> F[Usage Guidelines + History] +``` + +This flow: +1. Takes comma-separated tool names as input +2. Searches the vector store for exact matches +3. Returns tool memories with usage guidelines and call history + +### 3. Summary Tool Memory + +The `summary_tool_memory` flow analyzes historical data and generates comprehensive usage guidelines: + +```yaml +summary_tool_memory: + flow_content: summary_tool_memory_op >> update_vector_store_op + description: "Analyzes tool call history and generates comprehensive usage patterns" +``` + +```mermaid +graph LR + A[Tool Names] --> B[summary_tool_memory_op] + B --> C[Retrieve Tool Memory] + C --> D[Analyze Recent Calls] + D --> E[Calculate Statistics] + E --> F[Generate Guidelines] + F --> G[update_vector_store_op] + G --> H[Update Vector DB] +``` + +This flow: +1. Retrieves existing tool memories by tool name +2. Analyzes recent N tool calls (default: 20) +3. Calculates statistical metrics (success rate, avg score, costs) +4. Uses LLM to synthesize actionable usage guidelines +5. Updates the tool memory content with new insights + +## Complete Interaction Flow + +The following diagram illustrates how the three operations interact with the vector store and agent tool calls: + +```mermaid +graph LR + Agent[Agent] -->|1. Before tool call| Retrieve[retrieve_tool_memory] + Retrieve -->|Read| VectorStore[(Vector Store)] + VectorStore -->|Usage Guidelines| Agent + + Agent -->|2. Execute| Tool[Tool Call] + Tool -->|Result| Agent + + Agent -->|3. After tool call| Add[add_tool_call_result] + Add -->|Evaluate & Write| VectorStore + + Summary[summary_tool_memory] -->|4. Periodic| VectorStore + VectorStore -->|Read History| Summary + Summary -->|Update Guidelines| VectorStore + + style Agent fill:#e1f5ff + style Retrieve fill:#fff4e1 + style Add fill:#ffe1f5 + style Summary fill:#e1ffe1 + style VectorStore fill:#f0f0f0 +``` + +### Workflow Steps + +**1. retrieve_tool_memory** (Before tool execution) +- Agent queries usage guidelines from Vector Store +- Returns best practices and historical patterns + +**2. Tool Call Execution** +- Agent executes tool with recommended parameters +- Gets result (success/failure, output, costs) + +**3. add_tool_call_result** (After tool execution) +- Evaluates the tool call result +- Stores evaluated result to Vector Store + +**4. summary_tool_memory** (Periodic) +- Analyzes accumulated call history +- Generates comprehensive usage guidelines +- Updates Vector Store with new insights + +## Basic Usage + +Here's how to use Tool Memory in your application: + +### Step 1: Set Up Your Environment + +```python +import requests + +# API configuration +BASE_URL = "http://0.0.0.0:8002/" +WORKSPACE_ID = "your_workspace_id" +``` + +### Step 2: Record Tool Call Results + +After your agent executes a tool, record the invocation: + +```python +# Example: Record a web search tool call +tool_call_results = [ + { + "create_time": "2025-10-15 14:30:00", + "tool_name": "web_search", + "input": { + "query": "Python asyncio tutorial", + "max_results": 10, + "language": "en" + }, + "output": "Found 10 relevant results including official docs and tutorials", + "token_cost": 150, + "success": True, + "time_cost": 2.3 + } +] + +# Add the tool call result +response = requests.post( + url=f"{BASE_URL}add_tool_call_result", + json={ + "workspace_id": WORKSPACE_ID, + "tool_name": "web_search", + "tool_call_results": tool_call_results + } +) +``` + +### Step 3: Retrieve Tool Usage Guidelines + +Before using a tool, retrieve its usage guidelines: + +```python +# Retrieve memory for specific tools +response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": "web_search,file_reader" # Comma-separated + } +) + +memory_list = response.json().get("metadata", {}).get("memory_list", []) +for memory in memory_list: + print(f"Tool: {memory['when_to_use']}") + print(f"Guidelines: {memory['content']}") + print(f"Total Calls: {len(memory['tool_call_results'])}") +``` + +### Step 4: Generate Comprehensive Usage Guidelines + +After accumulating sufficient call history, generate synthesized guidelines: + +```python +# Summarize tool usage patterns +response = requests.post( + url=f"{BASE_URL}summary_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": "web_search" + } +) + +if response.json().get("success"): + print("Successfully generated usage guidelines") +``` + +## Complete Example Workflow + +Here's a complete example demonstrating the tool memory lifecycle: + +```python +import requests +from datetime import datetime + +BASE_URL = "http://0.0.0.0:8002/" +WORKSPACE_ID = "demo_workspace" + +def record_tool_usage(tool_name, input_params, output, success, time_cost, token_cost): + """Record a single tool invocation""" + tool_call_result = { + "create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": input_params, + "output": output, + "token_cost": token_cost, + "success": success, + "time_cost": time_cost + } + + response = requests.post( + url=f"{BASE_URL}add_tool_call_result", + json={ + "workspace_id": WORKSPACE_ID, + "tool_name": tool_name, + "tool_call_results": [tool_call_result] + } + ) + return response.json() + +def get_tool_guidelines(tool_name): + """Retrieve usage guidelines for a tool""" + response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": tool_name + } + ) + + result = response.json() + if result.get("success"): + memory_list = result.get("metadata", {}).get("memory_list", []) + if memory_list: + return memory_list[0] + return None + +def generate_guidelines(tool_name): + """Generate comprehensive usage guidelines""" + response = requests.post( + url=f"{BASE_URL}summary_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": tool_name + } + ) + return response.json() + +# Example usage +if __name__ == "__main__": + tool_name = "web_search" + + # 1. Record multiple tool invocations + print("Recording tool invocations...") + for i in range(5): + record_tool_usage( + tool_name=tool_name, + input_params={"query": f"test query {i}", "max_results": 10}, + output=f"Found results for query {i}", + success=True, + time_cost=2.0 + i * 0.5, + token_cost=100 + i * 20 + ) + + # 2. Generate usage guidelines + print("\nGenerating usage guidelines...") + generate_guidelines(tool_name) + + # 3. Retrieve and display guidelines + print("\nRetrieving guidelines...") + memory = get_tool_guidelines(tool_name) + if memory: + print(f"\nTool: {memory['when_to_use']}") + print(f"Guidelines:\n{memory['content']}") + print(f"\nTotal invocations: {len(memory['tool_call_results'])}") +``` + +## Use Cases + +### Use Case 1: Learning Optimal Parameters + +```mermaid +sequenceDiagram + participant Agent + participant ToolMemory + participant Tool + + Agent->>ToolMemory: retrieve_tool_memory("api_caller") + ToolMemory-->>Agent: Guidelines: Use timeout=30s, retry=3 + Agent->>Tool: Call with recommended params + Tool-->>Agent: Success (time: 2.5s) + Agent->>ToolMemory: add_tool_call_result(success=True) + ToolMemory->>ToolMemory: Update statistics +``` + +**Scenario**: An agent needs to call an external API. Tool Memory has learned from 50+ previous calls that: +- Setting `timeout=30s` achieves 95% success rate +- Using `retry=3` handles transient failures effectively +- Requests with `max_results > 100` often timeout + +The agent retrieves these guidelines before making the call, leading to higher success rates. + +### Use Case 2: Avoiding Common Pitfalls + +```mermaid +sequenceDiagram + participant Agent + participant ToolMemory + participant FileReader + + Agent->>ToolMemory: retrieve_tool_memory("file_reader") + ToolMemory-->>Agent: Warning: Large files (>10MB) cause timeouts + Agent->>Agent: Check file size first + Agent->>FileReader: Read file with streaming mode + FileReader-->>Agent: Success + Agent->>ToolMemory: add_tool_call_result(success=True) +``` + +**Scenario**: Tool Memory has recorded that the `file_reader` tool fails when: +- File paths contain special characters without escaping +- Files larger than 10MB are read without streaming mode +- Binary files are opened in text mode + +The agent retrieves these warnings and adjusts its approach accordingly. + +### Use Case 3: Performance Optimization + +```python +# Before using Tool Memory +average_time_cost = 5.2s +success_rate = 75% + +# After learning from Tool Memory +# - Use batch processing for multiple queries +# - Set appropriate timeout values +# - Cache frequently accessed data + +average_time_cost = 2.8s # 46% improvement +success_rate = 92% # 17% improvement +``` + +**Scenario**: Tool Memory analyzes 100+ invocations of a `database_query` tool and discovers: +- Batch queries are 3x faster than individual queries +- Connection pooling reduces overhead by 40% +- Queries during peak hours (2-4 PM) have higher failure rates + +The synthesized guidelines help the agent optimize its database interactions. + +## Managing Tool Memories + +### Delete a Workspace + +```python +response = requests.post( + url=f"{BASE_URL}vector_store", + json={ + "workspace_id": WORKSPACE_ID, + "action": "delete" + } +) +``` + +### Dump Memories to Disk + +```python +response = requests.post( + url=f"{BASE_URL}vector_store", + json={ + "workspace_id": WORKSPACE_ID, + "action": "dump", + "path": "./" + } +) +``` + +### Load Memories from Disk + +```python +response = requests.post( + url=f"{BASE_URL}vector_store", + json={ + "workspace_id": WORKSPACE_ID, + "action": "load", + "path": "./" + } +) +``` + +## Configuration Parameters + +### ParseToolCallResultOp Parameters + +Configure in `default.yaml`: + +```yaml +op: + parse_tool_call_result_op: + backend: parse_tool_call_result_op + llm: default + params: + max_history_tool_call_cnt: 100 # Max historical calls to retain + evaluation_sleep_interval: 1.0 # Delay between evaluations (seconds) +``` + +- `max_history_tool_call_cnt`: Limits the number of historical tool call results stored per tool. Older results are removed when this limit is exceeded. +- `evaluation_sleep_interval`: Controls the delay between concurrent evaluations to avoid rate limiting. + +### SummaryToolMemoryOp Parameters + +```yaml +op: + summary_tool_memory_op: + backend: summary_tool_memory_op + llm: default + params: + recent_call_count: 20 # Number of recent calls to analyze + summary_sleep_interval: 1.0 # Delay between summaries (seconds) +``` + +- `recent_call_count`: Number of most recent tool calls to analyze when generating guidelines. +- `summary_sleep_interval`: Controls the delay between concurrent summarizations. + +## Best Practices + +1. **Regular Recording**: + - Record every tool invocation, including failures + - Include detailed input parameters and output + - Capture performance metrics (time_cost, token_cost) + +2. **Periodic Summarization**: + - Generate guidelines after accumulating 20-50 tool calls + - Re-summarize when usage patterns change significantly + - Update guidelines when new tool versions are deployed + +3. **Retrieval Strategy**: + - Always retrieve guidelines before using unfamiliar tools + - Cache retrieved guidelines for the duration of a task + - Re-retrieve after tool memory updates + +4. **Quality Maintenance**: + - Monitor success rates and average scores + - Investigate tools with declining performance + - Clean up outdated memories when tools are deprecated + +5. **Parameter Tuning**: + - Adjust `max_history_tool_call_cnt` based on tool usage frequency + - Increase `recent_call_count` for tools with diverse usage patterns + - Reduce `evaluation_sleep_interval` if rate limiting is not a concern + +## Integration with Agent Workflows + +```mermaid +graph TB + A[Agent Receives Task] --> B{Tool Required?} + B -->|Yes| C[Retrieve Tool Memory] + C --> D[Apply Guidelines] + D --> E[Execute Tool] + E --> F[Record Result] + F --> G{Sufficient History?} + G -->|Yes| H[Generate Summary] + G -->|No| I[Continue] + H --> I + B -->|No| I[Process Task] + I --> J[Task Complete] +``` + +Tool Memory seamlessly integrates into agent workflows: +1. Before tool execution: Retrieve usage guidelines +2. During execution: Apply recommended parameters +3. After execution: Record results with evaluation +4. Periodically: Generate updated guidelines + +For more detailed examples, see the implementation in `reme_ai/summary/tool/` directory of the ReMe project. + diff --git a/docs/tool_memory/tool_retrieve_ops.md b/docs/tool_memory/tool_retrieve_ops.md new file mode 100644 index 00000000..38ac37e1 --- /dev/null +++ b/docs/tool_memory/tool_retrieve_ops.md @@ -0,0 +1,548 @@ +# Tool Memory Retrieval Operations + +## RetrieveToolMemoryOp + +### Purpose + +Retrieves tool memories from the vector database based on tool names, providing usage patterns, best practices, and historical call data. + +### Functionality + +- Accepts comma-separated tool names as input +- Searches the vector store for exact tool name matches +- Validates that retrieved memories are of type "tool" +- Returns complete tool memories including usage guidelines and call history +- Provides detailed logging for debugging and monitoring + +### Processing Flow + +```mermaid +graph TB + A[Receive Tool Names] --> B[Validate Input] + B --> C[Split by Comma] + C --> D[Trim Whitespace] + D --> E[For Each Tool Name] + E --> F[Search Vector Store] + F --> G{Results Found?} + G -->|Yes| H[Get Top Result] + G -->|No| I[Log Warning: Not Found] + H --> J{Type = tool?} + J -->|Yes| K{Name Matches?} + J -->|No| L[Log Warning: Wrong Type] + K -->|Yes| M[Add to Results] + K -->|No| N[Log Warning: Name Mismatch] + I --> O[Continue Next Tool] + L --> O + N --> O + M --> O + O --> P{More Tools?} + P -->|Yes| E + P -->|No| Q{Any Matches?} + Q -->|Yes| R[Return Memory List] + Q -->|No| S[Return Empty] +``` + +1. **Input Validation**: + - Check if `tool_names` parameter is provided + - Return error if empty + - Log workspace_id and tool count + +2. **Tool Name Processing**: + - Split input by comma delimiter + - Strip whitespace from each tool name + - Filter out empty strings + - Log the list of tools to retrieve + +3. **Vector Store Search**: + - For each tool name, search with `top_k=1` + - Use tool name as the query (exact match preferred) + - Retrieve the top matching result + +4. **Result Validation**: + - Verify result is of type `ToolMemory` + - Check that `when_to_use` field exactly matches tool name + - Log match details (memory_id, total_calls) + - Warn if no match or mismatch found + +5. **Response Preparation**: + - Collect all matched tool memories + - Set success status based on matches found + - Return memory list in metadata + +### Parameters + +This operation has no configurable parameters. It uses the default vector store configuration. + +### Input Schema + +```yaml +input_schema: + tool_names: + type: string + description: "Comma-separated tool names (e.g., 'tool_name1,tool_name2')" + required: true +``` + +### Output Format + +The operation sets the following in `context.response`: + +```python +{ + "success": True, + "answer": "Successfully retrieved 2 tool memories", + "metadata": { + "memory_list": [ + { + "workspace_id": "demo_workspace", + "memory_id": "abc123def456", + "memory_type": "tool", + "when_to_use": "web_search", + "content": "Core Function: The web_search tool retrieves...", + "score": 0.85, + "time_created": "2025-10-15 10:00:00", + "time_modified": "2025-10-15 14:30:00", + "author": "qwen3-30b-a3b-instruct-2507", + "tool_call_results": [ + { + "create_time": "2025-10-15 14:30:00", + "tool_name": "web_search", + "input": {...}, + "output": "...", + "summary": "...", + "evaluation": "...", + "score": 1.0, + "success": True, + "time_cost": 2.3, + "token_cost": 150 + } + ], + "metadata": {} + } + ] + } +} +``` + +### Usage Example + +#### Basic Retrieval + +```python +import requests + +BASE_URL = "http://0.0.0.0:8002/" +WORKSPACE_ID = "demo_workspace" + +# Retrieve memory for a single tool +response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": "web_search" + } +) + +result = response.json() +if result.get("success"): + memory_list = result.get("metadata", {}).get("memory_list", []) + for memory in memory_list: + print(f"Tool: {memory['when_to_use']}") + print(f"Guidelines:\n{memory['content']}") + print(f"Total Calls: {len(memory['tool_call_results'])}") +``` + +#### Multiple Tools Retrieval + +```python +# Retrieve memories for multiple tools at once +response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": "web_search,file_reader,api_caller,database_query" + } +) + +result = response.json() +if result.get("success"): + memory_list = result.get("metadata", {}).get("memory_list", []) + print(f"Retrieved {len(memory_list)} tool memories") + + for memory in memory_list: + print(f"\n{'='*60}") + print(f"Tool: {memory['when_to_use']}") + print(f"{'='*60}") + print(memory['content']) +``` + +#### Extracting Specific Information + +```python +def get_tool_statistics(tool_name): + """Get statistical information for a specific tool""" + response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": tool_name + } + ) + + result = response.json() + if not result.get("success"): + return None + + memory_list = result.get("metadata", {}).get("memory_list", []) + if not memory_list: + return None + + memory = memory_list[0] + tool_calls = memory['tool_call_results'] + + # Calculate statistics + total_calls = len(tool_calls) + successful_calls = sum(1 for call in tool_calls if call['success']) + avg_score = sum(call['score'] for call in tool_calls) / total_calls if total_calls > 0 else 0 + avg_time = sum(call['time_cost'] for call in tool_calls) / total_calls if total_calls > 0 else 0 + avg_tokens = sum(call['token_cost'] for call in tool_calls) / total_calls if total_calls > 0 else 0 + + return { + "tool_name": tool_name, + "total_calls": total_calls, + "success_rate": successful_calls / total_calls if total_calls > 0 else 0, + "avg_score": avg_score, + "avg_time_cost": avg_time, + "avg_token_cost": avg_tokens, + "guidelines": memory['content'] + } + +# Usage +stats = get_tool_statistics("web_search") +if stats: + print(f"Tool: {stats['tool_name']}") + print(f"Total Calls: {stats['total_calls']}") + print(f"Success Rate: {stats['success_rate']:.1%}") + print(f"Avg Score: {stats['avg_score']:.2f}") + print(f"Avg Time: {stats['avg_time_cost']:.2f}s") + print(f"Avg Tokens: {stats['avg_token_cost']:.0f}") +``` + +### Integration with Agent Workflows + +#### Pre-Execution Retrieval + +```python +def execute_tool_with_guidelines(tool_name, input_params): + """Execute a tool after retrieving its usage guidelines""" + + # 1. Retrieve tool memory + response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": tool_name + } + ) + + # 2. Extract guidelines + guidelines = "" + if response.json().get("success"): + memory_list = response.json().get("metadata", {}).get("memory_list", []) + if memory_list: + guidelines = memory_list[0]['content'] + print(f"Guidelines for {tool_name}:") + print(guidelines) + + # 3. Adjust parameters based on guidelines + # (This would be done by an LLM or rule-based system) + adjusted_params = adjust_parameters(input_params, guidelines) + + # 4. Execute tool + result = execute_tool(tool_name, adjusted_params) + + # 5. Record result + record_tool_call(tool_name, adjusted_params, result) + + return result +``` + +#### Batch Retrieval for Agent Initialization + +```python +def initialize_agent_with_tool_memories(available_tools): + """Load all tool memories at agent initialization""" + + # Retrieve all tool memories at once + tool_names = ",".join(available_tools) + response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": tool_names + } + ) + + # Build tool memory cache + tool_memory_cache = {} + if response.json().get("success"): + memory_list = response.json().get("metadata", {}).get("memory_list", []) + for memory in memory_list: + tool_memory_cache[memory['when_to_use']] = { + "guidelines": memory['content'], + "total_calls": len(memory['tool_call_results']), + "last_modified": memory['time_modified'] + } + + return tool_memory_cache + +# Usage +available_tools = ["web_search", "file_reader", "api_caller"] +tool_cache = initialize_agent_with_tool_memories(available_tools) + +# Agent can now quickly access guidelines +if "web_search" in tool_cache: + print(tool_cache["web_search"]["guidelines"]) +``` + +### Error Handling + +#### Tool Not Found + +```python +response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": "nonexistent_tool" + } +) + +result = response.json() +if not result.get("success"): + print(f"Error: {result.get('answer')}") + # Output: "No matching tool memories found" +``` + +#### Empty Tool Names + +```python +response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": "" + } +) + +result = response.json() +if not result.get("success"): + print(f"Error: {result.get('answer')}") + # Output: "tool_names is required" +``` + +#### Partial Matches + +```python +# Request 3 tools, but only 2 exist +response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": "web_search,file_reader,nonexistent_tool" + } +) + +result = response.json() +if result.get("success"): + memory_list = result.get("metadata", {}).get("memory_list", []) + print(f"Found {len(memory_list)} out of 3 requested tools") + # Output: "Found 2 out of 3 requested tools" +``` + +### Retrieval Workflow + +```mermaid +sequenceDiagram + participant Agent + participant RetrieveOp + participant VectorStore + + Agent->>RetrieveOp: retrieve_tool_memory(tool_names) + RetrieveOp->>RetrieveOp: Split and validate names + + loop For each tool name + RetrieveOp->>VectorStore: search(tool_name, top_k=1) + VectorStore-->>RetrieveOp: Return top result + RetrieveOp->>RetrieveOp: Validate type and name + alt Valid match + RetrieveOp->>RetrieveOp: Add to results + else No match + RetrieveOp->>RetrieveOp: Log warning + end + end + + RetrieveOp-->>Agent: Return memory_list + Agent->>Agent: Apply guidelines +``` + +### Use Cases + +#### Use Case 1: Pre-Execution Guidance + +**Scenario**: Before executing a tool, the agent retrieves usage guidelines to optimize parameters. + +```python +# Agent needs to search the web +tool_name = "web_search" +query = "machine learning basics" + +# Retrieve guidelines +response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": tool_name + } +) + +# Guidelines suggest: max_results=10-20, use filter_type="technical_docs" +# Agent adjusts parameters accordingly +optimized_params = { + "query": query, + "max_results": 15, # Within recommended range + "filter_type": "technical_docs", # As suggested + "language": "en" +} + +# Execute with optimized parameters +result = execute_tool(tool_name, optimized_params) +``` + +#### Use Case 2: Performance Monitoring + +**Scenario**: Monitor tool performance trends over time. + +```python +def monitor_tool_performance(tool_name): + """Monitor tool performance and detect degradation""" + response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": tool_name + } + ) + + if not response.json().get("success"): + return None + + memory = response.json()["metadata"]["memory_list"][0] + calls = memory["tool_call_results"] + + # Analyze recent vs historical performance + recent_calls = calls[-20:] # Last 20 calls + historical_calls = calls[:-20] if len(calls) > 20 else [] + + recent_success_rate = sum(1 for c in recent_calls if c['success']) / len(recent_calls) + historical_success_rate = (sum(1 for c in historical_calls if c['success']) / len(historical_calls) + if historical_calls else recent_success_rate) + + # Detect degradation + if recent_success_rate < historical_success_rate - 0.1: + print(f"Warning: {tool_name} performance degraded!") + print(f"Recent: {recent_success_rate:.1%}, Historical: {historical_success_rate:.1%}") + return "degraded" + + return "healthy" + +# Usage +status = monitor_tool_performance("web_search") +``` + +#### Use Case 3: Tool Selection + +**Scenario**: Choose the best tool for a task based on historical performance. + +```python +def select_best_tool(task_description, candidate_tools): + """Select the best tool based on historical performance""" + + # Retrieve memories for all candidate tools + tool_names = ",".join(candidate_tools) + response = requests.post( + url=f"{BASE_URL}retrieve_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": tool_names + } + ) + + if not response.json().get("success"): + return candidate_tools[0] # Default to first + + memory_list = response.json()["metadata"]["memory_list"] + + # Score each tool + tool_scores = {} + for memory in memory_list: + calls = memory["tool_call_results"] + if not calls: + continue + + # Calculate composite score + success_rate = sum(1 for c in calls if c['success']) / len(calls) + avg_score = sum(c['score'] for c in calls) / len(calls) + avg_time = sum(c['time_cost'] for c in calls) / len(calls) + + # Composite: prioritize success and quality, penalize slow tools + composite = (success_rate * 0.4 + avg_score * 0.4 + (1 / (1 + avg_time)) * 0.2) + tool_scores[memory['when_to_use']] = composite + + # Select best tool + if tool_scores: + best_tool = max(tool_scores, key=tool_scores.get) + print(f"Selected {best_tool} with score {tool_scores[best_tool]:.2f}") + return best_tool + + return candidate_tools[0] + +# Usage +best = select_best_tool( + "Search for technical documentation", + ["web_search", "doc_search", "api_search"] +) +``` + +## Best Practices + +1. **Retrieval Timing**: + - Retrieve guidelines before first use of a tool + - Cache retrieved memories for the duration of a task + - Re-retrieve after tool memory updates (post-summarization) + +2. **Batch Retrieval**: + - Retrieve multiple tool memories in a single request + - Use comma-separated tool names for efficiency + - Initialize agent with all available tool memories + +3. **Error Handling**: + - Always check `success` status in response + - Handle cases where tool memory doesn't exist + - Provide fallback behavior for missing guidelines + +4. **Guidelines Application**: + - Parse guidelines to extract parameter recommendations + - Use LLM to interpret guidelines in context + - Combine guidelines with task-specific requirements + +5. **Performance Optimization**: + - Cache retrieved memories to avoid repeated calls + - Invalidate cache after summarization updates + - Monitor retrieval latency and optimize if needed + +6. **Monitoring**: + - Log retrieval requests for debugging + - Track which tools are frequently retrieved + - Identify tools without memory (candidates for recording) + diff --git a/docs/tool_memory/tool_summary_ops.md b/docs/tool_memory/tool_summary_ops.md new file mode 100644 index 00000000..dd4992e9 --- /dev/null +++ b/docs/tool_memory/tool_summary_ops.md @@ -0,0 +1,465 @@ +# Tool Summary Operations + +## ParseToolCallResultOp + +### Purpose + +Evaluates individual tool invocations and adds them to the tool memory database with comprehensive assessments. + +### Functionality + +- Receives tool call results with input parameters, output, and metadata +- Uses LLM to evaluate each tool call based on success and parameter alignment +- Generates summary, evaluation, and score (0.0, 0.5, or 1.0) for each call +- Appends evaluated results to existing tool memory or creates new memory +- Maintains a sliding window of recent tool calls (configurable limit) + +### Processing Flow + +```mermaid +graph TB + A[Receive Tool Call Results] --> B[Validate Input] + B --> C[Search for Existing Memory] + C --> D{Memory Exists?} + D -->|Yes| E[Load Existing Memory] + D -->|No| F[Create New Memory] + E --> G[Concurrent Evaluation] + F --> G + G --> H[Evaluate Call #1] + G --> I[Evaluate Call #2] + G --> J[Evaluate Call #N] + H --> K[Generate Summary & Score] + I --> K + J --> K + K --> L[Append to Memory] + L --> M[Trim to Max History] + M --> N[Update Modified Time] + N --> O[Prepare for Vector Store] +``` + +1. **Input Validation**: + - Verify `tool_name` is provided + - Convert dict objects to `ToolCallResult` instances + - Check if `tool_call_results` list is not empty + +2. **Memory Lookup**: + - Search vector store for existing tool memory by tool name + - Verify exact match (memory type = "tool" and when_to_use = tool_name) + - Create new `ToolMemory` if no match found + +3. **Concurrent Evaluation**: + - Submit all tool call results for parallel evaluation + - Each evaluation uses LLM to analyze the call + - Generate structured evaluation with summary, assessment, and score + +4. **Memory Update**: + - Append evaluated results to tool memory + - Trim to `max_history_tool_call_cnt` if limit exceeded + - Update modification timestamp + - Prepare for vector store update + +### Parameters + +Configure in `default.yaml`: + +```yaml +op: + parse_tool_call_result_op: + backend: parse_tool_call_result_op + llm: default + params: + max_history_tool_call_cnt: 100 + evaluation_sleep_interval: 1.0 +``` + +- `max_history_tool_call_cnt` (integer, default: `100`): + - Maximum number of historical tool call results to retain per tool + - When exceeded, oldest results are removed (FIFO) + - Balances memory size with historical context + - Recommended: 50-200 depending on tool usage frequency + +- `evaluation_sleep_interval` (float, default: `1.0`): + - Delay in seconds between concurrent evaluations + - Prevents rate limiting when evaluating multiple calls + - Set to 0 for maximum speed (if no rate limits) + - Increase if encountering API throttling + +### Evaluation Criteria + +The LLM evaluates each tool call based on two dimensions: + +1. **Success Evaluation**: + - Check if the success flag indicates successful execution + - Verify output contains no error messages + - Assess if time and token costs are reasonable + - Identify any error indicators in the output + +2. **Parameter Alignment Evaluation**: + - Evaluate if output matches expected behavior given input + - Consider if input parameters are appropriate for the tool + - Check for parameter mismatches or unexpected behaviors + - Verify output is consistent with tool's intended purpose + +### Scoring Guidelines + +- **1.0 (Success)**: Tool executed successfully with good parameter alignment + - Example: Query returned relevant results, parameters were appropriate + +- **0.5 (Partial Success)**: Tool executed but with issues + - Example: Query succeeded but parameters were suboptimal (e.g., too generic) + - Example: Results returned but with warnings about invalid parameters + +- **0.0 (Failure)**: Tool execution failed or severe parameter misalignment + - Example: Timeout due to excessive max_results parameter + - Example: Error due to invalid parameter format + +### Output Format + +The operation sets the following in `context.response.metadata`: + +```python +{ + "deleted_memory_ids": ["memory_id_if_updating"], + "memory_list": [ + { + "memory_id": "abc123", + "when_to_use": "web_search", + "tool_call_results": [ + { + "create_time": "2025-10-15 14:30:00", + "tool_name": "web_search", + "input": {...}, + "output": "...", + "summary": "Successfully retrieved 10 relevant results", + "evaluation": "Good parameter alignment...", + "score": 1.0, + "success": True, + "time_cost": 2.3, + "token_cost": 150 + } + ] + } + ] +} +``` + +### Usage Example + +```python +import requests +from datetime import datetime + +BASE_URL = "http://0.0.0.0:8002/" +WORKSPACE_ID = "demo_workspace" + +# Prepare tool call results +tool_call_results = [ + { + "create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": "web_search", + "input": { + "query": "Python asyncio tutorial", + "max_results": 10, + "language": "en", + "filter_type": "technical_docs" + }, + "output": "Found 10 relevant results including official documentation and tutorials", + "token_cost": 150, + "success": True, + "time_cost": 2.3 + }, + { + "create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": "web_search", + "input": { + "query": "test", # Too generic + "max_results": 100, # Too many + "language": "unknown" # Invalid + }, + "output": "Warning: language 'unknown' not supported. Query too generic, limited results.", + "token_cost": 80, + "success": True, + "time_cost": 3.5 + } +] + +# Add tool call results +response = requests.post( + url=f"{BASE_URL}add_tool_call_result", + json={ + "workspace_id": WORKSPACE_ID, + "tool_name": "web_search", + "tool_call_results": tool_call_results + } +) + +result = response.json() +print(f"Success: {result.get('success')}") +print(f"Answer: {result.get('answer')}") + +# Check evaluated results +memory_list = result.get("metadata", {}).get("memory_list", []) +if memory_list: + tool_memory = memory_list[0] + for call_result in tool_memory["tool_call_results"]: + print(f"\nCall Summary: {call_result['summary']}") + print(f"Evaluation: {call_result['evaluation']}") + print(f"Score: {call_result['score']}") +``` + +## SummaryToolMemoryOp + +### Purpose + +Analyzes accumulated tool call history and generates comprehensive usage patterns, best practices, and recommendations. + +### Functionality + +- Retrieves existing tool memories from the vector store +- Analyzes the most recent N tool calls (configurable) +- Calculates statistical metrics (success rate, average scores, costs) +- Uses LLM to synthesize actionable usage guidelines +- Updates tool memory content with generated insights + +### Processing Flow + +```mermaid +graph TB + A[Receive Tool Names] --> B[Split by Comma] + B --> C[For Each Tool Name] + C --> D[Search Vector Store] + D --> E{Exact Match?} + E -->|Yes| F[Load Tool Memory] + E -->|No| G[Log Warning] + F --> H[Extract Recent N Calls] + H --> I[Calculate Statistics] + I --> J[Format Call Summaries] + J --> K[Format Statistics] + K --> L[Concurrent Summarization] + L --> M[LLM Analysis #1] + L --> N[LLM Analysis #2] + L --> O[LLM Analysis #N] + M --> P[Generate Guidelines] + N --> P + O --> P + P --> Q[Update Memory Content] + Q --> R[Update Modified Time] + R --> S[Prepare for Vector Store] +``` + +1. **Tool Name Processing**: + - Split comma-separated tool names + - Trim whitespace from each name + - Log the list of tools to process + +2. **Memory Retrieval**: + - Search vector store for each tool name + - Verify exact match (memory type and when_to_use) + - Skip tools without existing memory + +3. **Data Preparation**: + - Extract the most recent N tool call results + - Calculate statistical metrics: + - Total calls vs recent calls analyzed + - Success rate (overall and recent) + - Average score (overall and recent) + - Average time cost + - Average token cost + - Format call summaries as markdown + - Format statistics as markdown + +4. **Concurrent Summarization**: + - Submit all tools for parallel summarization + - Each summarization uses LLM to analyze patterns + - Generate structured usage guidelines + +5. **Memory Update**: + - Update tool memory content with new guidelines + - Update modification timestamp + - Prepare for vector store update + +### Parameters + +Configure in `default.yaml`: + +```yaml +op: + summary_tool_memory_op: + backend: summary_tool_memory_op + llm: default + params: + recent_call_count: 20 + summary_sleep_interval: 1.0 +``` + +- `recent_call_count` (integer, default: `20`): + - Number of most recent tool calls to analyze + - Focuses on recent usage patterns + - Recommended: 10-50 depending on tool usage frequency + - Higher values provide more context but may dilute recent patterns + +- `summary_sleep_interval` (float, default: `1.0`): + - Delay in seconds between concurrent summarizations + - Prevents rate limiting when summarizing multiple tools + - Set to 0 for maximum speed (if no rate limits) + - Increase if encountering API throttling + +### Statistical Metrics + +The operation calculates the following metrics: + +```python +{ + "total_calls": 100, # Total number of calls in history + "recent_calls": 20, # Number of recent calls analyzed + "success_rate": 0.85, # Overall success rate (85%) + "recent_success_rate": 0.90, # Recent success rate (90%) + "avg_score": 0.78, # Average evaluation score + "recent_avg_score": 0.82, # Recent average score + "avg_time_cost": 2.45, # Average time in seconds + "avg_token_cost": 125.3 # Average token consumption +} +``` + +### Generated Guidelines Structure + +The LLM generates guidelines following this structure: + +1. **Core Function**: What the tool does and when to use it +2. **Success Patterns**: Parameter patterns and scenarios that work well +3. **Common Issues**: Main pitfalls to avoid and why they fail +4. **Best Practices**: 2-3 actionable recommendations + +### Usage Example + +```python +import requests + +BASE_URL = "http://0.0.0.0:8002/" +WORKSPACE_ID = "demo_workspace" + +# After accumulating tool call history, generate guidelines +response = requests.post( + url=f"{BASE_URL}summary_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": "web_search,file_reader,api_caller" # Multiple tools + } +) + +result = response.json() +print(f"Success: {result.get('success')}") +print(f"Answer: {result.get('answer')}") + +# Display generated guidelines +memory_list = result.get("metadata", {}).get("memory_list", []) +for memory in memory_list: + print(f"\n{'='*60}") + print(f"Tool: {memory['when_to_use']}") + print(f"{'='*60}") + print(memory['content']) + + # Display statistics + stats = memory.get('metadata', {}).get('statistics', {}) + print(f"\nStatistics:") + print(f" Total Calls: {stats.get('total_calls', 0)}") + print(f" Success Rate: {stats.get('success_rate', 0):.1%}") + print(f" Avg Score: {stats.get('avg_score', 0):.2f}") +``` + +### Example Generated Guidelines + +``` +Core Function: +The web_search tool retrieves information from the internet based on query parameters. +Use it when you need up-to-date information, documentation, or external data. + +Success Patterns: +- Specific queries (e.g., "Python asyncio tutorial") achieve 95% success rate +- Setting max_results=10-20 balances quality and performance +- Using language="en" and filter_type="technical_docs" improves relevance +- Average successful call: 2.3s, 150 tokens + +Common Issues: +- Generic queries (e.g., "test") return poor results (score: 0.5) +- max_results > 50 often leads to timeouts (avg: 8.2s vs 2.3s) +- Invalid language codes default to English but add latency +- Missing filter_type returns mixed-quality results + +Best Practices: +1. Use specific, descriptive queries with clear intent +2. Set max_results=10-20 for optimal balance +3. Always specify language and filter_type for technical searches +``` + +### When to Run Summarization + +- **Initial Setup**: After accumulating 20-30 tool calls +- **Regular Updates**: Every 50-100 new calls +- **Pattern Changes**: When success rate changes significantly +- **Tool Updates**: After tool version changes or parameter updates +- **Performance Issues**: When investigating declining performance + +### Integration with Retrieval + +```mermaid +sequenceDiagram + participant Agent + participant AddOp as add_tool_call_result + participant SummaryOp as summary_tool_memory + participant RetrieveOp as retrieve_tool_memory + + loop Every Tool Call + Agent->>AddOp: Record tool call result + AddOp->>AddOp: Evaluate and store + end + + Note over Agent,SummaryOp: After 20+ calls + Agent->>SummaryOp: Generate guidelines + SummaryOp->>SummaryOp: Analyze patterns + SummaryOp->>SummaryOp: Update content + + Note over Agent,RetrieveOp: Before next use + Agent->>RetrieveOp: Get tool memory + RetrieveOp-->>Agent: Return guidelines + history + Agent->>Agent: Apply recommendations +``` + +The summarization operation works in conjunction with retrieval: +1. `add_tool_call_result` continuously records invocations +2. `summary_tool_memory` periodically generates guidelines +3. `retrieve_tool_memory` provides guidelines before tool use +4. Agent applies recommendations to improve success rates + +## Best Practices + +1. **Recording Strategy**: + - Record every tool invocation, including failures + - Include detailed input parameters and complete output + - Capture accurate performance metrics + - Add relevant metadata for context + +2. **Evaluation Quality**: + - Ensure LLM has sufficient context for evaluation + - Monitor score distribution (should not be all 1.0 or 0.0) + - Review evaluations periodically for quality + - Adjust evaluation prompts if needed + +3. **Summarization Timing**: + - Wait for 20-30 calls before first summarization + - Re-summarize after significant new data (50+ calls) + - Update when usage patterns change + - Regenerate after tool updates + +4. **Parameter Tuning**: + - Adjust `max_history_tool_call_cnt` based on tool usage frequency + - Increase `recent_call_count` for tools with diverse patterns + - Balance `sleep_interval` between speed and rate limits + - Monitor vector store size and adjust retention limits + +5. **Quality Maintenance**: + - Review generated guidelines for accuracy + - Validate statistical metrics match expectations + - Clean up deprecated tools from vector store + - Archive historical data before major changes + From 12f0b7d124fee9c94dd406871007d044601d605c Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Thu, 16 Oct 2025 23:31:50 +0800 Subject: [PATCH 06/13] feat(agent): implement mock search tools and tool memory benchmark - Added LLMMockSearchOp for simulating search operations with configurable complexity- Created SearchToolA, SearchToolB, and SearchToolC with distinct performance profiles- Implemented UseMockSearchOp for intelligent tool selection based on query analysis - Added test scripts and query datasets for evaluating tool memory effectiveness - Integrated tool memory service for storing and retrieving tool performance data - Created documentation for tool memory benchmark testing methodology - Refactored agent module structure and imports - Increased summary tool memory recent call count from20 to 30 --- cookbook/tool_memory/__init__.py | 0 cookbook/tool_memory/query.json | 139 +++ docs/tool_memory/tool_bench.md | 1063 +++++++++++++++++ reme_ai/__init__.py | 8 +- reme_ai/agent/__init__.py | 2 + reme_ai/{ => agent}/react/__init__.py | 0 reme_ai/{ => agent}/react/simple_react_op.py | 0 reme_ai/agent/tools/__init__.py | 3 + reme_ai/agent/tools/llm_mock_search_op.py | 297 +++++ .../agent/tools/llm_mock_search_prompt.yaml | 76 ++ reme_ai/agent/tools/mock_search_tools.py | 115 ++ reme_ai/agent/tools/test_use_mock_search.py | 132 ++ reme_ai/agent/tools/use_mock_search_op.py | 88 ++ .../agent/tools/use_mock_search_prompt.yaml | 64 + .../summary/tool/summary_tool_memory_op.py | 2 +- 15 files changed, 1984 insertions(+), 5 deletions(-) create mode 100644 cookbook/tool_memory/__init__.py create mode 100644 cookbook/tool_memory/query.json create mode 100644 docs/tool_memory/tool_bench.md create mode 100644 reme_ai/agent/__init__.py rename reme_ai/{ => agent}/react/__init__.py (100%) rename reme_ai/{ => agent}/react/simple_react_op.py (100%) create mode 100644 reme_ai/agent/tools/__init__.py create mode 100644 reme_ai/agent/tools/llm_mock_search_op.py create mode 100644 reme_ai/agent/tools/llm_mock_search_prompt.yaml create mode 100644 reme_ai/agent/tools/mock_search_tools.py create mode 100644 reme_ai/agent/tools/test_use_mock_search.py create mode 100644 reme_ai/agent/tools/use_mock_search_op.py create mode 100644 reme_ai/agent/tools/use_mock_search_prompt.yaml diff --git a/cookbook/tool_memory/__init__.py b/cookbook/tool_memory/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cookbook/tool_memory/query.json b/cookbook/tool_memory/query.json new file mode 100644 index 00000000..db250a72 --- /dev/null +++ b/cookbook/tool_memory/query.json @@ -0,0 +1,139 @@ +{ + "train": { + "simple": [ + "法国的首都是什么？", + "Python 是什么时候首次发布的？", + "地球上有几个大洲？", + "水的沸点是多少度？", + "谁发明了电话？", + "一年有多少天？", + "光速是多少？", + "氧气的化学符号是什么？", + "世界上最高的山峰是哪座？", + "太阳系有几颗行星？", + "地球的半径是多少公里？", + "元素周期表有多少个元素？", + "蒙娜丽莎是谁画的？", + "人类DNA有多少条染色体？", + "HTTP的默认端口号是多少？", + "世界上人口最多的国家是哪个？", + "一公里等于多少米？", + "谁提出了相对论？", + "圆周率π的前两位小数是多少？", + "中国的货币单位是什么？" + ], + "moderate": [ + "列举Python 3.10的主要特性", + "微服务架构有哪些好处？", + "解释区块链技术的工作原理", + "描述SQL和NoSQL数据库的主要区别", + "软件工程中常见的设计模式有哪些？", + "什么是RESTful API的设计原则？", + "解释Docker容器化技术的优势", + "云计算的三种服务模型是什么？", + "什么是机器学习中的过拟合问题？", + "解释Git中merge和rebase的区别", + "什么是持续集成和持续部署(CI/CD)？", + "描述OSI七层网络模型的作用", + "什么是JWT令牌认证机制？", + "解释虚拟内存的工作原理", + "Kubernetes的主要组件有哪些？", + "什么是函数式编程的核心概念？", + "解释负载均衡的常见策略", + "描述SOLID设计原则的含义", + "什么是异步编程的优势和挑战？", + "解释数据库索引的作用和类型" + ], + "complex": [ + "比较凯恩斯主义和奥地利学派的经济政策", + "分析可再生能源采用对环境的影响", + "解释量子力学和广义相对论之间的关系", + "讨论人工智能的历史演变过程", + "评估不同机器学习算法在NLP中的有效性", + "分析全球化对发展中国家经济的长期影响", + "比较不同民主制度模型的优缺点", + "探讨气候变化对生物多样性的影响机制", + "研究基因编辑技术的伦理问题和社会影响", + "分析区块链技术在金融领域的应用前景和挑战", + "评估不同神经网络架构在计算机视觉中的表现", + "探讨量子计算对现代密码学的威胁和机遇", + "分析人口老龄化对社会保障体系的影响", + "比较不同哲学流派对人工智能意识的观点", + "研究微生物组与人类健康之间的复杂关系", + "评估碳捕获技术在应对气候变化中的作用", + "分析5G技术对物联网发展的推动作用", + "探讨认知科学与人工智能的交叉研究领域", + "比较不同经济体制下的创新能力差异", + "研究纳米技术在医疗领域的应用和风险" + ] + }, + "test": { + "simple": [ + "日本的首都是什么？", + "Java 语言是哪一年发布的？", + "大西洋有多宽？", + "铁的熔点是多少度？", + "谁发明了汽车？", + "一周有多少小时？", + "声音在空气中的速度是多少？", + "氢气的化学符号是什么？", + "世界上最长的河流是哪条？", + "月球绕地球一周需要多少天？", + "标准大气压是多少帕？", + "人体有多少块骨头？", + "《星空》是谁画的？", + "成年人有多少颗牙齿？", + "HTTPS的默认端口号是多少？", + "世界上面积最大的国家是哪个？", + "一英里等于多少公里？", + "谁发现了万有引力定律？", + "黄金的化学元素符号是什么？", + "美国的货币单位是什么？" + ], + "moderate": [ + "列举TypeScript 5.0的主要新功能", + "单体架构和微服务架构的区别是什么？", + "解释分布式系统的CAP定理", + "描述关系型数据库的ACID特性", + "前端开发中常用的状态管理方案有哪些？", + "什么是GraphQL的核心优势？", + "解释Kubernetes容器编排的基本概念", + "边缘计算和雾计算有什么区别？", + "什么是神经网络中的梯度消失问题？", + "解释Git中revert和reset的区别", + "什么是DevOps的核心理念？", + "描述TCP/IP协议栈的层次结构", + "什么是OAuth 2.0授权框架？", + "解释进程和线程的本质区别", + "Service Mesh的主要功能是什么？", + "什么是响应式编程的基本原理？", + "解释缓存一致性的常见问题", + "描述DDD领域驱动设计的核心思想", + "什么是并发编程中的死锁问题？", + "解释数据库事务隔离级别的分类" + ], + "complex": [ + "比较货币主义和供给学派的经济理论差异", + "分析核能发展对能源转型的作用", + "解释弦理论与标准模型之间的联系", + "讨论云计算技术的发展历程和趋势", + "评估深度学习在计算机视觉中的应用效果", + "分析数字化转型对传统产业的影响", + "比较总统制和议会制的政治体制特点", + "探讨海洋酸化对海洋生态系统的影响", + "研究人工智能在医疗诊断中的伦理挑战", + "分析去中心化金融(DeFi)的发展机遇与风险", + "评估Transformer架构在自然语言处理中的优势", + "探讨后量子密码学的研究方向和应用", + "分析城市化进程对资源配置的影响", + "比较实用主义和理想主义对技术伦理的看法", + "研究肠道菌群与神经系统疾病的关联", + "评估直接空气捕获技术的经济可行性", + "分析边缘计算对云计算架构的影响", + "探讨神经语言学与自然语言处理的交叉应用", + "比较计划经济和市场经济的资源配置效率", + "研究量子传感技术在精密测量中的突破" + ] + } +} + diff --git a/docs/tool_memory/tool_bench.md b/docs/tool_memory/tool_bench.md new file mode 100644 index 00000000..16ace284 --- /dev/null +++ b/docs/tool_memory/tool_bench.md @@ -0,0 +1,1063 @@ +# Tool Memory 效果验证基准测试 + +## 概述 + +本文档设计了一个完整的实验方案来验证 Tool Memory 在提升 Agent 性能方面的有效性。实验通过对比 Agent 在有无 Tool Memory 支持下的表现，量化评估 Tool Memory 的实际效果。 + +## 实验设计 + +### 1. 环境构建 + +#### 1.1 三个 Mock 搜索工具 + +设计三个具有不同性能特征的模拟搜索工具： + +- **SearchToolA**: 快速但浅层搜索 +- **SearchToolB**: 平衡性能和质量 +- **SearchToolC**: 全面但成本高 + +每个工具的接口定义： +```python +def search_tool(query: str, scenario: str) -> dict: + """ + Args: + query: 搜索查询字符串 + scenario: 场景标识符 + + Returns: + { + "result": str, # 搜索结果 + "token_cost": int, # 消耗的 token 数 + "time_cost": float, # 耗时(秒) + "quality_score": float, # 质量评分 0-1 + "success": bool # 是否成功 + } + """ +``` + +#### 1.2 三个场景设计 + +**场景 1: 简单事实查询 (Simple Factual Queries)** +- 特征: 查询简短，答案直接明确 +- 最佳工具: SearchToolA (快速，质量足够) +- 示例查询: + - "法国的首都是什么？" + - "Python 是什么时候首次发布的？" + - "地球上有几个大洲？" + - "水的沸点是多少度？" + - "谁发明了电话？" + +**场景 2: 复杂研究查询 (Complex Research Queries)** +- 特征: 多维度问题，需要全面深入的搜索 +- 最佳工具: SearchToolC (高质量，值得成本) +- 示例查询: + - "比较凯恩斯主义和奥地利学派的经济政策" + - "分析可再生能源采用对环境的影响" + - "解释量子力学和广义相对论之间的关系" + - "讨论人工智能的历史演变过程" + - "评估不同机器学习算法在 NLP 中的有效性" + +**场景 3: 中等复杂度查询 (Moderate Complexity Queries)** +- 特征: 中等难度，需要平衡的性能 +- 最佳工具: SearchToolB (最优的成本-质量权衡) +- 示例查询: + - "列举 Python 3.10 的主要特性" + - "微服务架构有哪些好处？" + - "解释区块链技术的工作原理" + - "描述 SQL 和 NoSQL 数据库的主要区别" + - "软件工程中常见的设计模式有哪些？" + +#### 1.3 工具性能矩阵 + +| 工具 / 场景 | 场景 1 (简单) | 场景 2 (复杂) | 场景 3 (中等) | +|------------|--------------|--------------|--------------| +| **SearchToolA** | ⭐⭐⭐ 最佳 | ❌ 差 | ⚠️ 可接受 | +| **SearchToolB** | ⭐⭐ 良好 | ⭐⭐ 良好 | ⭐⭐⭐ 最佳 | +| **SearchToolC** | ⭐ 过度 | ⭐⭐⭐ 最佳 | ⭐⭐ 良好 | + +**详细性能指标：** + +``` +场景 1 (简单): +- SearchToolA: token_cost=100-200, time_cost=0.5-1s, quality=0.85-0.95, success_rate=0.95 +- SearchToolB: token_cost=300-500, time_cost=1-2s, quality=0.90-0.98, success_rate=0.98 +- SearchToolC: token_cost=800-1200, time_cost=3-5s, quality=0.90-0.95, success_rate=0.90 + +场景 2 (复杂): +- SearchToolA: token_cost=150-250, time_cost=0.5-1s, quality=0.40-0.60, success_rate=0.50 +- SearchToolB: token_cost=400-600, time_cost=1.5-2.5s, quality=0.70-0.85, success_rate=0.85 +- SearchToolC: token_cost=1000-1500, time_cost=3-6s, quality=0.90-0.98, success_rate=0.95 + +场景 3 (中等): +- SearchToolA: token_cost=120-220, time_cost=0.5-1s, quality=0.65-0.75, success_rate=0.75 +- SearchToolB: token_cost=350-550, time_cost=1-2s, quality=0.85-0.95, success_rate=0.95 +- SearchToolC: token_cost=900-1300, time_cost=3-5s, quality=0.85-0.92, success_rate=0.92 +``` + +### 2. 基准实验（无 Tool Memory） + +#### 2.1 实验设置 + +1. **Agent 配置**: + - LLM: GPT-4 或同等模型 + - 可用工具: SearchToolA, SearchToolB, SearchToolC + - 无历史性能数据 + +2. **测试数据集**: + - 每个场景 5 个查询 × 3 个场景 = 共 15 个查询 + - 随机顺序执行，避免学习偏差 + +3. **预期行为**: + - Agent 仅基于工具描述做决策 + - 无历史性能数据可参考 + - 可能出现次优工具选择 + +#### 2.2 执行流程 + +```python +# 伪代码 +for query in test_queries: + scenario = identify_scenario(query) + + # Agent 在没有 memory 指导的情况下选择工具 + selected_tool = agent.select_tool( + query=query, + available_tools=[SearchToolA, SearchToolB, SearchToolC] + ) + + # 执行工具 + result = selected_tool.execute(query, scenario) + + # 记录执行详情 + log_execution({ + "query": query, + "scenario": scenario, + "tool": selected_tool.name, + "token_cost": result.token_cost, + "time_cost": result.time_cost, + "quality_score": result.quality_score, + "success": result.success, + "timestamp": current_time() + }) +``` + +#### 2.3 数据采集 + +将执行日志保存到: `baseline_execution_log.jsonl` + +格式（对应 `ToolCallResult` schema）: +```json +{ + "create_time": "2025-10-16 10:30:00", + "tool_name": "SearchToolA", + "input": { + "query": "法国的首都是什么？", + "scenario": "simple" + }, + "output": "法国的首都是巴黎。巴黎是法国最大的城市...", + "token_cost": 150, + "success": true, + "time_cost": 0.8, + "summary": "成功返回法国首都的准确信息", + "evaluation": "回答准确简洁，符合简单查询的需求", + "score": 0.92, + "metadata": { + "experiment_id": "baseline_001", + "query_id": "q001", + "scenario": "simple" + } +} +``` + +#### 2.4 追踪指标 + +**按场景指标:** +- 总 token 成本 +- 总时间成本 +- 平均质量评分 +- 成功率 +- 工具选择分布 + +**总体指标:** +- 汇总 token 成本 +- 汇总时间成本 +- 总体平均质量 +- 总体成功率 +- 工具选择效率（最优选择百分比） + +### 3. 生成 Tool Memory + +#### 3.1 Tool Memory Service 集成 + +使用基准实验的执行日志生成 Tool Memory： + +```python +from reme_ai.service.task_memory_service import TaskMemoryService +from reme_ai.summary.tool import SummaryToolMemoryOp +from reme_ai.schema.memory import ToolMemory, ToolCallResult +import json + +# 初始化服务 +# 注意：当前版本使用 TaskMemoryService 来管理 ToolMemory +# ToolMemory 是通过 memory_type="tool" 来区分的 +tool_memory_service = TaskMemoryService( + vector_store_config={ + "vector_store_type": "chroma", + "persist_directory": "./memory_vector_store" + } +) + +# 处理基准执行日志 +tool_memories = {} # {tool_name: ToolMemory} + +with open("baseline_execution_log.jsonl", "r") as f: + for line in f: + record = json.loads(line) + tool_name = record["tool_name"] + + # 为每个工具创建或更新 ToolMemory + if tool_name not in tool_memories: + tool_memories[tool_name] = ToolMemory( + workspace_id="tool_bench_experiment", + memory_type="tool", + when_to_use=f"{tool_name} 的使用场景和性能统计", + content="", # 将在汇总时填充 + tool_call_results=[] + ) + + # 添加 ToolCallResult + tool_call_result = ToolCallResult(**record) + tool_memories[tool_name].tool_call_results.append(tool_call_result) + +# 使用 SummaryToolMemoryOp 生成 memory 内容 +summary_op = SummaryToolMemoryOp() +for tool_name, tool_memory in tool_memories.items(): + # 汇总工具性能 + summary_result = summary_op.summarize_tool_performance( + tool_memory=tool_memory + ) + + # 更新 content 和 when_to_use + tool_memory.content = summary_result["content"] + tool_memory.when_to_use = summary_result["when_to_use"] + + # 存储到向量数据库 + tool_memory_service.add_memory(tool_memory) +``` + +#### 3.2 预期 Tool Memory 内容 + +基于 `ToolMemory` schema，每个工具的 memory 应包含： + +**ToolMemory 结构:** +```python +{ + "workspace_id": "tool_bench_experiment", + "memory_id": "auto_generated_uuid", + "memory_type": "tool", + "when_to_use": "工具使用场景的简洁描述，用于检索匹配", + "content": "详细的工具性能分析和使用建议", + "score": 0.85, # 综合评分 + "time_created": "2025-10-16 10:30:00", + "time_modified": "2025-10-16 10:30:00", + "author": "tool_bench_system", + "tool_call_results": [ + # 所有的 ToolCallResult 记录 + ], + "metadata": { + "tool_name": "SearchToolA", + "total_calls": 15, + "scenarios_tested": ["simple", "complex", "moderate"] + } +} +``` + +**示例: SearchToolA 的 Tool Memory** + +```python +ToolMemory( + workspace_id="tool_bench_experiment", + memory_type="tool", + when_to_use="快速简单查询，事实性问题，需要低延迟响应的场景", + content=""" +## SearchToolA 性能分析 + +### 最佳使用场景 +- **简单事实查询 (场景1)**: + - 平均 token 成本: 150 + - 平均耗时: 0.8秒 + - 平均质量: 0.90 + - 成功率: 95% + - **推荐指数: ⭐⭐⭐⭐⭐** + +### 不适用场景 +- **复杂研究查询 (场景2)**: + - 平均质量仅 0.50，成功率 50% + - 无法提供深度分析 + - **强烈不推荐** + +- **中等复杂度查询 (场景3)**: + - 质量 0.70，可接受但不是最优 + - 建议考虑 SearchToolB + +### 统计数据 (最近20次调用) +- 总调用次数: 15 +- 平均 token 成本: 150 +- 平均耗时: 0.8秒 +- 平均质量评分: 0.72 +- 成功率: 73% + +### 使用建议 +1. 优先用于简单、直接的事实查询 +2. 需要快速响应时的首选 +3. 避免用于需要深度分析的复杂问题 +4. 成本最低，适合高频调用场景 +""", + score=0.72, + tool_call_results=[...], # 所有调用记录 + metadata={ + "tool_name": "SearchToolA", + "best_scenario": "simple", + "worst_scenario": "complex" + } +) +``` + +#### 3.3 使用 statistic() 方法 + +利用 `ToolMemory.statistic()` 方法获取统计信息： + +```python +# 获取最近20次调用的统计 +stats = tool_memory.statistic(recent_frequency=20) + +# 返回格式: +{ + "total_calls": 15, + "recent_calls_analyzed": 15, + "avg_token_cost": 150.5, + "success_rate": 0.7333, + "avg_time_cost": 0.850, + "avg_score": 0.720 +} +``` + +### 4. 增强实验（有 Tool Memory） + +#### 4.1 实验设置 + +1. **Agent 配置**: + - 同基准实验的 LLM + - 可用工具: SearchToolA, SearchToolB, SearchToolC + - **新增**: 可访问每个工具的 Tool Memory + +2. **测试数据集**: + - 与基准实验相同的 15 个查询 + - 相同的执行顺序以确保公平对比 + +3. **预期行为**: + - Agent 在选择工具前检索 Tool Memory + - 基于历史性能数据做出明智决策 + - 每个场景选择最优工具 + +#### 4.2 执行流程 + +```python +from reme_ai.retrieve.tool import RetrieveToolMemoryOp + +# 初始化检索操作 +retrieve_op = RetrieveToolMemoryOp() + +for query in test_queries: + scenario = identify_scenario(query) + + # 检索每个工具的 Tool Memory + tool_memories = {} + for tool in [SearchToolA, SearchToolB, SearchToolC]: + # 使用 RetrieveToolMemoryOp 检索相关 memory + memories = retrieve_op.retrieve( + workspace_id="tool_bench_experiment", + query=query, + tool_name=tool.name, + top_k=1 + ) + + if memories: + tool_memory = memories[0] + # 获取统计信息 + stats = tool_memory.statistic(recent_frequency=20) + tool_memories[tool.name] = { + "when_to_use": tool_memory.when_to_use, + "content": tool_memory.content, + "statistics": stats + } + + # Agent 基于 memory 选择工具 + selected_tool = agent.select_tool_with_memory( + query=query, + available_tools=[SearchToolA, SearchToolB, SearchToolC], + tool_memories=tool_memories + ) + + # 执行工具 + result = selected_tool.execute(query, scenario) + + # 记录执行详情（同基准实验格式） + tool_call_result = ToolCallResult( + create_time=current_time(), + tool_name=selected_tool.name, + input={"query": query, "scenario": scenario}, + output=result.output, + token_cost=result.token_cost, + success=result.success, + time_cost=result.time_cost, + summary=result.summary, + evaluation=result.evaluation, + score=result.score, + metadata={ + "experiment_id": "enhanced_001", + "memory_used": True, + "scenario": scenario + } + ) + + log_execution(tool_call_result.model_dump()) +``` + +#### 4.3 数据采集 + +保存执行日志到: `enhanced_execution_log.jsonl` + +格式与基准实验相同，但在 metadata 中增加 `memory_used: true` 标记。 + +### 5. 对比分析 + +#### 5.1 指标对比 + +创建两个实验的对比表： + +| 指标 | 基准实验 (无 Memory) | 增强实验 (有 Memory) | 改进幅度 | +|------|---------------------|---------------------|----------| +| **总 Token 成本** | X tokens | Y tokens | -Z% | +| **总时间成本** | X 秒 | Y 秒 | -Z% | +| **平均质量评分** | X.XX | Y.YY | +Z% | +| **总体成功率** | X% | Y% | +Z% | +| **最优工具选择率** | X% | Y% | +Z% | + +#### 5.2 按场景分析 + +使用 Python 脚本分析每个场景的性能： + +```python +import json +from collections import defaultdict +from reme_ai.schema.memory import ToolCallResult + +def analyze_by_scenario(log_file): + """分析执行日志，按场景统计""" + scenario_stats = defaultdict(lambda: { + "calls": [], + "tool_distribution": defaultdict(int) + }) + + with open(log_file, "r") as f: + for line in f: + record = json.loads(line) + scenario = record["metadata"]["scenario"] + + # 添加调用记录 + scenario_stats[scenario]["calls"].append( + ToolCallResult(**record) + ) + + # 统计工具分布 + scenario_stats[scenario]["tool_distribution"][ + record["tool_name"] + ] += 1 + + # 计算每个场景的指标 + results = {} + for scenario, data in scenario_stats.items(): + calls = data["calls"] + results[scenario] = { + "total_calls": len(calls), + "avg_token_cost": sum(c.token_cost for c in calls) / len(calls), + "avg_time_cost": sum(c.time_cost for c in calls) / len(calls), + "avg_quality": sum(c.score for c in calls) / len(calls), + "success_rate": sum(1 for c in calls if c.success) / len(calls), + "tool_distribution": dict(data["tool_distribution"]) + } + + return results + +# 分析两个实验 +baseline_results = analyze_by_scenario("baseline_execution_log.jsonl") +enhanced_results = analyze_by_scenario("enhanced_execution_log.jsonl") + +# 打印对比 +for scenario in ["simple", "complex", "moderate"]: + print(f"\n场景: {scenario}") + print(f"Token 成本: {baseline_results[scenario]['avg_token_cost']:.1f} -> " + f"{enhanced_results[scenario]['avg_token_cost']:.1f}") + print(f"时间成本: {baseline_results[scenario]['avg_time_cost']:.2f}s -> " + f"{enhanced_results[scenario]['avg_time_cost']:.2f}s") + print(f"质量评分: {baseline_results[scenario]['avg_quality']:.2f} -> " + f"{enhanced_results[scenario]['avg_quality']:.2f}") +``` + +#### 5.3 预期结果 + +**场景 1 (简单查询):** +- 基准: 可能混用三个工具，平均成本较高 +- 增强: 主要使用 SearchToolA，成本降低，质量保持 + +**场景 2 (复杂查询):** +- 基准: 可能误用 SearchToolA，质量和成功率低 +- 增强: 主要使用 SearchToolC，质量和成功率显著提升 + +**场景 3 (中等查询):** +- 基准: 工具选择随机，性能不稳定 +- 增强: 主要使用 SearchToolB，最优的成本-质量平衡 + +#### 5.4 统计显著性检验 + +```python +from scipy import stats +import numpy as np + +def significance_test(baseline_log, enhanced_log, metric="token_cost"): + """对指定指标进行 t 检验""" + baseline_values = [] + enhanced_values = [] + + with open(baseline_log) as f: + for line in f: + record = json.loads(line) + baseline_values.append(record[metric]) + + with open(enhanced_log) as f: + for line in f: + record = json.loads(line) + enhanced_values.append(record[metric]) + + # 执行 t 检验 + t_stat, p_value = stats.ttest_ind(baseline_values, enhanced_values) + + return { + "metric": metric, + "baseline_mean": np.mean(baseline_values), + "enhanced_mean": np.mean(enhanced_values), + "t_statistic": t_stat, + "p_value": p_value, + "significant": p_value < 0.05 + } + +# 测试各项指标 +for metric in ["token_cost", "time_cost", "score"]: + result = significance_test( + "baseline_execution_log.jsonl", + "enhanced_execution_log.jsonl", + metric + ) + print(f"\n{metric} 检验结果:") + print(f" 基准均值: {result['baseline_mean']:.2f}") + print(f" 增强均值: {result['enhanced_mean']:.2f}") + print(f" p-value: {result['p_value']:.4f}") + print(f" 显著性: {'是' if result['significant'] else '否'}") +``` + +#### 5.5 可视化 + +生成对比图表： + +```python +import matplotlib.pyplot as plt +import matplotlib +matplotlib.rcParams['font.sans-serif'] = ['Arial Unicode MS'] # 支持中文 + +def plot_comparison(): + """生成对比可视化图表""" + + # 1. 工具选择分布对比 + fig, axes = plt.subplots(1, 3, figsize=(15, 5)) + scenarios = ["simple", "complex", "moderate"] + + for idx, scenario in enumerate(scenarios): + baseline = baseline_results[scenario]["tool_distribution"] + enhanced = enhanced_results[scenario]["tool_distribution"] + + x = np.arange(3) + width = 0.35 + + axes[idx].bar(x - width/2, + [baseline.get(f"SearchTool{t}", 0) for t in ["A", "B", "C"]], + width, label='基准') + axes[idx].bar(x + width/2, + [enhanced.get(f"SearchTool{t}", 0) for t in ["A", "B", "C"]], + width, label='增强') + + axes[idx].set_title(f'场景: {scenario}') + axes[idx].set_xticks(x) + axes[idx].set_xticklabels(['ToolA', 'ToolB', 'ToolC']) + axes[idx].legend() + + plt.tight_layout() + plt.savefig('tool_selection_comparison.png') + + # 2. 性能指标对比 + metrics = ['Token成本', '时间成本', '质量评分', '成功率'] + # ... 更多可视化代码 +``` + +### 6. 实现指南 + +#### 6.1 目录结构 + +``` +cookbook/tool_memory/ +├── __init__.py +├── mock_tools.py # Mock 搜索工具实现 +├── scenarios.py # 场景定义和查询 +├── baseline_experiment.py # 运行基准实验 +├── generate_memory.py # 生成 Tool Memory +├── enhanced_experiment.py # 运行增强实验 +├── analysis.py # 对比分析 +├── visualization.py # 生成图表 +├── requirements.txt # 依赖项 +├── README.md # 实验说明 +└── data/ + ├── baseline_execution_log.jsonl + ├── enhanced_execution_log.jsonl + ├── tool_memories/ # 生成的 Tool Memory + └── analysis_results.json +``` + +#### 6.2 运行基准测试 + +```bash +# 步骤 1: 运行基准实验（无 Tool Memory） +python cookbook/tool_memory/baseline_experiment.py \ + --output data/baseline_execution_log.jsonl \ + --workspace-id tool_bench_experiment + +# 步骤 2: 生成 Tool Memory +python cookbook/tool_memory/generate_memory.py \ + --input data/baseline_execution_log.jsonl \ + --workspace-id tool_bench_experiment \ + --vector-store-path ./memory_vector_store + +# 步骤 3: 运行增强实验（有 Tool Memory） +python cookbook/tool_memory/enhanced_experiment.py \ + --output data/enhanced_execution_log.jsonl \ + --workspace-id tool_bench_experiment \ + --vector-store-path ./memory_vector_store + +# 步骤 4: 分析结果 +python cookbook/tool_memory/analysis.py \ + --baseline data/baseline_execution_log.jsonl \ + --enhanced data/enhanced_execution_log.jsonl \ + --output data/analysis_results.json + +# 步骤 5: 生成可视化 +python cookbook/tool_memory/visualization.py \ + --analysis data/analysis_results.json \ + --output-dir data/figures/ +``` + +#### 6.3 核心代码示例 + +**mock_tools.py:** +```python +import random +import time +from reme_ai.schema.memory import ToolCallResult + +class MockSearchTool: + """Mock 搜索工具基类""" + + def __init__(self, name: str, performance_matrix: dict): + self.name = name + self.performance_matrix = performance_matrix + + def execute(self, query: str, scenario: str) -> ToolCallResult: + """执行搜索并返回结果""" + perf = self.performance_matrix[scenario] + + # 模拟执行时间 + time_cost = random.uniform(perf["time_range"][0], perf["time_range"][1]) + time.sleep(time_cost) + + # 生成结果 + token_cost = random.randint(perf["token_range"][0], perf["token_range"][1]) + quality = random.uniform(perf["quality_range"][0], perf["quality_range"][1]) + success = random.random() < perf["success_rate"] + + result = ToolCallResult( + create_time=time.strftime("%Y-%m-%d %H:%M:%S"), + tool_name=self.name, + input={"query": query, "scenario": scenario}, + output=self._generate_output(query, success), + token_cost=token_cost, + success=success, + time_cost=time_cost, + summary=f"{'成功' if success else '失败'}执行查询: {query[:30]}...", + evaluation=self._generate_evaluation(quality, scenario), + score=quality if success else quality * 0.5, + metadata={"scenario": scenario} + ) + + return result + + def _generate_output(self, query: str, success: bool) -> str: + """生成模拟输出""" + if success: + return f"针对查询 '{query}' 的搜索结果...\n[模拟的详细内容]" + else: + return f"无法完成查询: {query}" + + def _generate_evaluation(self, quality: float, scenario: str) -> str: + """生成评估说明""" + if quality > 0.85: + return f"高质量回答，非常适合{scenario}场景" + elif quality > 0.70: + return f"良好回答，适合{scenario}场景" + else: + return f"回答质量不足，不适合{scenario}场景" + +# 创建三个工具实例 +SearchToolA = MockSearchTool("SearchToolA", { + "simple": { + "token_range": (100, 200), + "time_range": (0.5, 1.0), + "quality_range": (0.85, 0.95), + "success_rate": 0.95 + }, + "complex": { + "token_range": (150, 250), + "time_range": (0.5, 1.0), + "quality_range": (0.40, 0.60), + "success_rate": 0.50 + }, + "moderate": { + "token_range": (120, 220), + "time_range": (0.5, 1.0), + "quality_range": (0.65, 0.75), + "success_rate": 0.75 + } +}) + +SearchToolB = MockSearchTool("SearchToolB", { + "simple": { + "token_range": (300, 500), + "time_range": (1.0, 2.0), + "quality_range": (0.90, 0.98), + "success_rate": 0.98 + }, + "complex": { + "token_range": (400, 600), + "time_range": (1.5, 2.5), + "quality_range": (0.70, 0.85), + "success_rate": 0.85 + }, + "moderate": { + "token_range": (350, 550), + "time_range": (1.0, 2.0), + "quality_range": (0.85, 0.95), + "success_rate": 0.95 + } +}) + +SearchToolC = MockSearchTool("SearchToolC", { + "simple": { + "token_range": (800, 1200), + "time_range": (3.0, 5.0), + "quality_range": (0.90, 0.95), + "success_rate": 0.90 + }, + "complex": { + "token_range": (1000, 1500), + "time_range": (3.0, 6.0), + "quality_range": (0.90, 0.98), + "success_rate": 0.95 + }, + "moderate": { + "token_range": (900, 1300), + "time_range": (3.0, 5.0), + "quality_range": (0.85, 0.92), + "success_rate": 0.92 + } +}) + +TOOL_REGISTRY = { + "SearchToolA": SearchToolA, + "SearchToolB": SearchToolB, + "SearchToolC": SearchToolC +} +``` + +**scenarios.py:** +```python +"""场景和查询定义""" + +TEST_QUERIES = [ + # 场景 1: 简单事实查询 + {"query": "法国的首都是什么？", "scenario": "simple"}, + {"query": "Python 是什么时候首次发布的？", "scenario": "simple"}, + {"query": "地球上有几个大洲？", "scenario": "simple"}, + {"query": "水的沸点是多少度？", "scenario": "simple"}, + {"query": "谁发明了电话？", "scenario": "simple"}, + + # 场景 2: 复杂研究查询 + {"query": "比较凯恩斯主义和奥地利学派的经济政策", "scenario": "complex"}, + {"query": "分析可再生能源采用对环境的影响", "scenario": "complex"}, + {"query": "解释量子力学和广义相对论之间的关系", "scenario": "complex"}, + {"query": "讨论人工智能的历史演变过程", "scenario": "complex"}, + {"query": "评估不同机器学习算法在 NLP 中的有效性", "scenario": "complex"}, + + # 场景 3: 中等复杂度查询 + {"query": "列举 Python 3.10 的主要特性", "scenario": "moderate"}, + {"query": "微服务架构有哪些好处？", "scenario": "moderate"}, + {"query": "解释区块链技术的工作原理", "scenario": "moderate"}, + {"query": "描述 SQL 和 NoSQL 数据库的主要区别", "scenario": "moderate"}, + {"query": "软件工程中常见的设计模式有哪些？", "scenario": "moderate"} +] + +OPTIMAL_TOOL_MAPPING = { + "simple": "SearchToolA", + "complex": "SearchToolC", + "moderate": "SearchToolB" +} +``` + +#### 6.4 预期成果 + +**假设验证**: Tool Memory 应该带来： + +1. ✅ **降低 Token 成本**: 通过选择更高效的工具，降低 20-30% +2. ✅ **降低时间成本**: 通过最优工具选择，降低 15-25% +3. ✅ **提升质量评分**: 平均质量提升 10-15% +4. ✅ **提高成功率**: 总体成功率提升 5-10% +5. ✅ **优化工具选择**: 最优工具选择率从 33%（随机）提升到 60-80% + +**具体示例预期:** + +``` +基准实验 (15次调用): +- 总 Token 成本: 7,500 +- 总时间: 30秒 +- 平均质量: 0.75 +- 成功率: 78% +- 最优选择率: 40% (6/15) + +增强实验 (15次调用): +- 总 Token 成本: 5,250 (↓30%) +- 总时间: 22秒 (↓27%) +- 平均质量: 0.88 (↑17%) +- 成功率: 91% (↑13%) +- 最优选择率: 87% (13/15) (↑117%) +``` + +### 7. 进阶实验 + +#### 7.1 在线学习实验 + +测试 Tool Memory 的在线更新和学习能力： + +```python +def online_learning_experiment(): + """在线学习实验：从零开始逐步构建 Tool Memory""" + + # 初始化空 Tool Memory + tool_memories = { + "SearchToolA": ToolMemory(workspace_id="online_learning"), + "SearchToolB": ToolMemory(workspace_id="online_learning"), + "SearchToolC": ToolMemory(workspace_id="online_learning") + } + + learning_curve = [] + + for idx, test_case in enumerate(TEST_QUERIES): + # 基于当前 memory 选择工具 + selected_tool = agent.select_tool_with_memory( + query=test_case["query"], + tool_memories=tool_memories + ) + + # 执行并记录 + result = selected_tool.execute( + test_case["query"], + test_case["scenario"] + ) + + # 立即更新对应工具的 Tool Memory + tool_memories[selected_tool.name].tool_call_results.append(result) + + # 重新汇总 memory + update_tool_memory_content(tool_memories[selected_tool.name]) + + # 记录学习曲线 + is_optimal = (selected_tool.name == + OPTIMAL_TOOL_MAPPING[test_case["scenario"]]) + learning_curve.append({ + "call_index": idx + 1, + "is_optimal": is_optimal, + "cumulative_optimal_rate": sum(lc["is_optimal"] + for lc in learning_curve) / (idx + 1) + }) + + return learning_curve +``` + +#### 7.2 Memory 时效性实验 + +测试 Tool Memory 对工具性能变化的适应性： + +```python +def memory_decay_experiment(): + """测试当工具性能发生变化时，memory 的适应能力""" + + # 阶段 1: 使用原始性能矩阵，构建 Tool Memory + phase1_results = run_baseline_experiment() + tool_memories = generate_tool_memories(phase1_results) + + # 阶段 2: 改变工具性能（例如 ToolA 性能下降） + SearchToolA.performance_matrix["simple"]["quality_range"] = (0.50, 0.60) + SearchToolA.performance_matrix["simple"]["success_rate"] = 0.60 + + # 阶段 3: 使用旧 memory 运行新实验 + phase2_results = run_enhanced_experiment( + tool_memories=tool_memories, + update_memory=False # 不更新 memory + ) + + # 阶段 4: 允许 memory 更新，观察适应过程 + phase3_results = run_enhanced_experiment( + tool_memories=tool_memories, + update_memory=True, # 在线更新 memory + update_frequency=5 # 每5次调用更新一次 + ) + + return { + "phase1": phase1_results, # 原始性能，有 memory + "phase2": phase2_results, # 性能变化，旧 memory + "phase3": phase3_results # 性能变化，memory 适应 + } +``` + +#### 7.3 跨域迁移实验 + +测试 Tool Memory 在不同但相关场景间的迁移能力： + +```python +def cross_domain_experiment(): + """测试 Tool Memory 的跨域迁移能力""" + + # 定义新的场景：技术文档查询 + NEW_SCENARIOS = { + "api_reference": { + "queries": ["Python requests 库的 GET 方法参数", ...], + "optimal_tool": "SearchToolB" + }, + "troubleshooting": { + "queries": ["如何修复 CORS 错误", ...], + "optimal_tool": "SearchToolC" + }, + "quick_lookup": { + "queries": ["HTTP 状态码 404 的含义", ...], + "optimal_tool": "SearchToolA" + } + } + + # 使用原场景训练的 Tool Memory + tool_memories = load_tool_memories("tool_bench_experiment") + + # 在新场景测试 + transfer_results = run_experiment_with_new_scenarios( + scenarios=NEW_SCENARIOS, + tool_memories=tool_memories + ) + + # 对比：1) 无 memory, 2) 有旧 memory, 3) 新场景训练的 memory + return compare_transfer_effectiveness(transfer_results) +``` + +### 8. 数据分析和可视化 + +#### 8.1 生成分析报告 + +```python +def generate_report(): + """生成完整的实验分析报告""" + + report = { + "experiment_metadata": { + "date": datetime.now().isoformat(), + "total_queries": 15, + "scenarios": 3, + "tools": 3 + }, + "baseline_summary": analyze_experiment("baseline_execution_log.jsonl"), + "enhanced_summary": analyze_experiment("enhanced_execution_log.jsonl"), + "comparison": { + "token_cost_reduction": calculate_reduction("token_cost"), + "time_cost_reduction": calculate_reduction("time_cost"), + "quality_improvement": calculate_improvement("score"), + "success_rate_improvement": calculate_improvement("success"), + "optimal_selection_improvement": calculate_optimal_selection_rate() + }, + "statistical_tests": { + "token_cost_ttest": significance_test("token_cost"), + "time_cost_ttest": significance_test("time_cost"), + "quality_ttest": significance_test("score") + }, + "by_scenario_analysis": { + "simple": analyze_scenario("simple"), + "complex": analyze_scenario("complex"), + "moderate": analyze_scenario("moderate") + }, + "tool_memory_effectiveness": { + "SearchToolA": evaluate_memory_impact("SearchToolA"), + "SearchToolB": evaluate_memory_impact("SearchToolB"), + "SearchToolC": evaluate_memory_impact("SearchToolC") + } + } + + # 保存为 JSON + with open("data/analysis_results.json", "w", encoding="utf-8") as f: + json.dump(report, f, ensure_ascii=False, indent=2) + + # 生成 Markdown 报告 + generate_markdown_report(report) + + return report +``` + +#### 8.2 关键可视化图表 + +1. **工具选择分布热力图** +2. **成本-质量散点图** +3. **学习曲线图（在线学习实验）** +4. **Memory 影响力雷达图** +5. **时间线对比图** + +### 9. 结论 + +本基准测试框架提供了全面的 Tool Memory 效果验证方案，确保： + +- **公平对比**: 相同查询、相同工具、受控环境 +- **清晰指标**: token 成本、时间、质量、成功率的量化改进 +- **可重复性**: 详细步骤和数据收集规范 +- **统计严谨性**: 显著性检验和置信区间 +- **符合 Schema**: 完全基于 `reme_ai/schema/memory.py` 中的数据结构 + +**预期结论**: Tool Memory 应该展现出明显的优势，帮助 Agent 做出明智的工具选择决策，在效率和效果上都有可测量的提升。 + +--- + +## 参考资料 + +- Tool Memory Schema: `/reme_ai/schema/memory.py` +- Tool Memory 文档: `/docs/tool_memory/tool_memory.md` +- Tool Memory 实现: `/reme_ai/summary/tool/` +- Tool 检索操作: `/docs/tool_memory/tool_retrieve_ops.md` +- Tool 汇总操作: `/docs/tool_memory/tool_summary_ops.md` diff --git a/reme_ai/__init__.py b/reme_ai/__init__.py index 50195757..f07f9112 100644 --- a/reme_ai/__init__.py +++ b/reme_ai/__init__.py @@ -1,6 +1,6 @@ -from reme_ai import react -from reme_ai import retrieve -from reme_ai import summary -from reme_ai import vector_store +from . import agent +from . import retrieve +from . import summary +from . import vector_store __version__ = "0.1.9" diff --git a/reme_ai/agent/__init__.py b/reme_ai/agent/__init__.py new file mode 100644 index 00000000..bb1db659 --- /dev/null +++ b/reme_ai/agent/__init__.py @@ -0,0 +1,2 @@ +from . import react +from . import tools diff --git a/reme_ai/react/__init__.py b/reme_ai/agent/react/__init__.py similarity index 100% rename from reme_ai/react/__init__.py rename to reme_ai/agent/react/__init__.py diff --git a/reme_ai/react/simple_react_op.py b/reme_ai/agent/react/simple_react_op.py similarity index 100% rename from reme_ai/react/simple_react_op.py rename to reme_ai/agent/react/simple_react_op.py diff --git a/reme_ai/agent/tools/__init__.py b/reme_ai/agent/tools/__init__.py new file mode 100644 index 00000000..ec567ab5 --- /dev/null +++ b/reme_ai/agent/tools/__init__.py @@ -0,0 +1,3 @@ +from .llm_mock_search_op import LLMMockSearchOp +from .mock_search_tools import SearchToolA, SearchToolB, SearchToolC +from .use_mock_search_op import UseMockSearchOp diff --git a/reme_ai/agent/tools/llm_mock_search_op.py b/reme_ai/agent/tools/llm_mock_search_op.py new file mode 100644 index 00000000..eebef995 --- /dev/null +++ b/reme_ai/agent/tools/llm_mock_search_op.py @@ -0,0 +1,297 @@ +import asyncio +import json +import random +from typing import Dict, Any + +from loguru import logger + +from flowllm.context import FlowContext, C +from flowllm.enumeration.role import Role +from flowllm.op.base_async_tool_op import BaseAsyncToolOp +from flowllm.schema.message import Message +from flowllm.schema.tool_call import ToolCall + + +@C.register_op() +class LLMMockSearchOp(BaseAsyncToolOp): + """ + Mock search operation that uses LLM to classify queries and simulate different scenarios. + + Supports three query complexity levels: + - simple: Simple factual queries with short, direct answers + - medium: Medium complexity queries requiring balanced performance + - complex: Complex research queries requiring comprehensive, in-depth results + + Each scenario can be configured with: + - success_rate: Probability of successful response (vs "Service busy" error) + - extra_time: Extra sleep time in seconds to simulate latency + - relevance_ratio: Probability of returning relevant results (vs random query results) + """ + file_path: str = __file__ + + def __init__(self, + llm: str = "qwen3_30b_instruct", + simple_config: Dict[str, Any] = None, + medium_config: Dict[str, Any] = None, + complex_config: Dict[str, Any] = None, + **kwargs): + """ + Initialize the LLM Mock Search Op. + + Args: + llm: LLM model name to use for classification and content generation + simple_config: Configuration for simple queries + - success_rate: float (0-1), default 0.95 + - extra_time: float (seconds), default 0.5 + - relevance_ratio: float (0-1), default 0.98 + medium_config: Configuration for medium complexity queries + - success_rate: float (0-1), default 0.85 + - extra_time: float (seconds), default 1.0 + - relevance_ratio: float (0-1), default 0.90 + complex_config: Configuration for complex queries + - success_rate: float (0-1), default 0.70 + - extra_time: float (seconds), default 1.5 + - relevance_ratio: float (0-1), default 0.80 + """ + super().__init__(llm=llm, **kwargs) + + # Default configurations for each scenario + self.simple_config = { + "success_rate": 0.95, + "extra_time": 0.5, + "relevance_ratio": 0.98, + "content_length": "short" + } + if simple_config: + self.simple_config.update(simple_config) + + self.medium_config = { + "success_rate": 0.85, + "extra_time": 1.0, + "relevance_ratio": 0.90, + "content_length": "medium" + } + if medium_config: + self.medium_config.update(medium_config) + + self.complex_config = { + "success_rate": 0.70, + "extra_time": 1.5, + "relevance_ratio": 0.80, + "content_length": "long" + } + if complex_config: + self.complex_config.update(complex_config) + + def build_tool_call(self) -> ToolCall: + return ToolCall(**{ + "description": "Use search keywords to retrieve relevant information from the internet.", + "input_schema": { + "query": { + "type": "string", + "description": "search keyword or query", + "required": True + } + } + }) + + async def classify_query(self, query: str) -> str: + """ + Classify the query into simple, medium, or complex using LLM. + + Args: + query: The search query to classify + + Returns: + Classification result: "simple", "medium", or "complex" + """ + classification_prompt = self.prompt_format( + prompt_name="classification_prompt", + query=query + ) + + messages = [Message(role=Role.USER, content=classification_prompt)] + + response = await self.llm.achat(messages=messages) + classification = response.content.strip().lower() + + # Extract classification from response + if "simple" in classification: + return "simple" + elif "complex" in classification: + return "complex" + else: + return "medium" + + async def generate_search_result(self, query: str, complexity: str, config: Dict[str, Any]) -> str: + """ + Generate mock search results using LLM based on query complexity. + + Args: + query: The search query + complexity: Query complexity level + config: Configuration for this complexity level + + Returns: + Generated search result content + """ + content_length = config["content_length"] + + generation_prompt = self.prompt_format( + prompt_name="generation_prompt", + query=query, + complexity=complexity, + content_length=content_length + ) + + messages = [Message(role=Role.USER, content=generation_prompt)] + + response = await self.llm.achat(messages=messages) + return response.content + + async def generate_random_result(self) -> str: + """ + Generate a random/irrelevant search result to simulate low relevance. + + Returns: + Random search result content + """ + random_topics = [ + "the history of ancient civilizations", + "modern technology trends", + "climate change impacts", + "space exploration achievements", + "culinary traditions around the world", + "evolution of music genres", + "breakthroughs in medical science", + "architectural wonders", + "wildlife conservation efforts", + "developments in artificial intelligence" + ] + + random_query = random.choice(random_topics) + generation_prompt = self.prompt_format( + prompt_name="generation_prompt", + query=random_query, + complexity="simple", + content_length="short" + ) + + messages = [Message(role=Role.USER, content=generation_prompt)] + response = await self.llm.achat(messages=messages) + + return f"[Low Relevance Result]\n{response.content}" + + async def async_execute(self): + query: str = self.input_dict["query"] + logger.info(f"LLMMockSearchOp processing query: {query}") + + # Step 1: Classify the query + complexity = await self.classify_query(query) + logger.info(f"Query classified as: {complexity}") + + # Step 2: Get configuration for this complexity + if complexity == "simple": + config = self.simple_config + elif complexity == "medium": + config = self.medium_config + else: # complex + config = self.complex_config + + logger.info(f"Using config: {config}") + + # Step 3: Simulate extra time delay + extra_time = config["extra_time"] + await asyncio.sleep(extra_time) + logger.info(f"Simulated extra delay: {extra_time:.2f}s") + + # Step 4: Check success rate + if random.random() > config["success_rate"]: + error_message = "Search service is currently busy. Please try again later." + logger.warning(f"Simulated failure: {error_message}") + result_dict = { + "success": False, + "content": error_message, + "query": query, + "complexity": complexity + } + self.set_result(json.dumps(result_dict, ensure_ascii=False)) + return + + # Step 5: Check relevance ratio + if random.random() > config["relevance_ratio"]: + # Generate random/irrelevant result + logger.info("Generating low relevance result") + content = await self.generate_random_result() + result_dict = { + "success": False, + "content": content, + "query": query, + "complexity": complexity + } + else: + # Generate relevant result + logger.info("Generating relevant result") + content = await self.generate_search_result(query, complexity, config) + result_dict = { + "success": True, + "content": content, + "query": query, + "complexity": complexity + } + + self.set_result(json.dumps(result_dict, ensure_ascii=False)) + + +async def async_main(): + from flowllm.app import FlowLLMApp + + async with FlowLLMApp(load_default_config=True): + # Test with different query types + test_queries = [ + "What is the capital of France?", # Simple + "How does quantum computing work?", # Medium + "Analyze the impact of artificial intelligence on global economy, employment, and society", # Complex + ] + + # Custom configurations for testing + custom_simple = { + "success_rate": 1, + "extra_time": 0, + "relevance_ratio": 1 + } + + custom_medium = { + "success_rate": 1, + "extra_time": 0, + "relevance_ratio": 1 + } + + custom_complex = { + "success_rate": 1, + "extra_time": 0, + "relevance_ratio": 1 + } + + op = LLMMockSearchOp( + simple_config=custom_simple, + medium_config=custom_medium, + complex_config=custom_complex + ) + + for query in test_queries: + print(f"\n{'=' * 80}") + print(f"Testing query: {query}") + print(f"{'=' * 80}") + + context = FlowContext(query=query) + await op.async_call(context=context) + result = json.loads(context.llm_mock_search_result) + print(f"Success: {result['success']}") + print(f"Query: {result['query']}") + print(f"Complexity: {result['complexity']}") + print(f"Content:\n{result['content']}") + + +if __name__ == "__main__": + asyncio.run(async_main()) diff --git a/reme_ai/agent/tools/llm_mock_search_prompt.yaml b/reme_ai/agent/tools/llm_mock_search_prompt.yaml new file mode 100644 index 00000000..15670654 --- /dev/null +++ b/reme_ai/agent/tools/llm_mock_search_prompt.yaml @@ -0,0 +1,76 @@ +classification_prompt: | + You are a query complexity classifier. Analyze the following search query and classify it into one of three categories: + + 1. **simple** - Simple factual queries that: + - Ask for a single, direct fact + - Have a clear, unambiguous answer + - Require minimal context or explanation + - Examples: "What is the capital of France?", "Who invented the telephone?", "When did World War 2 end?" + + 2. **medium** - Medium complexity queries that: + - Require some explanation or context + - May involve multiple related facts + - Need balanced depth without being exhaustive + - Examples: "How does photosynthesis work?", "What are the main causes of climate change?", "Explain blockchain technology" + + 3. **complex** - Complex research queries that: + - Require comprehensive, multi-dimensional analysis + - Involve multiple subtopics or perspectives + - Need in-depth exploration and connections + - Examples: "Analyze the impact of AI on the global economy", "Compare different renewable energy solutions", "What are the geopolitical implications of space exploration?" + + Query to classify: {query} + + Respond with ONLY one word: simple, medium, or complex. + +classification_prompt_zh: | + 你是一个查询复杂度分类器。分析以下搜索查询并将其分类为以下三类之一： + + 1. **simple** - 简单事实查询： + - 询问单一、直接的事实 + - 有明确、无歧义的答案 + - 需要最少的上下文或解释 + - 示例："法国的首都是什么？"、"谁发明了电话？"、"第二次世界大战何时结束？" + + 2. **medium** - 中等复杂度查询： + - 需要一些解释或上下文 + - 可能涉及多个相关事实 + - 需要平衡的深度但不需要详尽无遗 + - 示例："光合作用如何工作？"、"气候变化的主要原因是什么？"、"解释区块链技术" + + 3. **complex** - 复杂研究查询： + - 需要全面、多维度的分析 + - 涉及多个子主题或观点 + - 需要深入探索和联系 + - 示例："分析人工智能对全球经济的影响"、"比较不同的可再生能源解决方案"、"太空探索的地缘政治影响是什么？" + + 要分类的查询：{query} + + 只用一个词回答：simple、medium 或 complex。 + +generation_prompt: | + You are a search engine generating mock search results. Generate a {content_length} response for the following query. + + Query: {query} + Complexity Level: {complexity} + + Instructions based on content length: + - **short**: Provide a concise answer in 1-3 sentences. Be direct and factual. + - **medium**: Provide a balanced answer in 2-4 paragraphs. Include key details and some context. + - **long**: Provide a comprehensive answer in 4-6 paragraphs. Include multiple perspectives, detailed explanations, and relevant context. + + Generate the search result content now: + +generation_prompt_zh: | + 你是一个搜索引擎，正在生成模拟搜索结果。为以下查询生成一个 {content_length} 的响应。 + + 查询：{query} + 复杂度级别：{complexity} + + 根据内容长度的指示： + - **short**（短）：提供 1-3 句话的简洁答案。要直接和事实性。 + - **medium**（中）：提供 2-4 段的平衡答案。包括关键细节和一些上下文。 + - **long**（长）：提供 4-6 段的全面答案。包括多个角度、详细解释和相关上下文。 + + 现在生成搜索结果内容： + diff --git a/reme_ai/agent/tools/mock_search_tools.py b/reme_ai/agent/tools/mock_search_tools.py new file mode 100644 index 00000000..68e7cae9 --- /dev/null +++ b/reme_ai/agent/tools/mock_search_tools.py @@ -0,0 +1,115 @@ +from flowllm.context import C +from flowllm.schema.tool_call import ToolCall + +from reme_ai.agent.tools.llm_mock_search_op import LLMMockSearchOp + + +@C.register_op() +class SearchToolA(LLMMockSearchOp): + def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs): + # Configure for fast but shallow performance + simple_config = { + "success_rate": 0.95, # High success rate for simple queries + "extra_time": 0.2, # Very fast (0.2-0.5s range) + "relevance_ratio": 0.92, # High relevance + "content_length": "short" # Concise answers + } + + medium_config = { + "success_rate": 0.75, # Lower success for medium queries + "extra_time": 0.3, # Still fast + "relevance_ratio": 0.70, # Moderate relevance + "content_length": "short" # Limited depth + } + + complex_config = { + "success_rate": 0.50, # Poor success rate for complex queries + "extra_time": 0.4, # Fast but insufficient + "relevance_ratio": 0.50, # Low relevance (often misses key aspects) + "content_length": "short" # Too shallow for complex topics + } + + super().__init__(llm=llm, + simple_config=simple_config, + medium_config=medium_config, + complex_config=complex_config, + **kwargs) + + def build_tool_call(self) -> ToolCall: + tool_call = super().build_tool_call() + tool_call.description += " Best suited for simple queries." + return tool_call + +@C.register_op() +class SearchToolB(LLMMockSearchOp): + def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs): + # Configure for balanced performance + simple_config = { + "success_rate": 0.8, # Very high success rate + "extra_time": 0.8, # Moderate speed (1.0-1.5s range) + "relevance_ratio": 0.8, # High relevance + "content_length": "medium" # More detailed than needed for simple + } + + medium_config = { + "success_rate": 0.8, # Excellent success rate + "extra_time": 1.0, # Balanced speed + "relevance_ratio": 0.8, # High relevance + "content_length": "medium" # Perfect depth for medium queries + } + + complex_config = { + "success_rate": 0.8, # Good success rate + "extra_time": 1.2, # Still reasonable speed + "relevance_ratio": 0.8, # Decent relevance but not exhaustive + "content_length": "medium" # Covers main points but lacks depth + } + + super().__init__(llm=llm, + simple_config=simple_config, + medium_config=medium_config, + complex_config=complex_config, + **kwargs) + + def build_tool_call(self) -> ToolCall: + tool_call = super().build_tool_call() + tool_call.description += " Best suited for medium complexity queries." + return tool_call + + +@C.register_op() +class SearchToolC(LLMMockSearchOp): + + def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs): + # Configure for comprehensive but costly performance + simple_config = { + "success_rate": 0.7, # Good but not optimal (over-processing) + "extra_time": 2.5, # Slow (3.0-4.0s range) + "relevance_ratio": 0.7, # High relevance but unnecessary depth + "content_length": "long" # Too detailed for simple queries + } + + medium_config = { + "success_rate": 0.7, # High success rate + "extra_time": 2.8, # Slow but thorough + "relevance_ratio": 0.7, # High relevance with extra context + "content_length": "long" # More depth than needed + } + + complex_config = { + "success_rate": 0.95, # Excellent success rate + "extra_time": 3.5, # Slow but comprehensive (3.5-5.0s range) + "relevance_ratio": 0.94, # Very high relevance + "content_length": "long" # Perfect depth for complex queries + } + + super().__init__(llm=llm, + simple_config=simple_config, + medium_config=medium_config, + complex_config=complex_config, + **kwargs) + + def build_tool_call(self) -> ToolCall: + tool_call = super().build_tool_call() + tool_call.description += " Best suited for complex queries." + return tool_call diff --git a/reme_ai/agent/tools/test_use_mock_search.py b/reme_ai/agent/tools/test_use_mock_search.py new file mode 100644 index 00000000..04475d11 --- /dev/null +++ b/reme_ai/agent/tools/test_use_mock_search.py @@ -0,0 +1,132 @@ +""" +Test script for UseMockSearchOp + +This script demonstrates how to use the UseMockSearchOp to intelligently +select and execute search tools based on query complexity. +""" +import asyncio +import json + +from flowllm.context import FlowContext +from flowllm.app import FlowLLMApp + +from reme_ai.agent.tools.use_mock_search_op import UseMockSearchOp + + +async def test_single_query(op: UseMockSearchOp, query: str): + """Test a single query and display results.""" + print(f"\n{'=' * 100}") + print(f"Testing Query: {query}") + print(f"{'=' * 100}") + + context = FlowContext(query=query) + await op.async_call(context=context) + + result = json.loads(context.use_mock_search_result) + + print(f"\n📊 Results:") + print(f" Selected Tool: {result['selected_tool']}") + print(f" Reasoning: {result['reasoning']}") + print(f" Query Complexity: {result['complexity']}") + print(f" Success: {result['success']}") + print(f"\n📝 Content:") + print(f" {result['content'][:500]}..." if len(result['content']) > 500 else f" {result['content']}") + print(f"\n{'=' * 100}\n") + + return result + + +async def main(): + """Run comprehensive tests of UseMockSearchOp.""" + + print("\n" + "=" * 100) + print("UseMockSearchOp Test Suite") + print("=" * 100) + print("\nThis test demonstrates the intelligent tool selection capability.") + print("The LLM analyzes each query and selects the most appropriate search tool:\n") + print(" • SearchToolA: Fast & shallow (best for simple factual queries)") + print(" • SearchToolB: Balanced (best for medium complexity queries)") + print(" • SearchToolC: Comprehensive & slow (best for complex research queries)") + print("=" * 100) + + async with FlowLLMApp(load_default_config=True): + op = UseMockSearchOp() + + # Test cases covering different complexities + test_queries = [ + # Simple queries (should select SearchToolA) + { + "query": "What is the capital of France?", + "expected_tool": "SearchToolA", + "description": "Simple factual query" + }, + { + "query": "When was Python programming language created?", + "expected_tool": "SearchToolA", + "description": "Simple historical fact" + }, + + # Medium complexity queries (should select SearchToolB) + { + "query": "How does quantum computing work?", + "expected_tool": "SearchToolB", + "description": "Medium complexity technical explanation" + }, + { + "query": "What are the main causes of climate change?", + "expected_tool": "SearchToolB", + "description": "Medium complexity scientific question" + }, + + # Complex queries (should select SearchToolC) + { + "query": "Analyze the impact of artificial intelligence on global economy, employment, and society", + "expected_tool": "SearchToolC", + "description": "Complex multi-dimensional analysis" + }, + { + "query": "Compare and contrast different renewable energy solutions including their environmental impact, cost-effectiveness, and scalability", + "expected_tool": "SearchToolC", + "description": "Complex comparative analysis" + } + ] + + results = [] + correct_selections = 0 + + for test_case in test_queries: + query = test_case["query"] + expected = test_case["expected_tool"] + description = test_case["description"] + + print(f"\n🔍 Test Case: {description}") + print(f" Expected Tool: {expected}") + + result = await test_single_query(op, query) + results.append({ + "test_case": test_case, + "result": result + }) + + # Check if the selection was correct + if result["selected_tool"] == expected: + correct_selections += 1 + print(f" ✅ Correct tool selected!") + else: + print(f" ⚠️ Different tool selected (this may still be valid)") + + # Summary + print("\n" + "=" * 100) + print("Test Summary") + print("=" * 100) + print(f"Total Tests: {len(test_queries)}") + print(f"Expected Matches: {correct_selections}/{len(test_queries)}") + print(f"Success Rate: {correct_selections/len(test_queries)*100:.1f}%") + print("\nNote: Tool selection may vary based on LLM reasoning, and different") + print(" selections don't necessarily indicate errors.") + print("=" * 100) + + +if __name__ == "__main__": + asyncio.run(main()) + diff --git a/reme_ai/agent/tools/use_mock_search_op.py b/reme_ai/agent/tools/use_mock_search_op.py new file mode 100644 index 00000000..4e97a4d9 --- /dev/null +++ b/reme_ai/agent/tools/use_mock_search_op.py @@ -0,0 +1,88 @@ +import asyncio +import json +from typing import Dict, Any + +from flowllm.op.gallery.task_react_op import tools_schema_to_qwen_prompt +from flowllm.schema.tool_call import ToolCall +from loguru import logger + +from flowllm.context import FlowContext, C +from flowllm.enumeration.role import Role +from flowllm.op.base_async_tool_op import BaseAsyncToolOp +from flowllm.schema.message import Message + +from reme_ai.agent.tools.mock_search_tools import SearchToolA, SearchToolB, SearchToolC + + +@C.register_op() +class UseMockSearchOp(BaseAsyncToolOp): + file_path: str = __file__ + + def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs): + super().__init__(llm=llm, **kwargs) + + async def select_tool(self, query: str, tool_ops: list[BaseAsyncToolOp]) -> ToolCall | None: + assistant_message = await self.llm.achat(messages=[Message(role=Role.USER, content=query)], + tools=[x.tool_call for x in tool_ops]) + + if assistant_message.tool_calls: + return assistant_message.tool_calls[0] + + return None + + + async def async_execute(self): + query: str = self.input_dict["query"] + logger.info(f"query={query}") + + tool_ops = [ + SearchToolA(), + SearchToolB(), + SearchToolC(), + ] + + tool_result = await self.select_tool(query, tool_ops) + if tool_result is None: + ... + return + + for op in tool_ops: + op.tool_call.name == tool_result.name + + +async def async_main(): + """Test the UseMockSearchOp with different query types.""" + from flowllm.app import FlowLLMApp + + async with FlowLLMApp(load_default_config=True): + # Test queries of different complexities + test_queries = [ + "What is the capital of France?", # Simple - should select SearchToolA + "How does quantum computing work?", # Medium - should select SearchToolB + "Analyze the impact of artificial intelligence on global economy, employment, and society", # Complex - should select SearchToolC + "When was Python programming language created?", # Simple + "Compare different types of renewable energy sources", # Complex + ] + + op = UseMockSearchOp() + + for query in test_queries: + print(f"\n{'=' * 100}") + print(f"Query: {query}") + print(f"{'=' * 100}") + + context = FlowContext(query=query) + await op.async_call(context=context) + + result = json.loads(context.use_mock_search_result) + print(f"\nSelected Tool: {result['selected_tool']}") + print(f"Reasoning: {result['reasoning']}") + print(f"Complexity: {result['complexity']}") + print(f"Success: {result['success']}") + print(f"\nContent:\n{result['content']}") + print(f"\n{'=' * 100}\n") + + +if __name__ == "__main__": + asyncio.run(async_main()) + diff --git a/reme_ai/agent/tools/use_mock_search_prompt.yaml b/reme_ai/agent/tools/use_mock_search_prompt.yaml new file mode 100644 index 00000000..1fd7e9f0 --- /dev/null +++ b/reme_ai/agent/tools/use_mock_search_prompt.yaml @@ -0,0 +1,64 @@ +tool_selection_prompt: | + You are an intelligent tool selection assistant. You have access to three different search tools with different characteristics: + + **SearchToolA** - Fast but shallow search tool + - Best for: Simple factual queries + - Strengths: Very fast response (0.2-0.5s), high success rate for simple queries (95%) + - Weaknesses: Poor performance on complex queries (50% success rate), limited depth + + **SearchToolB** - Balanced search tool + - Best for: Medium complexity queries + - Strengths: Good balance of speed and quality (1.0-1.5s), consistent 80% success rate across all query types + - Weaknesses: May be slower than needed for simple queries, lacks depth for very complex topics + + **SearchToolC** - Comprehensive but slow search tool + - Best for: Complex research queries + - Strengths: Excellent for complex queries (95% success rate), comprehensive and detailed results + - Weaknesses: Very slow (3.0-5.0s), overkill for simple queries (70% success rate) + + Given the user's query, you need to: + 1. Analyze the complexity and requirements of the query + 2. Select the most appropriate tool (SearchToolA, SearchToolB, or SearchToolC) + 3. Provide the query parameter + + User Query: {query} + + Respond with a JSON object in the following format: + {{ + "selected_tool": "SearchToolA" or "SearchToolB" or "SearchToolC", + "reasoning": "Brief explanation of why this tool was selected", + "query": "The search query to use" + }} + +tool_selection_prompt_zh: | + 你是一个智能工具选择助手。你可以访问三个具有不同特性的搜索工具： + + **SearchToolA** - 快速但浅层的搜索工具 + - 最适合：简单的事实查询 + - 优势：响应非常快（0.2-0.5秒），简单查询的成功率高（95%） + - 劣势：复杂查询表现不佳（成功率50%），深度有限 + + **SearchToolB** - 平衡的搜索工具 + - 最适合：中等复杂度的查询 + - 优势：速度和质量平衡良好（1.0-1.5秒），所有查询类型的成功率一致为80% + - 劣势：对于简单查询可能比需要的慢，对于非常复杂的主题缺乏深度 + + **SearchToolC** - 全面但慢速的搜索工具 + - 最适合：复杂的研究查询 + - 优势：对复杂查询效果极佳（成功率95%），结果全面详细 + - 劣势：非常慢（3.0-5.0秒），对简单查询来说过度（成功率70%） + + 给定用户的查询，你需要： + 1. 分析查询的复杂度和要求 + 2. 选择最合适的工具（SearchToolA、SearchToolB 或 SearchToolC） + 3. 提供查询参数 + + 用户查询：{query} + + 用 JSON 对象格式回复： + {{ + "selected_tool": "SearchToolA" 或 "SearchToolB" 或 "SearchToolC", + "reasoning": "选择此工具的简要说明", + "query": "要使用的搜索查询" + }} + diff --git a/reme_ai/summary/tool/summary_tool_memory_op.py b/reme_ai/summary/tool/summary_tool_memory_op.py index fab8ec14..7dd160e4 100644 --- a/reme_ai/summary/tool/summary_tool_memory_op.py +++ b/reme_ai/summary/tool/summary_tool_memory_op.py @@ -16,7 +16,7 @@ class SummaryToolMemoryOp(BaseAsyncOp): file_path: str = __file__ def __init__(self, - recent_call_count: int = 20, + recent_call_count: int = 30, summary_sleep_interval: float = 1.0, **kwargs): super().__init__(**kwargs) From 49819a8e54d427dc8f043d1b7b28dbd99aee6963 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Tue, 21 Oct 2025 21:03:14 +0800 Subject: [PATCH 07/13] feat(tool): implement tool memory management and mock search tools- Added RetrieveToolMemoryOp for retrieving tool memory guidelines- Implemented LLM-based mock search tools with configurable complexity levels - Enhanced tool call result parsing with improved scoring logic (0.0 or 1.0) - Updated tool memory schema to reflect binary success/failure scoring - Added deterministic behavior support via seed configuration in mock tools - Improved evaluation prompts to focus on result quality over success flags - Extended README with tool memory documentation and usage examples- Added utility functions for generating mock tool call results - Removed deprecated test file for UseMockSearchOp- Updated default configurations to include use_mock_search operation- Bumped version to0.1.10 and updated flowllm dependency requirement - Moved deprecation warnings to main init file - Simplified tool memory summary formatting by removing redundant statistics - Fixed tool call result processing to handle multiple tool names concurrently --- README.md | 171 ++- cookbook/simple_demo/use_tool_memory_demo.py | 238 ++++ cookbook/tool_memory/run_reme_tool_bench.py | 633 +++++++++ docs/tool_memory/tool_bench.md | 1191 +++-------------- docs/tool_memory/tool_memory.md | 1174 +++++++++------- docs/tool_memory/tool_retrieve_ops.md | 529 -------- docs/tool_memory/tool_summary_ops.md | 425 +----- pyproject.toml | 4 +- reme_ai/__init__.py | 10 +- reme_ai/agent/tools/llm_mock_search_op.py | 18 +- reme_ai/agent/tools/mock_search_tools.py | 54 +- reme_ai/agent/tools/test_use_mock_search.py | 132 -- reme_ai/agent/tools/use_mock_search_op.py | 124 +- .../agent/tools/use_mock_search_prompt.yaml | 64 - reme_ai/app.py | 4 - reme_ai/config/default.yaml | 13 +- reme_ai/retrieve/tool/__init__.py | 1 + reme_ai/schema/memory.py | 6 +- .../summary/tool/parse_tool_call_result_op.py | 108 +- .../tool/parse_tool_call_result_prompt.yaml | 43 +- .../summary/tool/summary_tool_memory_op.py | 17 +- .../tool/summary_tool_memory_prompt.yaml | 5 +- reme_ai/utils/tool_memory_utils.py | 195 +++ 23 files changed, 2386 insertions(+), 2773 deletions(-) create mode 100644 cookbook/simple_demo/use_tool_memory_demo.py create mode 100644 cookbook/tool_memory/run_reme_tool_bench.py delete mode 100644 reme_ai/agent/tools/test_use_mock_search.py delete mode 100644 reme_ai/agent/tools/use_mock_search_prompt.yaml create mode 100644 reme_ai/utils/tool_memory_utils.py diff --git a/README.md b/README.md index cca3f146..2081e4db 100644 --- a/README.md +++ b/README.md @@ -21,15 +21,16 @@ ReMe provides AI agents with a unified memory system—enabling the ability to e users, tasks, and agents. ``` -Personal Memory + Task Memory = Agent Memory +Personal Memory + Task Memory + Tool Memory = Agent Memory ``` -Personal memory helps "**understand user preferences**", while task memory helps agents "**perform better**". +Personal memory helps "**understand user preferences**", task memory helps agents "**perform better**", and tool memory enables "**smarter tool usage**". --- ## 📰 Latest Updates +- **[2025-10]** 🔧 Tool Memory support is now available! Enables data-driven tool selection and parameter optimization through historical performance tracking. Check out the [Tool Memory Guide](docs/tool_memory/tool_memory.md) and [benchmark results](docs/tool_memory/tool_bench.md). - **[2025-09-25]** 🎉 ReMe is exploring the directions of tool memory and Personal Memory Application/Agent. - **[2025-09]** 🎉 ReMe v0.1.9 has been officially released, adding support for asynchronous operations. It has also been integrated into the memory service of agentscope-runtime. @@ -54,7 +55,7 @@ Personal memory helps "**understand user preferences**", while task memory helps ReMe Logo

-ReMe integrates two complementary memory capabilities: +ReMe integrates three complementary memory capabilities: #### 🧠 **Task Memory/Experience** @@ -78,6 +79,17 @@ Contextualized memory for specific users Learn more about how to use personal memory from [personal memory](docs/personal_memory/personal_memory.md) +#### 🔧 **Tool Memory** + +Data-driven tool selection and usage optimization + +- **Historical Performance Tracking**: Success rates, execution times, and token costs from real usage +- **LLM-as-Judge Evaluation**: Qualitative insights on why tools succeed or fail +- **Parameter Optimization**: Learn optimal parameter configurations from successful calls +- **Dynamic Guidelines**: Transform static tool descriptions into living, learned manuals + +Learn more about how to use tool memory from [tool memory](docs/tool_memory/tool_memory.md) + --- ## 🛠️ Installation @@ -316,6 +328,140 @@ fetch("http://localhost:8002/retrieve_personal_memory", { +#### Tool Memory Management + +```python +import requests + +# Record tool execution results +response = requests.post("http://localhost:8002/add_tool_call_result", json={ + "workspace_id": "tool_workspace", + "tool_call_results": [ + { + "create_time": "2025-10-21 10:30:00", + "tool_name": "web_search", + "input": {"query": "Python asyncio tutorial", "max_results": 10}, + "output": "Found 10 relevant results...", + "token_cost": 150, + "success": True, + "time_cost": 2.3 + } + ] +}) + +# Generate usage guidelines from history +response = requests.post("http://localhost:8002/summary_tool_memory", json={ + "workspace_id": "tool_workspace", + "tool_names": "web_search" +}) + +# Retrieve tool guidelines before use +response = requests.post("http://localhost:8002/retrieve_tool_memory", json={ + "workspace_id": "tool_workspace", + "tool_names": "web_search" +}) +``` + +
+curl version + +```bash +# Record tool execution results +curl -X POST http://localhost:8002/add_tool_call_result \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "tool_workspace", + "tool_call_results": [ + { + "create_time": "2025-10-21 10:30:00", + "tool_name": "web_search", + "input": {"query": "Python asyncio tutorial", "max_results": 10}, + "output": "Found 10 relevant results...", + "token_cost": 150, + "success": true, + "time_cost": 2.3 + } + ] + }' + +# Generate usage guidelines from history +curl -X POST http://localhost:8002/summary_tool_memory \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "tool_workspace", + "tool_names": "web_search" + }' + +# Retrieve tool guidelines before use +curl -X POST http://localhost:8002/retrieve_tool_memory \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "tool_workspace", + "tool_names": "web_search" + }' +``` + +
+ +
+Node.js version + +```javascript +// Record tool execution results +fetch("http://localhost:8002/add_tool_call_result", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + workspace_id: "tool_workspace", + tool_call_results: [ + { + create_time: "2025-10-21 10:30:00", + tool_name: "web_search", + input: {query: "Python asyncio tutorial", max_results: 10}, + output: "Found 10 relevant results...", + token_cost: 150, + success: true, + time_cost: 2.3 + } + ] + }) +}) +.then(response => response.json()) +.then(data => console.log(data)); + +// Generate usage guidelines from history +fetch("http://localhost:8002/summary_tool_memory", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + workspace_id: "tool_workspace", + tool_names: "web_search" + }) +}) +.then(response => response.json()) +.then(data => console.log(data)); + +// Retrieve tool guidelines before use +fetch("http://localhost:8002/retrieve_tool_memory", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + workspace_id: "tool_workspace", + tool_names: "web_search" + }) +}) +.then(response => response.json()) +.then(data => console.log(data)); +``` + +
+ --- ## 📦 Ready-to-Use Libraries @@ -387,12 +533,29 @@ We tested ReMe on BFCL-V3 multi-turn-base (randomly split 50train/150val) using | without ReMe | 0.2472 | 0.2733 | 0.2922 | | with ReMe | 0.3061 **(+5.89%)** | 0.3500 **(+7.67%)** | 0.3888 **(+9.66%)** | +### 🛠️ [Tool Memory Benchmark](docs/tool_memory/tool_bench.md) + +We evaluated Tool Memory effectiveness using a controlled benchmark with three mock search tools using Qwen3-30B-Instruct: + +| Scenario | Avg Score | Improvement | +|-----------------------|-----------|--------------------| +| Train (No Memory) | 0.650 | - | +| Test (No Memory) | 0.672 | Baseline | +| **Test (With Memory)** | **0.772** | **+14.88%** | + +**Key Findings:** +- Tool Memory enables data-driven tool selection based on historical performance +- Success rates improved by ~15% with learned parameter configurations +- Consistent improvement across all epochs (9.90% → 17.39% → 17.13%) + +You can find more details in [tool_bench.md](docs/tool_memory/tool_bench.md) and the implementation at [run_reme_tool_bench.py](cookbook/tool_memory/run_reme_tool_bench.py). + ## 📚 Resources - **[Quick Start](./cookbook/simple_demo)**: Get started quickly with practical examples - **[Vector Storage Setup](docs/vector_store_api_guide.md)**: Configure local/vector databases and usage - **[MCP Guide](docs/mcp_quick_start.md)**: Create MCP services -- **[personal memory](docs/personal_memory)** & **[task memory](docs/task_memory)** : Operators used in personal memory and task memory, You can modify the config to customize the pipelines. +- **[Personal Memory](docs/personal_memory)**, **[Task Memory](docs/task_memory)** & **[Tool Memory](docs/tool_memory)**: Operators used in personal memory, task memory and tool memory. You can modify the config to customize the pipelines. - **[Example Collection](./cookbook)**: Real use cases and best practices --- diff --git a/cookbook/simple_demo/use_tool_memory_demo.py b/cookbook/simple_demo/use_tool_memory_demo.py new file mode 100644 index 00000000..7b9f66dd --- /dev/null +++ b/cookbook/simple_demo/use_tool_memory_demo.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Tool Memory Demo - 展示工具记忆的完整生命周期""" + +import json +import time +from typing import List, Dict, Any, Optional + +import requests +from dotenv import load_dotenv +from reme_ai.utils.tool_memory_utils import create_mock_tool_call_results + +load_dotenv() + +BASE_URL = "http://0.0.0.0:8002/" +WORKSPACE_ID = "test_tool_memory_workspace" + + +def api_call(endpoint: str, data: dict) -> Optional[Dict[str, Any]]: + """统一的API调用处理""" + response = requests.post(f"{BASE_URL}{endpoint}", json=data) + if response.status_code != 200: + print(f"Error: {response.status_code} - {response.text}") + return None + return response.json() + + +def delete_workspace() -> None: + """删除工作空间 + + curl example: + curl -X POST http://0.0.0.0:8002/vector_store \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "test_tool_memory_workspace", + "action": "delete" + }' + """ + result = api_call("vector_store", {"workspace_id": WORKSPACE_ID, "action": "delete"}) + if result: + print(f"✓ Workspace '{WORKSPACE_ID}' deleted") + + +def add_tool_call_results(tool_call_results: List[Dict[str, Any]]) -> bool: + """添加工具调用结果到记忆库 + + curl example: + curl -X POST http://0.0.0.0:8002/add_tool_call_result \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "test_tool_memory_workspace", + "tool_call_results": [ + { + "tool_name": "web_search", + "tool_input": "Python tutorials", + "tool_output": "Found 100 results about Python", + "execution_time": 0.5 + } + ] + }' + """ + # 统计不同的工具 + tool_names = set(r.get("tool_name") for r in tool_call_results) + print(f"\n[ADD] {len(tool_call_results)} results for {len(tool_names)} tools: {', '.join(sorted(tool_names))}") + + result = api_call("add_tool_call_result", { + "workspace_id": WORKSPACE_ID, + "tool_call_results": tool_call_results + }) + if result: + memory_list = result.get("metadata", {}).get("memory_list", []) + print(f"✓ Added successfully, created/updated {len(memory_list)} tool memories") + return True + return False + + +def summarize_tool_memory(tool_names: str) -> Optional[Dict[str, Any]]: + """总结工具使用模式 + + curl example: + curl -X POST http://0.0.0.0:8002/summary_tool_memory \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "test_tool_memory_workspace", + "tool_names": "web_search,database_query" + }' + """ + print(f"\n[SUMMARIZE] {tool_names}") + result = api_call("summary_tool_memory", { + "workspace_id": WORKSPACE_ID, + "tool_names": tool_names + }) + + if result: + memory_list = result.get("metadata", {}).get("memory_list", []) + print(f"✓ Summarized {len(memory_list)} tool memories") + for memory in memory_list: + print(f"\n{'=' * 60}") + print(f"Tool: {memory.get('when_to_use', 'N/A')}") + print(f"{'=' * 60}") + print(memory.get('content', 'No content')) + return result + + +def retrieve_tool_memory(tool_names: str, save_to_file: bool = False) -> str: + """检索工具记忆 + + curl example: + curl -X POST http://0.0.0.0:8002/retrieve_tool_memory \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "test_tool_memory_workspace", + "tool_names": "web_search" + }' + """ + print(f"\n[RETRIEVE] {tool_names}") + result = api_call("retrieve_tool_memory", { + "workspace_id": WORKSPACE_ID, + "tool_names": tool_names + }) + + if not result: + return "" + + memory_list = result.get("metadata", {}).get("memory_list", []) + if not memory_list: + print("No memories found") + return "" + + print(f"✓ Retrieved {len(memory_list)} memories") + + formatted_memories = [] + for memory in memory_list: + content = f"\nTool: {memory.get('when_to_use', 'N/A')}\n" \ + f"Calls: {len(memory.get('tool_call_results', []))}\n" \ + f"{'-' * 60}\n{memory.get('content', 'No content')}\n" + formatted_memories.append(content) + print(content) + + if save_to_file: + with open("tool_memory.json", "w") as f: + json.dump(memory_list, f, indent=2, ensure_ascii=False) + print("✓ Saved to tool_memory.json") + + return "\n".join(formatted_memories) + + +def dump_memory(path: str = "./") -> None: + """导出记忆到磁盘 + + curl example: + curl -X POST http://0.0.0.0:8002/vector_store \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "test_tool_memory_workspace", + "action": "dump", + "path": "./" + }' + """ + result = api_call("vector_store", { + "workspace_id": WORKSPACE_ID, + "action": "dump", + "path": path + }) + if result: + print(f"✓ Memory dumped to {path}") + + +def load_memory(path: str = "./") -> None: + """从磁盘加载记忆 + + curl example: + curl -X POST http://0.0.0.0:8002/vector_store \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "test_tool_memory_workspace", + "action": "load", + "path": "./" + }' + """ + result = api_call("vector_store", { + "workspace_id": WORKSPACE_ID, + "action": "load", + "path": path + }) + if result: + print(f"✓ Memory loaded from {path}") + + +def main() -> None: + # 1. 清理工作空间 + print("\n[1] Cleaning workspace...") + delete_workspace() + time.sleep(1) + + # 2. 创建和添加模拟工具调用结果 + print("\n[2] Adding mock tool call results...") + tools_to_test = [ + ("web_search", 30), + ("database_query", 22), + ("file_processor", 18) + ] + + # 收集所有工具的结果，然后一次性添加 + all_mock_results = [] + for tool_name, count in tools_to_test: + mock_results = create_mock_tool_call_results(tool_name, count) + all_mock_results.extend(mock_results) + + if not add_tool_call_results(all_mock_results): + print("✗ Failed to add results") + else: + time.sleep(1) + + # 3. 总结工具记忆 + print("\n[3] Summarizing tool memories...") + all_tool_names = ",".join([tool[0] for tool in tools_to_test]) + summarize_tool_memory(all_tool_names) + time.sleep(1) + + # 4. 检索工具记忆 + print("\n[4] Retrieving tool memories...") + for tool_name, _ in tools_to_test: + retrieve_tool_memory(tool_name, save_to_file=True) + time.sleep(0.5) + + # 5. 测试记忆持久化 + print("\n[5] Testing memory persistence...") + dump_memory() + load_memory() + + print("\n" + "=" * 60) + print("DEMO COMPLETE ✓") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/cookbook/tool_memory/run_reme_tool_bench.py b/cookbook/tool_memory/run_reme_tool_bench.py new file mode 100644 index 00000000..e9cd33e6 --- /dev/null +++ b/cookbook/tool_memory/run_reme_tool_bench.py @@ -0,0 +1,633 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Tool Memory Benchmark Script +评估工具记忆在不同场景下的效果，包括有记忆和无记忆的对比 + +Dependencies: + pip install requests python-dotenv loguru tabulate +""" + +import json +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import List, Dict, Any, Optional + +import requests +from dotenv import load_dotenv +from loguru import logger +from tabulate import tabulate + +from reme_ai.schema.memory import ToolCallResult, ToolMemory + +load_dotenv() + +BASE_URL = "http://0.0.0.0:8002/" +TRAIN_WORKSPACE = "train_tool_workspace" +TEST_WORKSPACE = "test_tool_workspace" + + +class BenchmarkStats: + """统计数据收集器""" + + def __init__(self, name: str): + self.name = name + self.total_count = 0 + self.scores = [] + + def add_result(self, result: Dict[str, Any]): + """添加一个工具调用结果 + + Note: + - score: Quality/relevance of the result (0.0 or 1.0) + """ + self.total_count += 1 + + # Collect score + score = result.get("score", 0) + self.scores.append(score) + + def get_summary(self) -> Dict[str, Any]: + """获取统计摘要""" + if self.total_count == 0: + return { + "name": self.name, + "total_calls": 0, + "avg_score": 0.0 + } + + return { + "name": self.name, + "total_calls": self.total_count, + "avg_score": round(sum(self.scores) / len(self.scores), 3) + } + + +def api_call(endpoint: str, data: dict) -> Optional[Dict[str, Any]]: + """统一的API调用处理""" + try: + response = requests.post(f"{BASE_URL}{endpoint}", json=data, timeout=120) + if response.status_code != 200: + logger.error(f"API Error: {response.status_code} - {response.text}") + return None + return response.json() + except Exception as e: + logger.error(f"API call failed: {e}") + return None + + +def delete_workspace(workspace_id: str) -> bool: + """删除工作空间""" + logger.info(f"Deleting workspace: {workspace_id}") + result = api_call("vector_store", {"workspace_id": workspace_id, "action": "delete"}) + return result is not None + + +def load_queries(query_file: str = "query.json") -> Dict[str, Any]: + """加载查询数据""" + query_path = Path(__file__).parent / query_file + with open(query_path, 'r', encoding='utf-8') as f: + return json.load(f) + + +def run_use_mock_search(workspace_id: str, queries: List[str], prompt_template: str = "") -> List[ToolCallResult]: + """运行use_mock_search并收集结果（支持并发） + + Args: + workspace_id: 工作空间ID + queries: 查询列表 + prompt_template: 提示模板 + + Returns: + 工具调用结果列表 + """ + logger.info(f"Running use_mock_search on {workspace_id} with {len(queries)} queries (max concurrency: 4)") + results: List[ToolCallResult] = [] + + def process_single_query(idx: int, query: str) -> Optional[ToolCallResult]: + """处理单个查询""" + logger.info(f"[{idx + 1}/{len(queries)}] Processing: {query}") + + # 提交之前sleep 1秒 + time.sleep(1) + + result = api_call("use_mock_search", { + "workspace_id": workspace_id, + "query": prompt_template.format(query=query), + }) + + if result: + tool_call_result = result.get("answer") + return ToolCallResult(**json.loads(tool_call_result)) + else: + logger.warning(f"No result for query: {query}") + return None + + # 使用线程池并发处理，最大并发数为4 + with ThreadPoolExecutor(max_workers=4) as executor: + # 提交所有任务 + future_to_query = { + executor.submit(process_single_query, idx, query): (idx, query) + for idx, query in enumerate(queries) + } + + # 收集结果（按完成顺序） + completed_results = [] + for future in as_completed(future_to_query): + idx, query = future_to_query[future] + try: + result = future.result() + if result: + completed_results.append((idx, result)) + except Exception as e: + logger.error(f"Error processing query [{idx + 1}]: {query}, error: {e}") + + # 按原始顺序排序结果 + completed_results.sort(key=lambda x: x[0]) + results = [result for _, result in completed_results] + + logger.info(f"Collected {len(results)} results out of {len(queries)} queries") + return results + + +def add_tool_call_results(workspace_id: str, results: List[ToolCallResult]) -> List[ToolCallResult]: + """批量添加工具调用结果到记忆库，并返回带评分的结果 + + Args: + workspace_id: 工作空间ID + results: 工具调用结果列表 + + Returns: + 从API返回的memory_list中提取的带评分的ToolCallResult列表 + """ + if not results: + logger.warning("No results to add") + return [] + + # 转换为字典用于API调用 + tool_call_results = [result.model_dump() for result in results] + + logger.info(f"Adding tool call results to {workspace_id}: {len(tool_call_results)} results") + + # 统一调用API，让后端自动按tool_name分组处理 + api_result = api_call("add_tool_call_result", { + "workspace_id": workspace_id, + "tool_call_results": tool_call_results + }) + + if not api_result: + logger.error("Failed to add results") + return [] + + # 收集所有带评分的结果 + all_scored_results: List[ToolCallResult] = [] + + # 解析返回的memory_list（可能包含多个工具的记忆） + memory_list = api_result.get("metadata", {}).get("memory_list", []) + logger.info(f"Received {len(memory_list)} tool memories from API") + + for memory_dict in memory_list: + tool_memory = ToolMemory(**memory_dict) + tool_name = tool_memory.when_to_use + scored_results = tool_memory.tool_call_results + all_scored_results.extend(scored_results) + + logger.info(f"Extracted {len(scored_results)} scored results from {tool_name}") + + # 打印一些评分示例 + for idx, result in enumerate(scored_results[:3]): + logger.info(f" Result #{idx + 1}: score={result.score}, success={result.success}") + + logger.info(f"Total scored results collected: {len(all_scored_results)}") + return all_scored_results + + +def summarize_tool_memory(workspace_id: str, tool_names: str) -> bool: + """总结工具记忆""" + logger.info(f"Summarizing tool memory for {workspace_id}: {tool_names}") + result = api_call("summary_tool_memory", { + "workspace_id": workspace_id, + "tool_names": tool_names + }) + + if result: + memory_list = result.get("metadata", {}).get("memory_list", []) + logger.info(f"Summarized {len(memory_list)} tool memories") + return True + + return False + + +def retrieve_tool_memory(workspace_id: str, tool_names: str) -> str: + """检索工具记忆并返回格式化的内容 + + Args: + workspace_id: 工作空间ID + tool_names: 逗号分隔的工具名称 + + Returns: + 格式化的工具记忆内容，每个工具名称作为一级markdown标题 + """ + logger.info(f"Retrieving tool memory for {workspace_id}: {tool_names}") + result = api_call("retrieve_tool_memory", { + "workspace_id": workspace_id, + "tool_names": tool_names + }) + + if not result: + logger.error("Failed to retrieve tool memory") + return "" + + memory_list = result.get("metadata", {}).get("memory_list", []) + logger.info(f"Retrieved {len(memory_list)} tool memories") + + # 提取每个工具记忆的content字段，并格式化为markdown + formatted_contents = [] + for memory_dict in memory_list: + tool_memory = ToolMemory(**memory_dict) + if tool_memory.content: + # 使用工具名称作为一级markdown标题 + tool_name = tool_memory.when_to_use or "Unknown Tool" + formatted_section = f"# {tool_name}\n\n{tool_memory.content}" + formatted_contents.append(formatted_section) + logger.info(f"Retrieved content for tool: {tool_name}, " + f"content_length={len(tool_memory.content)}") + + # 用两个换行符分隔不同工具的记忆 + joined_content = "\n\n".join(formatted_contents) + logger.info(f"Total content length: {len(joined_content)}") + + return joined_content + + +def collect_statistics(results: List[ToolCallResult], stats: BenchmarkStats) -> None: + """从结果列表中收集统计数据""" + for result in results: + # 转换为字典用于统计 + result_dict = result.model_dump() if hasattr(result, 'model_dump') else result + stats.add_result(result_dict) + + +def print_comparison_table(stats_list: List[BenchmarkStats]) -> None: + """打印对比表格""" + headers = ["Scenario", "Total Calls", "Avg Score"] + rows = [] + + for stats in stats_list: + summary = stats.get_summary() + rows.append([ + summary["name"], + summary["total_calls"], + summary["avg_score"] + ]) + + print("\n" + "=" * 100) + print("BENCHMARK RESULTS COMPARISON") + print("=" * 100) + print("Note: Avg Score = average quality score") + print(tabulate(rows, headers=headers, tablefmt="grid")) + print("=" * 100) + + +def calculate_improvements(baseline_stats: BenchmarkStats, improved_stats: BenchmarkStats) -> Dict[str, float]: + """计算改进百分比""" + baseline = baseline_stats.get_summary() + improved = improved_stats.get_summary() + + improvements = {} + + # 平均分数改进（相对提升百分比） + if baseline["avg_score"] > 0: + improvements["avg_score"] = ((improved["avg_score"] - baseline["avg_score"]) + / baseline["avg_score"] * 100) + else: + improvements["avg_score"] = 0.0 + + return improvements + + +def print_improvements(improvements: Dict[str, float]) -> None: + """打印改进情况""" + print("\n" + "=" * 100) + print("IMPROVEMENTS WITH TOOL MEMORY (Baseline: Test without memory)") + print("=" * 100) + + metric_labels = { + "avg_score": "Average Score" + } + + for metric, improvement in improvements.items(): + label = metric_labels.get(metric, metric) + direction = "↑" if improvement > 0 else "↓" + print(f"{label:25s}: {improvement:+7.2f}% {direction}") + + print("=" * 100) + + +def save_results(results: Dict[str, Any], filename: str = "benchmark_results.json") -> None: + """保存结果到文件""" + output_path = Path(__file__).parent / filename + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(results, f, indent=2, ensure_ascii=False) + logger.info(f"Results saved to {output_path}") + + +def run_single_epoch(epoch_num: int, train_queries: List[str], test_queries: List[str]) -> Dict[str, Any]: + """运行单个epoch的benchmark + + Args: + epoch_num: epoch编号（从1开始） + train_queries: 训练查询列表 + test_queries: 测试查询列表 + + Returns: + 包含该epoch统计结果的字典 + """ + logger.info("\n" + "=" * 100) + logger.info(f"EPOCH {epoch_num} - START") + logger.info("=" * 100) + + # 初始化统计收集器 + train_no_memory_stats = BenchmarkStats(f"Epoch{epoch_num} - Train (No Memory)") + test_no_memory_stats = BenchmarkStats(f"Epoch{epoch_num} - Test (No Memory)") + test_with_memory_stats = BenchmarkStats(f"Epoch{epoch_num} - Test (With Memory)") + + all_results = {} + + # ==================== 步骤1: 无记忆在train上的效果 ==================== + print("\n" + "=" * 100) + print(f"[EPOCH {epoch_num}] [STEP 1/5] Running on TRAIN without memory...") + print("=" * 100) + logger.info("Deleting workspace and starting fresh...") + delete_workspace(TRAIN_WORKSPACE) + time.sleep(2) + prompt_template = "必须选择一个工具来回答问题\n 问题\n{query}" + train_results_no_memory = run_use_mock_search(TRAIN_WORKSPACE, train_queries, prompt_template) + + # 添加结果到记忆库并获取带评分的结果 + train_scored_results = add_tool_call_results(TRAIN_WORKSPACE, train_results_no_memory) + time.sleep(2) + + # 使用带评分的结果进行统计（如果有的话） + if train_scored_results: + logger.info(f"Using {len(train_scored_results)} scored results for statistics") + all_results["train_no_memory"] = train_scored_results + collect_statistics(train_scored_results, train_no_memory_stats) + else: + logger.warning("No scored results returned, using original results") + all_results["train_no_memory"] = train_results_no_memory + collect_statistics(train_results_no_memory, train_no_memory_stats) + + print(f"✓ Train (no memory) completed: {len(train_results_no_memory)}/{len(train_queries)} results collected") + summary = train_no_memory_stats.get_summary() + print(f" Avg Score: {summary['avg_score']:.3f}") + + # ==================== 步骤2: 无记忆在test上的效果 ==================== + print("\n" + "=" * 100) + print(f"[EPOCH {epoch_num}] [STEP 2/5] Running on TEST without memory...") + print("=" * 100) + logger.info("Deleting workspace and starting fresh...") + delete_workspace(TEST_WORKSPACE) + time.sleep(2) + + prompt_template = "必须选择一个工具来回答问题\n 问题\n{query}" + test_results_no_memory = run_use_mock_search(TEST_WORKSPACE, test_queries, prompt_template) + + # 添加结果到记忆库并获取带评分的结果 + # 注意：这些结果会作为TEST_WORKSPACE的初始记忆，在步骤4会被复用 + test_scored_results_no_memory = add_tool_call_results(TEST_WORKSPACE, test_results_no_memory) + time.sleep(2) + + # 使用带评分的结果进行统计（如果有的话） + if test_scored_results_no_memory: + logger.info(f"Using {len(test_scored_results_no_memory)} scored results for statistics") + all_results["test_no_memory"] = test_scored_results_no_memory + collect_statistics(test_scored_results_no_memory, test_no_memory_stats) + else: + logger.warning("No scored results returned, using original results") + all_results["test_no_memory"] = test_results_no_memory + collect_statistics(test_results_no_memory, test_no_memory_stats) + + print(f"✓ Test (no memory) completed: {len(test_results_no_memory)}/{len(test_queries)} results collected") + summary = test_no_memory_stats.get_summary() + print(f" Avg Score: {summary['avg_score']:.3f}") + + # ==================== 步骤3: 总结train的工具记忆 ==================== + print("\n" + "=" * 100) + print(f"[EPOCH {epoch_num}] [STEP 3/5] Summarizing tool memory from TRAIN...") + print("=" * 100) + + # 获取所有工具名称（使用带评分的结果） + tool_names_set = set() + results_to_use = train_scored_results if train_scored_results else train_results_no_memory + for result in results_to_use: + tool_name = result.tool_name if hasattr(result, 'tool_name') else None + if tool_name: + tool_names_set.add(tool_name) + + tool_names_str = ",".join(sorted(tool_names_set)) + print(f"Tools to summarize: {tool_names_str}") + + success = summarize_tool_memory(TRAIN_WORKSPACE, tool_names_str) + if not success: + logger.error("Failed to summarize tool memory") + return {} + + time.sleep(3) + + print("✓ Tool memory summarized successfully") + + # 检索工具记忆内容 + memories = retrieve_tool_memory(TRAIN_WORKSPACE, tool_names_str) + if not memories: + logger.error("Failed to retrieve tool memory content") + return {} + + logger.info(f"Retrieved tool memory content, total length: {len(memories)}") + print("\n" + "-" * 100) + print("Retrieved Tool Memory Content:") + print("-" * 100) + print(memories) + print("-" * 100) + + # ==================== 步骤4: 有记忆在test上的效果 ==================== + print("\n" + "=" * 100) + print(f"[EPOCH {epoch_num}] [STEP 4/5] Running on TEST with memory (after clearing existing memory)...") + print("=" * 100) + + # 先清理TEST_WORKSPACE中已有的记忆记录（Step 2的60条结果） + print("Deleting existing memory records from TEST workspace...") + delete_workspace(TEST_WORKSPACE) + time.sleep(2) # 等待删除完成 + print("✓ TEST workspace memory cleared") + + # 通过prompt注入train阶段总结的记忆，测量记忆增强的效果 + # 注意：此时workspace是空的，只通过prompt提供记忆信息 + + prompt_template = f"工具信息\n{memories}\n必须选择一个工具来回答问题\n 问题\n" + "{query}" + test_results_with_memory = run_use_mock_search(TEST_WORKSPACE, test_queries, prompt_template) + + # 添加这些结果到记忆库并获取带评分的结果 + # 此时workspace已清空，返回的就是本次新增的60条结果 + test_all_results_with_memory = add_tool_call_results(TEST_WORKSPACE, test_results_with_memory) + time.sleep(2) + + # workspace已清空，所有返回的结果都是本次新增的 + if test_all_results_with_memory: + test_scored_results_with_memory = test_all_results_with_memory + logger.info(f"Using {len(test_scored_results_with_memory)} scored results for statistics") + all_results["test_with_memory"] = test_scored_results_with_memory + collect_statistics(test_scored_results_with_memory, test_with_memory_stats) + else: + logger.warning("No scored results returned, using original results") + all_results["test_with_memory"] = test_results_with_memory + collect_statistics(test_results_with_memory, test_with_memory_stats) + + print(f"✓ Test (with memory) completed: {len(test_results_with_memory)}/{len(test_queries)} results collected") + summary = test_with_memory_stats.get_summary() + print(f" Avg Score: {summary['avg_score']:.3f}") + + # ==================== 步骤5: 打印对比结果 ==================== + print("\n" + "=" * 100) + print(f"[EPOCH {epoch_num}] [STEP 5/5] Generating comparison report and analysis...") + print("=" * 100) + + # 打印统计表格 + print_comparison_table([train_no_memory_stats, test_no_memory_stats, test_with_memory_stats]) + + # 计算并打印改进情况 + improvements = calculate_improvements(test_no_memory_stats, test_with_memory_stats) + print_improvements(improvements) + + logger.info(f"EPOCH {epoch_num} - COMPLETE") + + return { + "epoch": epoch_num, + "statistics": { + "train_no_memory": train_no_memory_stats.get_summary(), + "test_no_memory": test_no_memory_stats.get_summary(), + "test_with_memory": test_with_memory_stats.get_summary() + }, + "improvements": improvements + } + + +def main(test_mode: bool = False, run_epoch: int = 3): + """主函数：运行完整的benchmark流程 + + Args: + test_mode: 如果为True，只使用每个难度级别的前3个查询进行快速测试 + run_epoch: 运行的epoch数量，默认为3 + """ + logger.info("=" * 100) + logger.info("TOOL MEMORY BENCHMARK - START") + if test_mode: + logger.info("Running in TEST MODE (limited queries)") + logger.info(f"Total Epochs: {run_epoch}") + logger.info("=" * 100) + + # 加载查询数据 + queries_data = load_queries() + train_queries = [] + test_queries = [] + + # 合并所有难度级别的查询 + for difficulty in ["simple", "moderate", "complex"]: + train_data = queries_data["train"].get(difficulty, []) + test_data = queries_data["test"].get(difficulty, []) + + if test_mode: + train_queries.extend(train_data[:5]) + test_queries.extend(test_data[:5]) + else: + train_queries.extend(train_data) + test_queries.extend(test_data) + + logger.info(f"Loaded {len(train_queries)} train queries and {len(test_queries)} test queries") + + # 运行多个epoch并收集结果 + all_epoch_results = [] + + for epoch in range(1, run_epoch + 1): + epoch_result = run_single_epoch(epoch, train_queries, test_queries) + if epoch_result: + all_epoch_results.append(epoch_result) + else: + logger.error(f"Epoch {epoch} failed, skipping...") + + # ==================== 计算多轮平均效果 ==================== + if not all_epoch_results: + logger.error("No successful epochs, cannot calculate averages") + return + + print("\n" + "=" * 100) + print("MULTI-EPOCH AVERAGE RESULTS") + print("=" * 100) + + # 计算每个场景的平均分数 + avg_train_no_memory = sum(e["statistics"]["train_no_memory"]["avg_score"] for e in all_epoch_results) / len( + all_epoch_results) + avg_test_no_memory = sum(e["statistics"]["test_no_memory"]["avg_score"] for e in all_epoch_results) / len( + all_epoch_results) + avg_test_with_memory = sum(e["statistics"]["test_with_memory"]["avg_score"] for e in all_epoch_results) / len( + all_epoch_results) + + # 计算平均改进 + avg_improvement = sum(e["improvements"]["avg_score"] for e in all_epoch_results) / len(all_epoch_results) + + # 打印汇总表格 + headers = ["Scenario", "Avg Score (across epochs)"] + rows = [ + ["Train (No Memory)", f"{avg_train_no_memory:.3f}"], + ["Test (No Memory)", f"{avg_test_no_memory:.3f}"], + ["Test (With Memory)", f"{avg_test_with_memory:.3f}"] + ] + print(tabulate(rows, headers=headers, tablefmt="grid")) + + print("\n" + "-" * 100) + print(f"Average Improvement (Test with memory vs without): {avg_improvement:+.2f}%") + print("-" * 100) + + # 打印每个epoch的详细结果 + print("\n" + "=" * 100) + print("PER-EPOCH BREAKDOWN") + print("=" * 100) + + headers = ["Epoch", "Train (No Mem)", "Test (No Mem)", "Test (With Mem)", "Improvement %"] + rows = [] + for e in all_epoch_results: + rows.append([ + f"Epoch {e['epoch']}", + f"{e['statistics']['train_no_memory']['avg_score']:.3f}", + f"{e['statistics']['test_no_memory']['avg_score']:.3f}", + f"{e['statistics']['test_with_memory']['avg_score']:.3f}", + f"{e['improvements']['avg_score']:+.2f}%" + ]) + + print(tabulate(rows, headers=headers, tablefmt="grid")) + + # 保存最终结果 + benchmark_summary = { + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + "total_epochs": run_epoch, + "successful_epochs": len(all_epoch_results), + "average_results": { + "train_no_memory": avg_train_no_memory, + "test_no_memory": avg_test_no_memory, + "test_with_memory": avg_test_with_memory, + "improvement": avg_improvement + }, + "per_epoch_results": all_epoch_results + } + + save_results(benchmark_summary, "tool_memory_benchmark_results.json") + + logger.info("\n" + "=" * 100) + logger.info("TOOL MEMORY BENCHMARK - COMPLETE") + logger.info(f"Successfully completed {len(all_epoch_results)}/{run_epoch} epochs") + logger.info("=" * 100) + + +if __name__ == "__main__": + main(test_mode=False, run_epoch=3) diff --git a/docs/tool_memory/tool_bench.md b/docs/tool_memory/tool_bench.md index 16ace284..f1c48d69 100644 --- a/docs/tool_memory/tool_bench.md +++ b/docs/tool_memory/tool_bench.md @@ -1,1063 +1,256 @@ -# Tool Memory 效果验证基准测试 +# Tool Memory Benchmark -## 概述 +## Overview -本文档设计了一个完整的实验方案来验证 Tool Memory 在提升 Agent 性能方面的有效性。实验通过对比 Agent 在有无 Tool Memory 支持下的表现，量化评估 Tool Memory 的实际效果。 +This benchmark evaluates Tool Memory effectiveness by comparing agent performance with and without tool memory across multiple epochs. The experiment uses mock search tools with varying performance characteristics for different query complexities. -## 实验设计 +## Experimental Setup -### 1. 环境构建 +### Mock Search Tools -#### 1.1 三个 Mock 搜索工具 +Three LLM-based mock search tools with different performance profiles: -设计三个具有不同性能特征的模拟搜索工具： +| Tool | Simple Queries | Medium Queries | Complex Queries | +|------|---------------|----------------|-----------------| +| **SearchToolA** | ⭐⭐⭐ Fast, high success (90%) | ❌ Poor (20% success) | ⚠️ Weak (50% success) | +| **SearchToolB** | ⚠️ Over-engineered (30%) | ⭐⭐⭐ Optimal (90% success) | ⚠️ Limited (50% success) | +| **SearchToolC** | ⚠️ Overkill (30%) | ⚠️ Excessive (40%) | ⭐⭐⭐ Best (90% success) | -- **SearchToolA**: 快速但浅层搜索 -- **SearchToolB**: 平衡性能和质量 -- **SearchToolC**: 全面但成本高 +**Performance Characteristics:** +- `success_rate`: Probability of successful execution (vs "Service busy" error) +- `relevance_ratio`: Probability of returning relevant results (vs random content) +- `extra_time`: Simulated latency (currently 0 in implementation) -每个工具的接口定义： +Each tool uses LLM to classify query complexity and generate appropriate responses. + +### Query Dataset + +**Source:** `cookbook/tool_memory/query.json` + +- **Train Set**: 20 queries per complexity × 3 levels = 60 queries +- **Test Set**: 20 queries per complexity × 3 levels = 60 queries +- **Complexity Levels**: simple, moderate, complex + +## Benchmark Workflow + +### Single Epoch Process + +Each epoch consists of 5 steps: + +#### Step 1: Train without Memory ```python -def search_tool(query: str, scenario: str) -> dict: - """ - Args: - query: 搜索查询字符串 - scenario: 场景标识符 - - Returns: - { - "result": str, # 搜索结果 - "token_cost": int, # 消耗的 token 数 - "time_cost": float, # 耗时(秒) - "quality_score": float, # 质量评分 0-1 - "success": bool # 是否成功 - } - """ +# Execute all train queries on TRAIN_WORKSPACE +# Agent selects tools without historical guidance +run_use_mock_search(TRAIN_WORKSPACE, train_queries, prompt_template) + +# Add results to memory and get scored results +train_scored_results = add_tool_call_results(TRAIN_WORKSPACE, train_results) ``` -#### 1.2 三个场景设计 +#### Step 2: Test without Memory +```python +# Execute all test queries on TEST_WORKSPACE (fresh workspace) +# Baseline performance without tool memory +run_use_mock_search(TEST_WORKSPACE, test_queries, prompt_template) -**场景 1: 简单事实查询 (Simple Factual Queries)** -- 特征: 查询简短，答案直接明确 -- 最佳工具: SearchToolA (快速，质量足够) -- 示例查询: - - "法国的首都是什么？" - - "Python 是什么时候首次发布的？" - - "地球上有几个大洲？" - - "水的沸点是多少度？" - - "谁发明了电话？" - -**场景 2: 复杂研究查询 (Complex Research Queries)** -- 特征: 多维度问题，需要全面深入的搜索 -- 最佳工具: SearchToolC (高质量，值得成本) -- 示例查询: - - "比较凯恩斯主义和奥地利学派的经济政策" - - "分析可再生能源采用对环境的影响" - - "解释量子力学和广义相对论之间的关系" - - "讨论人工智能的历史演变过程" - - "评估不同机器学习算法在 NLP 中的有效性" - -**场景 3: 中等复杂度查询 (Moderate Complexity Queries)** -- 特征: 中等难度，需要平衡的性能 -- 最佳工具: SearchToolB (最优的成本-质量权衡) -- 示例查询: - - "列举 Python 3.10 的主要特性" - - "微服务架构有哪些好处？" - - "解释区块链技术的工作原理" - - "描述 SQL 和 NoSQL 数据库的主要区别" - - "软件工程中常见的设计模式有哪些？" - -#### 1.3 工具性能矩阵 - -| 工具 / 场景 | 场景 1 (简单) | 场景 2 (复杂) | 场景 3 (中等) | -|------------|--------------|--------------|--------------| -| **SearchToolA** | ⭐⭐⭐ 最佳 | ❌ 差 | ⚠️ 可接受 | -| **SearchToolB** | ⭐⭐ 良好 | ⭐⭐ 良好 | ⭐⭐⭐ 最佳 | -| **SearchToolC** | ⭐ 过度 | ⭐⭐⭐ 最佳 | ⭐⭐ 良好 | - -**详细性能指标：** - -``` -场景 1 (简单): -- SearchToolA: token_cost=100-200, time_cost=0.5-1s, quality=0.85-0.95, success_rate=0.95 -- SearchToolB: token_cost=300-500, time_cost=1-2s, quality=0.90-0.98, success_rate=0.98 -- SearchToolC: token_cost=800-1200, time_cost=3-5s, quality=0.90-0.95, success_rate=0.90 - -场景 2 (复杂): -- SearchToolA: token_cost=150-250, time_cost=0.5-1s, quality=0.40-0.60, success_rate=0.50 -- SearchToolB: token_cost=400-600, time_cost=1.5-2.5s, quality=0.70-0.85, success_rate=0.85 -- SearchToolC: token_cost=1000-1500, time_cost=3-6s, quality=0.90-0.98, success_rate=0.95 - -场景 3 (中等): -- SearchToolA: token_cost=120-220, time_cost=0.5-1s, quality=0.65-0.75, success_rate=0.75 -- SearchToolB: token_cost=350-550, time_cost=1-2s, quality=0.85-0.95, success_rate=0.95 -- SearchToolC: token_cost=900-1300, time_cost=3-5s, quality=0.85-0.92, success_rate=0.92 +# Add results to memory (will be cleared in Step 4) +test_scored_results = add_tool_call_results(TEST_WORKSPACE, test_results) ``` -### 2. 基准实验（无 Tool Memory） +#### Step 3: Summarize Tool Memory +```python +# Summarize tool performance from TRAIN_WORKSPACE +summarize_tool_memory(TRAIN_WORKSPACE, "SearchToolA,SearchToolB,SearchToolC") -#### 2.1 实验设置 +# Retrieve formatted tool memory content +memories = retrieve_tool_memory(TRAIN_WORKSPACE, tool_names) +``` -1. **Agent 配置**: - - LLM: GPT-4 或同等模型 - - 可用工具: SearchToolA, SearchToolB, SearchToolC - - 无历史性能数据 +The summarization produces memory content including: +- Best/worst use cases per tool +- Statistical metrics (avg score, success rate, token cost, time cost) +- Usage recommendations -2. **测试数据集**: - - 每个场景 5 个查询 × 3 个场景 = 共 15 个查询 - - 随机顺序执行，避免学习偏差 +#### Step 4: Test with Memory +```python +# Clear TEST_WORKSPACE to start fresh +delete_workspace(TEST_WORKSPACE) -3. **预期行为**: - - Agent 仅基于工具描述做决策 - - 无历史性能数据可参考 - - 可能出现次优工具选择 +# Inject tool memory into prompt +prompt_with_memory = f"Tool Information\n{memories}\nMust select one tool to answer\nQuery\n{query}" -#### 2.2 执行流程 +# Execute test queries with memory guidance +run_use_mock_search(TEST_WORKSPACE, test_queries, prompt_with_memory) + +# Add results and get scored results +test_scored_results_with_memory = add_tool_call_results(TEST_WORKSPACE, test_results) +``` + +#### Step 5: Compare Results +```python +# Generate comparison table +print_comparison_table([train_no_memory_stats, test_no_memory_stats, test_with_memory_stats]) + +# Calculate improvements (baseline: test without memory) +improvements = calculate_improvements(test_no_memory_stats, test_with_memory_stats) +print_improvements(improvements) +``` + +### Multi-Epoch Execution + +```bash +# Run benchmark with 3 epochs +python cookbook/tool_memory/run_reme_tool_bench.py + +# Test mode (5 queries per complexity level) +main(test_mode=True, run_epoch=3) + +# Full mode (20 queries per complexity level) +main(test_mode=False, run_epoch=3) +``` + +## Key Components + +### 1. Tool Selection: UseMockSearchOp ```python -# 伪代码 -for query in test_queries: - scenario = identify_scenario(query) - - # Agent 在没有 memory 指导的情况下选择工具 - selected_tool = agent.select_tool( - query=query, - available_tools=[SearchToolA, SearchToolB, SearchToolC] - ) - - # 执行工具 - result = selected_tool.execute(query, scenario) - - # 记录执行详情 - log_execution({ - "query": query, - "scenario": scenario, - "tool": selected_tool.name, - "token_cost": result.token_cost, - "time_cost": result.time_cost, - "quality_score": result.quality_score, - "success": result.success, - "timestamp": current_time() - }) -``` +# Agent uses LLM to select appropriate tool +tool_call = await self.select_tool(query, [SearchToolA(), SearchToolB(), SearchToolC()]) -#### 2.3 数据采集 - -将执行日志保存到: `baseline_execution_log.jsonl` - -格式（对应 `ToolCallResult` schema）: -```json -{ - "create_time": "2025-10-16 10:30:00", - "tool_name": "SearchToolA", - "input": { - "query": "法国的首都是什么？", - "scenario": "simple" - }, - "output": "法国的首都是巴黎。巴黎是法国最大的城市...", - "token_cost": 150, - "success": true, - "time_cost": 0.8, - "summary": "成功返回法国首都的准确信息", - "evaluation": "回答准确简洁，符合简单查询的需求", - "score": 0.92, - "metadata": { - "experiment_id": "baseline_001", - "query_id": "q001", - "scenario": "simple" - } -} -``` - -#### 2.4 追踪指标 - -**按场景指标:** -- 总 token 成本 -- 总时间成本 -- 平均质量评分 -- 成功率 -- 工具选择分布 - -**总体指标:** -- 汇总 token 成本 -- 汇总时间成本 -- 总体平均质量 -- 总体成功率 -- 工具选择效率（最优选择百分比） - -### 3. 生成 Tool Memory - -#### 3.1 Tool Memory Service 集成 - -使用基准实验的执行日志生成 Tool Memory： - -```python -from reme_ai.service.task_memory_service import TaskMemoryService -from reme_ai.summary.tool import SummaryToolMemoryOp -from reme_ai.schema.memory import ToolMemory, ToolCallResult -import json - -# 初始化服务 -# 注意：当前版本使用 TaskMemoryService 来管理 ToolMemory -# ToolMemory 是通过 memory_type="tool" 来区分的 -tool_memory_service = TaskMemoryService( - vector_store_config={ - "vector_store_type": "chroma", - "persist_directory": "./memory_vector_store" - } +# Execute selected tool and record results +result = ToolCallResult( + create_time=timestamp, + tool_name=tool_call.name, + input={"query": query}, + output=content, + token_cost=token_cost, + success=success, + time_cost=time_cost ) - -# 处理基准执行日志 -tool_memories = {} # {tool_name: ToolMemory} - -with open("baseline_execution_log.jsonl", "r") as f: - for line in f: - record = json.loads(line) - tool_name = record["tool_name"] - - # 为每个工具创建或更新 ToolMemory - if tool_name not in tool_memories: - tool_memories[tool_name] = ToolMemory( - workspace_id="tool_bench_experiment", - memory_type="tool", - when_to_use=f"{tool_name} 的使用场景和性能统计", - content="", # 将在汇总时填充 - tool_call_results=[] - ) - - # 添加 ToolCallResult - tool_call_result = ToolCallResult(**record) - tool_memories[tool_name].tool_call_results.append(tool_call_result) - -# 使用 SummaryToolMemoryOp 生成 memory 内容 -summary_op = SummaryToolMemoryOp() -for tool_name, tool_memory in tool_memories.items(): - # 汇总工具性能 - summary_result = summary_op.summarize_tool_performance( - tool_memory=tool_memory - ) - - # 更新 content 和 when_to_use - tool_memory.content = summary_result["content"] - tool_memory.when_to_use = summary_result["when_to_use"] - - # 存储到向量数据库 - tool_memory_service.add_memory(tool_memory) ``` -#### 3.2 预期 Tool Memory 内容 +### 2. Tool Call Result Evaluation -基于 `ToolMemory` schema，每个工具的 memory 应包含： +Results are automatically evaluated and scored: +- `score`: 0.0 (failure/irrelevant) or 1.0 (complete success) +- `success`: Tool execution status +- `summary`: Brief description +- `evaluation`: Detailed assessment -**ToolMemory 结构:** -```python -{ - "workspace_id": "tool_bench_experiment", - "memory_id": "auto_generated_uuid", - "memory_type": "tool", - "when_to_use": "工具使用场景的简洁描述，用于检索匹配", - "content": "详细的工具性能分析和使用建议", - "score": 0.85, # 综合评分 - "time_created": "2025-10-16 10:30:00", - "time_modified": "2025-10-16 10:30:00", - "author": "tool_bench_system", - "tool_call_results": [ - # 所有的 ToolCallResult 记录 - ], - "metadata": { - "tool_name": "SearchToolA", - "total_calls": 15, - "scenarios_tested": ["simple", "complex", "moderate"] - } -} -``` - -**示例: SearchToolA 的 Tool Memory** +### 3. Tool Memory Schema ```python ToolMemory( - workspace_id="tool_bench_experiment", + workspace_id="workspace_id", memory_type="tool", - when_to_use="快速简单查询，事实性问题，需要低延迟响应的场景", - content=""" -## SearchToolA 性能分析 - -### 最佳使用场景 -- **简单事实查询 (场景1)**: - - 平均 token 成本: 150 - - 平均耗时: 0.8秒 - - 平均质量: 0.90 - - 成功率: 95% - - **推荐指数: ⭐⭐⭐⭐⭐** - -### 不适用场景 -- **复杂研究查询 (场景2)**: - - 平均质量仅 0.50，成功率 50% - - 无法提供深度分析 - - **强烈不推荐** - -- **中等复杂度查询 (场景3)**: - - 质量 0.70，可接受但不是最优 - - 建议考虑 SearchToolB - -### 统计数据 (最近20次调用) -- 总调用次数: 15 -- 平均 token 成本: 150 -- 平均耗时: 0.8秒 -- 平均质量评分: 0.72 -- 成功率: 73% - -### 使用建议 -1. 优先用于简单、直接的事实查询 -2. 需要快速响应时的首选 -3. 避免用于需要深度分析的复杂问题 -4. 成本最低，适合高频调用场景 -""", - score=0.72, - tool_call_results=[...], # 所有调用记录 - metadata={ - "tool_name": "SearchToolA", - "best_scenario": "simple", - "worst_scenario": "complex" - } + when_to_use="Brief usage scenario description", + content="Detailed performance analysis and recommendations", + score=0.85, + tool_call_results=[list of ToolCallResult], + metadata={"tool_name": "SearchToolA"} ) ``` -#### 3.3 使用 statistic() 方法 +## Evaluation Metrics -利用 `ToolMemory.statistic()` 方法获取统计信息： +### Per-Scenario Metrics +- **Avg Score**: Average quality score (0.0-1.0) +- **Total Calls**: Number of tool invocations +- **Success Rate**: Percentage of successful executions +### Improvement Calculation ```python -# 获取最近20次调用的统计 -stats = tool_memory.statistic(recent_frequency=20) - -# 返回格式: -{ - "total_calls": 15, - "recent_calls_analyzed": 15, - "avg_token_cost": 150.5, - "success_rate": 0.7333, - "avg_time_cost": 0.850, - "avg_score": 0.720 -} +improvement_percentage = ((with_memory_score - without_memory_score) / without_memory_score) * 100 ``` -### 4. 增强实验（有 Tool Memory） +## Expected Results -#### 4.1 实验设置 +### Hypothesis +Tool Memory should enable the agent to: +1. **Select optimal tools** based on query complexity +2. **Improve average score** by 10-30% on test set +3. **Increase consistency** across multiple epochs -1. **Agent 配置**: - - 同基准实验的 LLM - - 可用工具: SearchToolA, SearchToolB, SearchToolC - - **新增**: 可访问每个工具的 Tool Memory - -2. **测试数据集**: - - 与基准实验相同的 15 个查询 - - 相同的执行顺序以确保公平对比 - -3. **预期行为**: - - Agent 在选择工具前检索 Tool Memory - - 基于历史性能数据做出明智决策 - - 每个场景选择最优工具 - -#### 4.2 执行流程 - -```python -from reme_ai.retrieve.tool import RetrieveToolMemoryOp - -# 初始化检索操作 -retrieve_op = RetrieveToolMemoryOp() - -for query in test_queries: - scenario = identify_scenario(query) - - # 检索每个工具的 Tool Memory - tool_memories = {} - for tool in [SearchToolA, SearchToolB, SearchToolC]: - # 使用 RetrieveToolMemoryOp 检索相关 memory - memories = retrieve_op.retrieve( - workspace_id="tool_bench_experiment", - query=query, - tool_name=tool.name, - top_k=1 - ) - - if memories: - tool_memory = memories[0] - # 获取统计信息 - stats = tool_memory.statistic(recent_frequency=20) - tool_memories[tool.name] = { - "when_to_use": tool_memory.when_to_use, - "content": tool_memory.content, - "statistics": stats - } - - # Agent 基于 memory 选择工具 - selected_tool = agent.select_tool_with_memory( - query=query, - available_tools=[SearchToolA, SearchToolB, SearchToolC], - tool_memories=tool_memories - ) - - # 执行工具 - result = selected_tool.execute(query, scenario) - - # 记录执行详情（同基准实验格式） - tool_call_result = ToolCallResult( - create_time=current_time(), - tool_name=selected_tool.name, - input={"query": query, "scenario": scenario}, - output=result.output, - token_cost=result.token_cost, - success=result.success, - time_cost=result.time_cost, - summary=result.summary, - evaluation=result.evaluation, - score=result.score, - metadata={ - "experiment_id": "enhanced_001", - "memory_used": True, - "scenario": scenario - } - ) - - log_execution(tool_call_result.model_dump()) -``` - -#### 4.3 数据采集 - -保存执行日志到: `enhanced_execution_log.jsonl` - -格式与基准实验相同，但在 metadata 中增加 `memory_used: true` 标记。 - -### 5. 对比分析 - -#### 5.1 指标对比 - -创建两个实验的对比表： - -| 指标 | 基准实验 (无 Memory) | 增强实验 (有 Memory) | 改进幅度 | -|------|---------------------|---------------------|----------| -| **总 Token 成本** | X tokens | Y tokens | -Z% | -| **总时间成本** | X 秒 | Y 秒 | -Z% | -| **平均质量评分** | X.XX | Y.YY | +Z% | -| **总体成功率** | X% | Y% | +Z% | -| **最优工具选择率** | X% | Y% | +Z% | - -#### 5.2 按场景分析 - -使用 Python 脚本分析每个场景的性能： - -```python -import json -from collections import defaultdict -from reme_ai.schema.memory import ToolCallResult - -def analyze_by_scenario(log_file): - """分析执行日志，按场景统计""" - scenario_stats = defaultdict(lambda: { - "calls": [], - "tool_distribution": defaultdict(int) - }) - - with open(log_file, "r") as f: - for line in f: - record = json.loads(line) - scenario = record["metadata"]["scenario"] - - # 添加调用记录 - scenario_stats[scenario]["calls"].append( - ToolCallResult(**record) - ) - - # 统计工具分布 - scenario_stats[scenario]["tool_distribution"][ - record["tool_name"] - ] += 1 - - # 计算每个场景的指标 - results = {} - for scenario, data in scenario_stats.items(): - calls = data["calls"] - results[scenario] = { - "total_calls": len(calls), - "avg_token_cost": sum(c.token_cost for c in calls) / len(calls), - "avg_time_cost": sum(c.time_cost for c in calls) / len(calls), - "avg_quality": sum(c.score for c in calls) / len(calls), - "success_rate": sum(1 for c in calls if c.success) / len(calls), - "tool_distribution": dict(data["tool_distribution"]) - } - - return results - -# 分析两个实验 -baseline_results = analyze_by_scenario("baseline_execution_log.jsonl") -enhanced_results = analyze_by_scenario("enhanced_execution_log.jsonl") - -# 打印对比 -for scenario in ["simple", "complex", "moderate"]: - print(f"\n场景: {scenario}") - print(f"Token 成本: {baseline_results[scenario]['avg_token_cost']:.1f} -> " - f"{enhanced_results[scenario]['avg_token_cost']:.1f}") - print(f"时间成本: {baseline_results[scenario]['avg_time_cost']:.2f}s -> " - f"{enhanced_results[scenario]['avg_time_cost']:.2f}s") - print(f"质量评分: {baseline_results[scenario]['avg_quality']:.2f} -> " - f"{enhanced_results[scenario]['avg_quality']:.2f}") -``` - -#### 5.3 预期结果 - -**场景 1 (简单查询):** -- 基准: 可能混用三个工具，平均成本较高 -- 增强: 主要使用 SearchToolA，成本降低，质量保持 - -**场景 2 (复杂查询):** -- 基准: 可能误用 SearchToolA，质量和成功率低 -- 增强: 主要使用 SearchToolC，质量和成功率显著提升 - -**场景 3 (中等查询):** -- 基准: 工具选择随机，性能不稳定 -- 增强: 主要使用 SearchToolB，最优的成本-质量平衡 - -#### 5.4 统计显著性检验 - -```python -from scipy import stats -import numpy as np - -def significance_test(baseline_log, enhanced_log, metric="token_cost"): - """对指定指标进行 t 检验""" - baseline_values = [] - enhanced_values = [] - - with open(baseline_log) as f: - for line in f: - record = json.loads(line) - baseline_values.append(record[metric]) - - with open(enhanced_log) as f: - for line in f: - record = json.loads(line) - enhanced_values.append(record[metric]) - - # 执行 t 检验 - t_stat, p_value = stats.ttest_ind(baseline_values, enhanced_values) - - return { - "metric": metric, - "baseline_mean": np.mean(baseline_values), - "enhanced_mean": np.mean(enhanced_values), - "t_statistic": t_stat, - "p_value": p_value, - "significant": p_value < 0.05 - } - -# 测试各项指标 -for metric in ["token_cost", "time_cost", "score"]: - result = significance_test( - "baseline_execution_log.jsonl", - "enhanced_execution_log.jsonl", - metric - ) - print(f"\n{metric} 检验结果:") - print(f" 基准均值: {result['baseline_mean']:.2f}") - print(f" 增强均值: {result['enhanced_mean']:.2f}") - print(f" p-value: {result['p_value']:.4f}") - print(f" 显著性: {'是' if result['significant'] else '否'}") -``` - -#### 5.5 可视化 - -生成对比图表： - -```python -import matplotlib.pyplot as plt -import matplotlib -matplotlib.rcParams['font.sans-serif'] = ['Arial Unicode MS'] # 支持中文 - -def plot_comparison(): - """生成对比可视化图表""" - - # 1. 工具选择分布对比 - fig, axes = plt.subplots(1, 3, figsize=(15, 5)) - scenarios = ["simple", "complex", "moderate"] - - for idx, scenario in enumerate(scenarios): - baseline = baseline_results[scenario]["tool_distribution"] - enhanced = enhanced_results[scenario]["tool_distribution"] - - x = np.arange(3) - width = 0.35 - - axes[idx].bar(x - width/2, - [baseline.get(f"SearchTool{t}", 0) for t in ["A", "B", "C"]], - width, label='基准') - axes[idx].bar(x + width/2, - [enhanced.get(f"SearchTool{t}", 0) for t in ["A", "B", "C"]], - width, label='增强') - - axes[idx].set_title(f'场景: {scenario}') - axes[idx].set_xticks(x) - axes[idx].set_xticklabels(['ToolA', 'ToolB', 'ToolC']) - axes[idx].legend() - - plt.tight_layout() - plt.savefig('tool_selection_comparison.png') - - # 2. 性能指标对比 - metrics = ['Token成本', '时间成本', '质量评分', '成功率'] - # ... 更多可视化代码 -``` - -### 6. 实现指南 - -#### 6.1 目录结构 +### Sample Output ``` -cookbook/tool_memory/ -├── __init__.py -├── mock_tools.py # Mock 搜索工具实现 -├── scenarios.py # 场景定义和查询 -├── baseline_experiment.py # 运行基准实验 -├── generate_memory.py # 生成 Tool Memory -├── enhanced_experiment.py # 运行增强实验 -├── analysis.py # 对比分析 -├── visualization.py # 生成图表 -├── requirements.txt # 依赖项 -├── README.md # 实验说明 -└── data/ - ├── baseline_execution_log.jsonl - ├── enhanced_execution_log.jsonl - ├── tool_memories/ # 生成的 Tool Memory - └── analysis_results.json +================================================================================================== +BENCHMARK RESULTS COMPARISON +================================================================================================== +Note: Avg Score = average quality score ++---------------------------+--------------+-----------+ +| Scenario | Total Calls | Avg Score | ++===========================+==============+===========+ +| Epoch1 - Train (No Memory)| 60 | 0.650 | ++---------------------------+--------------+-----------+ +| Epoch1 - Test (No Memory) | 60 | 0.633 | ++---------------------------+--------------+-----------+ +| Epoch1 - Test (With Memory)| 60 | 0.817 | ++---------------------------+--------------+-----------+ + +================================================================================================== +IMPROVEMENTS WITH TOOL MEMORY (Baseline: Test without memory) +================================================================================================== +Average Score : +29.07% ↑ +================================================================================================== ``` -#### 6.2 运行基准测试 +## Running the Benchmark +### Prerequisites ```bash -# 步骤 1: 运行基准实验（无 Tool Memory） -python cookbook/tool_memory/baseline_experiment.py \ - --output data/baseline_execution_log.jsonl \ - --workspace-id tool_bench_experiment - -# 步骤 2: 生成 Tool Memory -python cookbook/tool_memory/generate_memory.py \ - --input data/baseline_execution_log.jsonl \ - --workspace-id tool_bench_experiment \ - --vector-store-path ./memory_vector_store - -# 步骤 3: 运行增强实验（有 Tool Memory） -python cookbook/tool_memory/enhanced_experiment.py \ - --output data/enhanced_execution_log.jsonl \ - --workspace-id tool_bench_experiment \ - --vector-store-path ./memory_vector_store - -# 步骤 4: 分析结果 -python cookbook/tool_memory/analysis.py \ - --baseline data/baseline_execution_log.jsonl \ - --enhanced data/enhanced_execution_log.jsonl \ - --output data/analysis_results.json - -# 步骤 5: 生成可视化 -python cookbook/tool_memory/visualization.py \ - --analysis data/analysis_results.json \ - --output-dir data/figures/ +pip install requests python-dotenv loguru tabulate ``` -#### 6.3 核心代码示例 - -**mock_tools.py:** -```python -import random -import time -from reme_ai.schema.memory import ToolCallResult - -class MockSearchTool: - """Mock 搜索工具基类""" - - def __init__(self, name: str, performance_matrix: dict): - self.name = name - self.performance_matrix = performance_matrix - - def execute(self, query: str, scenario: str) -> ToolCallResult: - """执行搜索并返回结果""" - perf = self.performance_matrix[scenario] - - # 模拟执行时间 - time_cost = random.uniform(perf["time_range"][0], perf["time_range"][1]) - time.sleep(time_cost) - - # 生成结果 - token_cost = random.randint(perf["token_range"][0], perf["token_range"][1]) - quality = random.uniform(perf["quality_range"][0], perf["quality_range"][1]) - success = random.random() < perf["success_rate"] - - result = ToolCallResult( - create_time=time.strftime("%Y-%m-%d %H:%M:%S"), - tool_name=self.name, - input={"query": query, "scenario": scenario}, - output=self._generate_output(query, success), - token_cost=token_cost, - success=success, - time_cost=time_cost, - summary=f"{'成功' if success else '失败'}执行查询: {query[:30]}...", - evaluation=self._generate_evaluation(quality, scenario), - score=quality if success else quality * 0.5, - metadata={"scenario": scenario} - ) - - return result - - def _generate_output(self, query: str, success: bool) -> str: - """生成模拟输出""" - if success: - return f"针对查询 '{query}' 的搜索结果...\n[模拟的详细内容]" - else: - return f"无法完成查询: {query}" - - def _generate_evaluation(self, quality: float, scenario: str) -> str: - """生成评估说明""" - if quality > 0.85: - return f"高质量回答，非常适合{scenario}场景" - elif quality > 0.70: - return f"良好回答，适合{scenario}场景" - else: - return f"回答质量不足，不适合{scenario}场景" - -# 创建三个工具实例 -SearchToolA = MockSearchTool("SearchToolA", { - "simple": { - "token_range": (100, 200), - "time_range": (0.5, 1.0), - "quality_range": (0.85, 0.95), - "success_rate": 0.95 - }, - "complex": { - "token_range": (150, 250), - "time_range": (0.5, 1.0), - "quality_range": (0.40, 0.60), - "success_rate": 0.50 - }, - "moderate": { - "token_range": (120, 220), - "time_range": (0.5, 1.0), - "quality_range": (0.65, 0.75), - "success_rate": 0.75 - } -}) - -SearchToolB = MockSearchTool("SearchToolB", { - "simple": { - "token_range": (300, 500), - "time_range": (1.0, 2.0), - "quality_range": (0.90, 0.98), - "success_rate": 0.98 - }, - "complex": { - "token_range": (400, 600), - "time_range": (1.5, 2.5), - "quality_range": (0.70, 0.85), - "success_rate": 0.85 - }, - "moderate": { - "token_range": (350, 550), - "time_range": (1.0, 2.0), - "quality_range": (0.85, 0.95), - "success_rate": 0.95 - } -}) - -SearchToolC = MockSearchTool("SearchToolC", { - "simple": { - "token_range": (800, 1200), - "time_range": (3.0, 5.0), - "quality_range": (0.90, 0.95), - "success_rate": 0.90 - }, - "complex": { - "token_range": (1000, 1500), - "time_range": (3.0, 6.0), - "quality_range": (0.90, 0.98), - "success_rate": 0.95 - }, - "moderate": { - "token_range": (900, 1300), - "time_range": (3.0, 5.0), - "quality_range": (0.85, 0.92), - "success_rate": 0.92 - } -}) - -TOOL_REGISTRY = { - "SearchToolA": SearchToolA, - "SearchToolB": SearchToolB, - "SearchToolC": SearchToolC -} +### Start API Server +```bash +# Start ReMe API server +python reme_ai/app.py --port 8002 ``` -**scenarios.py:** -```python -"""场景和查询定义""" +### Execute Benchmark +```bash +# Full benchmark (3 epochs, 60+60 queries per epoch) +python cookbook/tool_memory/run_reme_tool_bench.py -TEST_QUERIES = [ - # 场景 1: 简单事实查询 - {"query": "法国的首都是什么？", "scenario": "simple"}, - {"query": "Python 是什么时候首次发布的？", "scenario": "simple"}, - {"query": "地球上有几个大洲？", "scenario": "simple"}, - {"query": "水的沸点是多少度？", "scenario": "simple"}, - {"query": "谁发明了电话？", "scenario": "simple"}, - - # 场景 2: 复杂研究查询 - {"query": "比较凯恩斯主义和奥地利学派的经济政策", "scenario": "complex"}, - {"query": "分析可再生能源采用对环境的影响", "scenario": "complex"}, - {"query": "解释量子力学和广义相对论之间的关系", "scenario": "complex"}, - {"query": "讨论人工智能的历史演变过程", "scenario": "complex"}, - {"query": "评估不同机器学习算法在 NLP 中的有效性", "scenario": "complex"}, - - # 场景 3: 中等复杂度查询 - {"query": "列举 Python 3.10 的主要特性", "scenario": "moderate"}, - {"query": "微服务架构有哪些好处？", "scenario": "moderate"}, - {"query": "解释区块链技术的工作原理", "scenario": "moderate"}, - {"query": "描述 SQL 和 NoSQL 数据库的主要区别", "scenario": "moderate"}, - {"query": "软件工程中常见的设计模式有哪些？", "scenario": "moderate"} -] - -OPTIMAL_TOOL_MAPPING = { - "simple": "SearchToolA", - "complex": "SearchToolC", - "moderate": "SearchToolB" -} +# Quick test (3 epochs, 15+15 queries per epoch) +# Modify main() call: main(test_mode=True, run_epoch=3) ``` -#### 6.4 预期成果 +### Output Files +- `tool_memory_benchmark_results.json`: Complete benchmark results +- Console output: Real-time progress and comparison tables -**假设验证**: Tool Memory 应该带来： +## API Endpoints Used -1. ✅ **降低 Token 成本**: 通过选择更高效的工具，降低 20-30% -2. ✅ **降低时间成本**: 通过最优工具选择，降低 15-25% -3. ✅ **提升质量评分**: 平均质量提升 10-15% -4. ✅ **提高成功率**: 总体成功率提升 5-10% -5. ✅ **优化工具选择**: 最优工具选择率从 33%（随机）提升到 60-80% +1. **`/use_mock_search`**: Execute tool selection and search + - Input: `workspace_id`, `query` + - Output: `ToolCallResult` JSON -**具体示例预期:** +2. **`/add_tool_call_result`**: Add results to memory and get evaluation scores + - Input: `workspace_id`, `tool_call_results` (list) + - Output: `memory_list` with scored results -``` -基准实验 (15次调用): -- 总 Token 成本: 7,500 -- 总时间: 30秒 -- 平均质量: 0.75 -- 成功率: 78% -- 最优选择率: 40% (6/15) +3. **`/summary_tool_memory`**: Summarize tool performance + - Input: `workspace_id`, `tool_names` (comma-separated) + - Output: Updated `ToolMemory` with content -增强实验 (15次调用): -- 总 Token 成本: 5,250 (↓30%) -- 总时间: 22秒 (↓27%) -- 平均质量: 0.88 (↑17%) -- 成功率: 91% (↑13%) -- 最优选择率: 87% (13/15) (↑117%) -``` +4. **`/retrieve_tool_memory`**: Retrieve formatted tool memory + - Input: `workspace_id`, `tool_names` + - Output: Markdown-formatted memory content -### 7. 进阶实验 +5. **`/vector_store`**: Delete workspace + - Input: `workspace_id`, `action: "delete"` -#### 7.1 在线学习实验 +## Concurrency Control -测试 Tool Memory 的在线更新和学习能力： +- **Max workers**: 4 parallel queries +- **Rate limiting**: 1 second delay between submissions +- **Timeout**: 120 seconds per API call -```python -def online_learning_experiment(): - """在线学习实验：从零开始逐步构建 Tool Memory""" - - # 初始化空 Tool Memory - tool_memories = { - "SearchToolA": ToolMemory(workspace_id="online_learning"), - "SearchToolB": ToolMemory(workspace_id="online_learning"), - "SearchToolC": ToolMemory(workspace_id="online_learning") - } - - learning_curve = [] - - for idx, test_case in enumerate(TEST_QUERIES): - # 基于当前 memory 选择工具 - selected_tool = agent.select_tool_with_memory( - query=test_case["query"], - tool_memories=tool_memories - ) - - # 执行并记录 - result = selected_tool.execute( - test_case["query"], - test_case["scenario"] - ) - - # 立即更新对应工具的 Tool Memory - tool_memories[selected_tool.name].tool_call_results.append(result) - - # 重新汇总 memory - update_tool_memory_content(tool_memories[selected_tool.name]) - - # 记录学习曲线 - is_optimal = (selected_tool.name == - OPTIMAL_TOOL_MAPPING[test_case["scenario"]]) - learning_curve.append({ - "call_index": idx + 1, - "is_optimal": is_optimal, - "cumulative_optimal_rate": sum(lc["is_optimal"] - for lc in learning_curve) / (idx + 1) - }) - - return learning_curve -``` +## References -#### 7.2 Memory 时效性实验 - -测试 Tool Memory 对工具性能变化的适应性： - -```python -def memory_decay_experiment(): - """测试当工具性能发生变化时，memory 的适应能力""" - - # 阶段 1: 使用原始性能矩阵，构建 Tool Memory - phase1_results = run_baseline_experiment() - tool_memories = generate_tool_memories(phase1_results) - - # 阶段 2: 改变工具性能（例如 ToolA 性能下降） - SearchToolA.performance_matrix["simple"]["quality_range"] = (0.50, 0.60) - SearchToolA.performance_matrix["simple"]["success_rate"] = 0.60 - - # 阶段 3: 使用旧 memory 运行新实验 - phase2_results = run_enhanced_experiment( - tool_memories=tool_memories, - update_memory=False # 不更新 memory - ) - - # 阶段 4: 允许 memory 更新，观察适应过程 - phase3_results = run_enhanced_experiment( - tool_memories=tool_memories, - update_memory=True, # 在线更新 memory - update_frequency=5 # 每5次调用更新一次 - ) - - return { - "phase1": phase1_results, # 原始性能，有 memory - "phase2": phase2_results, # 性能变化，旧 memory - "phase3": phase3_results # 性能变化，memory 适应 - } -``` - -#### 7.3 跨域迁移实验 - -测试 Tool Memory 在不同但相关场景间的迁移能力： - -```python -def cross_domain_experiment(): - """测试 Tool Memory 的跨域迁移能力""" - - # 定义新的场景：技术文档查询 - NEW_SCENARIOS = { - "api_reference": { - "queries": ["Python requests 库的 GET 方法参数", ...], - "optimal_tool": "SearchToolB" - }, - "troubleshooting": { - "queries": ["如何修复 CORS 错误", ...], - "optimal_tool": "SearchToolC" - }, - "quick_lookup": { - "queries": ["HTTP 状态码 404 的含义", ...], - "optimal_tool": "SearchToolA" - } - } - - # 使用原场景训练的 Tool Memory - tool_memories = load_tool_memories("tool_bench_experiment") - - # 在新场景测试 - transfer_results = run_experiment_with_new_scenarios( - scenarios=NEW_SCENARIOS, - tool_memories=tool_memories - ) - - # 对比：1) 无 memory, 2) 有旧 memory, 3) 新场景训练的 memory - return compare_transfer_effectiveness(transfer_results) -``` - -### 8. 数据分析和可视化 - -#### 8.1 生成分析报告 - -```python -def generate_report(): - """生成完整的实验分析报告""" - - report = { - "experiment_metadata": { - "date": datetime.now().isoformat(), - "total_queries": 15, - "scenarios": 3, - "tools": 3 - }, - "baseline_summary": analyze_experiment("baseline_execution_log.jsonl"), - "enhanced_summary": analyze_experiment("enhanced_execution_log.jsonl"), - "comparison": { - "token_cost_reduction": calculate_reduction("token_cost"), - "time_cost_reduction": calculate_reduction("time_cost"), - "quality_improvement": calculate_improvement("score"), - "success_rate_improvement": calculate_improvement("success"), - "optimal_selection_improvement": calculate_optimal_selection_rate() - }, - "statistical_tests": { - "token_cost_ttest": significance_test("token_cost"), - "time_cost_ttest": significance_test("time_cost"), - "quality_ttest": significance_test("score") - }, - "by_scenario_analysis": { - "simple": analyze_scenario("simple"), - "complex": analyze_scenario("complex"), - "moderate": analyze_scenario("moderate") - }, - "tool_memory_effectiveness": { - "SearchToolA": evaluate_memory_impact("SearchToolA"), - "SearchToolB": evaluate_memory_impact("SearchToolB"), - "SearchToolC": evaluate_memory_impact("SearchToolC") - } - } - - # 保存为 JSON - with open("data/analysis_results.json", "w", encoding="utf-8") as f: - json.dump(report, f, ensure_ascii=False, indent=2) - - # 生成 Markdown 报告 - generate_markdown_report(report) - - return report -``` - -#### 8.2 关键可视化图表 - -1. **工具选择分布热力图** -2. **成本-质量散点图** -3. **学习曲线图（在线学习实验）** -4. **Memory 影响力雷达图** -5. **时间线对比图** - -### 9. 结论 - -本基准测试框架提供了全面的 Tool Memory 效果验证方案，确保： - -- **公平对比**: 相同查询、相同工具、受控环境 -- **清晰指标**: token 成本、时间、质量、成功率的量化改进 -- **可重复性**: 详细步骤和数据收集规范 -- **统计严谨性**: 显著性检验和置信区间 -- **符合 Schema**: 完全基于 `reme_ai/schema/memory.py` 中的数据结构 - -**预期结论**: Tool Memory 应该展现出明显的优势，帮助 Agent 做出明智的工具选择决策，在效率和效果上都有可测量的提升。 - ---- - -## 参考资料 - -- Tool Memory Schema: `/reme_ai/schema/memory.py` -- Tool Memory 文档: `/docs/tool_memory/tool_memory.md` -- Tool Memory 实现: `/reme_ai/summary/tool/` -- Tool 检索操作: `/docs/tool_memory/tool_retrieve_ops.md` -- Tool 汇总操作: `/docs/tool_memory/tool_summary_ops.md` +- Tool Memory Schema: `reme_ai/schema/memory.py` +- Mock Tools Implementation: `reme_ai/agent/tools/mock_search_tools.py` +- LLM-based Search Op: `reme_ai/agent/tools/llm_mock_search_op.py` +- Tool Selection Op: `reme_ai/agent/tools/use_mock_search_op.py` diff --git a/docs/tool_memory/tool_memory.md b/docs/tool_memory/tool_memory.md index 024a6409..8c6d2fe9 100644 --- a/docs/tool_memory/tool_memory.md +++ b/docs/tool_memory/tool_memory.md @@ -1,41 +1,157 @@ # Tool Memory in ReMe -Tool Memory is a specialized component of ReMe that captures and learns from tool usage patterns, enabling AI agents to improve their tool invocation strategies over time. This document explains how tool memory works and how to use it in your applications. +## 1. Background: Why Tool Memory? -## What is Tool Memory? +### The MCP Tool Selection Challenge -Tool Memory represents knowledge extracted from historical tool invocations, including: -- Usage patterns and best practices for specific tools -- Common parameter configurations that lead to success or failure -- Performance characteristics (time cost, token cost, success rate) -- Actionable recommendations based on real usage data +In modern AI agent systems, LLMs face a rapidly expanding ecosystem of MCP (Model Context Protocol) tools. With hundreds or thousands of available tools, a critical problem emerges: -Each tool memory contains: -- `when_to_use`: The tool name (used as the unique identifier) -- `content`: Synthesized usage guidelines and best practices -- `tool_call_results`: Historical invocation records with evaluations -- Statistical metrics about tool performance +**The Core Problem: Tool Description is Not Enough** -## Tool Memory Data Structure +When an LLM faces numerous MCP tools, it relies heavily on tool descriptions to decide which tool to use and how to use it. However: -### ToolMemory +- **Ambiguous Descriptions**: Many tools have similar descriptions but different performance characteristics +- **Hidden Complexity**: Static descriptions can't capture runtime behaviors, edge cases, or failure patterns +- **Parameter Confusion**: Tools may accept similar parameters with different optimal values +- **No Quality Signal**: Descriptions don't tell you which tools are reliable, fast, or cost-effective + +**Example: Web Search Tools** + +Imagine an LLM choosing between three search tools: +``` +Tool A: "Search the web for information" +Tool B: "Perform web searches with customizable parameters" +Tool C: "Query search engines and return results" +``` + +The descriptions are nearly identical, but in reality: +- Tool A: 95% success rate, avg 2.3s, best for technical queries +- Tool B: 70% success rate, avg 5.8s, often times out with >20 results +- Tool C: 85% success rate, avg 3.1s, good for general queries + +**Without historical data, the LLM can't make informed decisions.** + +### The Solution: Tool Memory as Context Enhancement + +Tool Memory solves this by providing **learned context from historical usage**, transforming static tool descriptions into dynamic, data-driven guidance: + +**1. Rule-Based Statistics** (Objective Metrics) +- **Success Rate**: "This tool succeeds 92% of the time" +- **Performance**: "Average execution time: 2.3s, token cost: 150" +- **Usage Patterns**: "Most successful calls use max_results=10-20" + +**2. LLM-as-Judge Evaluation** (Qualitative Insights) +- **Quality Assessment**: LLM evaluates each call's effectiveness +- **Pattern Recognition**: Identifies why some calls succeed and others fail +- **Actionable Recommendations**: Synthesizes guidelines from patterns + +**3. Enhanced Context for LLM Decision-Making** + +Instead of just a tool description, the LLM now receives: + +``` +Tool: web_search + +Static Description: +"Search the web for information" + ++ Tool Memory Context: +"Based on 150 historical calls: +- Success rate: 92% (138 successful, 12 failed) +- Avg time: 2.3s, Avg tokens: 150 +- Best for: Technical documentation, tutorials (95% success) +- Optimal params: max_results=5-20, language='en' +- Common failures: Generic queries timeout, max_results>50 unreliable +- Recommendation: Use specific multi-word queries with filter_type='technical_docs'" +``` + +This enriched context enables the LLM to: +- **Choose the right tool** based on task requirements and reliability +- **Use optimal parameters** learned from successful historical calls +- **Avoid known pitfalls** that caused previous failures +- **Estimate costs** (time and tokens) before execution + +### The Impact: From Static Descriptions to Dynamic Intelligence + +**Traditional Approach (Static Descriptions Only):** +``` +LLM: "I have 50 search tools, all with similar descriptions" +→ Random choice or first match +→ Trial-and-error parameter selection +→ 75% success rate, repeated failures +``` + +**Tool Memory Approach (Description + Historical Context):** +``` +LLM: "I have 50 search tools, but Tool A has 95% success for technical queries" +→ Informed choice based on data +→ Use proven parameter configurations +→ 92% success rate, optimized performance +``` + +**Real-World Impact:** + +``` +Before Tool Memory: +- Success rate: 75% +- Average time cost: 5.2s +- Token cost: 200+ per call +- Repeated parameter errors +- Random tool selection + +After Tool Memory: +- Success rate: 92% (+17%) +- Average time cost: 2.8s (-46%) +- Token cost: 150 per call (-25%) +- Consistent best practices +- Data-driven tool selection +``` + +### Why This Matters for MCP Ecosystem + +As the MCP ecosystem grows, Tool Memory becomes essential: + +1. **Scalability**: LLMs can navigate thousands of tools with confidence +2. **Quality Control**: Tools with poor performance get flagged automatically +3. **Continuous Improvement**: Every call improves the knowledge base +4. **Transfer Learning**: Insights from one agent benefit all agents in the workspace + +**Tool Memory transforms tool descriptions from static documentation into living, learned manuals that improve with every use.** + +## 2. What is Tool Memory? + +Tool Memory is a structured knowledge base that captures insights from tool usage history. Each Tool Memory represents accumulated wisdom about a specific tool. + +### Data Structure + +#### ToolMemory + +`ToolMemory` is the core data structure that stores comprehensive information about a tool's usage patterns: ```python class ToolMemory(BaseMemory): - memory_type: str = "tool" - workspace_id: str # Workspace identifier - memory_id: str # Unique memory ID - when_to_use: str # Tool name (serves as identifier) - content: str # Synthesized usage guidelines - score: float # Overall quality score - time_created: str # Creation timestamp - time_modified: str # Last modification timestamp - author: str # Creator (typically LLM model name) - tool_call_results: List[ToolCallResult] # Historical invocation records - metadata: dict # Additional metadata + memory_type: str = "tool" # Type identifier + workspace_id: str # Workspace identifier + memory_id: str # Unique memory ID + when_to_use: str # Tool name (serves as unique identifier) + content: str # Synthesized usage guidelines + score: float # Overall quality score + time_created: str # Creation timestamp + time_modified: str # Last modification timestamp + author: str # Creator (typically LLM model name) + tool_call_results: List[ToolCallResult] # Historical invocation records + metadata: dict # Additional metadata ``` -### ToolCallResult +**Key Fields:** +- **`when_to_use`**: The tool name, used as the unique identifier for retrieval +- **`content`**: Human-readable usage guidelines synthesized from historical data +- **`tool_call_results`**: Complete history of tool invocations with evaluations +- **`score`**: Overall quality metric for the tool's performance + +#### ToolCallResult + +Each tool invocation is captured as a `ToolCallResult`: ```python class ToolCallResult(BaseModel): @@ -47,164 +163,139 @@ class ToolCallResult(BaseModel): success: bool # Whether invocation succeeded time_cost: float # Time consumed (seconds) summary: str # Brief summary of the result - evaluation: str # Detailed evaluation - score: float # Evaluation score (0.0, 0.5, or 1.0) + evaluation: str # Detailed evaluation (generated by LLM) + score: float # Evaluation score (0.0 for failure, 1.0 for success) metadata: dict # Additional metadata ``` -Tool Memory learns from each tool invocation by evaluating the result and accumulating insights over time. +**Key Fields:** +- **`input`/`output`**: The complete I/O data for analysis +- **`summary`**: LLM-generated brief summary of what happened +- **`evaluation`**: LLM-generated detailed analysis of the call quality +- **`score`**: Binary evaluation (0.0 = failure, 1.0 = success) +- **Performance metrics**: `time_cost`, `token_cost`, `success` for statistical analysis -## Configuration Logic +### Tool Memory Lifecycle -Tool Memory in ReMe is configured through three main flows: +```mermaid +graph LR + A[Tool Call] --> B[Evaluate] + B --> C[Store Memory] + C --> D[(Vector Store)] + D --> E[Agent Retrieves] + E --> A + C -.Periodic.-> F[Summarize] + F --> C +``` -### 1. Add Tool Call Result +## 3. How Tool Memory Works: The Complete Flow -The `add_tool_call_result` flow processes individual tool invocations and adds them to memory: +Tool Memory operates through three complementary operations that work together to create a learning loop: +```mermaid +graph LR + A[Agent] -->|1. retrieve_tool_memory| B[(Vector Store)] + B -->|Guidelines| A + A -->|Execute Tool| C[Tool] + C -->|Result| A + A -->|add_tool_call_result| D[LLM Evaluate] + D -->|Store| B + B -->|Periodic| E[summary_tool_memory] + E -->|Update Guidelines| B +``` + +### Operation Flow + +**1. retrieve_tool_memory** (Before Execution) +- Agent queries: "How should I use `web_search` tool?" +- Retrieves stored guidelines and historical patterns +- Returns: Usage recommendations, parameter suggestions, common pitfalls + +**2. Tool Execution** +- Agent executes tool with informed parameters +- Collects: input, output, time_cost, token_cost, success status + +**3. add_tool_call_result** (After Execution) +- Submits execution data for evaluation +- LLM analyzes: Was it successful? What could be improved? +- Generates: summary, evaluation, score (0.0 or 1.0) +- Appends to tool's historical record in Vector Store + +**4. summary_tool_memory** (Periodic) +- Analyzes recent N tool calls (e.g., last 20-30) +- Calculates statistics: success rate, avg costs, avg score +- LLM synthesizes: Actionable usage guidelines +- Updates the `content` field with comprehensive guidance + +### Example Flow from Demo + +Based on `use_tool_memory_demo.py`, here's a typical workflow: + +```python +# Step 1: Add tool call results (accumulate history) +add_tool_call_results([ + {"tool_name": "web_search", "input": {...}, "output": "...", "success": True}, + {"tool_name": "web_search", "input": {...}, "output": "...", "success": False}, + # ... more results +]) + +# Step 2: Generate usage guidelines (periodic) +summarize_tool_memory("web_search") + +# Step 3: Retrieve guidelines before next use +memory = retrieve_tool_memory("web_search") +# Returns: +# "For web_search tool: +# - Use max_results=5-20 for optimal performance +# - Avoid generic queries, be specific +# - Language parameter 'en' has 95% success rate +# Statistics: 83% success, avg 2.3s, avg 150 tokens" + +# Step 4: Agent uses guidelines for better execution +execute_with_recommended_parameters() +``` + +## 4. Operation Details: How to Use Each Component + +### 4.1 `add_tool_call_result` + +**Purpose**: Evaluate and store tool call results into Tool Memory. + +**Flow**: ```yaml add_tool_call_result: flow_content: parse_tool_call_result_op >> update_vector_store_op description: "Evaluates and adds tool call results to the tool memory database" ``` -```mermaid -graph LR - A[Tool Call Results] --> B[parse_tool_call_result_op] - B --> C[Evaluate Each Call] - C --> D[Generate Summary & Score] - D --> E[Update/Create ToolMemory] - E --> F[update_vector_store_op] - F --> G[Store in Vector DB] -``` - -This flow: -1. Receives tool call results with input, output, and metadata -2. Evaluates each call using LLM (generates summary, evaluation, score) -3. Appends evaluated results to the tool's memory -4. Updates the vector store with the modified memory - -### 2. Retrieve Tool Memory - -The `retrieve_tool_memory` flow fetches usage guidelines for specific tools: +**Process**: +1. Receives raw tool call results +2. Uses LLM to evaluate each call (generates summary, evaluation, score) +3. Groups results by tool name +4. Creates or updates ToolMemory objects +5. Stores in Vector Store +**Configuration** (`default.yaml`): ```yaml -retrieve_tool_memory: - flow_content: retrieve_tool_memory_op - description: "Retrieves tool memories from the vector database based on tool names" +op: + parse_tool_call_result_op: + backend: parse_tool_call_result_op + llm: default + params: + max_history_tool_call_cnt: 100 # Max calls to retain per tool + evaluation_sleep_interval: 1.0 # Delay between evaluations (seconds) ``` -```mermaid -graph LR - A[Tool Names] --> B[retrieve_tool_memory_op] - B --> C[Search by Tool Name] - C --> D[Exact Match Check] - D --> E[Return ToolMemory] - E --> F[Usage Guidelines + History] -``` +#### Usage with curl -This flow: -1. Takes comma-separated tool names as input -2. Searches the vector store for exact matches -3. Returns tool memories with usage guidelines and call history - -### 3. Summary Tool Memory - -The `summary_tool_memory` flow analyzes historical data and generates comprehensive usage guidelines: - -```yaml -summary_tool_memory: - flow_content: summary_tool_memory_op >> update_vector_store_op - description: "Analyzes tool call history and generates comprehensive usage patterns" -``` - -```mermaid -graph LR - A[Tool Names] --> B[summary_tool_memory_op] - B --> C[Retrieve Tool Memory] - C --> D[Analyze Recent Calls] - D --> E[Calculate Statistics] - E --> F[Generate Guidelines] - F --> G[update_vector_store_op] - G --> H[Update Vector DB] -``` - -This flow: -1. Retrieves existing tool memories by tool name -2. Analyzes recent N tool calls (default: 20) -3. Calculates statistical metrics (success rate, avg score, costs) -4. Uses LLM to synthesize actionable usage guidelines -5. Updates the tool memory content with new insights - -## Complete Interaction Flow - -The following diagram illustrates how the three operations interact with the vector store and agent tool calls: - -```mermaid -graph LR - Agent[Agent] -->|1. Before tool call| Retrieve[retrieve_tool_memory] - Retrieve -->|Read| VectorStore[(Vector Store)] - VectorStore -->|Usage Guidelines| Agent - - Agent -->|2. Execute| Tool[Tool Call] - Tool -->|Result| Agent - - Agent -->|3. After tool call| Add[add_tool_call_result] - Add -->|Evaluate & Write| VectorStore - - Summary[summary_tool_memory] -->|4. Periodic| VectorStore - VectorStore -->|Read History| Summary - Summary -->|Update Guidelines| VectorStore - - style Agent fill:#e1f5ff - style Retrieve fill:#fff4e1 - style Add fill:#ffe1f5 - style Summary fill:#e1ffe1 - style VectorStore fill:#f0f0f0 -``` - -### Workflow Steps - -**1. retrieve_tool_memory** (Before tool execution) -- Agent queries usage guidelines from Vector Store -- Returns best practices and historical patterns - -**2. Tool Call Execution** -- Agent executes tool with recommended parameters -- Gets result (success/failure, output, costs) - -**3. add_tool_call_result** (After tool execution) -- Evaluates the tool call result -- Stores evaluated result to Vector Store - -**4. summary_tool_memory** (Periodic) -- Analyzes accumulated call history -- Generates comprehensive usage guidelines -- Updates Vector Store with new insights - -## Basic Usage - -Here's how to use Tool Memory in your application: - -### Step 1: Set Up Your Environment - -```python -import requests - -# API configuration -BASE_URL = "http://0.0.0.0:8002/" -WORKSPACE_ID = "your_workspace_id" -``` - -### Step 2: Record Tool Call Results - -After your agent executes a tool, record the invocation: - -```python -# Example: Record a web search tool call -tool_call_results = [ - { - "create_time": "2025-10-15 14:30:00", +```bash +curl -X POST http://0.0.0.0:8002/add_tool_call_result \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "my_workspace", + "tool_call_results": [ + { + "create_time": "2025-10-21 10:30:00", "tool_name": "web_search", "input": { "query": "Python asyncio tutorial", @@ -213,345 +304,518 @@ tool_call_results = [ }, "output": "Found 10 relevant results including official docs and tutorials", "token_cost": 150, - "success": True, + "success": true, "time_cost": 2.3 - } -] - -# Add the tool call result -response = requests.post( - url=f"{BASE_URL}add_tool_call_result", - json={ - "workspace_id": WORKSPACE_ID, + }, + { + "create_time": "2025-10-21 10:32:00", "tool_name": "web_search", - "tool_call_results": tool_call_results - } -) + "input": { + "query": "test", + "max_results": 100, + "language": "unknown" + }, + "output": "Error: Invalid language parameter", + "token_cost": 50, + "success": false, + "time_cost": 0.5 + } + ] + }' ``` -### Step 3: Retrieve Tool Usage Guidelines - -Before using a tool, retrieve its usage guidelines: - -```python -# Retrieve memory for specific tools -response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": "web_search,file_reader" # Comma-separated - } -) - -memory_list = response.json().get("metadata", {}).get("memory_list", []) -for memory in memory_list: - print(f"Tool: {memory['when_to_use']}") - print(f"Guidelines: {memory['content']}") - print(f"Total Calls: {len(memory['tool_call_results'])}") +**Response**: +```json +{ + "success": true, + "answer": "Successfully evaluated and stored 2 tool call results", + "metadata": { + "memory_list": [ + { + "when_to_use": "web_search", + "memory_id": "abc123...", + "tool_call_results": [ + { + "tool_name": "web_search", + "summary": "Successfully retrieved relevant Python asyncio documentation", + "evaluation": "Good parameter choices with appropriate max_results and language settings", + "score": 1.0, + ... + }, + { + "tool_name": "web_search", + "summary": "Failed due to invalid language parameter", + "evaluation": "Query too generic and language parameter not supported", + "score": 0.0, + ... + } + ] + } + ] + } +} ``` -### Step 4: Generate Comprehensive Usage Guidelines - -After accumulating sufficient call history, generate synthesized guidelines: - -```python -# Summarize tool usage patterns -response = requests.post( - url=f"{BASE_URL}summary_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": "web_search" - } -) - -if response.json().get("success"): - print("Successfully generated usage guidelines") -``` - -## Complete Example Workflow - -Here's a complete example demonstrating the tool memory lifecycle: +#### Usage with Python ```python import requests from datetime import datetime -BASE_URL = "http://0.0.0.0:8002/" -WORKSPACE_ID = "demo_workspace" - -def record_tool_usage(tool_name, input_params, output, success, time_cost, token_cost): - """Record a single tool invocation""" - tool_call_result = { - "create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - "tool_name": tool_name, - "input": input_params, - "output": output, - "token_cost": token_cost, - "success": success, - "time_cost": time_cost - } - +def add_tool_call_results(tool_call_results: list) -> dict: + """Add tool call results to Tool Memory""" response = requests.post( url=f"{BASE_URL}add_tool_call_result", json={ "workspace_id": WORKSPACE_ID, - "tool_name": tool_name, - "tool_call_results": [tool_call_result] + "tool_call_results": tool_call_results } ) return response.json() -def get_tool_guidelines(tool_name): - """Retrieve usage guidelines for a tool""" +# Example: Record a tool invocation +result = add_tool_call_results([{ + "create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": "web_search", + "input": {"query": "Python asyncio", "max_results": 10}, + "output": "Found 10 relevant results...", + "token_cost": 150, + "success": True, + "time_cost": 2.3 +}]) +``` + +**Complete examples**: See `cookbook/simple_demo/use_tool_memory_demo.py` for full working code. + +--- + +### 4.2 `retrieve_tool_memory` + +**Purpose**: Retrieve usage guidelines and historical data for specific tools. + +**Flow**: +```yaml +retrieve_tool_memory: + flow_content: retrieve_tool_memory_op + description: "Retrieves tool memories from the vector database based on tool names" +``` + +**Process**: +1. Takes comma-separated tool names as input +2. Searches Vector Store for exact matches (by `when_to_use` field) +3. Returns complete ToolMemory objects with: + - Usage guidelines (`content`) + - Historical call records (`tool_call_results`) + - Statistics and metadata + +#### Usage with curl + +```bash +# Retrieve single tool +curl -X POST http://0.0.0.0:8002/retrieve_tool_memory \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "my_workspace", + "tool_names": "web_search" + }' + +# Retrieve multiple tools (comma-separated) +curl -X POST http://0.0.0.0:8002/retrieve_tool_memory \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "my_workspace", + "tool_names": "web_search,database_query,file_processor" + }' +``` + +**Response**: +```json +{ + "success": true, + "answer": "Successfully retrieved 1 tool memories", + "metadata": { + "memory_list": [ + { + "memory_type": "tool", + "workspace_id": "my_workspace", + "memory_id": "abc123...", + "when_to_use": "web_search", + "content": "## Usage Guidelines\n\n**Best Practices:**\n- Use max_results between 5-20 for optimal performance\n- Always specify language parameter (en has 95% success rate)\n- Avoid generic single-word queries\n\n**Common Pitfalls:**\n- max_results > 50 often causes timeouts\n- Unknown language values default to 'en' with warning\n\n## Statistics\n- **Success Rate**: 83.33%\n- **Average Score**: 0.833\n- **Average Time Cost**: 2.345s\n- **Average Token Cost**: 156.7", + "score": 0.85, + "time_created": "2025-10-20 10:00:00", + "time_modified": "2025-10-21 10:35:00", + "author": "gpt-4", + "tool_call_results": [ + { + "create_time": "2025-10-21 10:30:00", + "tool_name": "web_search", + "input": {"query": "Python asyncio", "max_results": 10}, + "output": "Found 10 results...", + "summary": "Successfully retrieved relevant documentation", + "evaluation": "Good parameter choices...", + "score": 1.0, + "token_cost": 150, + "success": true, + "time_cost": 2.3 + } + // ... more historical calls + ] + } + ] + } +} +``` + +#### Usage with Python + +```python +import requests + +def retrieve_tool_memory(tool_names: str) -> dict: + """Retrieve tool memories by tool names""" response = requests.post( url=f"{BASE_URL}retrieve_tool_memory", json={ "workspace_id": WORKSPACE_ID, - "tool_names": tool_name - } - ) - - result = response.json() - if result.get("success"): - memory_list = result.get("metadata", {}).get("memory_list", []) - if memory_list: - return memory_list[0] - return None - -def generate_guidelines(tool_name): - """Generate comprehensive usage guidelines""" - response = requests.post( - url=f"{BASE_URL}summary_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": tool_name + "tool_names": tool_names } ) return response.json() -# Example usage -if __name__ == "__main__": - tool_name = "web_search" - - # 1. Record multiple tool invocations - print("Recording tool invocations...") - for i in range(5): - record_tool_usage( - tool_name=tool_name, - input_params={"query": f"test query {i}", "max_results": 10}, - output=f"Found results for query {i}", - success=True, - time_cost=2.0 + i * 0.5, - token_cost=100 + i * 20 - ) - - # 2. Generate usage guidelines - print("\nGenerating usage guidelines...") - generate_guidelines(tool_name) - - # 3. Retrieve and display guidelines - print("\nRetrieving guidelines...") - memory = get_tool_guidelines(tool_name) - if memory: - print(f"\nTool: {memory['when_to_use']}") - print(f"Guidelines:\n{memory['content']}") - print(f"\nTotal invocations: {len(memory['tool_call_results'])}") +# Example: Retrieve and use guidelines +result = retrieve_tool_memory("web_search") +if result['success']: + memory = result['metadata']['memory_list'][0] + print(f"Tool: {memory['when_to_use']}") + print(f"Guidelines:\n{memory['content']}") ``` -## Use Cases +**Complete examples**: See `cookbook/simple_demo/use_tool_memory_demo.py` for full working code. -### Use Case 1: Learning Optimal Parameters +--- -```mermaid -sequenceDiagram - participant Agent - participant ToolMemory - participant Tool - - Agent->>ToolMemory: retrieve_tool_memory("api_caller") - ToolMemory-->>Agent: Guidelines: Use timeout=30s, retry=3 - Agent->>Tool: Call with recommended params - Tool-->>Agent: Success (time: 2.5s) - Agent->>ToolMemory: add_tool_call_result(success=True) - ToolMemory->>ToolMemory: Update statistics -``` +### 4.3 `summary_tool_memory` -**Scenario**: An agent needs to call an external API. Tool Memory has learned from 50+ previous calls that: -- Setting `timeout=30s` achieves 95% success rate -- Using `retry=3` handles transient failures effectively -- Requests with `max_results > 100` often timeout - -The agent retrieves these guidelines before making the call, leading to higher success rates. - -### Use Case 2: Avoiding Common Pitfalls - -```mermaid -sequenceDiagram - participant Agent - participant ToolMemory - participant FileReader - - Agent->>ToolMemory: retrieve_tool_memory("file_reader") - ToolMemory-->>Agent: Warning: Large files (>10MB) cause timeouts - Agent->>Agent: Check file size first - Agent->>FileReader: Read file with streaming mode - FileReader-->>Agent: Success - Agent->>ToolMemory: add_tool_call_result(success=True) -``` - -**Scenario**: Tool Memory has recorded that the `file_reader` tool fails when: -- File paths contain special characters without escaping -- Files larger than 10MB are read without streaming mode -- Binary files are opened in text mode - -The agent retrieves these warnings and adjusts its approach accordingly. - -### Use Case 3: Performance Optimization - -```python -# Before using Tool Memory -average_time_cost = 5.2s -success_rate = 75% - -# After learning from Tool Memory -# - Use batch processing for multiple queries -# - Set appropriate timeout values -# - Cache frequently accessed data - -average_time_cost = 2.8s # 46% improvement -success_rate = 92% # 17% improvement -``` - -**Scenario**: Tool Memory analyzes 100+ invocations of a `database_query` tool and discovers: -- Batch queries are 3x faster than individual queries -- Connection pooling reduces overhead by 40% -- Queries during peak hours (2-4 PM) have higher failure rates - -The synthesized guidelines help the agent optimize its database interactions. - -## Managing Tool Memories - -### Delete a Workspace - -```python -response = requests.post( - url=f"{BASE_URL}vector_store", - json={ - "workspace_id": WORKSPACE_ID, - "action": "delete" - } -) -``` - -### Dump Memories to Disk - -```python -response = requests.post( - url=f"{BASE_URL}vector_store", - json={ - "workspace_id": WORKSPACE_ID, - "action": "dump", - "path": "./" - } -) -``` - -### Load Memories from Disk - -```python -response = requests.post( - url=f"{BASE_URL}vector_store", - json={ - "workspace_id": WORKSPACE_ID, - "action": "load", - "path": "./" - } -) -``` - -## Configuration Parameters - -### ParseToolCallResultOp Parameters - -Configure in `default.yaml`: +**Purpose**: Analyze historical tool calls and generate comprehensive usage guidelines. +**Flow**: ```yaml -op: - parse_tool_call_result_op: - backend: parse_tool_call_result_op - llm: default - params: - max_history_tool_call_cnt: 100 # Max historical calls to retain - evaluation_sleep_interval: 1.0 # Delay between evaluations (seconds) +summary_tool_memory: + flow_content: summary_tool_memory_op >> update_vector_store_op + description: "Analyzes tool call history and generates comprehensive usage patterns" ``` -- `max_history_tool_call_cnt`: Limits the number of historical tool call results stored per tool. Older results are removed when this limit is exceeded. -- `evaluation_sleep_interval`: Controls the delay between concurrent evaluations to avoid rate limiting. - -### SummaryToolMemoryOp Parameters +**Process**: +1. Retrieves existing ToolMemory by tool name +2. Analyzes recent N tool calls (default: 30) +3. Calculates statistics: + - Success rate + - Average score + - Average time cost + - Average token cost +4. Uses LLM to synthesize actionable guidelines from call summaries +5. Appends statistics to guidelines +6. Updates ToolMemory content in Vector Store +**Configuration** (`default.yaml`): ```yaml op: summary_tool_memory_op: backend: summary_tool_memory_op llm: default params: - recent_call_count: 20 # Number of recent calls to analyze + recent_call_count: 30 # Number of recent calls to analyze summary_sleep_interval: 1.0 # Delay between summaries (seconds) ``` -- `recent_call_count`: Number of most recent tool calls to analyze when generating guidelines. -- `summary_sleep_interval`: Controls the delay between concurrent summarizations. +#### Usage with curl -## Best Practices +```bash +# Summarize single tool +curl -X POST http://0.0.0.0:8002/summary_tool_memory \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "my_workspace", + "tool_names": "web_search" + }' -1. **Regular Recording**: - - Record every tool invocation, including failures - - Include detailed input parameters and output - - Capture performance metrics (time_cost, token_cost) - -2. **Periodic Summarization**: - - Generate guidelines after accumulating 20-50 tool calls - - Re-summarize when usage patterns change significantly - - Update guidelines when new tool versions are deployed - -3. **Retrieval Strategy**: - - Always retrieve guidelines before using unfamiliar tools - - Cache retrieved guidelines for the duration of a task - - Re-retrieve after tool memory updates - -4. **Quality Maintenance**: - - Monitor success rates and average scores - - Investigate tools with declining performance - - Clean up outdated memories when tools are deprecated - -5. **Parameter Tuning**: - - Adjust `max_history_tool_call_cnt` based on tool usage frequency - - Increase `recent_call_count` for tools with diverse usage patterns - - Reduce `evaluation_sleep_interval` if rate limiting is not a concern - -## Integration with Agent Workflows - -```mermaid -graph TB - A[Agent Receives Task] --> B{Tool Required?} - B -->|Yes| C[Retrieve Tool Memory] - C --> D[Apply Guidelines] - D --> E[Execute Tool] - E --> F[Record Result] - F --> G{Sufficient History?} - G -->|Yes| H[Generate Summary] - G -->|No| I[Continue] - H --> I - B -->|No| I[Process Task] - I --> J[Task Complete] +# Summarize multiple tools (comma-separated) +curl -X POST http://0.0.0.0:8002/summary_tool_memory \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "my_workspace", + "tool_names": "web_search,database_query,file_processor" + }' ``` -Tool Memory seamlessly integrates into agent workflows: -1. Before tool execution: Retrieve usage guidelines -2. During execution: Apply recommended parameters -3. After execution: Record results with evaluation -4. Periodically: Generate updated guidelines +**Response**: +```json +{ + "success": true, + "answer": "Successfully summarized 1 tool memories", + "metadata": { + "memory_list": [ + { + "memory_type": "tool", + "when_to_use": "web_search", + "content": "## Usage Guidelines\n\n**Optimal Parameters:**\n- Set max_results between 5-20 for best balance of coverage and speed\n- Always specify language='en' for technical queries (95% success rate)\n- Use filter_type='technical_docs' for development-related searches\n\n**Success Patterns:**\n- Specific, multi-word queries perform significantly better than generic terms\n- Queries with clear intent (e.g., 'Python asyncio tutorial') return high-quality results\n- Technical terms and version numbers improve result relevance\n\n**Common Failures:**\n- Generic single-word queries (e.g., 'test') return poor results\n- max_results > 50 increases timeout risk (5 failures observed)\n- Invalid language codes cause fallback to default with warnings\n\n**Performance Insights:**\n- Typical response time: 1.5-3.5s for successful queries\n- Timeout threshold: 10s (consider simplifying complex queries)\n- Token cost scales with result count: ~150 tokens for 10 results\n\n**Recommendations:**\n1. Always validate language parameter before calling\n2. Start with max_results=10, adjust based on needs\n3. For time-sensitive operations, set timeout < 5s\n4. Monitor token costs for high-frequency usage\n\n## Statistics\n- **Success Rate**: 83.33%\n- **Average Score**: 0.833\n- **Average Time Cost**: 2.345s\n- **Average Token Cost**: 156.7", + "memory_id": "abc123...", + "time_modified": "2025-10-21 10:40:00", + ... + } + ] + } +} +``` -For more detailed examples, see the implementation in `reme_ai/summary/tool/` directory of the ReMe project. +#### Usage with Python +```python +import requests + +def summarize_tool_memory(tool_names: str) -> dict: + """Generate comprehensive usage guidelines for tools""" + response = requests.post( + url=f"{BASE_URL}summary_tool_memory", + json={ + "workspace_id": WORKSPACE_ID, + "tool_names": tool_names + } + ) + return response.json() + +# Example: Generate guidelines +result = summarize_tool_memory("web_search") +if result['success']: + memory = result['metadata']['memory_list'][0] + print(f"Tool: {memory['when_to_use']}") + print(f"Guidelines:\n{memory['content']}") +``` + +**Complete examples**: See `cookbook/simple_demo/use_tool_memory_demo.py` for full working code. + +--- + +## 5. Best Practices + +### When to Record Tool Calls +- **Always**: Record every tool invocation, including failures +- **Include**: Complete input parameters, output, and performance metrics +- **Timing**: Record immediately after tool execution completes + +### When to Generate Summaries +- **Initial**: After accumulating 20-30 tool calls for meaningful patterns +- **Periodic**: Re-summarize every 50-100 new calls or weekly +- **Trigger-based**: When success rate drops or patterns change significantly + +### When to Retrieve Guidelines +- **Before first use**: Always retrieve before using an unfamiliar tool +- **Before critical operations**: Check latest guidelines for important tasks +- **After updates**: Re-retrieve when tool memory has been updated + +### Performance Tuning + +**For High-Volume Tools** (>100 calls/day): +```yaml +op: + parse_tool_call_result_op: + params: + max_history_tool_call_cnt: 200 # Keep more history + evaluation_sleep_interval: 0.5 # Faster evaluation + + summary_tool_memory_op: + params: + recent_call_count: 50 # Analyze more calls +``` + +**For Low-Volume Tools** (<20 calls/day): +```yaml +op: + parse_tool_call_result_op: + params: + max_history_tool_call_cnt: 50 # Less history needed + evaluation_sleep_interval: 1.0 # Standard rate + + summary_tool_memory_op: + params: + recent_call_count: 20 # Analyze fewer calls +``` + +### Quality Maintenance + +1. **Monitor Metrics**: +```python + memory = retrieve_tool_memory("web_search")['metadata']['memory_list'][0] + stats = ToolMemory(**memory).statistic(recent_frequency=30) + + print(f"Success Rate: {stats['success_rate']:.2%}") + print(f"Avg Score: {stats['avg_score']:.2f}") + + if stats['success_rate'] < 0.7: + print("⚠️ Low success rate - investigate tool issues") + ``` + +2. **Clean Old Memories**: + - Delete tool memories for deprecated tools + - Reset memories when tool behavior changes significantly + +3. **Validate Guidelines**: + - Periodically review generated guidelines for accuracy + - Test recommended parameters in production scenarios + +## 6. Memory Management + +### Delete Workspace +```bash +curl -X POST http://0.0.0.0:8002/vector_store \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "my_workspace", + "action": "delete" + }' +``` + +```python +def delete_workspace(workspace_id: str): + response = requests.post( + url=f"{BASE_URL}vector_store", + json={"workspace_id": workspace_id, "action": "delete"} + ) + return response.json() +``` + +### Dump Memories to Disk +```bash +curl -X POST http://0.0.0.0:8002/vector_store \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "my_workspace", + "action": "dump", + "path": "./memory_backup/" + }' +``` + +```python +def dump_memory(workspace_id: str, path: str = "./"): + response = requests.post( + url=f"{BASE_URL}vector_store", + json={"workspace_id": workspace_id, "action": "dump", "path": path} + ) + return response.json() +``` + +### Load Memories from Disk +```bash +curl -X POST http://0.0.0.0:8002/vector_store \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "my_workspace", + "action": "load", + "path": "./memory_backup/" + }' +``` + +```python +def load_memory(workspace_id: str, path: str = "./"): + response = requests.post( + url=f"{BASE_URL}vector_store", + json={"workspace_id": workspace_id, "action": "load", "path": path} + ) + return response.json() +``` + +## 7. Complete Working Example + +For a complete, runnable example demonstrating the full Tool Memory lifecycle, see: + +**`cookbook/simple_demo/use_tool_memory_demo.py`** + +This demo includes: +- **Workspace management**: Clean, delete, dump, and load operations +- **Tool call recording**: Adding 30+ mock tool invocations with various scenarios +- **Summarization**: Generating usage guidelines from historical data +- **Retrieval**: Fetching and displaying tool memories +- **Statistics**: Analyzing success rates, costs, and performance + +Run the demo: +```bash +cd cookbook/simple_demo +python use_tool_memory_demo.py +``` + +**Key Workflow Steps:** +1. **Clean workspace**: Remove existing data +2. **Add tool calls**: Record 30+ invocations (success/failure scenarios) +3. **Generate guidelines**: LLM analyzes patterns and creates recommendations +4. **Retrieve memory**: Get usage guidelines for agent consumption +5. **Persistence**: Test dump/load operations + +## 8. Advanced Use Cases + +### Use Case 1: Adaptive Parameter Tuning + +Retrieve tool memory statistics and adapt parameters based on historical performance: +- If `avg_time_cost > 5s`: Increase timeout +- If `success_rate < 80%`: Enable retry logic +- If `avg_token_cost` high: Reduce result limits + +### Use Case 2: Multi-Tool Workflow Optimization + +Retrieve memories for multiple tools at once and optimize workflow order based on: +- Success rates: Execute reliable tools first +- Time costs: Parallelize slow operations +- Token costs: Budget-aware tool selection + +### Use Case 3: Automated Quality Monitoring + +Periodically check tool memory statistics and alert on: +- Success rate degradation +- Increasing time/token costs +- Unusual failure patterns + +**Implementation examples**: See `cookbook/simple_demo/use_tool_memory_demo.py` and the ToolBench evaluation scripts. + +## 9. Benchmark Results + +### Tool Memory Performance Evaluation + +We evaluated Tool Memory effectiveness using a controlled benchmark with three mock search tools, each optimized for different query complexity levels (simple, moderate, complex). The benchmark compares agent performance with and without tool memory guidance across multiple epochs. + +**Experimental Settings:** +- **Model**: Qwen3-30B-Instruct with default parameters +- **Task**: Single-turn tool selection and invocation +- **Dataset**: 60 training queries + 60 test queries per epoch +- **Tools**: 3 mock search tools with varying performance profiles +- **Metrics**: Average quality score (0.0-1.0) based on LLM evaluation +- **Baseline**: Test set performance without tool memory +- **Replication**: Results averaged across 3 independent experimental runs + +**Results (averaged across 3 epochs):** + +| Scenario | Avg Score | Improvement | +|----------|-----------|-------------| +| Train (No Memory) | 0.650 | - | +| Test (No Memory) | 0.672 | Baseline | +| **Test (With Memory)** | **0.772** | **+14.88%** | + +**Key Findings:** +- **Consistent improvement**: Tool Memory boosted test performance by ~15% on average +- **Knowledge transfer**: Training data successfully informed test-time tool selection +- **Stability**: Improvement remained consistent across all 3 epochs (9.90% → 17.39% → 17.13%) + +The benchmark demonstrates that Tool Memory enables agents to make data-driven tool selection decisions, significantly improving task success rates compared to relying solely on static tool descriptions. + +**Benchmark Resources:** +- **Design Documentation**: [`docs/tool_memory/tool_bench.md`](tool_bench.md) - Complete benchmark methodology and workflow +- **Implementation**: [`cookbook/tool_memory/run_reme_tool_bench.py`](../../cookbook/tool_memory/run_reme_tool_bench.py) - Full benchmark script +- **Query Dataset**: [`cookbook/tool_memory/query.json`](../../cookbook/tool_memory/query.json) - 60 train + 60 test queries across 3 complexity levels + +--- + +## 10. References + +- **Implementation**: See `reme_ai/summary/tool/` and `reme_ai/retrieve/tool/` +- **Demo**: `cookbook/simple_demo/use_tool_memory_demo.py` +- **Benchmark**: `cookbook/tool_memory/run_reme_tool_bench.py` +- **Schema**: `reme_ai/schema/memory.py` +- **Utilities**: `reme_ai/utils/tool_memory_utils.py` diff --git a/docs/tool_memory/tool_retrieve_ops.md b/docs/tool_memory/tool_retrieve_ops.md index 38ac37e1..f9a257f9 100644 --- a/docs/tool_memory/tool_retrieve_ops.md +++ b/docs/tool_memory/tool_retrieve_ops.md @@ -12,537 +12,8 @@ Retrieves tool memories from the vector database based on tool names, providing - Searches the vector store for exact tool name matches - Validates that retrieved memories are of type "tool" - Returns complete tool memories including usage guidelines and call history -- Provides detailed logging for debugging and monitoring - -### Processing Flow - -```mermaid -graph TB - A[Receive Tool Names] --> B[Validate Input] - B --> C[Split by Comma] - C --> D[Trim Whitespace] - D --> E[For Each Tool Name] - E --> F[Search Vector Store] - F --> G{Results Found?} - G -->|Yes| H[Get Top Result] - G -->|No| I[Log Warning: Not Found] - H --> J{Type = tool?} - J -->|Yes| K{Name Matches?} - J -->|No| L[Log Warning: Wrong Type] - K -->|Yes| M[Add to Results] - K -->|No| N[Log Warning: Name Mismatch] - I --> O[Continue Next Tool] - L --> O - N --> O - M --> O - O --> P{More Tools?} - P -->|Yes| E - P -->|No| Q{Any Matches?} - Q -->|Yes| R[Return Memory List] - Q -->|No| S[Return Empty] -``` - -1. **Input Validation**: - - Check if `tool_names` parameter is provided - - Return error if empty - - Log workspace_id and tool count - -2. **Tool Name Processing**: - - Split input by comma delimiter - - Strip whitespace from each tool name - - Filter out empty strings - - Log the list of tools to retrieve - -3. **Vector Store Search**: - - For each tool name, search with `top_k=1` - - Use tool name as the query (exact match preferred) - - Retrieve the top matching result - -4. **Result Validation**: - - Verify result is of type `ToolMemory` - - Check that `when_to_use` field exactly matches tool name - - Log match details (memory_id, total_calls) - - Warn if no match or mismatch found - -5. **Response Preparation**: - - Collect all matched tool memories - - Set success status based on matches found - - Return memory list in metadata ### Parameters This operation has no configurable parameters. It uses the default vector store configuration. -### Input Schema - -```yaml -input_schema: - tool_names: - type: string - description: "Comma-separated tool names (e.g., 'tool_name1,tool_name2')" - required: true -``` - -### Output Format - -The operation sets the following in `context.response`: - -```python -{ - "success": True, - "answer": "Successfully retrieved 2 tool memories", - "metadata": { - "memory_list": [ - { - "workspace_id": "demo_workspace", - "memory_id": "abc123def456", - "memory_type": "tool", - "when_to_use": "web_search", - "content": "Core Function: The web_search tool retrieves...", - "score": 0.85, - "time_created": "2025-10-15 10:00:00", - "time_modified": "2025-10-15 14:30:00", - "author": "qwen3-30b-a3b-instruct-2507", - "tool_call_results": [ - { - "create_time": "2025-10-15 14:30:00", - "tool_name": "web_search", - "input": {...}, - "output": "...", - "summary": "...", - "evaluation": "...", - "score": 1.0, - "success": True, - "time_cost": 2.3, - "token_cost": 150 - } - ], - "metadata": {} - } - ] - } -} -``` - -### Usage Example - -#### Basic Retrieval - -```python -import requests - -BASE_URL = "http://0.0.0.0:8002/" -WORKSPACE_ID = "demo_workspace" - -# Retrieve memory for a single tool -response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": "web_search" - } -) - -result = response.json() -if result.get("success"): - memory_list = result.get("metadata", {}).get("memory_list", []) - for memory in memory_list: - print(f"Tool: {memory['when_to_use']}") - print(f"Guidelines:\n{memory['content']}") - print(f"Total Calls: {len(memory['tool_call_results'])}") -``` - -#### Multiple Tools Retrieval - -```python -# Retrieve memories for multiple tools at once -response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": "web_search,file_reader,api_caller,database_query" - } -) - -result = response.json() -if result.get("success"): - memory_list = result.get("metadata", {}).get("memory_list", []) - print(f"Retrieved {len(memory_list)} tool memories") - - for memory in memory_list: - print(f"\n{'='*60}") - print(f"Tool: {memory['when_to_use']}") - print(f"{'='*60}") - print(memory['content']) -``` - -#### Extracting Specific Information - -```python -def get_tool_statistics(tool_name): - """Get statistical information for a specific tool""" - response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": tool_name - } - ) - - result = response.json() - if not result.get("success"): - return None - - memory_list = result.get("metadata", {}).get("memory_list", []) - if not memory_list: - return None - - memory = memory_list[0] - tool_calls = memory['tool_call_results'] - - # Calculate statistics - total_calls = len(tool_calls) - successful_calls = sum(1 for call in tool_calls if call['success']) - avg_score = sum(call['score'] for call in tool_calls) / total_calls if total_calls > 0 else 0 - avg_time = sum(call['time_cost'] for call in tool_calls) / total_calls if total_calls > 0 else 0 - avg_tokens = sum(call['token_cost'] for call in tool_calls) / total_calls if total_calls > 0 else 0 - - return { - "tool_name": tool_name, - "total_calls": total_calls, - "success_rate": successful_calls / total_calls if total_calls > 0 else 0, - "avg_score": avg_score, - "avg_time_cost": avg_time, - "avg_token_cost": avg_tokens, - "guidelines": memory['content'] - } - -# Usage -stats = get_tool_statistics("web_search") -if stats: - print(f"Tool: {stats['tool_name']}") - print(f"Total Calls: {stats['total_calls']}") - print(f"Success Rate: {stats['success_rate']:.1%}") - print(f"Avg Score: {stats['avg_score']:.2f}") - print(f"Avg Time: {stats['avg_time_cost']:.2f}s") - print(f"Avg Tokens: {stats['avg_token_cost']:.0f}") -``` - -### Integration with Agent Workflows - -#### Pre-Execution Retrieval - -```python -def execute_tool_with_guidelines(tool_name, input_params): - """Execute a tool after retrieving its usage guidelines""" - - # 1. Retrieve tool memory - response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": tool_name - } - ) - - # 2. Extract guidelines - guidelines = "" - if response.json().get("success"): - memory_list = response.json().get("metadata", {}).get("memory_list", []) - if memory_list: - guidelines = memory_list[0]['content'] - print(f"Guidelines for {tool_name}:") - print(guidelines) - - # 3. Adjust parameters based on guidelines - # (This would be done by an LLM or rule-based system) - adjusted_params = adjust_parameters(input_params, guidelines) - - # 4. Execute tool - result = execute_tool(tool_name, adjusted_params) - - # 5. Record result - record_tool_call(tool_name, adjusted_params, result) - - return result -``` - -#### Batch Retrieval for Agent Initialization - -```python -def initialize_agent_with_tool_memories(available_tools): - """Load all tool memories at agent initialization""" - - # Retrieve all tool memories at once - tool_names = ",".join(available_tools) - response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": tool_names - } - ) - - # Build tool memory cache - tool_memory_cache = {} - if response.json().get("success"): - memory_list = response.json().get("metadata", {}).get("memory_list", []) - for memory in memory_list: - tool_memory_cache[memory['when_to_use']] = { - "guidelines": memory['content'], - "total_calls": len(memory['tool_call_results']), - "last_modified": memory['time_modified'] - } - - return tool_memory_cache - -# Usage -available_tools = ["web_search", "file_reader", "api_caller"] -tool_cache = initialize_agent_with_tool_memories(available_tools) - -# Agent can now quickly access guidelines -if "web_search" in tool_cache: - print(tool_cache["web_search"]["guidelines"]) -``` - -### Error Handling - -#### Tool Not Found - -```python -response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": "nonexistent_tool" - } -) - -result = response.json() -if not result.get("success"): - print(f"Error: {result.get('answer')}") - # Output: "No matching tool memories found" -``` - -#### Empty Tool Names - -```python -response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": "" - } -) - -result = response.json() -if not result.get("success"): - print(f"Error: {result.get('answer')}") - # Output: "tool_names is required" -``` - -#### Partial Matches - -```python -# Request 3 tools, but only 2 exist -response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": "web_search,file_reader,nonexistent_tool" - } -) - -result = response.json() -if result.get("success"): - memory_list = result.get("metadata", {}).get("memory_list", []) - print(f"Found {len(memory_list)} out of 3 requested tools") - # Output: "Found 2 out of 3 requested tools" -``` - -### Retrieval Workflow - -```mermaid -sequenceDiagram - participant Agent - participant RetrieveOp - participant VectorStore - - Agent->>RetrieveOp: retrieve_tool_memory(tool_names) - RetrieveOp->>RetrieveOp: Split and validate names - - loop For each tool name - RetrieveOp->>VectorStore: search(tool_name, top_k=1) - VectorStore-->>RetrieveOp: Return top result - RetrieveOp->>RetrieveOp: Validate type and name - alt Valid match - RetrieveOp->>RetrieveOp: Add to results - else No match - RetrieveOp->>RetrieveOp: Log warning - end - end - - RetrieveOp-->>Agent: Return memory_list - Agent->>Agent: Apply guidelines -``` - -### Use Cases - -#### Use Case 1: Pre-Execution Guidance - -**Scenario**: Before executing a tool, the agent retrieves usage guidelines to optimize parameters. - -```python -# Agent needs to search the web -tool_name = "web_search" -query = "machine learning basics" - -# Retrieve guidelines -response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": tool_name - } -) - -# Guidelines suggest: max_results=10-20, use filter_type="technical_docs" -# Agent adjusts parameters accordingly -optimized_params = { - "query": query, - "max_results": 15, # Within recommended range - "filter_type": "technical_docs", # As suggested - "language": "en" -} - -# Execute with optimized parameters -result = execute_tool(tool_name, optimized_params) -``` - -#### Use Case 2: Performance Monitoring - -**Scenario**: Monitor tool performance trends over time. - -```python -def monitor_tool_performance(tool_name): - """Monitor tool performance and detect degradation""" - response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": tool_name - } - ) - - if not response.json().get("success"): - return None - - memory = response.json()["metadata"]["memory_list"][0] - calls = memory["tool_call_results"] - - # Analyze recent vs historical performance - recent_calls = calls[-20:] # Last 20 calls - historical_calls = calls[:-20] if len(calls) > 20 else [] - - recent_success_rate = sum(1 for c in recent_calls if c['success']) / len(recent_calls) - historical_success_rate = (sum(1 for c in historical_calls if c['success']) / len(historical_calls) - if historical_calls else recent_success_rate) - - # Detect degradation - if recent_success_rate < historical_success_rate - 0.1: - print(f"Warning: {tool_name} performance degraded!") - print(f"Recent: {recent_success_rate:.1%}, Historical: {historical_success_rate:.1%}") - return "degraded" - - return "healthy" - -# Usage -status = monitor_tool_performance("web_search") -``` - -#### Use Case 3: Tool Selection - -**Scenario**: Choose the best tool for a task based on historical performance. - -```python -def select_best_tool(task_description, candidate_tools): - """Select the best tool based on historical performance""" - - # Retrieve memories for all candidate tools - tool_names = ",".join(candidate_tools) - response = requests.post( - url=f"{BASE_URL}retrieve_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": tool_names - } - ) - - if not response.json().get("success"): - return candidate_tools[0] # Default to first - - memory_list = response.json()["metadata"]["memory_list"] - - # Score each tool - tool_scores = {} - for memory in memory_list: - calls = memory["tool_call_results"] - if not calls: - continue - - # Calculate composite score - success_rate = sum(1 for c in calls if c['success']) / len(calls) - avg_score = sum(c['score'] for c in calls) / len(calls) - avg_time = sum(c['time_cost'] for c in calls) / len(calls) - - # Composite: prioritize success and quality, penalize slow tools - composite = (success_rate * 0.4 + avg_score * 0.4 + (1 / (1 + avg_time)) * 0.2) - tool_scores[memory['when_to_use']] = composite - - # Select best tool - if tool_scores: - best_tool = max(tool_scores, key=tool_scores.get) - print(f"Selected {best_tool} with score {tool_scores[best_tool]:.2f}") - return best_tool - - return candidate_tools[0] - -# Usage -best = select_best_tool( - "Search for technical documentation", - ["web_search", "doc_search", "api_search"] -) -``` - -## Best Practices - -1. **Retrieval Timing**: - - Retrieve guidelines before first use of a tool - - Cache retrieved memories for the duration of a task - - Re-retrieve after tool memory updates (post-summarization) - -2. **Batch Retrieval**: - - Retrieve multiple tool memories in a single request - - Use comma-separated tool names for efficiency - - Initialize agent with all available tool memories - -3. **Error Handling**: - - Always check `success` status in response - - Handle cases where tool memory doesn't exist - - Provide fallback behavior for missing guidelines - -4. **Guidelines Application**: - - Parse guidelines to extract parameter recommendations - - Use LLM to interpret guidelines in context - - Combine guidelines with task-specific requirements - -5. **Performance Optimization**: - - Cache retrieved memories to avoid repeated calls - - Invalidate cache after summarization updates - - Monitor retrieval latency and optimize if needed - -6. **Monitoring**: - - Log retrieval requests for debugging - - Track which tools are frequently retrieved - - Identify tools without memory (candidates for recording) - diff --git a/docs/tool_memory/tool_summary_ops.md b/docs/tool_memory/tool_summary_ops.md index dd4992e9..5b949ed5 100644 --- a/docs/tool_memory/tool_summary_ops.md +++ b/docs/tool_memory/tool_summary_ops.md @@ -10,202 +10,19 @@ Evaluates individual tool invocations and adds them to the tool memory database - Receives tool call results with input parameters, output, and metadata - Uses LLM to evaluate each tool call based on success and parameter alignment -- Generates summary, evaluation, and score (0.0, 0.5, or 1.0) for each call +- Generates summary, evaluation, and score (0.0 or 1.0) for each call - Appends evaluated results to existing tool memory or creates new memory - Maintains a sliding window of recent tool calls (configurable limit) -### Processing Flow - -```mermaid -graph TB - A[Receive Tool Call Results] --> B[Validate Input] - B --> C[Search for Existing Memory] - C --> D{Memory Exists?} - D -->|Yes| E[Load Existing Memory] - D -->|No| F[Create New Memory] - E --> G[Concurrent Evaluation] - F --> G - G --> H[Evaluate Call #1] - G --> I[Evaluate Call #2] - G --> J[Evaluate Call #N] - H --> K[Generate Summary & Score] - I --> K - J --> K - K --> L[Append to Memory] - L --> M[Trim to Max History] - M --> N[Update Modified Time] - N --> O[Prepare for Vector Store] -``` - -1. **Input Validation**: - - Verify `tool_name` is provided - - Convert dict objects to `ToolCallResult` instances - - Check if `tool_call_results` list is not empty - -2. **Memory Lookup**: - - Search vector store for existing tool memory by tool name - - Verify exact match (memory type = "tool" and when_to_use = tool_name) - - Create new `ToolMemory` if no match found - -3. **Concurrent Evaluation**: - - Submit all tool call results for parallel evaluation - - Each evaluation uses LLM to analyze the call - - Generate structured evaluation with summary, assessment, and score - -4. **Memory Update**: - - Append evaluated results to tool memory - - Trim to `max_history_tool_call_cnt` if limit exceeded - - Update modification timestamp - - Prepare for vector store update - ### Parameters -Configure in `default.yaml`: - -```yaml -op: - parse_tool_call_result_op: - backend: parse_tool_call_result_op - llm: default - params: - max_history_tool_call_cnt: 100 - evaluation_sleep_interval: 1.0 -``` - -- `max_history_tool_call_cnt` (integer, default: `100`): +- `op.parse_tool_call_result_op.params.max_history_tool_call_cnt` (integer, default: `100`): - Maximum number of historical tool call results to retain per tool - When exceeded, oldest results are removed (FIFO) - - Balances memory size with historical context - - Recommended: 50-200 depending on tool usage frequency -- `evaluation_sleep_interval` (float, default: `1.0`): +- `op.parse_tool_call_result_op.params.evaluation_sleep_interval` (float, default: `1.0`): - Delay in seconds between concurrent evaluations - Prevents rate limiting when evaluating multiple calls - - Set to 0 for maximum speed (if no rate limits) - - Increase if encountering API throttling - -### Evaluation Criteria - -The LLM evaluates each tool call based on two dimensions: - -1. **Success Evaluation**: - - Check if the success flag indicates successful execution - - Verify output contains no error messages - - Assess if time and token costs are reasonable - - Identify any error indicators in the output - -2. **Parameter Alignment Evaluation**: - - Evaluate if output matches expected behavior given input - - Consider if input parameters are appropriate for the tool - - Check for parameter mismatches or unexpected behaviors - - Verify output is consistent with tool's intended purpose - -### Scoring Guidelines - -- **1.0 (Success)**: Tool executed successfully with good parameter alignment - - Example: Query returned relevant results, parameters were appropriate - -- **0.5 (Partial Success)**: Tool executed but with issues - - Example: Query succeeded but parameters were suboptimal (e.g., too generic) - - Example: Results returned but with warnings about invalid parameters - -- **0.0 (Failure)**: Tool execution failed or severe parameter misalignment - - Example: Timeout due to excessive max_results parameter - - Example: Error due to invalid parameter format - -### Output Format - -The operation sets the following in `context.response.metadata`: - -```python -{ - "deleted_memory_ids": ["memory_id_if_updating"], - "memory_list": [ - { - "memory_id": "abc123", - "when_to_use": "web_search", - "tool_call_results": [ - { - "create_time": "2025-10-15 14:30:00", - "tool_name": "web_search", - "input": {...}, - "output": "...", - "summary": "Successfully retrieved 10 relevant results", - "evaluation": "Good parameter alignment...", - "score": 1.0, - "success": True, - "time_cost": 2.3, - "token_cost": 150 - } - ] - } - ] -} -``` - -### Usage Example - -```python -import requests -from datetime import datetime - -BASE_URL = "http://0.0.0.0:8002/" -WORKSPACE_ID = "demo_workspace" - -# Prepare tool call results -tool_call_results = [ - { - "create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - "tool_name": "web_search", - "input": { - "query": "Python asyncio tutorial", - "max_results": 10, - "language": "en", - "filter_type": "technical_docs" - }, - "output": "Found 10 relevant results including official documentation and tutorials", - "token_cost": 150, - "success": True, - "time_cost": 2.3 - }, - { - "create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - "tool_name": "web_search", - "input": { - "query": "test", # Too generic - "max_results": 100, # Too many - "language": "unknown" # Invalid - }, - "output": "Warning: language 'unknown' not supported. Query too generic, limited results.", - "token_cost": 80, - "success": True, - "time_cost": 3.5 - } -] - -# Add tool call results -response = requests.post( - url=f"{BASE_URL}add_tool_call_result", - json={ - "workspace_id": WORKSPACE_ID, - "tool_name": "web_search", - "tool_call_results": tool_call_results - } -) - -result = response.json() -print(f"Success: {result.get('success')}") -print(f"Answer: {result.get('answer')}") - -# Check evaluated results -memory_list = result.get("metadata", {}).get("memory_list", []) -if memory_list: - tool_memory = memory_list[0] - for call_result in tool_memory["tool_call_results"]: - print(f"\nCall Summary: {call_result['summary']}") - print(f"Evaluation: {call_result['evaluation']}") - print(f"Score: {call_result['score']}") -``` ## SummaryToolMemoryOp @@ -221,245 +38,13 @@ Analyzes accumulated tool call history and generates comprehensive usage pattern - Uses LLM to synthesize actionable usage guidelines - Updates tool memory content with generated insights -### Processing Flow - -```mermaid -graph TB - A[Receive Tool Names] --> B[Split by Comma] - B --> C[For Each Tool Name] - C --> D[Search Vector Store] - D --> E{Exact Match?} - E -->|Yes| F[Load Tool Memory] - E -->|No| G[Log Warning] - F --> H[Extract Recent N Calls] - H --> I[Calculate Statistics] - I --> J[Format Call Summaries] - J --> K[Format Statistics] - K --> L[Concurrent Summarization] - L --> M[LLM Analysis #1] - L --> N[LLM Analysis #2] - L --> O[LLM Analysis #N] - M --> P[Generate Guidelines] - N --> P - O --> P - P --> Q[Update Memory Content] - Q --> R[Update Modified Time] - R --> S[Prepare for Vector Store] -``` - -1. **Tool Name Processing**: - - Split comma-separated tool names - - Trim whitespace from each name - - Log the list of tools to process - -2. **Memory Retrieval**: - - Search vector store for each tool name - - Verify exact match (memory type and when_to_use) - - Skip tools without existing memory - -3. **Data Preparation**: - - Extract the most recent N tool call results - - Calculate statistical metrics: - - Total calls vs recent calls analyzed - - Success rate (overall and recent) - - Average score (overall and recent) - - Average time cost - - Average token cost - - Format call summaries as markdown - - Format statistics as markdown - -4. **Concurrent Summarization**: - - Submit all tools for parallel summarization - - Each summarization uses LLM to analyze patterns - - Generate structured usage guidelines - -5. **Memory Update**: - - Update tool memory content with new guidelines - - Update modification timestamp - - Prepare for vector store update - ### Parameters -Configure in `default.yaml`: - -```yaml -op: - summary_tool_memory_op: - backend: summary_tool_memory_op - llm: default - params: - recent_call_count: 20 - summary_sleep_interval: 1.0 -``` - -- `recent_call_count` (integer, default: `20`): +- `op.summary_tool_memory_op.params.recent_call_count` (integer, default: `30`): - Number of most recent tool calls to analyze - Focuses on recent usage patterns - - Recommended: 10-50 depending on tool usage frequency - - Higher values provide more context but may dilute recent patterns -- `summary_sleep_interval` (float, default: `1.0`): +- `op.summary_tool_memory_op.params.summary_sleep_interval` (float, default: `1.0`): - Delay in seconds between concurrent summarizations - Prevents rate limiting when summarizing multiple tools - - Set to 0 for maximum speed (if no rate limits) - - Increase if encountering API throttling - -### Statistical Metrics - -The operation calculates the following metrics: - -```python -{ - "total_calls": 100, # Total number of calls in history - "recent_calls": 20, # Number of recent calls analyzed - "success_rate": 0.85, # Overall success rate (85%) - "recent_success_rate": 0.90, # Recent success rate (90%) - "avg_score": 0.78, # Average evaluation score - "recent_avg_score": 0.82, # Recent average score - "avg_time_cost": 2.45, # Average time in seconds - "avg_token_cost": 125.3 # Average token consumption -} -``` - -### Generated Guidelines Structure - -The LLM generates guidelines following this structure: - -1. **Core Function**: What the tool does and when to use it -2. **Success Patterns**: Parameter patterns and scenarios that work well -3. **Common Issues**: Main pitfalls to avoid and why they fail -4. **Best Practices**: 2-3 actionable recommendations - -### Usage Example - -```python -import requests - -BASE_URL = "http://0.0.0.0:8002/" -WORKSPACE_ID = "demo_workspace" - -# After accumulating tool call history, generate guidelines -response = requests.post( - url=f"{BASE_URL}summary_tool_memory", - json={ - "workspace_id": WORKSPACE_ID, - "tool_names": "web_search,file_reader,api_caller" # Multiple tools - } -) - -result = response.json() -print(f"Success: {result.get('success')}") -print(f"Answer: {result.get('answer')}") - -# Display generated guidelines -memory_list = result.get("metadata", {}).get("memory_list", []) -for memory in memory_list: - print(f"\n{'='*60}") - print(f"Tool: {memory['when_to_use']}") - print(f"{'='*60}") - print(memory['content']) - - # Display statistics - stats = memory.get('metadata', {}).get('statistics', {}) - print(f"\nStatistics:") - print(f" Total Calls: {stats.get('total_calls', 0)}") - print(f" Success Rate: {stats.get('success_rate', 0):.1%}") - print(f" Avg Score: {stats.get('avg_score', 0):.2f}") -``` - -### Example Generated Guidelines - -``` -Core Function: -The web_search tool retrieves information from the internet based on query parameters. -Use it when you need up-to-date information, documentation, or external data. - -Success Patterns: -- Specific queries (e.g., "Python asyncio tutorial") achieve 95% success rate -- Setting max_results=10-20 balances quality and performance -- Using language="en" and filter_type="technical_docs" improves relevance -- Average successful call: 2.3s, 150 tokens - -Common Issues: -- Generic queries (e.g., "test") return poor results (score: 0.5) -- max_results > 50 often leads to timeouts (avg: 8.2s vs 2.3s) -- Invalid language codes default to English but add latency -- Missing filter_type returns mixed-quality results - -Best Practices: -1. Use specific, descriptive queries with clear intent -2. Set max_results=10-20 for optimal balance -3. Always specify language and filter_type for technical searches -``` - -### When to Run Summarization - -- **Initial Setup**: After accumulating 20-30 tool calls -- **Regular Updates**: Every 50-100 new calls -- **Pattern Changes**: When success rate changes significantly -- **Tool Updates**: After tool version changes or parameter updates -- **Performance Issues**: When investigating declining performance - -### Integration with Retrieval - -```mermaid -sequenceDiagram - participant Agent - participant AddOp as add_tool_call_result - participant SummaryOp as summary_tool_memory - participant RetrieveOp as retrieve_tool_memory - - loop Every Tool Call - Agent->>AddOp: Record tool call result - AddOp->>AddOp: Evaluate and store - end - - Note over Agent,SummaryOp: After 20+ calls - Agent->>SummaryOp: Generate guidelines - SummaryOp->>SummaryOp: Analyze patterns - SummaryOp->>SummaryOp: Update content - - Note over Agent,RetrieveOp: Before next use - Agent->>RetrieveOp: Get tool memory - RetrieveOp-->>Agent: Return guidelines + history - Agent->>Agent: Apply recommendations -``` - -The summarization operation works in conjunction with retrieval: -1. `add_tool_call_result` continuously records invocations -2. `summary_tool_memory` periodically generates guidelines -3. `retrieve_tool_memory` provides guidelines before tool use -4. Agent applies recommendations to improve success rates - -## Best Practices - -1. **Recording Strategy**: - - Record every tool invocation, including failures - - Include detailed input parameters and complete output - - Capture accurate performance metrics - - Add relevant metadata for context - -2. **Evaluation Quality**: - - Ensure LLM has sufficient context for evaluation - - Monitor score distribution (should not be all 1.0 or 0.0) - - Review evaluations periodically for quality - - Adjust evaluation prompts if needed - -3. **Summarization Timing**: - - Wait for 20-30 calls before first summarization - - Re-summarize after significant new data (50+ calls) - - Update when usage patterns change - - Regenerate after tool updates - -4. **Parameter Tuning**: - - Adjust `max_history_tool_call_cnt` based on tool usage frequency - - Increase `recent_call_count` for tools with diverse patterns - - Balance `sleep_interval` between speed and rate limits - - Monitor vector store size and adjust retention limits - -5. **Quality Maintenance**: - - Review generated guidelines for accuracy - - Validate statistical metrics match expectations - - Clean up deprecated tools from vector store - - Archive historical data before major changes diff --git a/pyproject.toml b/pyproject.toml index fb6edcb0..5025a3d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "reme_ai" -version = "0.1.9" +version = "0.1.10" description = "Remember me" authors = [ { name = "jinli.yl", email = "jinli.yl@alibaba-inc.com" }, @@ -24,7 +24,7 @@ classifiers = [ keywords = ["llm", "memory", "experience", "memoryscope", "ai", "mcp", "http"] dependencies = [ - "flowllm==0.1.9", + "flowllm>=0.1.11.1", ] [tool.setuptools.packages.find] diff --git a/reme_ai/__init__.py b/reme_ai/__init__.py index f07f9112..7afad95e 100644 --- a/reme_ai/__init__.py +++ b/reme_ai/__init__.py @@ -1,6 +1,14 @@ +import warnings + +from pydantic.warnings import PydanticDeprecatedSince20 + +warnings.filterwarnings("ignore", category=DeprecationWarning, module="websockets") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="uvicorn") +warnings.filterwarnings("ignore", category=PydanticDeprecatedSince20) + from . import agent from . import retrieve from . import summary from . import vector_store -__version__ = "0.1.9" +__version__ = "0.1.10" diff --git a/reme_ai/agent/tools/llm_mock_search_op.py b/reme_ai/agent/tools/llm_mock_search_op.py index eebef995..7ba53415 100644 --- a/reme_ai/agent/tools/llm_mock_search_op.py +++ b/reme_ai/agent/tools/llm_mock_search_op.py @@ -34,6 +34,7 @@ class LLMMockSearchOp(BaseAsyncToolOp): simple_config: Dict[str, Any] = None, medium_config: Dict[str, Any] = None, complex_config: Dict[str, Any] = None, + seed: int = 0, **kwargs): """ Initialize the LLM Mock Search Op. @@ -52,8 +53,13 @@ class LLMMockSearchOp(BaseAsyncToolOp): - success_rate: float (0-1), default 0.70 - extra_time: float (seconds), default 1.5 - relevance_ratio: float (0-1), default 0.80 + seed: Random seed for deterministic behavior, default 0 """ super().__init__(llm=llm, **kwargs) + + # Set random seed for deterministic behavior + self.seed = seed + random.seed(self.seed) # Default configurations for each scenario self.simple_config = { @@ -221,13 +227,16 @@ class LLMMockSearchOp(BaseAsyncToolOp): # Step 5: Check relevance ratio if random.random() > config["relevance_ratio"]: # Generate random/irrelevant result - logger.info("Generating low relevance result") + # NOTE: success=True because technically the tool executed without errors, + # but the content is irrelevant (low quality), which should result in score=0.0 during evaluation + logger.info("Generating low relevance result (success=True but low quality)") content = await self.generate_random_result() result_dict = { - "success": False, + "success": True, # Technical execution succeeded "content": content, "query": query, - "complexity": complexity + "complexity": complexity, + "is_relevant": False # Mark as irrelevant for debugging } else: # Generate relevant result @@ -237,7 +246,8 @@ class LLMMockSearchOp(BaseAsyncToolOp): "success": True, "content": content, "query": query, - "complexity": complexity + "complexity": complexity, + "is_relevant": True # Mark as relevant for debugging } self.set_result(json.dumps(result_dict, ensure_ascii=False)) diff --git a/reme_ai/agent/tools/mock_search_tools.py b/reme_ai/agent/tools/mock_search_tools.py index 68e7cae9..140fa895 100644 --- a/reme_ai/agent/tools/mock_search_tools.py +++ b/reme_ai/agent/tools/mock_search_tools.py @@ -9,23 +9,23 @@ class SearchToolA(LLMMockSearchOp): def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs): # Configure for fast but shallow performance simple_config = { - "success_rate": 0.95, # High success rate for simple queries - "extra_time": 0.2, # Very fast (0.2-0.5s range) - "relevance_ratio": 0.92, # High relevance + "success_rate": 0.9, # High success rate for simple queries + "extra_time": 0, # Very fast (0.2-0.5s range) + "relevance_ratio": 0.9, # High relevance "content_length": "short" # Concise answers } medium_config = { - "success_rate": 0.75, # Lower success for medium queries - "extra_time": 0.3, # Still fast - "relevance_ratio": 0.70, # Moderate relevance + "success_rate": 0.2, # Lower success for medium queries + "extra_time": 0, # Still fast + "relevance_ratio": 0.2, # Moderate relevance "content_length": "short" # Limited depth } complex_config = { - "success_rate": 0.50, # Poor success rate for complex queries - "extra_time": 0.4, # Fast but insufficient - "relevance_ratio": 0.50, # Low relevance (often misses key aspects) + "success_rate": 0.5, # Poor success rate for complex queries + "extra_time": 0, # Fast but insufficient + "relevance_ratio": 0.5, # Low relevance (often misses key aspects) "content_length": "short" # Too shallow for complex topics } @@ -45,23 +45,23 @@ class SearchToolB(LLMMockSearchOp): def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs): # Configure for balanced performance simple_config = { - "success_rate": 0.8, # Very high success rate - "extra_time": 0.8, # Moderate speed (1.0-1.5s range) - "relevance_ratio": 0.8, # High relevance + "success_rate": 0.3, # Very high success rate + "extra_time": 0, # Moderate speed (1.0-1.5s range) + "relevance_ratio": 0.3, # High relevance "content_length": "medium" # More detailed than needed for simple } medium_config = { - "success_rate": 0.8, # Excellent success rate - "extra_time": 1.0, # Balanced speed - "relevance_ratio": 0.8, # High relevance + "success_rate": 0.9, # Excellent success rate + "extra_time": 0, # Balanced speed + "relevance_ratio": 0.9, # High relevance "content_length": "medium" # Perfect depth for medium queries } complex_config = { - "success_rate": 0.8, # Good success rate - "extra_time": 1.2, # Still reasonable speed - "relevance_ratio": 0.8, # Decent relevance but not exhaustive + "success_rate": 0.5, # Good success rate + "extra_time": 0, # Still reasonable speed + "relevance_ratio": 0.5, # Decent relevance but not exhaustive "content_length": "medium" # Covers main points but lacks depth } @@ -83,23 +83,23 @@ class SearchToolC(LLMMockSearchOp): def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs): # Configure for comprehensive but costly performance simple_config = { - "success_rate": 0.7, # Good but not optimal (over-processing) - "extra_time": 2.5, # Slow (3.0-4.0s range) - "relevance_ratio": 0.7, # High relevance but unnecessary depth + "success_rate": 0.3, # Good but not optimal (over-processing) + "extra_time": 0, # Slow (3.0-4.0s range) + "relevance_ratio": 0.3, # High relevance but unnecessary depth "content_length": "long" # Too detailed for simple queries } medium_config = { - "success_rate": 0.7, # High success rate - "extra_time": 2.8, # Slow but thorough - "relevance_ratio": 0.7, # High relevance with extra context + "success_rate": 0.4, # High success rate + "extra_time": 0, # Slow but thorough + "relevance_ratio": 0.4, # High relevance with extra context "content_length": "long" # More depth than needed } complex_config = { - "success_rate": 0.95, # Excellent success rate - "extra_time": 3.5, # Slow but comprehensive (3.5-5.0s range) - "relevance_ratio": 0.94, # Very high relevance + "success_rate": 0.9, # Excellent success rate + "extra_time": 0, # Slow but comprehensive (3.5-5.0s range) + "relevance_ratio": 0.9, # Very high relevance "content_length": "long" # Perfect depth for complex queries } diff --git a/reme_ai/agent/tools/test_use_mock_search.py b/reme_ai/agent/tools/test_use_mock_search.py deleted file mode 100644 index 04475d11..00000000 --- a/reme_ai/agent/tools/test_use_mock_search.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -Test script for UseMockSearchOp - -This script demonstrates how to use the UseMockSearchOp to intelligently -select and execute search tools based on query complexity. -""" -import asyncio -import json - -from flowllm.context import FlowContext -from flowllm.app import FlowLLMApp - -from reme_ai.agent.tools.use_mock_search_op import UseMockSearchOp - - -async def test_single_query(op: UseMockSearchOp, query: str): - """Test a single query and display results.""" - print(f"\n{'=' * 100}") - print(f"Testing Query: {query}") - print(f"{'=' * 100}") - - context = FlowContext(query=query) - await op.async_call(context=context) - - result = json.loads(context.use_mock_search_result) - - print(f"\n📊 Results:") - print(f" Selected Tool: {result['selected_tool']}") - print(f" Reasoning: {result['reasoning']}") - print(f" Query Complexity: {result['complexity']}") - print(f" Success: {result['success']}") - print(f"\n📝 Content:") - print(f" {result['content'][:500]}..." if len(result['content']) > 500 else f" {result['content']}") - print(f"\n{'=' * 100}\n") - - return result - - -async def main(): - """Run comprehensive tests of UseMockSearchOp.""" - - print("\n" + "=" * 100) - print("UseMockSearchOp Test Suite") - print("=" * 100) - print("\nThis test demonstrates the intelligent tool selection capability.") - print("The LLM analyzes each query and selects the most appropriate search tool:\n") - print(" • SearchToolA: Fast & shallow (best for simple factual queries)") - print(" • SearchToolB: Balanced (best for medium complexity queries)") - print(" • SearchToolC: Comprehensive & slow (best for complex research queries)") - print("=" * 100) - - async with FlowLLMApp(load_default_config=True): - op = UseMockSearchOp() - - # Test cases covering different complexities - test_queries = [ - # Simple queries (should select SearchToolA) - { - "query": "What is the capital of France?", - "expected_tool": "SearchToolA", - "description": "Simple factual query" - }, - { - "query": "When was Python programming language created?", - "expected_tool": "SearchToolA", - "description": "Simple historical fact" - }, - - # Medium complexity queries (should select SearchToolB) - { - "query": "How does quantum computing work?", - "expected_tool": "SearchToolB", - "description": "Medium complexity technical explanation" - }, - { - "query": "What are the main causes of climate change?", - "expected_tool": "SearchToolB", - "description": "Medium complexity scientific question" - }, - - # Complex queries (should select SearchToolC) - { - "query": "Analyze the impact of artificial intelligence on global economy, employment, and society", - "expected_tool": "SearchToolC", - "description": "Complex multi-dimensional analysis" - }, - { - "query": "Compare and contrast different renewable energy solutions including their environmental impact, cost-effectiveness, and scalability", - "expected_tool": "SearchToolC", - "description": "Complex comparative analysis" - } - ] - - results = [] - correct_selections = 0 - - for test_case in test_queries: - query = test_case["query"] - expected = test_case["expected_tool"] - description = test_case["description"] - - print(f"\n🔍 Test Case: {description}") - print(f" Expected Tool: {expected}") - - result = await test_single_query(op, query) - results.append({ - "test_case": test_case, - "result": result - }) - - # Check if the selection was correct - if result["selected_tool"] == expected: - correct_selections += 1 - print(f" ✅ Correct tool selected!") - else: - print(f" ⚠️ Different tool selected (this may still be valid)") - - # Summary - print("\n" + "=" * 100) - print("Test Summary") - print("=" * 100) - print(f"Total Tests: {len(test_queries)}") - print(f"Expected Matches: {correct_selections}/{len(test_queries)}") - print(f"Success Rate: {correct_selections/len(test_queries)*100:.1f}%") - print("\nNote: Tool selection may vary based on LLM reasoning, and different") - print(" selections don't necessarily indicate errors.") - print("=" * 100) - - -if __name__ == "__main__": - asyncio.run(main()) - diff --git a/reme_ai/agent/tools/use_mock_search_op.py b/reme_ai/agent/tools/use_mock_search_op.py index 4e97a4d9..5c043d7a 100644 --- a/reme_ai/agent/tools/use_mock_search_op.py +++ b/reme_ai/agent/tools/use_mock_search_op.py @@ -1,17 +1,18 @@ import asyncio +import datetime import json -from typing import Dict, Any -from flowllm.op.gallery.task_react_op import tools_schema_to_qwen_prompt -from flowllm.schema.tool_call import ToolCall -from loguru import logger - -from flowllm.context import FlowContext, C +from flowllm.context import C from flowllm.enumeration.role import Role from flowllm.op.base_async_tool_op import BaseAsyncToolOp from flowllm.schema.message import Message +from flowllm.schema.tool_call import ToolCall +from flowllm.utils.timer import Timer +from flowllm.utils.token_utils import TokenCounter +from loguru import logger from reme_ai.agent.tools.mock_search_tools import SearchToolA, SearchToolB, SearchToolC +from reme_ai.schema.memory import ToolCallResult @C.register_op() @@ -19,18 +20,30 @@ class UseMockSearchOp(BaseAsyncToolOp): file_path: str = __file__ def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs): - super().__init__(llm=llm, **kwargs) + super().__init__(llm=llm, save_answer=True, **kwargs) + + def build_tool_call(self) -> ToolCall: + return ToolCall(**{ + "description": "Intelligently selects and executes the most appropriate search tool based on query complexity. " + "Automatically tracks performance metrics and records tool usage for optimization.", + "input_schema": { + "query": { + "type": "string", + "description": "query", + "required": True + } + } + }) async def select_tool(self, query: str, tool_ops: list[BaseAsyncToolOp]) -> ToolCall | None: assistant_message = await self.llm.achat(messages=[Message(role=Role.USER, content=query)], tools=[x.tool_call for x in tool_ops]) - + logger.info(f"assistant_message={assistant_message.model_dump_json()}") if assistant_message.tool_calls: return assistant_message.tool_calls[0] return None - async def async_execute(self): query: str = self.input_dict["query"] logger.info(f"query={query}") @@ -41,46 +54,85 @@ class UseMockSearchOp(BaseAsyncToolOp): SearchToolC(), ] - tool_result = await self.select_tool(query, tool_ops) - if tool_result is None: - ... + # Step 1: Select the appropriate tool using LLM + tool_call = await self.select_tool(query, tool_ops) + + if tool_call is None: + # No tool selected + error_result = ToolCallResult( + create_time=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + tool_name="None", + input={"query": query}, + output="No appropriate tool was selected for the query", + token_cost=0, + success=False, + time_cost=0.0 + ) + self.set_result(error_result.model_dump_json()) return + # Step 2: Execute the selected tool + selected_op = None for op in tool_ops: - op.tool_call.name == tool_result.name + if op.tool_call.name == tool_call.name: + selected_op = op + break + + if selected_op is None: + # Tool not found (should not happen) + error_result = ToolCallResult( + create_time=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + tool_name=tool_call.name, + input=tool_call.arguments, + output=f"Tool {tool_call.name} not found in available tools", + token_cost=0, + success=False, + time_cost=0.0 + ) + self.set_result(error_result.model_dump_json()) + return + + # Step 3: Execute the tool with timer + timer = Timer("tool execute") + with timer: + await selected_op.async_call(query=query) + selected_op_output = json.loads(selected_op.output) + content = selected_op_output["content"] + success = selected_op_output["success"] + token_cost = TokenCounter().count(content) + + time_cost = timer.time_cost + + # Create ToolCallResult + tool_call_result = ToolCallResult( + create_time=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + tool_name=tool_call.name, + input={"query": query}, + output=content, + token_cost=token_cost, + success=success, + time_cost=round(time_cost, 3) + ) + + self.set_result(tool_call_result.model_dump_json()) async def async_main(): - """Test the UseMockSearchOp with different query types.""" from flowllm.app import FlowLLMApp async with FlowLLMApp(load_default_config=True): - # Test queries of different complexities test_queries = [ - "What is the capital of France?", # Simple - should select SearchToolA - "How does quantum computing work?", # Medium - should select SearchToolB - "Analyze the impact of artificial intelligence on global economy, employment, and society", # Complex - should select SearchToolC - "When was Python programming language created?", # Simple - "Compare different types of renewable energy sources", # Complex + "What is the capital of France?", + "How does quantum computing work?", + "Analyze the impact of artificial intelligence on global economy, employment, and society", + "When was Python programming language created?", + "Compare different types of renewable energy sources", ] - op = UseMockSearchOp() - for query in test_queries: - print(f"\n{'=' * 100}") - print(f"Query: {query}") - print(f"{'=' * 100}") - - context = FlowContext(query=query) - await op.async_call(context=context) - - result = json.loads(context.use_mock_search_result) - print(f"\nSelected Tool: {result['selected_tool']}") - print(f"Reasoning: {result['reasoning']}") - print(f"Complexity: {result['complexity']}") - print(f"Success: {result['success']}") - print(f"\nContent:\n{result['content']}") - print(f"\n{'=' * 100}\n") + op = UseMockSearchOp() + await op.async_call(query=query) + print(op.output) if __name__ == "__main__": diff --git a/reme_ai/agent/tools/use_mock_search_prompt.yaml b/reme_ai/agent/tools/use_mock_search_prompt.yaml deleted file mode 100644 index 1fd7e9f0..00000000 --- a/reme_ai/agent/tools/use_mock_search_prompt.yaml +++ /dev/null @@ -1,64 +0,0 @@ -tool_selection_prompt: | - You are an intelligent tool selection assistant. You have access to three different search tools with different characteristics: - - **SearchToolA** - Fast but shallow search tool - - Best for: Simple factual queries - - Strengths: Very fast response (0.2-0.5s), high success rate for simple queries (95%) - - Weaknesses: Poor performance on complex queries (50% success rate), limited depth - - **SearchToolB** - Balanced search tool - - Best for: Medium complexity queries - - Strengths: Good balance of speed and quality (1.0-1.5s), consistent 80% success rate across all query types - - Weaknesses: May be slower than needed for simple queries, lacks depth for very complex topics - - **SearchToolC** - Comprehensive but slow search tool - - Best for: Complex research queries - - Strengths: Excellent for complex queries (95% success rate), comprehensive and detailed results - - Weaknesses: Very slow (3.0-5.0s), overkill for simple queries (70% success rate) - - Given the user's query, you need to: - 1. Analyze the complexity and requirements of the query - 2. Select the most appropriate tool (SearchToolA, SearchToolB, or SearchToolC) - 3. Provide the query parameter - - User Query: {query} - - Respond with a JSON object in the following format: - {{ - "selected_tool": "SearchToolA" or "SearchToolB" or "SearchToolC", - "reasoning": "Brief explanation of why this tool was selected", - "query": "The search query to use" - }} - -tool_selection_prompt_zh: | - 你是一个智能工具选择助手。你可以访问三个具有不同特性的搜索工具： - - **SearchToolA** - 快速但浅层的搜索工具 - - 最适合：简单的事实查询 - - 优势：响应非常快（0.2-0.5秒），简单查询的成功率高（95%） - - 劣势：复杂查询表现不佳（成功率50%），深度有限 - - **SearchToolB** - 平衡的搜索工具 - - 最适合：中等复杂度的查询 - - 优势：速度和质量平衡良好（1.0-1.5秒），所有查询类型的成功率一致为80% - - 劣势：对于简单查询可能比需要的慢，对于非常复杂的主题缺乏深度 - - **SearchToolC** - 全面但慢速的搜索工具 - - 最适合：复杂的研究查询 - - 优势：对复杂查询效果极佳（成功率95%），结果全面详细 - - 劣势：非常慢（3.0-5.0秒），对简单查询来说过度（成功率70%） - - 给定用户的查询，你需要： - 1. 分析查询的复杂度和要求 - 2. 选择最合适的工具（SearchToolA、SearchToolB 或 SearchToolC） - 3. 提供查询参数 - - 用户查询：{query} - - 用 JSON 对象格式回复： - {{ - "selected_tool": "SearchToolA" 或 "SearchToolB" 或 "SearchToolC", - "reasoning": "选择此工具的简要说明", - "query": "要使用的搜索查询" - }} - diff --git a/reme_ai/app.py b/reme_ai/app.py index f4931b25..9945b730 100644 --- a/reme_ai/app.py +++ b/reme_ai/app.py @@ -1,8 +1,4 @@ import sys -import warnings - -warnings.filterwarnings("ignore", category=DeprecationWarning, module="websockets") -warnings.filterwarnings("ignore", category=DeprecationWarning, module="uvicorn") from flowllm.app import FlowLLMApp diff --git a/reme_ai/config/default.yaml b/reme_ai/config/default.yaml index e5b91518..992b0117 100644 --- a/reme_ai/config/default.yaml +++ b/reme_ai/config/default.yaml @@ -62,10 +62,6 @@ flow: flow_content: parse_tool_call_result_op >> update_vector_store_op description: "Evaluates and adds tool call results to the tool memory database, creating new memory or updating existing memory for the specified tool" input_schema: - tool_name: - type: string - description: "The name of the tool to add call results for" - required: true tool_call_results: type: array description: "List of tool call result objects, each containing: tool_name, input, output, success, time_cost, token_cost, create_time" @@ -80,6 +76,15 @@ flow: description: "Comma-separated tool names to summarize (e.g., 'tool_name1,tool_name2')" required: true + use_mock_search: + flow_content: use_mock_search_op + description: "Simulates intelligent search tool selection and execution based on query complexity, with automatic tool memory recording" + input_schema: + query: + type: string + description: "User search query to process" + required: true + summary_task_memory_simple: flow_content: simple_summary_op >> update_vector_store_op description: "Summarizes conversation trajectories or messages into memories using a simplified approach" diff --git a/reme_ai/retrieve/tool/__init__.py b/reme_ai/retrieve/tool/__init__.py index e69de29b..c29f58ae 100644 --- a/reme_ai/retrieve/tool/__init__.py +++ b/reme_ai/retrieve/tool/__init__.py @@ -0,0 +1 @@ +from .retrieve_tool_memory_op import RetrieveToolMemoryOp diff --git a/reme_ai/schema/memory.py b/reme_ai/schema/memory.py index a1dd6a31..b0f63609 100644 --- a/reme_ai/schema/memory.py +++ b/reme_ai/schema/memory.py @@ -117,7 +117,7 @@ class ToolCallResult(BaseModel): time_cost: float = Field(default=0, description="Time consumed by the tool invocation, in seconds") summary: str = Field(default="", description="Brief summary of the tool call result") evaluation: str = Field(default="", description="Detailed evaluation for the tool invocation") - score: float = Field(default=0, description="Score of the Evaluation (0.0, 0.5, or 1.0)") + score: float = Field(default=0, description="Score of the Evaluation (0.0 for failure, 1.0 for complete success)") metadata: dict = Field(default_factory=dict) @@ -193,11 +193,9 @@ class ToolMemory(BaseMemory): avg_score = total_score / recent_calls_count if recent_calls_count > 0 else 0.0 return { - "total_calls": total_calls, - "recent_calls_analyzed": recent_calls_count, "avg_token_cost": round(avg_token_cost, 2), - "success_rate": round(success_rate, 4), "avg_time_cost": round(avg_time_cost, 3), + "success_rate": round(success_rate, 4), "avg_score": round(avg_score, 3) } diff --git a/reme_ai/summary/tool/parse_tool_call_result_op.py b/reme_ai/summary/tool/parse_tool_call_result_op.py index 1a28fdbc..75137ee5 100644 --- a/reme_ai/summary/tool/parse_tool_call_result_op.py +++ b/reme_ai/summary/tool/parse_tool_call_result_op.py @@ -1,4 +1,5 @@ import asyncio +from collections import defaultdict from typing import List from flowllm import C, BaseAsyncOp @@ -44,18 +45,19 @@ class ParseToolCallResultOp(BaseAsyncOp): tool_call_result.evaluation = eval_data.get("evaluation", "") tool_call_result.score = float(eval_data.get("score", 0.0)) - # 验证 score 是否符合 3 档要求 (0.0, 0.5, 1.0) - if tool_call_result.score not in [0.0, 0.5, 1.0]: - logger.warning(f"Score {tool_call_result.score} not in [0.0, 0.5, 1.0], rounding to nearest") - if tool_call_result.score < 0.25: + # 验证 score 是否符合 2 档要求 (0.0, 1.0) + if tool_call_result.score not in [0.0, 1.0]: + if tool_call_result.score < 0.5: tool_call_result.score = 0.0 - elif tool_call_result.score < 0.75: - tool_call_result.score = 0.5 else: tool_call_result.score = 1.0 - logger.info(f"Evaluated tool call {index}: tool_name={tool_call_result.tool_name}, " - f"score={tool_call_result.score}, summary={tool_call_result.summary[:50]}...") + # 打印完整的prompt和result + logger.info(f"\n{'='*80}\nLLM Evaluation [Index {index}]\n{'='*80}\n" + f"PROMPT:\n{prompt}\n\n" + f"RESULT:\n{content}\n" + f"{'='*80}\n") + return tool_call_result # 调用 LLM 进行评估 @@ -66,79 +68,68 @@ class ParseToolCallResultOp(BaseAsyncOp): async def async_execute(self): tool_call_results: list = self.context.get("tool_call_results", []) tool_call_results = [ToolCallResult(**x) if isinstance(x, dict) else x for x in tool_call_results] - tool_name: str = self.context.get("tool_name", "") workspace_id: str = self.context.workspace_id - logger.info(f"workspace_id={workspace_id} count={len(tool_call_results)} tool_name={tool_name}") - - if not tool_name: - logger.warning("tool_name is empty, skipping processing") - self.context.response.answer = "tool_name is required" - self.context.response.success = False - return if not tool_call_results: - logger.info("No valid tool_call_results to process") self.context.response.answer = "No valid tool_call_results" self.context.response.success = False return - # 并发评估所有 tool_call_results - logger.info(f"Starting concurrent evaluation of {len(tool_call_results)} tool call results") - # 使用基类的 submit_async_task 提交所有评估任务 for index, tool_call_result in enumerate(tool_call_results): self.submit_async_task(self._evaluate_single_tool_call, tool_call_result, index) # 使用基类的 join_async_task 等待所有任务完成 # 注意: 基类已经过滤掉异常,返回的只包含成功的结果 - tool_call_results = await self.join_async_task(return_exceptions=True) - logger.info(f"Completed evaluation of {len(tool_call_results)} tool call results") + evaluated_results = await self.join_async_task(return_exceptions=True) - nodes: List[VectorNode] = await self.vector_store.async_search(query=tool_name, - workspace_id=workspace_id, - top_k=1) + tool_results_by_name = defaultdict(list) + for result in evaluated_results: + tool_results_by_name[result.tool_name].append(result) - tool_memory: ToolMemory | None = None - exist_node: bool = False + # 处理每个 tool_name 的结果 + all_memory_list = [] + all_deleted_memory_ids = [] - if nodes: - top_node = nodes[0] - memory: ToolMemory = vector_node_to_memory(top_node) + for tool_name, tool_call_results in tool_results_by_name.items(): + nodes: List[VectorNode] = await self.vector_store.async_search(query=tool_name, + workspace_id=workspace_id, + top_k=1) - # 确保是 ToolMemory 类型且 when_to_use 与 tool_name 匹配 - if isinstance(memory, ToolMemory) and memory.when_to_use == tool_name: - tool_memory = memory - exist_node = True - logger.info(f"Found existing tool_memory for tool_name={tool_name}, memory_id={tool_memory.memory_id}") - else: - logger.info(f"Top result does not match tool_name={tool_name}, will create new memory") + tool_memory: ToolMemory | None = None + exist_node: bool = False - # 如果没有找到匹配的 memory，创建新的 - if tool_memory is None: - tool_memory = ToolMemory(workspace_id=workspace_id, when_to_use=tool_name) - logger.info(f"Created new tool_memory for tool_name={tool_name}, memory_id={tool_memory.memory_id}") + if nodes: + top_node = nodes[0] + memory: ToolMemory = vector_node_to_memory(top_node) - tool_memory.tool_call_results.extend(tool_call_results) + # 确保是 ToolMemory 类型且 when_to_use 与 tool_name 匹配 + if isinstance(memory, ToolMemory) and memory.when_to_use == tool_name: + tool_memory = memory + exist_node = True - # 保留最近的 n 个 - if len(tool_memory.tool_call_results) > self.max_history_tool_call_cnt: - tool_memory.tool_call_results = tool_memory.tool_call_results[-self.max_history_tool_call_cnt:] - logger.info(f"Trimmed tool_call_results to {self.max_history_tool_call_cnt} most recent entries") + # 如果没有找到匹配的 memory，创建新的 + if tool_memory is None: + tool_memory = ToolMemory(workspace_id=workspace_id, when_to_use=tool_name) - # 更新修改时间 - tool_memory.update_modified_time() + tool_memory.tool_call_results.extend(tool_call_results) - # 4. 将更新后的 memory 保存到向量数据库 - # 如果是更新现有的 memory，需要先删除旧的 - deleted_memory_ids = [] - if exist_node: - deleted_memory_ids = [tool_memory.memory_id] + # 保留最近的 n 个 + if len(tool_memory.tool_call_results) > self.max_history_tool_call_cnt: + tool_memory.tool_call_results = tool_memory.tool_call_results[-self.max_history_tool_call_cnt:] + + # 更新修改时间 + tool_memory.update_modified_time() + + # 如果是更新现有的 memory，需要先删除旧的 + if exist_node: + all_deleted_memory_ids.append(tool_memory.memory_id) + + all_memory_list.append(tool_memory) # 设置返回结果 - self.context.response.metadata["deleted_memory_ids"] = deleted_memory_ids - self.context.response.metadata["memory_list"] = [tool_memory] - - logger.info(f"Updated tool_memory: tool_name={tool_name}, total_results={len(tool_memory.tool_call_results)}") + self.context.response.metadata["deleted_memory_ids"] = all_deleted_memory_ids + self.context.response.metadata["memory_list"] = all_memory_list async def main(): @@ -166,10 +157,9 @@ async def main(): "time_cost": 2.3 } ] - tool_name = "test_tool" workspace_id = "test_workspace1" - await op.async_call(tool_call_results=tool_call_results, tool_name=tool_name, workspace_id=workspace_id) + await op.async_call(tool_call_results=tool_call_results, workspace_id=workspace_id) logger.info(f"Response: {op.context.response.model_dump_json()}") diff --git a/reme_ai/summary/tool/parse_tool_call_result_prompt.yaml b/reme_ai/summary/tool/parse_tool_call_result_prompt.yaml index acc530ce..b8b3d740 100644 --- a/reme_ai/summary/tool/parse_tool_call_result_prompt.yaml +++ b/reme_ai/summary/tool/parse_tool_call_result_prompt.yaml @@ -14,17 +14,23 @@ evaluate_tool_call_prompt: | ## Evaluation Criteria: - ### 1. Success Evaluation - - Check if the success flag indicates successful execution - - Verify if the output contains error messages or indicates failure - - Consider if time and token costs are reasonable - - Assess if there are any error indicators in the output + ### Important: Score independently from the success flag + The `success_flag` indicates whether the tool executed without technical errors. + The `score` should evaluate the QUALITY and RELEVANCE of the result. - ### 2. Parameter Alignment Evaluation - - Evaluate if the output matches what would be expected given the input parameters - - Consider if the input parameters are appropriate for the tool - - Check for any parameter mismatches or unexpected behaviors - - Verify the output is consistent with the tool's intended purpose + A tool can execute successfully (success=True) but still produce low-quality or irrelevant results (score=0.0). + + ### Evaluation Dimensions: + + 1. **Technical Execution** (reflected in success_flag): + - Did the tool run without errors? + - Are there error messages or exceptions in the output? + + 2. **Result Quality** (what you should score): + - Is the output relevant to the input query/parameters? + - Does the output provide meaningful and useful information? + - Is the result appropriate given the input parameters? + - Does the output match the tool's intended purpose? ## Response Format: Please provide your evaluation in the following JSON format: @@ -32,15 +38,20 @@ evaluate_tool_call_prompt: | ```json {{ "summary": "A brief one-sentence summary of the tool call result", - "evaluation": "A brief evaluation (2-3 sentences) explaining the success status and parameter alignment.", + "evaluation": "A brief evaluation (2-3 sentences) explaining the result quality and relevance, NOT just repeating the success flag.", "score": 1.0 }} ``` - ## Scoring Guidelines: - - **1.0**: Success - Tool executed successfully with good parameter alignment - - **0.5**: Partial Success - Tool executed but with parameter alignment issues or minor errors - - **0.0**: Failure - Tool execution failed or severe parameter misalignment + ## Scoring Guidelines (Focus on Result Quality): + - **1.0**: High Quality - The output is relevant, useful, and appropriate for the given input parameters + - **0.0**: Low Quality - The output is irrelevant, incorrect, unhelpful, or inappropriate for the input - Please evaluate carefully and provide a score of exactly 0.0, 0.5, or 1.0. + ## Examples: + - success=True, but output is "No results found" for a reasonable query → score=0.0 (technically succeeded but unhelpful) + - success=True, but output contains generic/irrelevant information → score=0.0 (poor quality) + - success=True, and output provides relevant, useful information → score=1.0 (good quality) + - success=False, with error messages → score=0.0 (failed execution) + + Please evaluate carefully and provide a score of exactly 0.0 or 1.0 based on OUTPUT QUALITY, not just the success flag. diff --git a/reme_ai/summary/tool/summary_tool_memory_op.py b/reme_ai/summary/tool/summary_tool_memory_op.py index 7dd160e4..1292eb38 100644 --- a/reme_ai/summary/tool/summary_tool_memory_op.py +++ b/reme_ai/summary/tool/summary_tool_memory_op.py @@ -45,12 +45,8 @@ class SummaryToolMemoryOp(BaseAsyncOp): @staticmethod def _format_statistics_markdown(statistics: dict) -> str: """Format statistics as markdown.""" - lines = [f"- **Total Calls**: {statistics.get('total_calls', 0)}", - f"- **Recent Calls**: {statistics.get('recent_calls', 0)}", - f"- **Success Rate**: {statistics.get('success_rate', 0):.2%}", - f"- **Recent Success Rate**: {statistics.get('recent_success_rate', 0):.2%}", + lines = [f"- **Success Rate**: {statistics.get('success_rate', 0):.2%}", f"- **Average Score**: {statistics.get('avg_score', 0):.3f}", - f"- **Recent Average Score**: {statistics.get('recent_avg_score', 0):.3f}", f"- **Average Time Cost**: {statistics.get('avg_time_cost', 0):.3f}s", f"- **Average Token Cost**: {statistics.get('avg_token_cost', 0):.1f}"] @@ -73,15 +69,18 @@ class SummaryToolMemoryOp(BaseAsyncOp): call_summaries_md = self._format_call_summaries_markdown(recent_calls) statistics_md = self._format_statistics_markdown(statistics) + # Don't include statistics in prompt - only call summaries prompt = self.prompt_format(prompt_name="summarize_tool_usage_prompt", tool_name=tool_memory.when_to_use, - call_summaries=call_summaries_md, - statistics=statistics_md) + call_summaries=call_summaries_md) def parse_summary(message: Message) -> ToolMemory: content = message.content.strip() # Extract content from txt code block - tool_memory.content = extract_content(content, "txt") + llm_summary = extract_content(content, "txt") + + # Append statistics markdown to LLM result + tool_memory.content = f"{llm_summary}\n\n## Statistics\n{statistics_md}" # Update modified time tool_memory.update_modified_time() @@ -334,7 +333,7 @@ async def main(): logger.info(f"\n工具名称: {summarized_memory.when_to_use}") logger.info(f"\n统计信息:") stats = summarized_memory.statistic(recent_frequency=30) - logger.info(f" 总调用次数: {stats['total_calls']}") + logger.info(f" 总调用次数: {len(summarized_memory.tool_call_results)}") logger.info(f" 成功率: {stats['success_rate']:.1%}") logger.info(f" 平均评分: {stats['avg_score']:.2f}") logger.info(f" 平均耗时: {stats['avg_time_cost']:.2f}s") diff --git a/reme_ai/summary/tool/summary_tool_memory_prompt.yaml b/reme_ai/summary/tool/summary_tool_memory_prompt.yaml index 9fd1e075..79fdf62c 100644 --- a/reme_ai/summary/tool/summary_tool_memory_prompt.yaml +++ b/reme_ai/summary/tool/summary_tool_memory_prompt.yaml @@ -7,14 +7,11 @@ summarize_tool_usage_prompt: | ## Recent Tool Call History: {call_summaries} - ## Statistical Summary: - {statistics} - ## Your Task: Based on the tool call history, generate a concise and logical tool usage description following this structure: 1. **Core Function**: What this tool does and when to use it - 2. **Success Patterns**: Parameter patterns and usage scenarios that work well (with key metrics) + 2. **Success Patterns**: Parameter patterns and usage scenarios that work well 3. **Common Issues**: Main pitfalls to avoid and why they fail 4. **Best Practices**: 2-3 actionable recommendations diff --git a/reme_ai/utils/tool_memory_utils.py b/reme_ai/utils/tool_memory_utils.py new file mode 100644 index 00000000..7adc00d0 --- /dev/null +++ b/reme_ai/utils/tool_memory_utils.py @@ -0,0 +1,195 @@ +import random +from datetime import datetime, timedelta +from typing import List, Dict, Any + + +def create_mock_tool_call_results(tool_name: str, count: int = 30) -> List[Dict[str, Any]]: + """ + Create mock tool call results for testing + + Args: + tool_name: Name of the tool to create results for + count: Number of mock results to create + + Returns: + List of tool call result dictionaries + """ + base_time = datetime.now() - timedelta(days=7) + tool_call_results = [] + + if tool_name == "web_search": + # Scenario 1: Successful searches (15 calls) + success_queries = [ + "Python asyncio tutorial", "machine learning basics", "React hooks guide", + "Docker best practices", "SQL optimization tips", "Git workflow strategies", + "RESTful API design", "microservices architecture", "Redis caching patterns", + "Kubernetes deployment", "GraphQL advantages", "MongoDB schema design", + "JWT authentication", "OAuth2 flow", "WebSocket real-time" + ] + + for i, query in enumerate(success_queries): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=i * 2)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "query": query, + "max_results": random.randint(5, 20), + "language": "en", + "filter_type": "technical_docs" + }, + "output": f"Found {random.randint(8, 20)} relevant results for '{query}'. Top results include official documentation, tutorials, and best practice guides.", + "token_cost": random.randint(100, 300), + "success": True, + "time_cost": round(random.uniform(1.5, 3.5), 2) + }) + + # Scenario 2: Poor parameters (8 calls) + for i in range(8): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=30 + i * 3)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "query": f"test query {i}", + "max_results": 100, + "language": "unknown", + }, + "output": f"Warning: language 'unknown' not supported, using default. Query too generic, returning limited results. Found {random.randint(2, 5)} results.", + "token_cost": random.randint(50, 150), + "success": True, + "time_cost": round(random.uniform(2.0, 4.0), 2) + }) + + # Scenario 3: Timeouts or failures (5 calls) + for i in range(5): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=54 + i * 4)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "query": f"extremely complex query with many conditions {i}", + "max_results": 50, + "language": "en", + "filter_type": "all" + }, + "output": "Error: Request timeout after 10 seconds. Try simplifying the query or reducing max_results.", + "token_cost": 20, + "success": False, + "time_cost": 10.0 + }) + + # Scenario 4: Empty results (2 calls) + for i in range(2): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=74 + i * 5)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "query": f"xyzabc123nonexistent{i}", + "max_results": 10, + "language": "en", + }, + "output": "No results found for the given query. Please try different keywords.", + "token_cost": 30, + "success": True, + "time_cost": 1.2 + }) + + elif tool_name == "database_query": + # Scenario 1: Successful queries (12 calls) + for i in range(12): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=i * 3)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "table": f"users_{i % 3}", + "query": f"SELECT * FROM table WHERE id > {i * 10}", + "limit": random.randint(10, 100) + }, + "output": f"Query executed successfully. Returned {random.randint(5, 50)} rows in {round(random.uniform(0.1, 0.5), 3)}s.", + "token_cost": random.randint(20, 80), + "success": True, + "time_cost": round(random.uniform(0.1, 0.5), 3) + }) + + # Scenario 2: Slow queries (6 calls) + for i in range(6): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=36 + i * 4)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "table": "large_table", + "query": f"SELECT * FROM large_table WHERE name LIKE '%pattern%' ORDER BY created_at", + "limit": 1000 + }, + "output": f"Query executed but took longer than expected. Returned {random.randint(100, 1000)} rows. Consider adding indexes.", + "token_cost": random.randint(50, 150), + "success": True, + "time_cost": round(random.uniform(5.0, 10.0), 2) + }) + + # Scenario 3: Query errors (4 calls) + for i in range(4): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=60 + i * 5)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "table": "invalid_table", + "query": f"SELECT * FROM invalid_table WHERE bad_column = {i}", + "limit": 10 + }, + "output": f"Error: Table 'invalid_table' does not exist or column 'bad_column' not found.", + "token_cost": 10, + "success": False, + "time_cost": 0.05 + }) + + elif tool_name == "file_processor": + # Scenario 1: Successful file processing (10 calls) + file_types = ["csv", "json", "xml", "txt", "pdf"] + for i in range(10): + file_type = file_types[i % len(file_types)] + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=i * 4)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "file_path": f"/data/file_{i}.{file_type}", + "operation": "read", + "encoding": "utf-8" + }, + "output": f"Successfully processed {file_type.upper()} file. Size: {random.randint(100, 5000)}KB, Records: {random.randint(100, 10000)}", + "token_cost": random.randint(50, 200), + "success": True, + "time_cost": round(random.uniform(1.0, 3.0), 2) + }) + + # Scenario 2: Large file warnings (5 calls) + for i in range(5): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=40 + i * 6)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "file_path": f"/data/large_file_{i}.csv", + "operation": "read", + "encoding": "utf-8" + }, + "output": f"Warning: Large file detected ({random.randint(50, 200)}MB). Processing may take longer. Consider using streaming mode.", + "token_cost": random.randint(200, 500), + "success": True, + "time_cost": round(random.uniform(10.0, 30.0), 2) + }) + + # Scenario 3: File not found (3 calls) + for i in range(3): + tool_call_results.append({ + "create_time": (base_time + timedelta(hours=70 + i * 8)).strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "file_path": f"/invalid/path/file_{i}.txt", + "operation": "read", + "encoding": "utf-8" + }, + "output": "Error: File not found. Please check the file path and ensure the file exists.", + "token_cost": 10, + "success": False, + "time_cost": 0.01 + }) + + return tool_call_results[:count] From 6aa81559c020f86956b4ba039312913f1061f96c Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Tue, 21 Oct 2025 22:56:24 +0800 Subject: [PATCH 08/13] docs(mkdocs): add tool memory section and deployment instructions - Added new Tool Memory section with overview, retrieve ops, summary ops, and benchmark pages - Included SOP Memory section with making SOP memories page - Added deployment instructions for mkdocs including build, serve, and gh-deploy commands --- mkdocs.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 40ebab03..f885c4c6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -25,6 +25,12 @@ nav: - Retrieve Ops: task_memory/task_retrieve_ops.md - Summary Ops: task_memory/task_summary_ops.md + - Tool Memory: + - Overview: tool_memory/tool_memory.md + - Retrieve Ops: tool_memory/tool_retrieve_ops.md + - Summary Ops: tool_memory/tool_summary_ops.md + - Benchmark: tool_memory/tool_bench.md + - SOP Memory: - Making SOP Memories: sop_memory/making_sop_memories.md @@ -62,3 +68,9 @@ markdown_extensions: shadcn.extensions.echarts.alpha: shadcn.extensions.codexec: shadcn.extensions.iconify: + + +# pip install mkdocs-shadcn +# mkdocs build +# mkdocs serve +# mkdocs gh-deploy --force From 594495de9e48f70e14411de75a7d64e7c330b591 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 22 Oct 2025 11:19:43 +0800 Subject: [PATCH 09/13] feat(memory): add is_summarized flag to track tool call summarization - Add is_summarized boolean field to ToolCallResult schema - Implement logic to mark tool calls as summarized after processing - Skip summarization for tools with all recent calls already summarized - Update summary operation to process only tools with unsummarized calls - Enhance logging to show summarization and skip statistics - Modify response to include count of summarized vs skipped tools - Add documentation for smart skip logic and is_summarized feature - Include test scenarios for skip logic and incremental summarization --- docs/tool_memory/tool_summary_ops.md | 32 ++++ reme_ai/schema/memory.py | 1 + .../summary/tool/summary_tool_memory_op.py | 148 +++++++++++++++--- 3 files changed, 163 insertions(+), 18 deletions(-) diff --git a/docs/tool_memory/tool_summary_ops.md b/docs/tool_memory/tool_summary_ops.md index 5b949ed5..b02b2d09 100644 --- a/docs/tool_memory/tool_summary_ops.md +++ b/docs/tool_memory/tool_summary_ops.md @@ -33,18 +33,50 @@ Analyzes accumulated tool call history and generates comprehensive usage pattern ### Functionality - Retrieves existing tool memories from the vector store +- **Intelligently skips tools** where all recent calls have already been summarized (using `is_summarized` flag) - Analyzes the most recent N tool calls (configurable) - Calculates statistical metrics (success rate, average scores, costs) - Uses LLM to synthesize actionable usage guidelines - Updates tool memory content with generated insights +- **Marks processed calls** as summarized to avoid redundant processing in future runs + +### Smart Skip Logic + +To optimize costs and performance, `SummaryToolMemoryOp` tracks which tool call results have been included in a summary: + +- **Skip Condition**: If all recent N calls are already summarized (`is_summarized=True`), the tool is skipped entirely +- **Trigger Condition**: If at least 1 recent call is new (`is_summarized=False`), re-summarization is triggered +- **Automatic Marking**: After successful summarization, all processed calls are marked with `is_summarized=True` + +**Example Behavior**: +``` +Run 1: 30 new calls → Summarize all 30, mark as summarized +Run 2: Same 30 calls → Skip (all already summarized) ✓ Cost savings +Run 3: 30 old + 1 new → Re-summarize all 31, mark new call as summarized +``` + +This ensures summaries stay fresh while avoiding unnecessary LLM calls. ### Parameters - `op.summary_tool_memory_op.params.recent_call_count` (integer, default: `30`): - Number of most recent tool calls to analyze + - Also determines the window for checking summarization status - Focuses on recent usage patterns - `op.summary_tool_memory_op.params.summary_sleep_interval` (float, default: `1.0`): - Delay in seconds between concurrent summarizations - Prevents rate limiting when summarizing multiple tools +### Return Value + +The operation returns a response message indicating: +- Number of tools summarized (had new unsummarized calls) +- Number of tools skipped (all recent calls already summarized) + +Example: `"Successfully processed 5 tool memories: 2 summarized, 3 skipped (already up-to-date)"` + +### See Also + +For detailed implementation and examples, see [is_summarized Feature Documentation](is_summarized_feature.md). + diff --git a/reme_ai/schema/memory.py b/reme_ai/schema/memory.py index b0f63609..821f74e4 100644 --- a/reme_ai/schema/memory.py +++ b/reme_ai/schema/memory.py @@ -118,6 +118,7 @@ class ToolCallResult(BaseModel): summary: str = Field(default="", description="Brief summary of the tool call result") evaluation: str = Field(default="", description="Detailed evaluation for the tool invocation") score: float = Field(default=0, description="Score of the Evaluation (0.0 for failure, 1.0 for complete success)") + is_summarized: bool = Field(default=False, description="Whether this tool call has been included in a summary") metadata: dict = Field(default_factory=dict) diff --git a/reme_ai/summary/tool/summary_tool_memory_op.py b/reme_ai/summary/tool/summary_tool_memory_op.py index 1292eb38..70976bc5 100644 --- a/reme_ai/summary/tool/summary_tool_memory_op.py +++ b/reme_ai/summary/tool/summary_tool_memory_op.py @@ -62,6 +62,11 @@ class SummaryToolMemoryOp(BaseAsyncOp): logger.warning(f"No tool call results found for tool: {tool_memory.when_to_use}") return tool_memory + # Log how many unsummarized calls we're processing + unsummarized_count = sum(1 for call in recent_calls if not call.is_summarized) + logger.info(f"Summarizing tool {tool_memory.when_to_use}: " + f"{unsummarized_count}/{len(recent_calls)} unsummarized calls") + # Get statistics statistics = tool_memory.statistic(recent_frequency=self.recent_call_count) @@ -82,11 +87,16 @@ class SummaryToolMemoryOp(BaseAsyncOp): # Append statistics markdown to LLM result tool_memory.content = f"{llm_summary}\n\n## Statistics\n{statistics_md}" + # Mark all recent calls as summarized + for call in recent_calls: + call.is_summarized = True + # Update modified time tool_memory.update_modified_time() logger.info(f"Summarized tool {index}: tool_name={tool_memory.when_to_use}, " - f"content_length={len(tool_memory.content)}") + f"content_length={len(tool_memory.content)}, " + f"marked {len(recent_calls)} calls as summarized") return tool_memory # Call LLM to generate summary @@ -137,23 +147,46 @@ class SummaryToolMemoryOp(BaseAsyncOp): self.context.response.success = False return - # Concurrently summarize all tool memories - logger.info(f"Starting concurrent summarization of {len(matched_tool_memories)} tool memories") + # Check which tools need summarization + tools_need_summary = [] + tools_skipped = [] + + for tool_memory in matched_tool_memories: + recent_calls = tool_memory.tool_call_results[-self.recent_call_count:] + unsummarized_count = sum(1 for call in recent_calls if not call.is_summarized) + + if unsummarized_count > 0: + tools_need_summary.append(tool_memory) + else: + tools_skipped.append(tool_memory) + logger.info(f"Skipping tool {tool_memory.when_to_use}: all recent {len(recent_calls)} calls already summarized") - # 使用基类的 submit_async_task 提交所有总结任务 - for index, tool_memory in enumerate(matched_tool_memories): - self.submit_async_task(self._summarize_single_tool, tool_memory, index) + # Concurrently summarize tool memories that need it + if tools_need_summary: + logger.info(f"Starting concurrent summarization of {len(tools_need_summary)} tool memories") - # 使用基类的 join_async_task 等待所有任务完成 - # 注意: 基类已经过滤掉异常,返回的只包含成功的结果 - valid_summarized_memories = await self.join_async_task(return_exceptions=True) - logger.info(f"Completed summarization of {len(valid_summarized_memories)} tool memories") + # 使用基类的 submit_async_task 提交所有总结任务 + for index, tool_memory in enumerate(tools_need_summary): + self.submit_async_task(self._summarize_single_tool, tool_memory, index) + + # 使用基类的 join_async_task 等待所有任务完成 + # 注意: 基类已经过滤掉异常,返回的只包含成功的结果 + valid_summarized_memories = await self.join_async_task(return_exceptions=True) + logger.info(f"Completed summarization of {len(valid_summarized_memories)} tool memories") + else: + valid_summarized_memories = [] + logger.info("All tool memories are up-to-date, no summarization needed") + + # Combine summarized and skipped memories + all_memories = valid_summarized_memories + tools_skipped # Set response - self.context.response.answer = f"Successfully summarized {len(valid_summarized_memories)} tool memories" + self.context.response.answer = (f"Successfully processed {len(all_memories)} tool memories: " + f"{len(valid_summarized_memories)} summarized, " + f"{len(tools_skipped)} skipped (already up-to-date)") self.context.response.success = True - self.context.response.metadata["memory_list"] = valid_summarized_memories - self.context.response.metadata["deleted_memory_ids"] = [m.memory_id for m in valid_summarized_memories] + self.context.response.metadata["memory_list"] = all_memories + self.context.response.metadata["deleted_memory_ids"] = [m.memory_id for m in all_memories] # Log summary for each tool for memory in valid_summarized_memories: @@ -339,11 +372,90 @@ async def main(): logger.info(f" 平均耗时: {stats['avg_time_cost']:.2f}s") logger.info(f" 平均Token消耗: {stats['avg_token_cost']:.1f}") - logger.info(f"\n" + "=" * 60) - logger.info("Summary 生成的使用指南 (从30条分散记录中提取):") - logger.info("=" * 60) - logger.info(summarized_memory.content) - logger.info("=" * 60) + logger.info(f"\n" + "=" * 60) + logger.info("Summary 生成的使用指南 (从30条分散记录中提取):") + logger.info("=" * 60) + logger.info(summarized_memory.content) + logger.info("=" * 60) + + # ===== 第五步: 验证跳过逻辑 - 再次运行应该跳过 ===== + logger.info("\n" + "=" * 80) + logger.info("步骤5: 验证跳过逻辑 - 再次运行 SummaryToolMemoryOp 应该跳过已总结的记录") + logger.info("=" * 80) + + # 需要先保存更新后的 memory (带有 is_summarized=True 标记) + from reme_ai.vector_store.update_vector_store_op import UpdateVectorStoreOp + update_op = UpdateVectorStoreOp() + await update_op.async_call( + memory_list=summarized_memories, + workspace_id=workspace_id + ) + logger.info("✓ 已保存带有 is_summarized 标记的 tool memory") + + # 再次运行总结 + summary_op2 = SummaryToolMemoryOp( + recent_call_count=30, + summary_sleep_interval=0.5 + ) + await summary_op2.async_call( + tool_names=tool_name, + workspace_id=workspace_id + ) + + if summary_op2.context.response.success: + logger.info("✓ 第二次总结完成 (预期应该跳过)") + else: + logger.info(f"第二次总结失败: {summary_op2.context.response.answer}") + + # ===== 第六步: 添加新记录并验证增量总结 ===== + logger.info("\n" + "=" * 80) + logger.info("步骤6: 添加 1 条新记录,验证会触发重新总结") + logger.info("=" * 80) + + # 添加一条新的工具调用记录 + new_tool_call = { + "create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "tool_name": tool_name, + "input": { + "query": "FastAPI async best practices", + "max_results": 15, + "language": "en", + "filter_type": "technical_docs" + }, + "output": "Found 12 excellent resources on FastAPI async patterns including official docs, real-world examples, and performance tips.", + "token_cost": 180, + "success": True, + "time_cost": 2.1 + } + + # 使用 ParseToolCallResultOp 评估新记录 + new_pipeline = ParseToolCallResultOp(evaluation_sleep_interval=0.1) >> UpdateVectorStoreOp() + await new_pipeline.async_call( + tool_call_results=[new_tool_call], + tool_name=tool_name, + workspace_id=workspace_id + ) + logger.info("✓ 添加并评估了 1 条新记录") + + # 第三次运行总结 - 这次应该会执行 + summary_op3 = SummaryToolMemoryOp( + recent_call_count=30, + summary_sleep_interval=0.5 + ) + await summary_op3.async_call( + tool_names=tool_name, + workspace_id=workspace_id + ) + + if summary_op3.context.response.success: + logger.info("✓ 第三次总结完成 (因为有新记录)") + summarized_memories3 = summary_op3.context.response.metadata.get("memory_list", []) + if summarized_memories3: + summarized_memory3 = summarized_memories3[0] + unsummarized = sum(1 for call in summarized_memory3.tool_call_results[-30:] if not call.is_summarized) + logger.info(f" 最近30条中未总结的记录数: {unsummarized} (应该为 0)") + else: + logger.info(f"第三次总结失败: {summary_op3.context.response.answer}") if __name__ == "__main__": From 19735750c7587b2356a6ef8f957d882d2c90923c Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Wed, 22 Oct 2025 11:54:48 +0800 Subject: [PATCH 10/13] Apply suggestion from @gemini-code-assist[bot] Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2081e4db..1e265762 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Personal memory helps "**understand user preferences**", task memory helps agent ## 📰 Latest Updates - **[2025-10]** 🔧 Tool Memory support is now available! Enables data-driven tool selection and parameter optimization through historical performance tracking. Check out the [Tool Memory Guide](docs/tool_memory/tool_memory.md) and [benchmark results](docs/tool_memory/tool_bench.md). -- **[2025-09-25]** 🎉 ReMe is exploring the directions of tool memory and Personal Memory Application/Agent. +- **[2025-09-25]** 🎉 ReMe is now exploring tool memory and its applications for Personal Memory Agents. - **[2025-09]** 🎉 ReMe v0.1.9 has been officially released, adding support for asynchronous operations. It has also been integrated into the memory service of agentscope-runtime. - **[2025-09]** 🎉 ReMe v0.1 officially released, integrating task memory and personal memory. If you want to use the From 1be95d0fce261c1e9ed637043842b7be4143c4ed Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 22 Oct 2025 11:56:04 +0800 Subject: [PATCH 11/13] docs(readme): remove outdated update entry - Removed September 25, 2025 entry about ReMe exploring tool memory- Kept October 2025 entry about Tool Memory support release- Maintained chronological order of latest updates --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 1e265762..b09e29f4 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,6 @@ Personal memory helps "**understand user preferences**", task memory helps agent ## 📰 Latest Updates - **[2025-10]** 🔧 Tool Memory support is now available! Enables data-driven tool selection and parameter optimization through historical performance tracking. Check out the [Tool Memory Guide](docs/tool_memory/tool_memory.md) and [benchmark results](docs/tool_memory/tool_bench.md). -- **[2025-09-25]** 🎉 ReMe is now exploring tool memory and its applications for Personal Memory Agents. - **[2025-09]** 🎉 ReMe v0.1.9 has been officially released, adding support for asynchronous operations. It has also been integrated into the memory service of agentscope-runtime. - **[2025-09]** 🎉 ReMe v0.1 officially released, integrating task memory and personal memory. If you want to use the From d3d928fff46b5ff86c8496fee05b9ffb1c584e09 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 22 Oct 2025 11:57:04 +0800 Subject: [PATCH 12/13] docs(readme): remove outdated benchmark result line- Removed inconsistent improvement data across epochs - Updated key findings to reflect current tool memory performance- Maintained reference links to detailed documentation and implementation --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index b09e29f4..aa57e57f 100644 --- a/README.md +++ b/README.md @@ -545,7 +545,6 @@ We evaluated Tool Memory effectiveness using a controlled benchmark with three m **Key Findings:** - Tool Memory enables data-driven tool selection based on historical performance - Success rates improved by ~15% with learned parameter configurations -- Consistent improvement across all epochs (9.90% → 17.39% → 17.13%) You can find more details in [tool_bench.md](docs/tool_memory/tool_bench.md) and the implementation at [run_reme_tool_bench.py](cookbook/tool_memory/run_reme_tool_bench.py). From d9938c6a43e6ba437849b89688c457360b0af4ee Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 22 Oct 2025 12:11:20 +0800 Subject: [PATCH 13/13] feat(docs): add tool memory documentation and update ReMe memory model - Introduce tool memory as a new memory type in ReMe framework - Update agent memory formula to include tool memory- Add comprehensive tool memory documentation with usage examples - Include tool memory API usage guides for Python, curl, and Node.js - Add tool memory benchmark results showing 14.88% improvement- Create new tool memory demo and benchmark resources - Update documentation structure to reflect three memory capabilities- Add TODO list with planned features including automatic tool exploration- Fix mermaid diagram numbering in tool memory documentation - Remove outdated latest updates section from main documentation - Update resource links to include tool memory references --- README.md | 4 +- README_ZH.md | 412 -------------------------------- docs/index.md | 190 +++++++++++++-- docs/todo.md | 25 ++ docs/tool_memory/tool_memory.md | 8 +- mkdocs.yml | 1 + 6 files changed, 199 insertions(+), 441 deletions(-) delete mode 100644 README_ZH.md create mode 100644 docs/todo.md diff --git a/README.md b/README.md index aa57e57f..db587c1a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -English | [**中文**](./README_ZH.md) -

ReMe Logo

@@ -551,6 +549,8 @@ You can find more details in [tool_bench.md](docs/tool_memory/tool_bench.md) and ## 📚 Resources - **[Quick Start](./cookbook/simple_demo)**: Get started quickly with practical examples + - [Tool Memory Demo](cookbook/simple_demo/use_tool_memory_demo.py): Complete lifecycle demonstration of tool memory + - [Tool Memory Benchmark](cookbook/tool_memory/run_reme_tool_bench.py): Evaluate tool memory effectiveness - **[Vector Storage Setup](docs/vector_store_api_guide.md)**: Configure local/vector databases and usage - **[MCP Guide](docs/mcp_quick_start.md)**: Create MCP services - **[Personal Memory](docs/personal_memory)**, **[Task Memory](docs/task_memory)** & **[Tool Memory](docs/tool_memory)**: Operators used in personal memory, task memory and tool memory. You can modify the config to customize the pipelines. diff --git a/README_ZH.md b/README_ZH.md deleted file mode 100644 index 3e3b15bb..00000000 --- a/README_ZH.md +++ /dev/null @@ -1,412 +0,0 @@ -中文 | [**English**](./README.md) - -

- ReMe Logo -

- -

- Python Version - PyPI Version - License - GitHub Stars -

- -

- ReMe (formerly MemoryScope)：为Agent设计的记忆管理框架
- Remember Me, Refine Me. -

- ---- -ReMe为AI智能体提供了统一的记忆与经验系统——在跨用户、跨任务、跨智能体下抽取、复用和分享记忆的能力。 - -``` -个性化记忆 (Personal Memory) + 任务经验 (Task Memory)= agent记忆 -``` - -个性化记忆能够"**理解用户偏好**"，任务记忆让agent"**做得更好**"， - ---- - -## 📰 最新动态 - -- **[2025-09]** 🎉 ReMe v0.1 - 正式发布，整合任务记忆与个人记忆。如果想使用原始的memoryscope项目，你可以在[MemoryScope](https://github.com/modelscope/Reme/tree/memoryscope_branch) - 中找到。 -- **[2025-09]** 🧪 我们在appworld, bfcl(v3) - 以及frozenlake环境验证了任务记忆抽取与复用在Agent中的效果，更多信息请查看 [appworld exp](docs/cookbook/appworld/quickstart.md), [bfcl exp](docs/cookbook/bfcl/quickstart.md) - 和 [frozenlake exp](docs/cookbook/frozenlake/quickstart.md)。 -- **[2025-08]** 🚀 MCP协议支持已上线-> [MCP指南](docs/mcp_quick_start.md)。 -- **[2025-06]** 🚀 多后端向量存储支持 (Elasticsearch & ChromaDB) -> [向量数据库指南](docs/vector_store_api_guide.md)。 -- **[2024-09]** 🧠 [MemoryScope](https://github.com/modelscope/Reme/tree/memoryscope_branch) v0.1 发布，个性化和时间感知的记忆存储与使用。 - ---- - -## ✨ 功能设计 - -

- ReMe Logo -

- -ReMe整合两种互补的记忆能力： - -#### 🧠 **任务经验 (Task Memory/Experience)** -跨智能体复用的程序性知识 -- **成功模式识别**：识别有效策略并理解其根本原理 -- **失败分析学习**：从错误中学习，避免重复同样的问题 -- **对比模式**：不同采样轨迹通过对比得到更有价值的经验 -- **验证模式**：经过验证模块确认抽取记忆的有效性 - -你可以从[task memory](docs/task_memory/task_memory.md)了解更多如何使用task memory的方法 - -#### 👤 **个人记忆 (Personal Memory)** -特定用户的情境化记忆 -- **个体偏好**：用户的习惯、偏好和交互风格 -- **情境适应**：基于时间和上下文的智能记忆管理 -- **渐进学习**：通过长期交互逐步建立深度理解 -- **时间感知**：检索和整合时都具备时间敏感性 - -你可以从[personal memory](docs/personal_memory/personal_memory.md)了解更多如何使用personal memory的方法 - - ---- - -## 🛠️ 安装 - -### 从PyPI安装（推荐） -```bash -pip install reme-ai -``` - -### 从源码安装 -```bash -git clone https://github.com/modelscope/ReMe.git -cd ReMe -pip install . -``` - -### 环境配置 - -复制 `example.env` 为 .env并修改其中对应参数： - -```bash -FLOW_APP_NAME=ReMe -FLOW_LLM_API_KEY=sk-xxxx -FLOW_LLM_BASE_URL=https://xxxx/v1 -FLOW_EMBEDDING_API_KEY=sk-xxxx -FLOW_EMBEDDING_BASE_URL=https://xxxx/v1 -``` - ---- - -## 🚀 快速开始 - -### HTTP服务启动 -```bash -reme \ - backend=http \ - http.port=8002 \ - llm.default.model_name=qwen3-30b-a3b-thinking-2507 \ - embedding_model.default.model_name=text-embedding-v4 \ - vector_store.default.backend=local -``` - -### MCP服务器支持 -```bash -reme \ - backend=mcp \ - mcp.transport=stdio \ - llm.default.model_name=qwen3-30b-a3b-thinking-2507 \ - embedding_model.default.model_name=text-embedding-v4 \ - vector_store.default.backend=local -``` - -### 核心API使用 - -#### 任务记忆管理 -```python -import requests - -# 经验总结器：从执行轨迹学习 -response = requests.post("http://localhost:8002/summary_task_memory", json={ - "workspace_id": "task_workspace", - "trajectories": [ - {"messages": [{"role": "user", "content": "帮我制定项目计划"}], "score": 1.0} - ] -}) - -# 经验检索器：获取相关经验 -response = requests.post("http://localhost:8002/retrieve_task_memory", json={ - "workspace_id": "task_workspace", - "query": "如何高效管理项目进度？", - "top_k": 1 -}) -``` - -
-curl 版本 - -```bash -# 经验总结器：从执行轨迹学习 -curl -X POST http://localhost:8002/summary_task_memory \ - -H "Content-Type: application/json" \ - -d '{ - "workspace_id": "task_workspace", - "trajectories": [ - {"messages": [{"role": "user", "content": "帮我制定项目计划"}], "score": 1.0} - ] - }' - -# 经验检索器：获取相关经验 -curl -X POST http://localhost:8002/retrieve_task_memory \ - -H "Content-Type: application/json" \ - -d '{ - "workspace_id": "task_workspace", - "query": "如何高效管理项目进度？", - "top_k": 1 - }' -``` -
- -
-Node.js 版本 - -```javascript -// 经验总结器：从执行轨迹学习 -fetch("http://localhost:8002/summary_task_memory", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - workspace_id: "task_workspace", - trajectories: [ - {messages: [{role: "user", content: "帮我制定项目计划"}], score: 1.0} - ] - }) -}) -.then(response => response.json()) -.then(data => console.log(data)); - -// 经验检索器：获取相关经验 -fetch("http://localhost:8002/retrieve_task_memory", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - workspace_id: "task_workspace", - query: "如何高效管理项目进度？", - top_k: 1 - }) -}) -.then(response => response.json()) -.then(data => console.log(data)); -``` -
- -#### 个人记忆管理 -```python -# 记忆整合：从用户交互中学习 -response = requests.post("http://localhost:8002/summary_personal_memory", json={ - "workspace_id": "task_workspace", - "trajectories": [ - {"messages": - [ - {"role": "user", "content": "我喜欢早上喝咖啡工作"}, - {"role": "assistant", "content": "了解，您习惯早上用咖啡提神来开始工作"} - ] - } - ] -}) - -# 记忆检索：获取个人记忆片段 -response = requests.post("http://localhost:8002/retrieve_personal_memory", json={ - "workspace_id": "task_workspace", - "query": "用户的工作习惯是什么？", - "top_k": 5 -}) -``` - -
-curl 版本 - -```bash -# 记忆整合：从用户交互中学习 -curl -X POST http://localhost:8002/summary_personal_memory \ - -H "Content-Type: application/json" \ - -d '{ - "workspace_id": "task_workspace", - "trajectories": [ - {"messages": [ - {"role": "user", "content": "我喜欢早上喝咖啡工作"}, - {"role": "assistant", "content": "了解，您习惯早上用咖啡提神来开始工作"} - ]} - ] - }' - -# 记忆检索：获取个人记忆片段 -curl -X POST http://localhost:8002/retrieve_personal_memory \ - -H "Content-Type: application/json" \ - -d '{ - "workspace_id": "task_workspace", - "query": "用户的工作习惯是什么？", - "top_k": 5 - }' -``` -
- -
-Node.js 版本 - -```javascript -// 记忆整合：从用户交互中学习 -fetch("http://localhost:8002/summary_personal_memory", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - workspace_id: "task_workspace", - trajectories: [ - {messages: [ - {role: "user", content: "我喜欢早上喝咖啡工作"}, - {role: "assistant", content: "了解，您习惯早上用咖啡提神来开始工作"} - ]} - ] - }) -}) -.then(response => response.json()) -.then(data => console.log(data)); - -// 记忆检索：获取个人记忆片段 -fetch("http://localhost:8002/retrieve_personal_memory", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - workspace_id: "task_workspace", - query: "用户的工作习惯是什么？", - top_k: 5 - }) -}) -.then(response => response.json()) -.then(data => console.log(data)); -``` -
- ---- - -## 📦 即用型经验库 - -ReMe提供预构建的经验库，智能体可以立即使用经过验证的最佳实践： - -### 可用经验库 - -- **`appworld.jsonl`**：Appworld智能体交互的记忆库，涵盖复杂任务规划和执行模式 -- **`bfcl_v3.jsonl`**：BFCL工具调用的工作记忆库 - -### 快速使用 -```python -# 加载预构建经验 -response = requests.post("http://localhost:8002/vector_store", json={ - "workspace_id": "appworld", - "action": "load", - "path": "./docs/library/" -}) - -# 查询相关经验 -response = requests.post("http://localhost:8002/retrieve_task_memory", json={ - "workspace_id": "appworld", - "query": "如何导航到设置并更新用户资料？", - "top_k": 1 -}) -``` - -## 🧪 实验 - -### 🌍 [Appworld 实验](docs/cookbook/appworld/quickstart.md) - -我们在 Appworld 上使用 qwen3-8b 测试 ReMe： - -| 方法 | pass@1 | pass@2 | pass@4 | -|--------------|-------------------|-------------------|-------------------| -| without ReMe | 0.083 | 0.140 | 0.228 | -| with ReMe | 0.109 **(+2.6%)** | 0.175 **(+3.5%)** | 0.281 **(+5.3%)** | - -Pass@K 衡量的是在生成的 K 个样本中，至少有一个成功完成任务（score=1）的概率。 -当前实验使用的是一个内部的 AppWorld 环境，可能存在轻微差异。 - -你可以在 [quickstart.md](docs/cookbook/appworld/quickstart.md) 中找到复现实验的更多细节。 - - -### 🧊 [Frozenlake 实验](docs/cookbook/frozenlake/quickstart.md) - -| 不使用ReMe | 使用ReMe | -|:--------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------:| -|

GIF 1

|

GIF 2

| - -我们在 100 个随机 frozenlake 地图上使用 qwen3-8b 进行测试： - -| 方法 | pass rate | -|--------------|------------------| -| without ReMe | 0.66 | -| with ReMe | 0.72 **(+6.0%)** | - -你可以在 [quickstart.md](docs/cookbook/frozenlake/quickstart.md) 中找到复现实验的更多细节。 - -### 🔧 [BFCL-V3 实验](docs/cookbook/bfcl/quickstart.md) - -我们在 BFCL-V3 multi-turn-base (随机划分50train/150val) 上使用 qwen3-8b 测试 ReMe： - -| 方法 | pass@1 | pass@2 | pass@4 | -|--------------|---------------------|---------------------|---------------------| -| without ReMe | 0.2472 | 0.2733 | 0.2922 | -| with ReMe | 0.3061 **(+5.89%)** | 0.3500 **(+7.67%)** | 0.3888 **(+9.66%)** | - -## 📚 相关资源 - -- **[快速开始](./cookbook/simple_demo)**：通过实际示例快速上手 -- **[向量存储设置](docs/vector_store_api_guide.md)**：配置本地/向量数据库以及使用 -- **[mcp指南](docs/mcp_quick_start.md)**：创建mcp服务 -- **[个性化记忆](docs/personal_memory)** 与 [任务记忆](docs/task_memory): 个性化记忆与任务记忆中分别使用的算子及其含义，你可以修改config以自定义链路 -- **[示例集合](./cookbook)**：实际用例和最佳实践 - ---- - -## 🤝 贡献 - -我们相信最好的记忆系统来自集体智慧。欢迎贡献👉[指南](docs/contribution.md)： - -### 代码贡献 -- 新操作和工具开发 -- 后端实现和优化 -- API增强和新端点 - -### 文档改进 -- 使用示例和教程 -- 最佳实践指南 - ---- - -## 📄 引用 - -```bibtex -@software{ReMe2025, - title = {ReMe: Memory Management Framework for Agents}, - author = {Li Yu, Jiaji Deng, Zouying Cao}, - url = {https://github.com/modelscope/ReMe}, - year = {2025} -} -``` - ---- - -## ⚖️ 许可证 - -本项目采用Apache License 2.0许可证 - 详情请参阅[LICENSE](./LICENSE)文件。 - ---- - -## Star 历史 -[![Star History Chart](https://api.star-history.com/svg?repos=modelscope/ReMe&type=Date)](https://www.star-history.com/#modelscope/ReMe&Date) diff --git a/docs/index.md b/docs/index.md index 75a9894a..222d74a2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,29 +26,10 @@ ReMe provides AI agents with a unified memory system—enabling the ability to e users, tasks, and agents. ``` -Personal Memory + Task Memory = Agent Memory +Personal Memory + Task Memory + Tool Memory = Agent Memory ``` -Personal memory helps "**understand user preferences**", while task memory helps agents "**perform better**". - ---- - -## 📰 Latest Updates - -- **[2025-09]** 🎉 ReMe v0.1.8 has been officially released, adding support for asynchronous operations. It has also been - integrated into the memory service of agentscope-runtime. -- **[2025-09]** 🎉 ReMe v0.1 officially released, integrating task memory and personal memory. If you want to use the - original memoryscope project, you can find it - in [MemoryScope](https://github.com/modelscope/Reme/tree/memoryscope_branch). -- **[2025-09]** 🧪 We validated the effectiveness of task memory extraction and reuse in agents in appworld, bfcl(v3), - and frozenlake environments. For more information, - check [appworld exp](./cookbook/appworld/quickstart.md), [bfcl exp](./cookbook/bfcl/quickstart.md), - and [frozenlake exp](./cookbook/frozenlake/quickstart.md). -- **[2025-08]** 🚀 MCP protocol support is now available -> [MCP Quick Start](mcp_quick_start.md). -- **[2025-06]** 🚀 Multiple backend vector storage support (Elasticsearch & - ChromaDB) -> [Vector DB quick start](vector_store_api_guide.md). -- **[2024-09]** 🧠 [MemoryScope](https://github.com/modelscope/Reme/tree/memoryscope_branch) v0.1 released, - personalized and time-aware memory storage and usage. +Personal memory helps "**understand user preferences**", task memory helps agents "**perform better**", and tool memory enables "**smarter tool usage**". --- @@ -58,7 +39,7 @@ Personal memory helps "**understand user preferences**", while task memory helps ReMe Logo

-ReMe integrates two complementary memory capabilities: +ReMe integrates three complementary memory capabilities: #### 🧠 **Task Memory/Experience** @@ -82,6 +63,17 @@ Contextualized memory for specific users Learn more about how to use personal memory from [personal memory](personal_memory/personal_memory.md) +#### 🔧 **Tool Memory** + +Data-driven tool selection and usage optimization + +- **Historical Performance Tracking**: Success rates, execution times, and token costs from real usage +- **LLM-as-Judge Evaluation**: Qualitative insights on why tools succeed or fail +- **Parameter Optimization**: Learn optimal parameter configurations from successful calls +- **Dynamic Guidelines**: Transform static tool descriptions into living, learned manuals + +Learn more about how to use tool memory from [tool memory](tool_memory/tool_memory.md) + --- ## 🛠️ Installation @@ -320,6 +312,140 @@ fetch("http://localhost:8002/retrieve_personal_memory", { +#### Tool Memory Management + +```python +import requests + +# Record tool execution results +response = requests.post("http://localhost:8002/add_tool_call_result", json={ + "workspace_id": "tool_workspace", + "tool_call_results": [ + { + "create_time": "2025-10-21 10:30:00", + "tool_name": "web_search", + "input": {"query": "Python asyncio tutorial", "max_results": 10}, + "output": "Found 10 relevant results...", + "token_cost": 150, + "success": True, + "time_cost": 2.3 + } + ] +}) + +# Generate usage guidelines from history +response = requests.post("http://localhost:8002/summary_tool_memory", json={ + "workspace_id": "tool_workspace", + "tool_names": "web_search" +}) + +# Retrieve tool guidelines before use +response = requests.post("http://localhost:8002/retrieve_tool_memory", json={ + "workspace_id": "tool_workspace", + "tool_names": "web_search" +}) +``` + +
+curl version + +```bash +# Record tool execution results +curl -X POST http://localhost:8002/add_tool_call_result \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "tool_workspace", + "tool_call_results": [ + { + "create_time": "2025-10-21 10:30:00", + "tool_name": "web_search", + "input": {"query": "Python asyncio tutorial", "max_results": 10}, + "output": "Found 10 relevant results...", + "token_cost": 150, + "success": true, + "time_cost": 2.3 + } + ] + }' + +# Generate usage guidelines from history +curl -X POST http://localhost:8002/summary_tool_memory \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "tool_workspace", + "tool_names": "web_search" + }' + +# Retrieve tool guidelines before use +curl -X POST http://localhost:8002/retrieve_tool_memory \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "tool_workspace", + "tool_names": "web_search" + }' +``` + +
+ +
+Node.js version + +```javascript +// Record tool execution results +fetch("http://localhost:8002/add_tool_call_result", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + workspace_id: "tool_workspace", + tool_call_results: [ + { + create_time: "2025-10-21 10:30:00", + tool_name: "web_search", + input: {query: "Python asyncio tutorial", max_results: 10}, + output: "Found 10 relevant results...", + token_cost: 150, + success: true, + time_cost: 2.3 + } + ] + }) +}) +.then(response => response.json()) +.then(data => console.log(data)); + +// Generate usage guidelines from history +fetch("http://localhost:8002/summary_tool_memory", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + workspace_id: "tool_workspace", + tool_names: "web_search" + }) +}) +.then(response => response.json()) +.then(data => console.log(data)); + +// Retrieve tool guidelines before use +fetch("http://localhost:8002/retrieve_tool_memory", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + workspace_id: "tool_workspace", + tool_names: "web_search" + }) +}) +.then(response => response.json()) +.then(data => console.log(data)); +``` + +
+ --- ## 📦 Ready-to-Use Libraries @@ -391,12 +517,30 @@ We tested ReMe on BFCL-V3 multi-turn-base (randomly split 50train/150val) using | without ReMe | 0.2472 | 0.2733 | 0.2922 | | with ReMe | 0.3061 **(+5.89%)** | 0.3500 **(+7.67%)** | 0.3888 **(+9.66%)** | +### 🛠️ [Tool Memory Benchmark](tool_memory/tool_bench.md) + +We evaluated Tool Memory effectiveness using a controlled benchmark with three mock search tools using Qwen3-30B-Instruct: + +| Scenario | Avg Score | Improvement | +|-----------------------|-----------|--------------------| +| Train (No Memory) | 0.650 | - | +| Test (No Memory) | 0.672 | Baseline | +| **Test (With Memory)** | **0.772** | **+14.88%** | + +**Key Findings:** +- Tool Memory enables data-driven tool selection based on historical performance +- Success rates improved by ~15% with learned parameter configurations + +You can find more details in [tool_bench.md](tool_memory/tool_bench.md) and the implementation at [run_reme_tool_bench.py](https://github.com/modelscope/ReMe/tree/main/cookbook/tool_memory/run_reme_tool_bench.py). + ## 📚 Resources - **[Quick Start](https://github.com/modelscope/ReMe/tree/main/cookbook/simple_demo)**: Get started quickly with practical examples + - [Tool Memory Demo](https://github.com/modelscope/ReMe/tree/main/cookbook/simple_demo/use_tool_memory_demo.py): Complete lifecycle demonstration of tool memory + - [Tool Memory Benchmark](https://github.com/modelscope/ReMe/tree/main/cookbook/tool_memory/run_reme_tool_bench.py): Evaluate tool memory effectiveness - **[Vector Storage Setup](vector_store_api_guide.md)**: Configure local/vector databases and usage - **[MCP Guide](mcp_quick_start.md)**: Create MCP services -- **[personal memory](personal_memory/personal_memory.md)** & **[task memory](task_memory/task_memory.md)** : Operators used in personal memory and task memory, You can modify the config to customize the pipelines. +- **[Personal Memory](personal_memory/personal_memory.md)**, **[Task Memory](task_memory/task_memory.md)** & **[Tool Memory](tool_memory/tool_memory.md)**: Operators used in personal memory, task memory and tool memory. You can modify the config to customize the pipelines. - **[Example Collection](./cookbook/appworld/quickstart.md)**: Real use cases and best practices --- diff --git a/docs/todo.md b/docs/todo.md new file mode 100644 index 00000000..8e44c695 --- /dev/null +++ b/docs/todo.md @@ -0,0 +1,25 @@ +# TODO + +## Planned Features + +### 1. Automatic Tool Exploration Mode +Add an automatic tool exploration mode that generates tool memory by: +- Automatically discovering and testing available tools +- Learning tool usage patterns and best practices +- Building a comprehensive tool memory database from exploration results + +### 2. Desktop Pet Personal Assistant +Build a desktop pet personal assistant with: +- Interactive desktop companion interface +- Personalized assistance capabilities +- Integration with ReMe's memory system + +### 3. Task Memory Research Implementation +We are currently working on implementing features based on task memory research papers. Coming soon. + +### 4. Mem-Agent Exploration +We are exploring mem-agent to implement agentic memory pathways: +- Investigating agent-driven memory management +- Developing autonomous memory retrieval and storage mechanisms +- Building more intelligent memory update strategies + diff --git a/docs/tool_memory/tool_memory.md b/docs/tool_memory/tool_memory.md index 8c6d2fe9..3d1df28c 100644 --- a/docs/tool_memory/tool_memory.md +++ b/docs/tool_memory/tool_memory.md @@ -194,14 +194,14 @@ Tool Memory operates through three complementary operations that work together t ```mermaid graph LR - A[Agent] -->|1. retrieve_tool_memory| B[(Vector Store)] + A[Agent] -->|1 retrieve_tool_memory| B[(Vector Store)] B -->|Guidelines| A - A -->|Execute Tool| C[Tool] + A -->|2 Execute Tool| C[Tool] C -->|Result| A - A -->|add_tool_call_result| D[LLM Evaluate] + A -->|3 add_tool_call_result| D[LLM Evaluate] D -->|Store| B B -->|Periodic| E[summary_tool_memory] - E -->|Update Guidelines| B + E -->|4 Update Guidelines| B ``` ### Operation Flow diff --git a/mkdocs.yml b/mkdocs.yml index f885c4c6..e1839772 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,6 +43,7 @@ nav: - BFCL: cookbook/bfcl/quickstart.md - FrozenLake: cookbook/frozenlake/quickstart.md + - TODO: todo.md - Contributions: contribution.md