mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-10-07 03:00:27 +00:00
Jiaji
GitHub
commit
691eec954a
42 changed files with 4569 additions and 464 deletions
176
README.md
176
README.md
|
|
@ -1,5 +1,3 @@
|
|||
English | [**中文**](./README_ZH.md)
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/figure/reme_logo.png" alt="ReMe Logo" width="50%">
|
||||
</p>
|
||||
|
|
@ -21,16 +19,17 @@ ReMe provides AI agents with a unified memory system—enabling the ability to e
|
|||
users, tasks, and agents.
|
||||
|
||||
```
|
||||
Personal Memory + Task Memory = Agent Memory
|
||||
Personal Memory + Task Memory + Tool Memory = Agent Memory
|
||||
```
|
||||
|
||||
Personal memory helps "**understand user preferences**", while task memory helps agents "**perform better**".
|
||||
Personal memory helps "**understand user preferences**", task memory helps agents "**perform better**", and tool memory enables "**smarter tool usage**".
|
||||
|
||||
---
|
||||
|
||||
## 📰 Latest Updates
|
||||
|
||||
- **[2025-09]** 🎉 ReMe v0.1.8 has been officially released, adding support for asynchronous operations. It has also been
|
||||
- **[2025-10]** 🔧 Tool Memory support is now available! Enables data-driven tool selection and parameter optimization through historical performance tracking. Check out the [Tool Memory Guide](docs/tool_memory/tool_memory.md) and [benchmark results](docs/tool_memory/tool_bench.md).
|
||||
- **[2025-09]** 🎉 ReMe v0.1.9 has been officially released, adding support for asynchronous operations. It has also been
|
||||
integrated into the memory service of agentscope-runtime.
|
||||
- **[2025-09]** 🎉 ReMe v0.1 officially released, integrating task memory and personal memory. If you want to use the
|
||||
original memoryscope project, you can find it
|
||||
|
|
@ -53,7 +52,7 @@ Personal memory helps "**understand user preferences**", while task memory helps
|
|||
<img src="docs/figure/reme_structure.jpg" alt="ReMe Logo" width="100%">
|
||||
</p>
|
||||
|
||||
ReMe integrates two complementary memory capabilities:
|
||||
ReMe integrates three complementary memory capabilities:
|
||||
|
||||
#### 🧠 **Task Memory/Experience**
|
||||
|
||||
|
|
@ -77,6 +76,17 @@ Contextualized memory for specific users
|
|||
|
||||
Learn more about how to use personal memory from [personal memory](docs/personal_memory/personal_memory.md)
|
||||
|
||||
#### 🔧 **Tool Memory**
|
||||
|
||||
Data-driven tool selection and usage optimization
|
||||
|
||||
- **Historical Performance Tracking**: Success rates, execution times, and token costs from real usage
|
||||
- **LLM-as-Judge Evaluation**: Qualitative insights on why tools succeed or fail
|
||||
- **Parameter Optimization**: Learn optimal parameter configurations from successful calls
|
||||
- **Dynamic Guidelines**: Transform static tool descriptions into living, learned manuals
|
||||
|
||||
Learn more about how to use tool memory from [tool memory](docs/tool_memory/tool_memory.md)
|
||||
|
||||
---
|
||||
|
||||
## 🛠️ Installation
|
||||
|
|
@ -315,6 +325,140 @@ fetch("http://localhost:8002/retrieve_personal_memory", {
|
|||
|
||||
</details>
|
||||
|
||||
#### Tool Memory Management
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
# Record tool execution results
|
||||
response = requests.post("http://localhost:8002/add_tool_call_result", json={
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_call_results": [
|
||||
{
|
||||
"create_time": "2025-10-21 10:30:00",
|
||||
"tool_name": "web_search",
|
||||
"input": {"query": "Python asyncio tutorial", "max_results": 10},
|
||||
"output": "Found 10 relevant results...",
|
||||
"token_cost": 150,
|
||||
"success": True,
|
||||
"time_cost": 2.3
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
# Generate usage guidelines from history
|
||||
response = requests.post("http://localhost:8002/summary_tool_memory", json={
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_names": "web_search"
|
||||
})
|
||||
|
||||
# Retrieve tool guidelines before use
|
||||
response = requests.post("http://localhost:8002/retrieve_tool_memory", json={
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_names": "web_search"
|
||||
})
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary>curl version</summary>
|
||||
|
||||
```bash
|
||||
# Record tool execution results
|
||||
curl -X POST http://localhost:8002/add_tool_call_result \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_call_results": [
|
||||
{
|
||||
"create_time": "2025-10-21 10:30:00",
|
||||
"tool_name": "web_search",
|
||||
"input": {"query": "Python asyncio tutorial", "max_results": 10},
|
||||
"output": "Found 10 relevant results...",
|
||||
"token_cost": 150,
|
||||
"success": true,
|
||||
"time_cost": 2.3
|
||||
}
|
||||
]
|
||||
}'
|
||||
|
||||
# Generate usage guidelines from history
|
||||
curl -X POST http://localhost:8002/summary_tool_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_names": "web_search"
|
||||
}'
|
||||
|
||||
# Retrieve tool guidelines before use
|
||||
curl -X POST http://localhost:8002/retrieve_tool_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_names": "web_search"
|
||||
}'
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Node.js version</summary>
|
||||
|
||||
```javascript
|
||||
// Record tool execution results
|
||||
fetch("http://localhost:8002/add_tool_call_result", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workspace_id: "tool_workspace",
|
||||
tool_call_results: [
|
||||
{
|
||||
create_time: "2025-10-21 10:30:00",
|
||||
tool_name: "web_search",
|
||||
input: {query: "Python asyncio tutorial", max_results: 10},
|
||||
output: "Found 10 relevant results...",
|
||||
token_cost: 150,
|
||||
success: true,
|
||||
time_cost: 2.3
|
||||
}
|
||||
]
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => console.log(data));
|
||||
|
||||
// Generate usage guidelines from history
|
||||
fetch("http://localhost:8002/summary_tool_memory", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workspace_id: "tool_workspace",
|
||||
tool_names: "web_search"
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => console.log(data));
|
||||
|
||||
// Retrieve tool guidelines before use
|
||||
fetch("http://localhost:8002/retrieve_tool_memory", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workspace_id: "tool_workspace",
|
||||
tool_names: "web_search"
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => console.log(data));
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
## 📦 Ready-to-Use Libraries
|
||||
|
|
@ -386,12 +530,30 @@ We tested ReMe on BFCL-V3 multi-turn-base (randomly split 50train/150val) using
|
|||
| without ReMe | 0.2472 | 0.2733 | 0.2922 |
|
||||
| with ReMe | 0.3061 **(+5.89%)** | 0.3500 **(+7.67%)** | 0.3888 **(+9.66%)** |
|
||||
|
||||
### 🛠️ [Tool Memory Benchmark](docs/tool_memory/tool_bench.md)
|
||||
|
||||
We evaluated Tool Memory effectiveness using a controlled benchmark with three mock search tools using Qwen3-30B-Instruct:
|
||||
|
||||
| Scenario | Avg Score | Improvement |
|
||||
|-----------------------|-----------|--------------------|
|
||||
| Train (No Memory) | 0.650 | - |
|
||||
| Test (No Memory) | 0.672 | Baseline |
|
||||
| **Test (With Memory)** | **0.772** | **+14.88%** |
|
||||
|
||||
**Key Findings:**
|
||||
- Tool Memory enables data-driven tool selection based on historical performance
|
||||
- Success rates improved by ~15% with learned parameter configurations
|
||||
|
||||
You can find more details in [tool_bench.md](docs/tool_memory/tool_bench.md) and the implementation at [run_reme_tool_bench.py](cookbook/tool_memory/run_reme_tool_bench.py).
|
||||
|
||||
## 📚 Resources
|
||||
|
||||
- **[Quick Start](./cookbook/simple_demo)**: Get started quickly with practical examples
|
||||
- [Tool Memory Demo](cookbook/simple_demo/use_tool_memory_demo.py): Complete lifecycle demonstration of tool memory
|
||||
- [Tool Memory Benchmark](cookbook/tool_memory/run_reme_tool_bench.py): Evaluate tool memory effectiveness
|
||||
- **[Vector Storage Setup](docs/vector_store_api_guide.md)**: Configure local/vector databases and usage
|
||||
- **[MCP Guide](docs/mcp_quick_start.md)**: Create MCP services
|
||||
- **[personal memory](docs/personal_memory)** & **[task memory](docs/task_memory)** : Operators used in personal memory and task memory, You can modify the config to customize the pipelines.
|
||||
- **[Personal Memory](docs/personal_memory)**, **[Task Memory](docs/task_memory)** & **[Tool Memory](docs/tool_memory)**: Operators used in personal memory, task memory and tool memory. You can modify the config to customize the pipelines.
|
||||
- **[Example Collection](./cookbook)**: Real use cases and best practices
|
||||
|
||||
---
|
||||
|
|
|
|||
412
README_ZH.md
412
README_ZH.md
|
|
@ -1,412 +0,0 @@
|
|||
中文 | [**English**](./README.md)
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/figure/reme_logo.png" alt="ReMe Logo" width="50%">
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/badge/python-3.12+-blue" alt="Python Version"></a>
|
||||
<a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/badge/pypi-v0.1-blue?logo=pypi" alt="PyPI Version"></a>
|
||||
<a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-black" alt="License"></a>
|
||||
<a href="https://github.com/modelscope/ReMe"><img src="https://img.shields.io/github/stars/modelscope/ReMe?style=social" alt="GitHub Stars"></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<strong>ReMe (formerly MemoryScope)：为Agent设计的记忆管理框架</strong><br>
|
||||
<em>Remember Me, Refine Me.</em>
|
||||
</p>
|
||||
|
||||
---
|
||||
ReMe为AI智能体提供了统一的记忆与经验系统——在跨用户、跨任务、跨智能体下抽取、复用和分享记忆的能力。
|
||||
|
||||
```
|
||||
个性化记忆 (Personal Memory) + 任务经验 (Task Memory)= agent记忆
|
||||
```
|
||||
|
||||
个性化记忆能够"**理解用户偏好**"，任务记忆让agent"**做得更好**"，
|
||||
|
||||
---
|
||||
|
||||
## 📰 最新动态
|
||||
|
||||
- **[2025-09]** 🎉 ReMe v0.1
|
||||
正式发布，整合任务记忆与个人记忆。如果想使用原始的memoryscope项目，你可以在[MemoryScope](https://github.com/modelscope/Reme/tree/memoryscope_branch)
|
||||
中找到。
|
||||
- **[2025-09]** 🧪 我们在appworld, bfcl(v3)
|
||||
以及frozenlake环境验证了任务记忆抽取与复用在Agent中的效果，更多信息请查看 [appworld exp](docs/cookbook/appworld/quickstart.md), [bfcl exp](docs/cookbook/bfcl/quickstart.md)
|
||||
和 [frozenlake exp](docs/cookbook/frozenlake/quickstart.md)。
|
||||
- **[2025-08]** 🚀 MCP协议支持已上线-> [MCP指南](docs/mcp_quick_start.md)。
|
||||
- **[2025-06]** 🚀 多后端向量存储支持 (Elasticsearch & ChromaDB) -> [向量数据库指南](docs/vector_store_api_guide.md)。
|
||||
- **[2024-09]** 🧠 [MemoryScope](https://github.com/modelscope/Reme/tree/memoryscope_branch) v0.1 发布，个性化和时间感知的记忆存储与使用。
|
||||
|
||||
---
|
||||
|
||||
## ✨ 功能设计
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/figure/reme_structure.jpg" alt="ReMe Logo" width="100%">
|
||||
</p>
|
||||
|
||||
ReMe整合两种互补的记忆能力：
|
||||
|
||||
#### 🧠 **任务经验 (Task Memory/Experience)**
|
||||
跨智能体复用的程序性知识
|
||||
- **成功模式识别**：识别有效策略并理解其根本原理
|
||||
- **失败分析学习**：从错误中学习，避免重复同样的问题
|
||||
- **对比模式**：不同采样轨迹通过对比得到更有价值的经验
|
||||
- **验证模式**：经过验证模块确认抽取记忆的有效性
|
||||
|
||||
你可以从[task memory](docs/task_memory/task_memory.md)了解更多如何使用task memory的方法
|
||||
|
||||
#### 👤 **个人记忆 (Personal Memory)**
|
||||
特定用户的情境化记忆
|
||||
- **个体偏好**：用户的习惯、偏好和交互风格
|
||||
- **情境适应**：基于时间和上下文的智能记忆管理
|
||||
- **渐进学习**：通过长期交互逐步建立深度理解
|
||||
- **时间感知**：检索和整合时都具备时间敏感性
|
||||
|
||||
你可以从[personal memory](docs/personal_memory/personal_memory.md)了解更多如何使用personal memory的方法
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 🛠️ 安装
|
||||
|
||||
### 从PyPI安装（推荐）
|
||||
```bash
|
||||
pip install reme-ai
|
||||
```
|
||||
|
||||
### 从源码安装
|
||||
```bash
|
||||
git clone https://github.com/modelscope/ReMe.git
|
||||
cd ReMe
|
||||
pip install .
|
||||
```
|
||||
|
||||
### 环境配置
|
||||
|
||||
复制 `example.env` 为 .env并修改其中对应参数：
|
||||
|
||||
```bash
|
||||
FLOW_APP_NAME=ReMe
|
||||
FLOW_LLM_API_KEY=sk-xxxx
|
||||
FLOW_LLM_BASE_URL=https://xxxx/v1
|
||||
FLOW_EMBEDDING_API_KEY=sk-xxxx
|
||||
FLOW_EMBEDDING_BASE_URL=https://xxxx/v1
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🚀 快速开始
|
||||
|
||||
### HTTP服务启动
|
||||
```bash
|
||||
reme \
|
||||
backend=http \
|
||||
http.port=8002 \
|
||||
llm.default.model_name=qwen3-30b-a3b-thinking-2507 \
|
||||
embedding_model.default.model_name=text-embedding-v4 \
|
||||
vector_store.default.backend=local
|
||||
```
|
||||
|
||||
### MCP服务器支持
|
||||
```bash
|
||||
reme \
|
||||
backend=mcp \
|
||||
mcp.transport=stdio \
|
||||
llm.default.model_name=qwen3-30b-a3b-thinking-2507 \
|
||||
embedding_model.default.model_name=text-embedding-v4 \
|
||||
vector_store.default.backend=local
|
||||
```
|
||||
|
||||
### 核心API使用
|
||||
|
||||
#### 任务记忆管理
|
||||
```python
|
||||
import requests
|
||||
|
||||
# 经验总结器：从执行轨迹学习
|
||||
response = requests.post("http://localhost:8002/summary_task_memory", json={
|
||||
"workspace_id": "task_workspace",
|
||||
"trajectories": [
|
||||
{"messages": [{"role": "user", "content": "帮我制定项目计划"}], "score": 1.0}
|
||||
]
|
||||
})
|
||||
|
||||
# 经验检索器：获取相关经验
|
||||
response = requests.post("http://localhost:8002/retrieve_task_memory", json={
|
||||
"workspace_id": "task_workspace",
|
||||
"query": "如何高效管理项目进度？",
|
||||
"top_k": 1
|
||||
})
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary>curl 版本</summary>
|
||||
|
||||
```bash
|
||||
# 经验总结器：从执行轨迹学习
|
||||
curl -X POST http://localhost:8002/summary_task_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "task_workspace",
|
||||
"trajectories": [
|
||||
{"messages": [{"role": "user", "content": "帮我制定项目计划"}], "score": 1.0}
|
||||
]
|
||||
}'
|
||||
|
||||
# 经验检索器：获取相关经验
|
||||
curl -X POST http://localhost:8002/retrieve_task_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "task_workspace",
|
||||
"query": "如何高效管理项目进度？",
|
||||
"top_k": 1
|
||||
}'
|
||||
```
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Node.js 版本</summary>
|
||||
|
||||
```javascript
|
||||
// 经验总结器：从执行轨迹学习
|
||||
fetch("http://localhost:8002/summary_task_memory", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workspace_id: "task_workspace",
|
||||
trajectories: [
|
||||
{messages: [{role: "user", content: "帮我制定项目计划"}], score: 1.0}
|
||||
]
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => console.log(data));
|
||||
|
||||
// 经验检索器：获取相关经验
|
||||
fetch("http://localhost:8002/retrieve_task_memory", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workspace_id: "task_workspace",
|
||||
query: "如何高效管理项目进度？",
|
||||
top_k: 1
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => console.log(data));
|
||||
```
|
||||
</details>
|
||||
|
||||
#### 个人记忆管理
|
||||
```python
|
||||
# 记忆整合：从用户交互中学习
|
||||
response = requests.post("http://localhost:8002/summary_personal_memory", json={
|
||||
"workspace_id": "task_workspace",
|
||||
"trajectories": [
|
||||
{"messages":
|
||||
[
|
||||
{"role": "user", "content": "我喜欢早上喝咖啡工作"},
|
||||
{"role": "assistant", "content": "了解，您习惯早上用咖啡提神来开始工作"}
|
||||
]
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
# 记忆检索：获取个人记忆片段
|
||||
response = requests.post("http://localhost:8002/retrieve_personal_memory", json={
|
||||
"workspace_id": "task_workspace",
|
||||
"query": "用户的工作习惯是什么？",
|
||||
"top_k": 5
|
||||
})
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary>curl 版本</summary>
|
||||
|
||||
```bash
|
||||
# 记忆整合：从用户交互中学习
|
||||
curl -X POST http://localhost:8002/summary_personal_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "task_workspace",
|
||||
"trajectories": [
|
||||
{"messages": [
|
||||
{"role": "user", "content": "我喜欢早上喝咖啡工作"},
|
||||
{"role": "assistant", "content": "了解，您习惯早上用咖啡提神来开始工作"}
|
||||
]}
|
||||
]
|
||||
}'
|
||||
|
||||
# 记忆检索：获取个人记忆片段
|
||||
curl -X POST http://localhost:8002/retrieve_personal_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "task_workspace",
|
||||
"query": "用户的工作习惯是什么？",
|
||||
"top_k": 5
|
||||
}'
|
||||
```
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Node.js 版本</summary>
|
||||
|
||||
```javascript
|
||||
// 记忆整合：从用户交互中学习
|
||||
fetch("http://localhost:8002/summary_personal_memory", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workspace_id: "task_workspace",
|
||||
trajectories: [
|
||||
{messages: [
|
||||
{role: "user", content: "我喜欢早上喝咖啡工作"},
|
||||
{role: "assistant", content: "了解，您习惯早上用咖啡提神来开始工作"}
|
||||
]}
|
||||
]
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => console.log(data));
|
||||
|
||||
// 记忆检索：获取个人记忆片段
|
||||
fetch("http://localhost:8002/retrieve_personal_memory", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workspace_id: "task_workspace",
|
||||
query: "用户的工作习惯是什么？",
|
||||
top_k: 5
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => console.log(data));
|
||||
```
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
## 📦 即用型经验库
|
||||
|
||||
ReMe提供预构建的经验库，智能体可以立即使用经过验证的最佳实践：
|
||||
|
||||
### 可用经验库
|
||||
|
||||
- **`appworld.jsonl`**：Appworld智能体交互的记忆库，涵盖复杂任务规划和执行模式
|
||||
- **`bfcl_v3.jsonl`**：BFCL工具调用的工作记忆库
|
||||
|
||||
### 快速使用
|
||||
```python
|
||||
# 加载预构建经验
|
||||
response = requests.post("http://localhost:8002/vector_store", json={
|
||||
"workspace_id": "appworld",
|
||||
"action": "load",
|
||||
"path": "./docs/library/"
|
||||
})
|
||||
|
||||
# 查询相关经验
|
||||
response = requests.post("http://localhost:8002/retrieve_task_memory", json={
|
||||
"workspace_id": "appworld",
|
||||
"query": "如何导航到设置并更新用户资料？",
|
||||
"top_k": 1
|
||||
})
|
||||
```
|
||||
|
||||
## 🧪 实验
|
||||
|
||||
### 🌍 [Appworld 实验](docs/cookbook/appworld/quickstart.md)
|
||||
|
||||
我们在 Appworld 上使用 qwen3-8b 测试 ReMe：
|
||||
|
||||
| 方法 | pass@1 | pass@2 | pass@4 |
|
||||
|--------------|-------------------|-------------------|-------------------|
|
||||
| without ReMe | 0.083 | 0.140 | 0.228 |
|
||||
| with ReMe | 0.109 **(+2.6%)** | 0.175 **(+3.5%)** | 0.281 **(+5.3%)** |
|
||||
|
||||
Pass@K 衡量的是在生成的 K 个样本中，至少有一个成功完成任务（score=1）的概率。
|
||||
当前实验使用的是一个内部的 AppWorld 环境，可能存在轻微差异。
|
||||
|
||||
你可以在 [quickstart.md](docs/cookbook/appworld/quickstart.md) 中找到复现实验的更多细节。
|
||||
|
||||
|
||||
### 🧊 [Frozenlake 实验](docs/cookbook/frozenlake/quickstart.md)
|
||||
|
||||
| 不使用ReMe | 使用ReMe |
|
||||
|:--------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------:|
|
||||
| <p align="center"><img src="docs/figure/frozenlake_failure.gif" alt="GIF 1" width="30%"></p> | <p align="center"><img src="docs/figure/frozenlake_success.gif" alt="GIF 2" width="30%"></p> |
|
||||
|
||||
我们在 100 个随机 frozenlake 地图上使用 qwen3-8b 进行测试：
|
||||
|
||||
| 方法 | pass rate |
|
||||
|--------------|------------------|
|
||||
| without ReMe | 0.66 |
|
||||
| with ReMe | 0.72 **(+6.0%)** |
|
||||
|
||||
你可以在 [quickstart.md](docs/cookbook/frozenlake/quickstart.md) 中找到复现实验的更多细节。
|
||||
|
||||
### 🔧 [BFCL-V3 实验](docs/cookbook/bfcl/quickstart.md)
|
||||
|
||||
我们在 BFCL-V3 multi-turn-base (随机划分50train/150val) 上使用 qwen3-8b 测试 ReMe：
|
||||
|
||||
| 方法 | pass@1 | pass@2 | pass@4 |
|
||||
|--------------|---------------------|---------------------|---------------------|
|
||||
| without ReMe | 0.2472 | 0.2733 | 0.2922 |
|
||||
| with ReMe | 0.3061 **(+5.89%)** | 0.3500 **(+7.67%)** | 0.3888 **(+9.66%)** |
|
||||
|
||||
## 📚 相关资源
|
||||
|
||||
- **[快速开始](./cookbook/simple_demo)**：通过实际示例快速上手
|
||||
- **[向量存储设置](docs/vector_store_api_guide.md)**：配置本地/向量数据库以及使用
|
||||
- **[mcp指南](docs/mcp_quick_start.md)**：创建mcp服务
|
||||
- **[个性化记忆](docs/personal_memory)** 与 [任务记忆](docs/task_memory): 个性化记忆与任务记忆中分别使用的算子及其含义，你可以修改config以自定义链路
|
||||
- **[示例集合](./cookbook)**：实际用例和最佳实践
|
||||
|
||||
---
|
||||
|
||||
## 🤝 贡献
|
||||
|
||||
我们相信最好的记忆系统来自集体智慧。欢迎贡献👉[指南](docs/contribution.md)：
|
||||
|
||||
### 代码贡献
|
||||
- 新操作和工具开发
|
||||
- 后端实现和优化
|
||||
- API增强和新端点
|
||||
|
||||
### 文档改进
|
||||
- 使用示例和教程
|
||||
- 最佳实践指南
|
||||
|
||||
---
|
||||
|
||||
## 📄 引用
|
||||
|
||||
```bibtex
|
||||
@software{ReMe2025,
|
||||
title = {ReMe: Memory Management Framework for Agents},
|
||||
author = {Li Yu, Jiaji Deng, Zouying Cao},
|
||||
url = {https://github.com/modelscope/ReMe},
|
||||
year = {2025}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ⚖️ 许可证
|
||||
|
||||
本项目采用Apache License 2.0许可证 - 详情请参阅[LICENSE](./LICENSE)文件。
|
||||
|
||||
---
|
||||
|
||||
## Star 历史
|
||||
[](https://www.star-history.com/#modelscope/ReMe&Date)
|
||||
238
cookbook/simple_demo/use_tool_memory_demo.py
Normal file
238
cookbook/simple_demo/use_tool_memory_demo.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Tool Memory Demo - 展示工具记忆的完整生命周期"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
from reme_ai.utils.tool_memory_utils import create_mock_tool_call_results
|
||||
|
||||
load_dotenv()
|
||||
|
||||
BASE_URL = "http://0.0.0.0:8002/"
|
||||
WORKSPACE_ID = "test_tool_memory_workspace"
|
||||
|
||||
|
||||
def api_call(endpoint: str, data: dict) -> Optional[Dict[str, Any]]:
|
||||
"""统一的API调用处理"""
|
||||
response = requests.post(f"{BASE_URL}{endpoint}", json=data)
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code} - {response.text}")
|
||||
return None
|
||||
return response.json()
|
||||
|
||||
|
||||
def delete_workspace() -> None:
|
||||
"""删除工作空间
|
||||
|
||||
curl example:
|
||||
curl -X POST http://0.0.0.0:8002/vector_store \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "test_tool_memory_workspace",
|
||||
"action": "delete"
|
||||
}'
|
||||
"""
|
||||
result = api_call("vector_store", {"workspace_id": WORKSPACE_ID, "action": "delete"})
|
||||
if result:
|
||||
print(f"✓ Workspace '{WORKSPACE_ID}' deleted")
|
||||
|
||||
|
||||
def add_tool_call_results(tool_call_results: List[Dict[str, Any]]) -> bool:
|
||||
"""添加工具调用结果到记忆库
|
||||
|
||||
curl example:
|
||||
curl -X POST http://0.0.0.0:8002/add_tool_call_result \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "test_tool_memory_workspace",
|
||||
"tool_call_results": [
|
||||
{
|
||||
"tool_name": "web_search",
|
||||
"tool_input": "Python tutorials",
|
||||
"tool_output": "Found 100 results about Python",
|
||||
"execution_time": 0.5
|
||||
}
|
||||
]
|
||||
}'
|
||||
"""
|
||||
# 统计不同的工具
|
||||
tool_names = set(r.get("tool_name") for r in tool_call_results)
|
||||
print(f"\n[ADD] {len(tool_call_results)} results for {len(tool_names)} tools: {', '.join(sorted(tool_names))}")
|
||||
|
||||
result = api_call("add_tool_call_result", {
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"tool_call_results": tool_call_results
|
||||
})
|
||||
if result:
|
||||
memory_list = result.get("metadata", {}).get("memory_list", [])
|
||||
print(f"✓ Added successfully, created/updated {len(memory_list)} tool memories")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def summarize_tool_memory(tool_names: str) -> Optional[Dict[str, Any]]:
|
||||
"""总结工具使用模式
|
||||
|
||||
curl example:
|
||||
curl -X POST http://0.0.0.0:8002/summary_tool_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "test_tool_memory_workspace",
|
||||
"tool_names": "web_search,database_query"
|
||||
}'
|
||||
"""
|
||||
print(f"\n[SUMMARIZE] {tool_names}")
|
||||
result = api_call("summary_tool_memory", {
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"tool_names": tool_names
|
||||
})
|
||||
|
||||
if result:
|
||||
memory_list = result.get("metadata", {}).get("memory_list", [])
|
||||
print(f"✓ Summarized {len(memory_list)} tool memories")
|
||||
for memory in memory_list:
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Tool: {memory.get('when_to_use', 'N/A')}")
|
||||
print(f"{'=' * 60}")
|
||||
print(memory.get('content', 'No content'))
|
||||
return result
|
||||
|
||||
|
||||
def retrieve_tool_memory(tool_names: str, save_to_file: bool = False) -> str:
|
||||
"""检索工具记忆
|
||||
|
||||
curl example:
|
||||
curl -X POST http://0.0.0.0:8002/retrieve_tool_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "test_tool_memory_workspace",
|
||||
"tool_names": "web_search"
|
||||
}'
|
||||
"""
|
||||
print(f"\n[RETRIEVE] {tool_names}")
|
||||
result = api_call("retrieve_tool_memory", {
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"tool_names": tool_names
|
||||
})
|
||||
|
||||
if not result:
|
||||
return ""
|
||||
|
||||
memory_list = result.get("metadata", {}).get("memory_list", [])
|
||||
if not memory_list:
|
||||
print("No memories found")
|
||||
return ""
|
||||
|
||||
print(f"✓ Retrieved {len(memory_list)} memories")
|
||||
|
||||
formatted_memories = []
|
||||
for memory in memory_list:
|
||||
content = f"\nTool: {memory.get('when_to_use', 'N/A')}\n" \
|
||||
f"Calls: {len(memory.get('tool_call_results', []))}\n" \
|
||||
f"{'-' * 60}\n{memory.get('content', 'No content')}\n"
|
||||
formatted_memories.append(content)
|
||||
print(content)
|
||||
|
||||
if save_to_file:
|
||||
with open("tool_memory.json", "w") as f:
|
||||
json.dump(memory_list, f, indent=2, ensure_ascii=False)
|
||||
print("✓ Saved to tool_memory.json")
|
||||
|
||||
return "\n".join(formatted_memories)
|
||||
|
||||
|
||||
def dump_memory(path: str = "./") -> None:
|
||||
"""导出记忆到磁盘
|
||||
|
||||
curl example:
|
||||
curl -X POST http://0.0.0.0:8002/vector_store \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "test_tool_memory_workspace",
|
||||
"action": "dump",
|
||||
"path": "./"
|
||||
}'
|
||||
"""
|
||||
result = api_call("vector_store", {
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"action": "dump",
|
||||
"path": path
|
||||
})
|
||||
if result:
|
||||
print(f"✓ Memory dumped to {path}")
|
||||
|
||||
|
||||
def load_memory(path: str = "./") -> None:
|
||||
"""从磁盘加载记忆
|
||||
|
||||
curl example:
|
||||
curl -X POST http://0.0.0.0:8002/vector_store \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "test_tool_memory_workspace",
|
||||
"action": "load",
|
||||
"path": "./"
|
||||
}'
|
||||
"""
|
||||
result = api_call("vector_store", {
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"action": "load",
|
||||
"path": path
|
||||
})
|
||||
if result:
|
||||
print(f"✓ Memory loaded from {path}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
# 1. 清理工作空间
|
||||
print("\n[1] Cleaning workspace...")
|
||||
delete_workspace()
|
||||
time.sleep(1)
|
||||
|
||||
# 2. 创建和添加模拟工具调用结果
|
||||
print("\n[2] Adding mock tool call results...")
|
||||
tools_to_test = [
|
||||
("web_search", 30),
|
||||
("database_query", 22),
|
||||
("file_processor", 18)
|
||||
]
|
||||
|
||||
# 收集所有工具的结果，然后一次性添加
|
||||
all_mock_results = []
|
||||
for tool_name, count in tools_to_test:
|
||||
mock_results = create_mock_tool_call_results(tool_name, count)
|
||||
all_mock_results.extend(mock_results)
|
||||
|
||||
if not add_tool_call_results(all_mock_results):
|
||||
print("✗ Failed to add results")
|
||||
else:
|
||||
time.sleep(1)
|
||||
|
||||
# 3. 总结工具记忆
|
||||
print("\n[3] Summarizing tool memories...")
|
||||
all_tool_names = ",".join([tool[0] for tool in tools_to_test])
|
||||
summarize_tool_memory(all_tool_names)
|
||||
time.sleep(1)
|
||||
|
||||
# 4. 检索工具记忆
|
||||
print("\n[4] Retrieving tool memories...")
|
||||
for tool_name, _ in tools_to_test:
|
||||
retrieve_tool_memory(tool_name, save_to_file=True)
|
||||
time.sleep(0.5)
|
||||
|
||||
# 5. 测试记忆持久化
|
||||
print("\n[5] Testing memory persistence...")
|
||||
dump_memory()
|
||||
load_memory()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("DEMO COMPLETE ✓")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
0
cookbook/tool_memory/__init__.py
Normal file
0
cookbook/tool_memory/__init__.py
Normal file
139
cookbook/tool_memory/query.json
Normal file
139
cookbook/tool_memory/query.json
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
{
|
||||
"train": {
|
||||
"simple": [
|
||||
"法国的首都是什么？",
|
||||
"Python 是什么时候首次发布的？",
|
||||
"地球上有几个大洲？",
|
||||
"水的沸点是多少度？",
|
||||
"谁发明了电话？",
|
||||
"一年有多少天？",
|
||||
"光速是多少？",
|
||||
"氧气的化学符号是什么？",
|
||||
"世界上最高的山峰是哪座？",
|
||||
"太阳系有几颗行星？",
|
||||
"地球的半径是多少公里？",
|
||||
"元素周期表有多少个元素？",
|
||||
"蒙娜丽莎是谁画的？",
|
||||
"人类DNA有多少条染色体？",
|
||||
"HTTP的默认端口号是多少？",
|
||||
"世界上人口最多的国家是哪个？",
|
||||
"一公里等于多少米？",
|
||||
"谁提出了相对论？",
|
||||
"圆周率π的前两位小数是多少？",
|
||||
"中国的货币单位是什么？"
|
||||
],
|
||||
"moderate": [
|
||||
"列举Python 3.10的主要特性",
|
||||
"微服务架构有哪些好处？",
|
||||
"解释区块链技术的工作原理",
|
||||
"描述SQL和NoSQL数据库的主要区别",
|
||||
"软件工程中常见的设计模式有哪些？",
|
||||
"什么是RESTful API的设计原则？",
|
||||
"解释Docker容器化技术的优势",
|
||||
"云计算的三种服务模型是什么？",
|
||||
"什么是机器学习中的过拟合问题？",
|
||||
"解释Git中merge和rebase的区别",
|
||||
"什么是持续集成和持续部署(CI/CD)？",
|
||||
"描述OSI七层网络模型的作用",
|
||||
"什么是JWT令牌认证机制？",
|
||||
"解释虚拟内存的工作原理",
|
||||
"Kubernetes的主要组件有哪些？",
|
||||
"什么是函数式编程的核心概念？",
|
||||
"解释负载均衡的常见策略",
|
||||
"描述SOLID设计原则的含义",
|
||||
"什么是异步编程的优势和挑战？",
|
||||
"解释数据库索引的作用和类型"
|
||||
],
|
||||
"complex": [
|
||||
"比较凯恩斯主义和奥地利学派的经济政策",
|
||||
"分析可再生能源采用对环境的影响",
|
||||
"解释量子力学和广义相对论之间的关系",
|
||||
"讨论人工智能的历史演变过程",
|
||||
"评估不同机器学习算法在NLP中的有效性",
|
||||
"分析全球化对发展中国家经济的长期影响",
|
||||
"比较不同民主制度模型的优缺点",
|
||||
"探讨气候变化对生物多样性的影响机制",
|
||||
"研究基因编辑技术的伦理问题和社会影响",
|
||||
"分析区块链技术在金融领域的应用前景和挑战",
|
||||
"评估不同神经网络架构在计算机视觉中的表现",
|
||||
"探讨量子计算对现代密码学的威胁和机遇",
|
||||
"分析人口老龄化对社会保障体系的影响",
|
||||
"比较不同哲学流派对人工智能意识的观点",
|
||||
"研究微生物组与人类健康之间的复杂关系",
|
||||
"评估碳捕获技术在应对气候变化中的作用",
|
||||
"分析5G技术对物联网发展的推动作用",
|
||||
"探讨认知科学与人工智能的交叉研究领域",
|
||||
"比较不同经济体制下的创新能力差异",
|
||||
"研究纳米技术在医疗领域的应用和风险"
|
||||
]
|
||||
},
|
||||
"test": {
|
||||
"simple": [
|
||||
"日本的首都是什么？",
|
||||
"Java 语言是哪一年发布的？",
|
||||
"大西洋有多宽？",
|
||||
"铁的熔点是多少度？",
|
||||
"谁发明了汽车？",
|
||||
"一周有多少小时？",
|
||||
"声音在空气中的速度是多少？",
|
||||
"氢气的化学符号是什么？",
|
||||
"世界上最长的河流是哪条？",
|
||||
"月球绕地球一周需要多少天？",
|
||||
"标准大气压是多少帕？",
|
||||
"人体有多少块骨头？",
|
||||
"《星空》是谁画的？",
|
||||
"成年人有多少颗牙齿？",
|
||||
"HTTPS的默认端口号是多少？",
|
||||
"世界上面积最大的国家是哪个？",
|
||||
"一英里等于多少公里？",
|
||||
"谁发现了万有引力定律？",
|
||||
"黄金的化学元素符号是什么？",
|
||||
"美国的货币单位是什么？"
|
||||
],
|
||||
"moderate": [
|
||||
"列举TypeScript 5.0的主要新功能",
|
||||
"单体架构和微服务架构的区别是什么？",
|
||||
"解释分布式系统的CAP定理",
|
||||
"描述关系型数据库的ACID特性",
|
||||
"前端开发中常用的状态管理方案有哪些？",
|
||||
"什么是GraphQL的核心优势？",
|
||||
"解释Kubernetes容器编排的基本概念",
|
||||
"边缘计算和雾计算有什么区别？",
|
||||
"什么是神经网络中的梯度消失问题？",
|
||||
"解释Git中revert和reset的区别",
|
||||
"什么是DevOps的核心理念？",
|
||||
"描述TCP/IP协议栈的层次结构",
|
||||
"什么是OAuth 2.0授权框架？",
|
||||
"解释进程和线程的本质区别",
|
||||
"Service Mesh的主要功能是什么？",
|
||||
"什么是响应式编程的基本原理？",
|
||||
"解释缓存一致性的常见问题",
|
||||
"描述DDD领域驱动设计的核心思想",
|
||||
"什么是并发编程中的死锁问题？",
|
||||
"解释数据库事务隔离级别的分类"
|
||||
],
|
||||
"complex": [
|
||||
"比较货币主义和供给学派的经济理论差异",
|
||||
"分析核能发展对能源转型的作用",
|
||||
"解释弦理论与标准模型之间的联系",
|
||||
"讨论云计算技术的发展历程和趋势",
|
||||
"评估深度学习在计算机视觉中的应用效果",
|
||||
"分析数字化转型对传统产业的影响",
|
||||
"比较总统制和议会制的政治体制特点",
|
||||
"探讨海洋酸化对海洋生态系统的影响",
|
||||
"研究人工智能在医疗诊断中的伦理挑战",
|
||||
"分析去中心化金融(DeFi)的发展机遇与风险",
|
||||
"评估Transformer架构在自然语言处理中的优势",
|
||||
"探讨后量子密码学的研究方向和应用",
|
||||
"分析城市化进程对资源配置的影响",
|
||||
"比较实用主义和理想主义对技术伦理的看法",
|
||||
"研究肠道菌群与神经系统疾病的关联",
|
||||
"评估直接空气捕获技术的经济可行性",
|
||||
"分析边缘计算对云计算架构的影响",
|
||||
"探讨神经语言学与自然语言处理的交叉应用",
|
||||
"比较计划经济和市场经济的资源配置效率",
|
||||
"研究量子传感技术在精密测量中的突破"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
633
cookbook/tool_memory/run_reme_tool_bench.py
Normal file
633
cookbook/tool_memory/run_reme_tool_bench.py
Normal file
|
|
@ -0,0 +1,633 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Tool Memory Benchmark Script
|
||||
评估工具记忆在不同场景下的效果，包括有记忆和无记忆的对比
|
||||
|
||||
Dependencies:
|
||||
pip install requests python-dotenv loguru tabulate
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
from loguru import logger
|
||||
from tabulate import tabulate
|
||||
|
||||
from reme_ai.schema.memory import ToolCallResult, ToolMemory
|
||||
|
||||
load_dotenv()
|
||||
|
||||
BASE_URL = "http://0.0.0.0:8002/"
|
||||
TRAIN_WORKSPACE = "train_tool_workspace"
|
||||
TEST_WORKSPACE = "test_tool_workspace"
|
||||
|
||||
|
||||
class BenchmarkStats:
|
||||
"""统计数据收集器"""
|
||||
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self.total_count = 0
|
||||
self.scores = []
|
||||
|
||||
def add_result(self, result: Dict[str, Any]):
|
||||
"""添加一个工具调用结果
|
||||
|
||||
Note:
|
||||
- score: Quality/relevance of the result (0.0 or 1.0)
|
||||
"""
|
||||
self.total_count += 1
|
||||
|
||||
# Collect score
|
||||
score = result.get("score", 0)
|
||||
self.scores.append(score)
|
||||
|
||||
def get_summary(self) -> Dict[str, Any]:
|
||||
"""获取统计摘要"""
|
||||
if self.total_count == 0:
|
||||
return {
|
||||
"name": self.name,
|
||||
"total_calls": 0,
|
||||
"avg_score": 0.0
|
||||
}
|
||||
|
||||
return {
|
||||
"name": self.name,
|
||||
"total_calls": self.total_count,
|
||||
"avg_score": round(sum(self.scores) / len(self.scores), 3)
|
||||
}
|
||||
|
||||
|
||||
def api_call(endpoint: str, data: dict) -> Optional[Dict[str, Any]]:
|
||||
"""统一的API调用处理"""
|
||||
try:
|
||||
response = requests.post(f"{BASE_URL}{endpoint}", json=data, timeout=120)
|
||||
if response.status_code != 200:
|
||||
logger.error(f"API Error: {response.status_code} - {response.text}")
|
||||
return None
|
||||
return response.json()
|
||||
except Exception as e:
|
||||
logger.error(f"API call failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def delete_workspace(workspace_id: str) -> bool:
|
||||
"""删除工作空间"""
|
||||
logger.info(f"Deleting workspace: {workspace_id}")
|
||||
result = api_call("vector_store", {"workspace_id": workspace_id, "action": "delete"})
|
||||
return result is not None
|
||||
|
||||
|
||||
def load_queries(query_file: str = "query.json") -> Dict[str, Any]:
|
||||
"""加载查询数据"""
|
||||
query_path = Path(__file__).parent / query_file
|
||||
with open(query_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def run_use_mock_search(workspace_id: str, queries: List[str], prompt_template: str = "") -> List[ToolCallResult]:
|
||||
"""运行use_mock_search并收集结果（支持并发）
|
||||
|
||||
Args:
|
||||
workspace_id: 工作空间ID
|
||||
queries: 查询列表
|
||||
prompt_template: 提示模板
|
||||
|
||||
Returns:
|
||||
工具调用结果列表
|
||||
"""
|
||||
logger.info(f"Running use_mock_search on {workspace_id} with {len(queries)} queries (max concurrency: 4)")
|
||||
results: List[ToolCallResult] = []
|
||||
|
||||
def process_single_query(idx: int, query: str) -> Optional[ToolCallResult]:
|
||||
"""处理单个查询"""
|
||||
logger.info(f"[{idx + 1}/{len(queries)}] Processing: {query}")
|
||||
|
||||
# 提交之前sleep 1秒
|
||||
time.sleep(1)
|
||||
|
||||
result = api_call("use_mock_search", {
|
||||
"workspace_id": workspace_id,
|
||||
"query": prompt_template.format(query=query),
|
||||
})
|
||||
|
||||
if result:
|
||||
tool_call_result = result.get("answer")
|
||||
return ToolCallResult(**json.loads(tool_call_result))
|
||||
else:
|
||||
logger.warning(f"No result for query: {query}")
|
||||
return None
|
||||
|
||||
# 使用线程池并发处理，最大并发数为4
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
# 提交所有任务
|
||||
future_to_query = {
|
||||
executor.submit(process_single_query, idx, query): (idx, query)
|
||||
for idx, query in enumerate(queries)
|
||||
}
|
||||
|
||||
# 收集结果（按完成顺序）
|
||||
completed_results = []
|
||||
for future in as_completed(future_to_query):
|
||||
idx, query = future_to_query[future]
|
||||
try:
|
||||
result = future.result()
|
||||
if result:
|
||||
completed_results.append((idx, result))
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing query [{idx + 1}]: {query}, error: {e}")
|
||||
|
||||
# 按原始顺序排序结果
|
||||
completed_results.sort(key=lambda x: x[0])
|
||||
results = [result for _, result in completed_results]
|
||||
|
||||
logger.info(f"Collected {len(results)} results out of {len(queries)} queries")
|
||||
return results
|
||||
|
||||
|
||||
def add_tool_call_results(workspace_id: str, results: List[ToolCallResult]) -> List[ToolCallResult]:
|
||||
"""批量添加工具调用结果到记忆库，并返回带评分的结果
|
||||
|
||||
Args:
|
||||
workspace_id: 工作空间ID
|
||||
results: 工具调用结果列表
|
||||
|
||||
Returns:
|
||||
从API返回的memory_list中提取的带评分的ToolCallResult列表
|
||||
"""
|
||||
if not results:
|
||||
logger.warning("No results to add")
|
||||
return []
|
||||
|
||||
# 转换为字典用于API调用
|
||||
tool_call_results = [result.model_dump() for result in results]
|
||||
|
||||
logger.info(f"Adding tool call results to {workspace_id}: {len(tool_call_results)} results")
|
||||
|
||||
# 统一调用API，让后端自动按tool_name分组处理
|
||||
api_result = api_call("add_tool_call_result", {
|
||||
"workspace_id": workspace_id,
|
||||
"tool_call_results": tool_call_results
|
||||
})
|
||||
|
||||
if not api_result:
|
||||
logger.error("Failed to add results")
|
||||
return []
|
||||
|
||||
# 收集所有带评分的结果
|
||||
all_scored_results: List[ToolCallResult] = []
|
||||
|
||||
# 解析返回的memory_list（可能包含多个工具的记忆）
|
||||
memory_list = api_result.get("metadata", {}).get("memory_list", [])
|
||||
logger.info(f"Received {len(memory_list)} tool memories from API")
|
||||
|
||||
for memory_dict in memory_list:
|
||||
tool_memory = ToolMemory(**memory_dict)
|
||||
tool_name = tool_memory.when_to_use
|
||||
scored_results = tool_memory.tool_call_results
|
||||
all_scored_results.extend(scored_results)
|
||||
|
||||
logger.info(f"Extracted {len(scored_results)} scored results from {tool_name}")
|
||||
|
||||
# 打印一些评分示例
|
||||
for idx, result in enumerate(scored_results[:3]):
|
||||
logger.info(f" Result #{idx + 1}: score={result.score}, success={result.success}")
|
||||
|
||||
logger.info(f"Total scored results collected: {len(all_scored_results)}")
|
||||
return all_scored_results
|
||||
|
||||
|
||||
def summarize_tool_memory(workspace_id: str, tool_names: str) -> bool:
|
||||
"""总结工具记忆"""
|
||||
logger.info(f"Summarizing tool memory for {workspace_id}: {tool_names}")
|
||||
result = api_call("summary_tool_memory", {
|
||||
"workspace_id": workspace_id,
|
||||
"tool_names": tool_names
|
||||
})
|
||||
|
||||
if result:
|
||||
memory_list = result.get("metadata", {}).get("memory_list", [])
|
||||
logger.info(f"Summarized {len(memory_list)} tool memories")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def retrieve_tool_memory(workspace_id: str, tool_names: str) -> str:
|
||||
"""检索工具记忆并返回格式化的内容
|
||||
|
||||
Args:
|
||||
workspace_id: 工作空间ID
|
||||
tool_names: 逗号分隔的工具名称
|
||||
|
||||
Returns:
|
||||
格式化的工具记忆内容，每个工具名称作为一级markdown标题
|
||||
"""
|
||||
logger.info(f"Retrieving tool memory for {workspace_id}: {tool_names}")
|
||||
result = api_call("retrieve_tool_memory", {
|
||||
"workspace_id": workspace_id,
|
||||
"tool_names": tool_names
|
||||
})
|
||||
|
||||
if not result:
|
||||
logger.error("Failed to retrieve tool memory")
|
||||
return ""
|
||||
|
||||
memory_list = result.get("metadata", {}).get("memory_list", [])
|
||||
logger.info(f"Retrieved {len(memory_list)} tool memories")
|
||||
|
||||
# 提取每个工具记忆的content字段，并格式化为markdown
|
||||
formatted_contents = []
|
||||
for memory_dict in memory_list:
|
||||
tool_memory = ToolMemory(**memory_dict)
|
||||
if tool_memory.content:
|
||||
# 使用工具名称作为一级markdown标题
|
||||
tool_name = tool_memory.when_to_use or "Unknown Tool"
|
||||
formatted_section = f"# {tool_name}\n\n{tool_memory.content}"
|
||||
formatted_contents.append(formatted_section)
|
||||
logger.info(f"Retrieved content for tool: {tool_name}, "
|
||||
f"content_length={len(tool_memory.content)}")
|
||||
|
||||
# 用两个换行符分隔不同工具的记忆
|
||||
joined_content = "\n\n".join(formatted_contents)
|
||||
logger.info(f"Total content length: {len(joined_content)}")
|
||||
|
||||
return joined_content
|
||||
|
||||
|
||||
def collect_statistics(results: List[ToolCallResult], stats: BenchmarkStats) -> None:
|
||||
"""从结果列表中收集统计数据"""
|
||||
for result in results:
|
||||
# 转换为字典用于统计
|
||||
result_dict = result.model_dump() if hasattr(result, 'model_dump') else result
|
||||
stats.add_result(result_dict)
|
||||
|
||||
|
||||
def print_comparison_table(stats_list: List[BenchmarkStats]) -> None:
|
||||
"""打印对比表格"""
|
||||
headers = ["Scenario", "Total Calls", "Avg Score"]
|
||||
rows = []
|
||||
|
||||
for stats in stats_list:
|
||||
summary = stats.get_summary()
|
||||
rows.append([
|
||||
summary["name"],
|
||||
summary["total_calls"],
|
||||
summary["avg_score"]
|
||||
])
|
||||
|
||||
print("\n" + "=" * 100)
|
||||
print("BENCHMARK RESULTS COMPARISON")
|
||||
print("=" * 100)
|
||||
print("Note: Avg Score = average quality score")
|
||||
print(tabulate(rows, headers=headers, tablefmt="grid"))
|
||||
print("=" * 100)
|
||||
|
||||
|
||||
def calculate_improvements(baseline_stats: BenchmarkStats, improved_stats: BenchmarkStats) -> Dict[str, float]:
|
||||
"""计算改进百分比"""
|
||||
baseline = baseline_stats.get_summary()
|
||||
improved = improved_stats.get_summary()
|
||||
|
||||
improvements = {}
|
||||
|
||||
# 平均分数改进（相对提升百分比）
|
||||
if baseline["avg_score"] > 0:
|
||||
improvements["avg_score"] = ((improved["avg_score"] - baseline["avg_score"])
|
||||
/ baseline["avg_score"] * 100)
|
||||
else:
|
||||
improvements["avg_score"] = 0.0
|
||||
|
||||
return improvements
|
||||
|
||||
|
||||
def print_improvements(improvements: Dict[str, float]) -> None:
|
||||
"""打印改进情况"""
|
||||
print("\n" + "=" * 100)
|
||||
print("IMPROVEMENTS WITH TOOL MEMORY (Baseline: Test without memory)")
|
||||
print("=" * 100)
|
||||
|
||||
metric_labels = {
|
||||
"avg_score": "Average Score"
|
||||
}
|
||||
|
||||
for metric, improvement in improvements.items():
|
||||
label = metric_labels.get(metric, metric)
|
||||
direction = "↑" if improvement > 0 else "↓"
|
||||
print(f"{label:25s}: {improvement:+7.2f}% {direction}")
|
||||
|
||||
print("=" * 100)
|
||||
|
||||
|
||||
def save_results(results: Dict[str, Any], filename: str = "benchmark_results.json") -> None:
|
||||
"""保存结果到文件"""
|
||||
output_path = Path(__file__).parent / filename
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(results, f, indent=2, ensure_ascii=False)
|
||||
logger.info(f"Results saved to {output_path}")
|
||||
|
||||
|
||||
def run_single_epoch(epoch_num: int, train_queries: List[str], test_queries: List[str]) -> Dict[str, Any]:
|
||||
"""运行单个epoch的benchmark
|
||||
|
||||
Args:
|
||||
epoch_num: epoch编号（从1开始）
|
||||
train_queries: 训练查询列表
|
||||
test_queries: 测试查询列表
|
||||
|
||||
Returns:
|
||||
包含该epoch统计结果的字典
|
||||
"""
|
||||
logger.info("\n" + "=" * 100)
|
||||
logger.info(f"EPOCH {epoch_num} - START")
|
||||
logger.info("=" * 100)
|
||||
|
||||
# 初始化统计收集器
|
||||
train_no_memory_stats = BenchmarkStats(f"Epoch{epoch_num} - Train (No Memory)")
|
||||
test_no_memory_stats = BenchmarkStats(f"Epoch{epoch_num} - Test (No Memory)")
|
||||
test_with_memory_stats = BenchmarkStats(f"Epoch{epoch_num} - Test (With Memory)")
|
||||
|
||||
all_results = {}
|
||||
|
||||
# ==================== 步骤1: 无记忆在train上的效果 ====================
|
||||
print("\n" + "=" * 100)
|
||||
print(f"[EPOCH {epoch_num}] [STEP 1/5] Running on TRAIN without memory...")
|
||||
print("=" * 100)
|
||||
logger.info("Deleting workspace and starting fresh...")
|
||||
delete_workspace(TRAIN_WORKSPACE)
|
||||
time.sleep(2)
|
||||
prompt_template = "必须选择一个工具来回答问题\n 问题\n{query}"
|
||||
train_results_no_memory = run_use_mock_search(TRAIN_WORKSPACE, train_queries, prompt_template)
|
||||
|
||||
# 添加结果到记忆库并获取带评分的结果
|
||||
train_scored_results = add_tool_call_results(TRAIN_WORKSPACE, train_results_no_memory)
|
||||
time.sleep(2)
|
||||
|
||||
# 使用带评分的结果进行统计（如果有的话）
|
||||
if train_scored_results:
|
||||
logger.info(f"Using {len(train_scored_results)} scored results for statistics")
|
||||
all_results["train_no_memory"] = train_scored_results
|
||||
collect_statistics(train_scored_results, train_no_memory_stats)
|
||||
else:
|
||||
logger.warning("No scored results returned, using original results")
|
||||
all_results["train_no_memory"] = train_results_no_memory
|
||||
collect_statistics(train_results_no_memory, train_no_memory_stats)
|
||||
|
||||
print(f"✓ Train (no memory) completed: {len(train_results_no_memory)}/{len(train_queries)} results collected")
|
||||
summary = train_no_memory_stats.get_summary()
|
||||
print(f" Avg Score: {summary['avg_score']:.3f}")
|
||||
|
||||
# ==================== 步骤2: 无记忆在test上的效果 ====================
|
||||
print("\n" + "=" * 100)
|
||||
print(f"[EPOCH {epoch_num}] [STEP 2/5] Running on TEST without memory...")
|
||||
print("=" * 100)
|
||||
logger.info("Deleting workspace and starting fresh...")
|
||||
delete_workspace(TEST_WORKSPACE)
|
||||
time.sleep(2)
|
||||
|
||||
prompt_template = "必须选择一个工具来回答问题\n 问题\n{query}"
|
||||
test_results_no_memory = run_use_mock_search(TEST_WORKSPACE, test_queries, prompt_template)
|
||||
|
||||
# 添加结果到记忆库并获取带评分的结果
|
||||
# 注意：这些结果会作为TEST_WORKSPACE的初始记忆，在步骤4会被复用
|
||||
test_scored_results_no_memory = add_tool_call_results(TEST_WORKSPACE, test_results_no_memory)
|
||||
time.sleep(2)
|
||||
|
||||
# 使用带评分的结果进行统计（如果有的话）
|
||||
if test_scored_results_no_memory:
|
||||
logger.info(f"Using {len(test_scored_results_no_memory)} scored results for statistics")
|
||||
all_results["test_no_memory"] = test_scored_results_no_memory
|
||||
collect_statistics(test_scored_results_no_memory, test_no_memory_stats)
|
||||
else:
|
||||
logger.warning("No scored results returned, using original results")
|
||||
all_results["test_no_memory"] = test_results_no_memory
|
||||
collect_statistics(test_results_no_memory, test_no_memory_stats)
|
||||
|
||||
print(f"✓ Test (no memory) completed: {len(test_results_no_memory)}/{len(test_queries)} results collected")
|
||||
summary = test_no_memory_stats.get_summary()
|
||||
print(f" Avg Score: {summary['avg_score']:.3f}")
|
||||
|
||||
# ==================== 步骤3: 总结train的工具记忆 ====================
|
||||
print("\n" + "=" * 100)
|
||||
print(f"[EPOCH {epoch_num}] [STEP 3/5] Summarizing tool memory from TRAIN...")
|
||||
print("=" * 100)
|
||||
|
||||
# 获取所有工具名称（使用带评分的结果）
|
||||
tool_names_set = set()
|
||||
results_to_use = train_scored_results if train_scored_results else train_results_no_memory
|
||||
for result in results_to_use:
|
||||
tool_name = result.tool_name if hasattr(result, 'tool_name') else None
|
||||
if tool_name:
|
||||
tool_names_set.add(tool_name)
|
||||
|
||||
tool_names_str = ",".join(sorted(tool_names_set))
|
||||
print(f"Tools to summarize: {tool_names_str}")
|
||||
|
||||
success = summarize_tool_memory(TRAIN_WORKSPACE, tool_names_str)
|
||||
if not success:
|
||||
logger.error("Failed to summarize tool memory")
|
||||
return {}
|
||||
|
||||
time.sleep(3)
|
||||
|
||||
print("✓ Tool memory summarized successfully")
|
||||
|
||||
# 检索工具记忆内容
|
||||
memories = retrieve_tool_memory(TRAIN_WORKSPACE, tool_names_str)
|
||||
if not memories:
|
||||
logger.error("Failed to retrieve tool memory content")
|
||||
return {}
|
||||
|
||||
logger.info(f"Retrieved tool memory content, total length: {len(memories)}")
|
||||
print("\n" + "-" * 100)
|
||||
print("Retrieved Tool Memory Content:")
|
||||
print("-" * 100)
|
||||
print(memories)
|
||||
print("-" * 100)
|
||||
|
||||
# ==================== 步骤4: 有记忆在test上的效果 ====================
|
||||
print("\n" + "=" * 100)
|
||||
print(f"[EPOCH {epoch_num}] [STEP 4/5] Running on TEST with memory (after clearing existing memory)...")
|
||||
print("=" * 100)
|
||||
|
||||
# 先清理TEST_WORKSPACE中已有的记忆记录（Step 2的60条结果）
|
||||
print("Deleting existing memory records from TEST workspace...")
|
||||
delete_workspace(TEST_WORKSPACE)
|
||||
time.sleep(2) # 等待删除完成
|
||||
print("✓ TEST workspace memory cleared")
|
||||
|
||||
# 通过prompt注入train阶段总结的记忆，测量记忆增强的效果
|
||||
# 注意：此时workspace是空的，只通过prompt提供记忆信息
|
||||
|
||||
prompt_template = f"工具信息\n{memories}\n必须选择一个工具来回答问题\n 问题\n" + "{query}"
|
||||
test_results_with_memory = run_use_mock_search(TEST_WORKSPACE, test_queries, prompt_template)
|
||||
|
||||
# 添加这些结果到记忆库并获取带评分的结果
|
||||
# 此时workspace已清空，返回的就是本次新增的60条结果
|
||||
test_all_results_with_memory = add_tool_call_results(TEST_WORKSPACE, test_results_with_memory)
|
||||
time.sleep(2)
|
||||
|
||||
# workspace已清空，所有返回的结果都是本次新增的
|
||||
if test_all_results_with_memory:
|
||||
test_scored_results_with_memory = test_all_results_with_memory
|
||||
logger.info(f"Using {len(test_scored_results_with_memory)} scored results for statistics")
|
||||
all_results["test_with_memory"] = test_scored_results_with_memory
|
||||
collect_statistics(test_scored_results_with_memory, test_with_memory_stats)
|
||||
else:
|
||||
logger.warning("No scored results returned, using original results")
|
||||
all_results["test_with_memory"] = test_results_with_memory
|
||||
collect_statistics(test_results_with_memory, test_with_memory_stats)
|
||||
|
||||
print(f"✓ Test (with memory) completed: {len(test_results_with_memory)}/{len(test_queries)} results collected")
|
||||
summary = test_with_memory_stats.get_summary()
|
||||
print(f" Avg Score: {summary['avg_score']:.3f}")
|
||||
|
||||
# ==================== 步骤5: 打印对比结果 ====================
|
||||
print("\n" + "=" * 100)
|
||||
print(f"[EPOCH {epoch_num}] [STEP 5/5] Generating comparison report and analysis...")
|
||||
print("=" * 100)
|
||||
|
||||
# 打印统计表格
|
||||
print_comparison_table([train_no_memory_stats, test_no_memory_stats, test_with_memory_stats])
|
||||
|
||||
# 计算并打印改进情况
|
||||
improvements = calculate_improvements(test_no_memory_stats, test_with_memory_stats)
|
||||
print_improvements(improvements)
|
||||
|
||||
logger.info(f"EPOCH {epoch_num} - COMPLETE")
|
||||
|
||||
return {
|
||||
"epoch": epoch_num,
|
||||
"statistics": {
|
||||
"train_no_memory": train_no_memory_stats.get_summary(),
|
||||
"test_no_memory": test_no_memory_stats.get_summary(),
|
||||
"test_with_memory": test_with_memory_stats.get_summary()
|
||||
},
|
||||
"improvements": improvements
|
||||
}
|
||||
|
||||
|
||||
def main(test_mode: bool = False, run_epoch: int = 3):
|
||||
"""主函数：运行完整的benchmark流程
|
||||
|
||||
Args:
|
||||
test_mode: 如果为True，只使用每个难度级别的前3个查询进行快速测试
|
||||
run_epoch: 运行的epoch数量，默认为3
|
||||
"""
|
||||
logger.info("=" * 100)
|
||||
logger.info("TOOL MEMORY BENCHMARK - START")
|
||||
if test_mode:
|
||||
logger.info("Running in TEST MODE (limited queries)")
|
||||
logger.info(f"Total Epochs: {run_epoch}")
|
||||
logger.info("=" * 100)
|
||||
|
||||
# 加载查询数据
|
||||
queries_data = load_queries()
|
||||
train_queries = []
|
||||
test_queries = []
|
||||
|
||||
# 合并所有难度级别的查询
|
||||
for difficulty in ["simple", "moderate", "complex"]:
|
||||
train_data = queries_data["train"].get(difficulty, [])
|
||||
test_data = queries_data["test"].get(difficulty, [])
|
||||
|
||||
if test_mode:
|
||||
train_queries.extend(train_data[:5])
|
||||
test_queries.extend(test_data[:5])
|
||||
else:
|
||||
train_queries.extend(train_data)
|
||||
test_queries.extend(test_data)
|
||||
|
||||
logger.info(f"Loaded {len(train_queries)} train queries and {len(test_queries)} test queries")
|
||||
|
||||
# 运行多个epoch并收集结果
|
||||
all_epoch_results = []
|
||||
|
||||
for epoch in range(1, run_epoch + 1):
|
||||
epoch_result = run_single_epoch(epoch, train_queries, test_queries)
|
||||
if epoch_result:
|
||||
all_epoch_results.append(epoch_result)
|
||||
else:
|
||||
logger.error(f"Epoch {epoch} failed, skipping...")
|
||||
|
||||
# ==================== 计算多轮平均效果 ====================
|
||||
if not all_epoch_results:
|
||||
logger.error("No successful epochs, cannot calculate averages")
|
||||
return
|
||||
|
||||
print("\n" + "=" * 100)
|
||||
print("MULTI-EPOCH AVERAGE RESULTS")
|
||||
print("=" * 100)
|
||||
|
||||
# 计算每个场景的平均分数
|
||||
avg_train_no_memory = sum(e["statistics"]["train_no_memory"]["avg_score"] for e in all_epoch_results) / len(
|
||||
all_epoch_results)
|
||||
avg_test_no_memory = sum(e["statistics"]["test_no_memory"]["avg_score"] for e in all_epoch_results) / len(
|
||||
all_epoch_results)
|
||||
avg_test_with_memory = sum(e["statistics"]["test_with_memory"]["avg_score"] for e in all_epoch_results) / len(
|
||||
all_epoch_results)
|
||||
|
||||
# 计算平均改进
|
||||
avg_improvement = sum(e["improvements"]["avg_score"] for e in all_epoch_results) / len(all_epoch_results)
|
||||
|
||||
# 打印汇总表格
|
||||
headers = ["Scenario", "Avg Score (across epochs)"]
|
||||
rows = [
|
||||
["Train (No Memory)", f"{avg_train_no_memory:.3f}"],
|
||||
["Test (No Memory)", f"{avg_test_no_memory:.3f}"],
|
||||
["Test (With Memory)", f"{avg_test_with_memory:.3f}"]
|
||||
]
|
||||
print(tabulate(rows, headers=headers, tablefmt="grid"))
|
||||
|
||||
print("\n" + "-" * 100)
|
||||
print(f"Average Improvement (Test with memory vs without): {avg_improvement:+.2f}%")
|
||||
print("-" * 100)
|
||||
|
||||
# 打印每个epoch的详细结果
|
||||
print("\n" + "=" * 100)
|
||||
print("PER-EPOCH BREAKDOWN")
|
||||
print("=" * 100)
|
||||
|
||||
headers = ["Epoch", "Train (No Mem)", "Test (No Mem)", "Test (With Mem)", "Improvement %"]
|
||||
rows = []
|
||||
for e in all_epoch_results:
|
||||
rows.append([
|
||||
f"Epoch {e['epoch']}",
|
||||
f"{e['statistics']['train_no_memory']['avg_score']:.3f}",
|
||||
f"{e['statistics']['test_no_memory']['avg_score']:.3f}",
|
||||
f"{e['statistics']['test_with_memory']['avg_score']:.3f}",
|
||||
f"{e['improvements']['avg_score']:+.2f}%"
|
||||
])
|
||||
|
||||
print(tabulate(rows, headers=headers, tablefmt="grid"))
|
||||
|
||||
# 保存最终结果
|
||||
benchmark_summary = {
|
||||
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"total_epochs": run_epoch,
|
||||
"successful_epochs": len(all_epoch_results),
|
||||
"average_results": {
|
||||
"train_no_memory": avg_train_no_memory,
|
||||
"test_no_memory": avg_test_no_memory,
|
||||
"test_with_memory": avg_test_with_memory,
|
||||
"improvement": avg_improvement
|
||||
},
|
||||
"per_epoch_results": all_epoch_results
|
||||
}
|
||||
|
||||
save_results(benchmark_summary, "tool_memory_benchmark_results.json")
|
||||
|
||||
logger.info("\n" + "=" * 100)
|
||||
logger.info("TOOL MEMORY BENCHMARK - COMPLETE")
|
||||
logger.info(f"Successfully completed {len(all_epoch_results)}/{run_epoch} epochs")
|
||||
logger.info("=" * 100)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(test_mode=False, run_epoch=3)
|
||||
257
docs/index.md
257
docs/index.md
|
|
@ -26,11 +26,11 @@ show_datetime: true
|
|||
ReMe provides AI agents with a unified memory system—enabling the ability to extract, reuse, and share memories across
|
||||
users, tasks, and agents.
|
||||
|
||||
!!! info "Personal Memory + Task Memory = Agent Memory"
|
||||
|
||||
Personal memory helps "**understand user preferences**", while task memory helps agents "**perform better**".
|
||||
|
||||
```
|
||||
Personal Memory + Task Memory + Tool Memory = Agent Memory
|
||||
```
|
||||
|
||||
Personal memory helps "**understand user preferences**", task memory helps agents "**perform better**", and tool memory enables "**smarter tool usage**".
|
||||
|
||||
## Architecture Design
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ Personal memory helps "**understand user preferences**", while task memory helps
|
|||
<img src="figure/reme_structure.jpg" alt="ReMe Logo" width="100%">
|
||||
</p>
|
||||
|
||||
ReMe integrates two complementary memory capabilities:
|
||||
ReMe integrates three complementary memory capabilities:
|
||||
|
||||
!!! note "Task Memory/Experience"
|
||||
|
||||
|
|
@ -59,10 +59,18 @@ ReMe integrates two complementary memory capabilities:
|
|||
|
||||
Contextualized memory for specific users
|
||||
|
||||
- **Individual Preferences**: User habits, preferences, and interaction styles
|
||||
- **Contextual Adaptation**: Intelligent memory management based on time and context
|
||||
- **Progressive Learning**: Gradually build deep understanding through long-term interaction
|
||||
- **Time Awareness**: Time sensitivity in both retrieval and integration
|
||||
#### 🔧 **Tool Memory**
|
||||
|
||||
Data-driven tool selection and usage optimization
|
||||
|
||||
- **Historical Performance Tracking**: Success rates, execution times, and token costs from real usage
|
||||
- **LLM-as-Judge Evaluation**: Qualitative insights on why tools succeed or fail
|
||||
- **Parameter Optimization**: Learn optimal parameter configurations from successful calls
|
||||
- **Dynamic Guidelines**: Transform static tool descriptions into living, learned manuals
|
||||
|
||||
Learn more about how to use tool memory from [tool memory](tool_memory/tool_memory.md)
|
||||
|
||||
---
|
||||
|
||||
Learn more about how to use personal memory from [personal memory](personal_memory/personal_memory.md)
|
||||
|
||||
|
|
@ -303,15 +311,236 @@ fetch("http://localhost:8002/retrieve_personal_memory", {
|
|||
|
||||
</details>
|
||||
|
||||
#### Tool Memory Management
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
# Record tool execution results
|
||||
response = requests.post("http://localhost:8002/add_tool_call_result", json={
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_call_results": [
|
||||
{
|
||||
"create_time": "2025-10-21 10:30:00",
|
||||
"tool_name": "web_search",
|
||||
"input": {"query": "Python asyncio tutorial", "max_results": 10},
|
||||
"output": "Found 10 relevant results...",
|
||||
"token_cost": 150,
|
||||
"success": True,
|
||||
"time_cost": 2.3
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
# Generate usage guidelines from history
|
||||
response = requests.post("http://localhost:8002/summary_tool_memory", json={
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_names": "web_search"
|
||||
})
|
||||
|
||||
# Retrieve tool guidelines before use
|
||||
response = requests.post("http://localhost:8002/retrieve_tool_memory", json={
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_names": "web_search"
|
||||
})
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary>curl version</summary>
|
||||
|
||||
```bash
|
||||
# Record tool execution results
|
||||
curl -X POST http://localhost:8002/add_tool_call_result \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_call_results": [
|
||||
{
|
||||
"create_time": "2025-10-21 10:30:00",
|
||||
"tool_name": "web_search",
|
||||
"input": {"query": "Python asyncio tutorial", "max_results": 10},
|
||||
"output": "Found 10 relevant results...",
|
||||
"token_cost": 150,
|
||||
"success": true,
|
||||
"time_cost": 2.3
|
||||
}
|
||||
]
|
||||
}'
|
||||
|
||||
# Generate usage guidelines from history
|
||||
curl -X POST http://localhost:8002/summary_tool_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_names": "web_search"
|
||||
}'
|
||||
|
||||
# Retrieve tool guidelines before use
|
||||
curl -X POST http://localhost:8002/retrieve_tool_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "tool_workspace",
|
||||
"tool_names": "web_search"
|
||||
}'
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Node.js version</summary>
|
||||
|
||||
```javascript
|
||||
// Record tool execution results
|
||||
fetch("http://localhost:8002/add_tool_call_result", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workspace_id: "tool_workspace",
|
||||
tool_call_results: [
|
||||
{
|
||||
create_time: "2025-10-21 10:30:00",
|
||||
tool_name: "web_search",
|
||||
input: {query: "Python asyncio tutorial", max_results: 10},
|
||||
output: "Found 10 relevant results...",
|
||||
token_cost: 150,
|
||||
success: true,
|
||||
time_cost: 2.3
|
||||
}
|
||||
]
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => console.log(data));
|
||||
|
||||
// Generate usage guidelines from history
|
||||
fetch("http://localhost:8002/summary_tool_memory", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workspace_id: "tool_workspace",
|
||||
tool_names: "web_search"
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => console.log(data));
|
||||
|
||||
// Retrieve tool guidelines before use
|
||||
fetch("http://localhost:8002/retrieve_tool_memory", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workspace_id: "tool_workspace",
|
||||
tool_names: "web_search"
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => console.log(data));
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
## Resources
|
||||
- **[Personal memory](personal_memory/personal_memory.md)** & **[Task memory](task_memory/task_memory.md)** : Operators used in personal memory and task memory, You can modify the config to customize the pipelines.
|
||||
- **[Example Collection](cookbook/experiment_overview.md)**: Real use cases and best practices
|
||||
- **[Library](./library/library.md)**: Directly use existing task memory/experience for your tasks, and you can also contribute more task memory/experience to us.
|
||||
- **[Contribution](contribution.md)**: welcome to your contributions!
|
||||
## 📦 Ready-to-Use Libraries
|
||||
|
||||
ReMe provides pre-built memory libraries that agents can immediately use with verified best practices:
|
||||
|
||||
### Available Libraries
|
||||
|
||||
- **`appworld.jsonl`**: Memory library for Appworld agent interactions, covering complex task planning and execution
|
||||
patterns
|
||||
- **`bfcl_v3.jsonl`**: Working memory library for BFCL tool calls
|
||||
|
||||
### Quick Usage
|
||||
|
||||
```python
|
||||
# Load pre-built memories
|
||||
response = requests.post("http://localhost:8002/vector_store", json={
|
||||
"workspace_id": "appworld",
|
||||
"action": "load",
|
||||
"path": "./docs/library/"
|
||||
})
|
||||
|
||||
# Query relevant memories
|
||||
response = requests.post("http://localhost:8002/retrieve_task_memory", json={
|
||||
"workspace_id": "appworld",
|
||||
"query": "How to navigate to settings and update user profile?",
|
||||
"top_k": 1
|
||||
})
|
||||
```
|
||||
|
||||
## 🧪 Experiments
|
||||
|
||||
### 🌍 [Appworld Experiment](cookbook/appworld/quickstart.md)
|
||||
|
||||
We tested ReMe on Appworld using qwen3-8b:
|
||||
|
||||
| Method | pass@1 | pass@2 | pass@4 |
|
||||
|--------------|-------------------|-------------------|-------------------|
|
||||
| without ReMe | 0.083 | 0.140 | 0.228 |
|
||||
| with ReMe | 0.109 **(+2.6%)** | 0.175 **(+3.5%)** | 0.281 **(+5.3%)** |
|
||||
|
||||
Pass@K measures the probability that at least one of the K generated samples successfully completes the task (
|
||||
score=1).
|
||||
The current experiment uses an internal AppWorld environment, which may have slight differences.
|
||||
|
||||
You can find more details on reproducing the experiment in [quickstart.md](cookbook/appworld/quickstart.md).
|
||||
|
||||
### 🧊 [Frozenlake Experiment](./cookbook/frozenlake/quickstart.md)
|
||||
|
||||
| without ReMe | with ReMe |
|
||||
|:-------------------------------------------------------------------------------------------:|:-------------------------------------------------------------------------------------------:|
|
||||
| <p align="center"><img src="figure/frozenlake_failure.gif" alt="GIF 1" width="30%"></p> | <p align="center"><img src="figure/frozenlake_success.gif" alt="GIF 2" width="30%"></p> |
|
||||
|
||||
We tested on 100 random frozenlake maps using qwen3-8b:
|
||||
|
||||
| Method | pass rate |
|
||||
|--------------|------------------|
|
||||
| without ReMe | 0.66 |
|
||||
| with ReMe | 0.72 **(+6.0%)** |
|
||||
|
||||
You can find more details on reproducing the experiment in [quickstart.md](cookbook/frozenlake/quickstart.md).
|
||||
|
||||
### 🔧 [BFCL-V3 Experiment](./cookbook/bfcl/quickstart.md)
|
||||
|
||||
We tested ReMe on BFCL-V3 multi-turn-base (randomly split 50train/150val) using qwen3-8b:
|
||||
|
||||
| Method | pass@1 | pass@2 | pass@4 |
|
||||
|--------------|---------------------|---------------------|---------------------|
|
||||
| without ReMe | 0.2472 | 0.2733 | 0.2922 |
|
||||
| with ReMe | 0.3061 **(+5.89%)** | 0.3500 **(+7.67%)** | 0.3888 **(+9.66%)** |
|
||||
|
||||
### 🛠️ [Tool Memory Benchmark](tool_memory/tool_bench.md)
|
||||
|
||||
We evaluated Tool Memory effectiveness using a controlled benchmark with three mock search tools using Qwen3-30B-Instruct:
|
||||
|
||||
| Scenario | Avg Score | Improvement |
|
||||
|-----------------------|-----------|--------------------|
|
||||
| Train (No Memory) | 0.650 | - |
|
||||
| Test (No Memory) | 0.672 | Baseline |
|
||||
| **Test (With Memory)** | **0.772** | **+14.88%** |
|
||||
|
||||
**Key Findings:**
|
||||
- Tool Memory enables data-driven tool selection based on historical performance
|
||||
- Success rates improved by ~15% with learned parameter configurations
|
||||
|
||||
You can find more details in [tool_bench.md](tool_memory/tool_bench.md) and the implementation at [run_reme_tool_bench.py](https://github.com/modelscope/ReMe/tree/main/cookbook/tool_memory/run_reme_tool_bench.py).
|
||||
|
||||
## 📚 Resources
|
||||
|
||||
- **[Quick Start](https://github.com/modelscope/ReMe/tree/main/cookbook/simple_demo)**: Get started quickly with practical examples
|
||||
- [Tool Memory Demo](https://github.com/modelscope/ReMe/tree/main/cookbook/simple_demo/use_tool_memory_demo.py): Complete lifecycle demonstration of tool memory
|
||||
- [Tool Memory Benchmark](https://github.com/modelscope/ReMe/tree/main/cookbook/tool_memory/run_reme_tool_bench.py): Evaluate tool memory effectiveness
|
||||
- **[Vector Storage Setup](vector_store_api_guide.md)**: Configure local/vector databases and usage
|
||||
- **[MCP Guide](mcp_quick_start.md)**: Create MCP services
|
||||
- **[Personal Memory](personal_memory/personal_memory.md)**, **[Task Memory](task_memory/task_memory.md)** & **[Tool Memory](tool_memory/tool_memory.md)**: Operators used in personal memory, task memory and tool memory. You can modify the config to customize the pipelines.
|
||||
- **[Example Collection](./cookbook/appworld/quickstart.md)**: Real use cases and best practices
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -199,4 +199,4 @@ ReMe also provides additional task memory operations:
|
|||
- `record_task_memory`: Update frequency and utility attributes of retrieved memories
|
||||
- `delete_task_memory`: Delete memories based on utility/frequency thresholds
|
||||
|
||||
For more detailed examples, see the `use_task_memory_demo.py` file in the cookbook directory of the ReMe project.
|
||||
For more detailed examples, see the `use_task_memory_demo.py` file in the cookbook directory of the ReMe project.
|
||||
|
|
@ -70,4 +70,4 @@ An alternative to RewriteMemoryOp that merges multiple memories into a single re
|
|||
|
||||
- Collects the content from all memories in the memory list
|
||||
- Formats them into a single response with a standard structure
|
||||
- Adds a prompt to consider the helpful parts when answering the question
|
||||
- Adds a prompt to consider the helpful parts when answering the question
|
||||
|
|
@ -160,4 +160,4 @@ A simplified version of comparative memory extraction.
|
|||
|
||||
### Parameters
|
||||
|
||||
No specific parameters beyond the LLM configuration.
|
||||
No specific parameters beyond the LLM configuration.
|
||||
25
docs/todo.md
Normal file
25
docs/todo.md
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# TODO
|
||||
|
||||
## Planned Features
|
||||
|
||||
### 1. Automatic Tool Exploration Mode
|
||||
Add an automatic tool exploration mode that generates tool memory by:
|
||||
- Automatically discovering and testing available tools
|
||||
- Learning tool usage patterns and best practices
|
||||
- Building a comprehensive tool memory database from exploration results
|
||||
|
||||
### 2. Desktop Pet Personal Assistant
|
||||
Build a desktop pet personal assistant with:
|
||||
- Interactive desktop companion interface
|
||||
- Personalized assistance capabilities
|
||||
- Integration with ReMe's memory system
|
||||
|
||||
### 3. Task Memory Research Implementation
|
||||
We are currently working on implementing features based on task memory research papers. Coming soon.
|
||||
|
||||
### 4. Mem-Agent Exploration
|
||||
We are exploring mem-agent to implement agentic memory pathways:
|
||||
- Investigating agent-driven memory management
|
||||
- Developing autonomous memory retrieval and storage mechanisms
|
||||
- Building more intelligent memory update strategies
|
||||
|
||||
256
docs/tool_memory/tool_bench.md
Normal file
256
docs/tool_memory/tool_bench.md
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
# Tool Memory Benchmark
|
||||
|
||||
## Overview
|
||||
|
||||
This benchmark evaluates Tool Memory effectiveness by comparing agent performance with and without tool memory across multiple epochs. The experiment uses mock search tools with varying performance characteristics for different query complexities.
|
||||
|
||||
## Experimental Setup
|
||||
|
||||
### Mock Search Tools
|
||||
|
||||
Three LLM-based mock search tools with different performance profiles:
|
||||
|
||||
| Tool | Simple Queries | Medium Queries | Complex Queries |
|
||||
|------|---------------|----------------|-----------------|
|
||||
| **SearchToolA** | ⭐⭐⭐ Fast, high success (90%) | ❌ Poor (20% success) | ⚠️ Weak (50% success) |
|
||||
| **SearchToolB** | ⚠️ Over-engineered (30%) | ⭐⭐⭐ Optimal (90% success) | ⚠️ Limited (50% success) |
|
||||
| **SearchToolC** | ⚠️ Overkill (30%) | ⚠️ Excessive (40%) | ⭐⭐⭐ Best (90% success) |
|
||||
|
||||
**Performance Characteristics:**
|
||||
- `success_rate`: Probability of successful execution (vs "Service busy" error)
|
||||
- `relevance_ratio`: Probability of returning relevant results (vs random content)
|
||||
- `extra_time`: Simulated latency (currently 0 in implementation)
|
||||
|
||||
Each tool uses LLM to classify query complexity and generate appropriate responses.
|
||||
|
||||
### Query Dataset
|
||||
|
||||
**Source:** `cookbook/tool_memory/query.json`
|
||||
|
||||
- **Train Set**: 20 queries per complexity × 3 levels = 60 queries
|
||||
- **Test Set**: 20 queries per complexity × 3 levels = 60 queries
|
||||
- **Complexity Levels**: simple, moderate, complex
|
||||
|
||||
## Benchmark Workflow
|
||||
|
||||
### Single Epoch Process
|
||||
|
||||
Each epoch consists of 5 steps:
|
||||
|
||||
#### Step 1: Train without Memory
|
||||
```python
|
||||
# Execute all train queries on TRAIN_WORKSPACE
|
||||
# Agent selects tools without historical guidance
|
||||
run_use_mock_search(TRAIN_WORKSPACE, train_queries, prompt_template)
|
||||
|
||||
# Add results to memory and get scored results
|
||||
train_scored_results = add_tool_call_results(TRAIN_WORKSPACE, train_results)
|
||||
```
|
||||
|
||||
#### Step 2: Test without Memory
|
||||
```python
|
||||
# Execute all test queries on TEST_WORKSPACE (fresh workspace)
|
||||
# Baseline performance without tool memory
|
||||
run_use_mock_search(TEST_WORKSPACE, test_queries, prompt_template)
|
||||
|
||||
# Add results to memory (will be cleared in Step 4)
|
||||
test_scored_results = add_tool_call_results(TEST_WORKSPACE, test_results)
|
||||
```
|
||||
|
||||
#### Step 3: Summarize Tool Memory
|
||||
```python
|
||||
# Summarize tool performance from TRAIN_WORKSPACE
|
||||
summarize_tool_memory(TRAIN_WORKSPACE, "SearchToolA,SearchToolB,SearchToolC")
|
||||
|
||||
# Retrieve formatted tool memory content
|
||||
memories = retrieve_tool_memory(TRAIN_WORKSPACE, tool_names)
|
||||
```
|
||||
|
||||
The summarization produces memory content including:
|
||||
- Best/worst use cases per tool
|
||||
- Statistical metrics (avg score, success rate, token cost, time cost)
|
||||
- Usage recommendations
|
||||
|
||||
#### Step 4: Test with Memory
|
||||
```python
|
||||
# Clear TEST_WORKSPACE to start fresh
|
||||
delete_workspace(TEST_WORKSPACE)
|
||||
|
||||
# Inject tool memory into prompt
|
||||
prompt_with_memory = f"Tool Information\n{memories}\nMust select one tool to answer\nQuery\n{query}"
|
||||
|
||||
# Execute test queries with memory guidance
|
||||
run_use_mock_search(TEST_WORKSPACE, test_queries, prompt_with_memory)
|
||||
|
||||
# Add results and get scored results
|
||||
test_scored_results_with_memory = add_tool_call_results(TEST_WORKSPACE, test_results)
|
||||
```
|
||||
|
||||
#### Step 5: Compare Results
|
||||
```python
|
||||
# Generate comparison table
|
||||
print_comparison_table([train_no_memory_stats, test_no_memory_stats, test_with_memory_stats])
|
||||
|
||||
# Calculate improvements (baseline: test without memory)
|
||||
improvements = calculate_improvements(test_no_memory_stats, test_with_memory_stats)
|
||||
print_improvements(improvements)
|
||||
```
|
||||
|
||||
### Multi-Epoch Execution
|
||||
|
||||
```bash
|
||||
# Run benchmark with 3 epochs
|
||||
python cookbook/tool_memory/run_reme_tool_bench.py
|
||||
|
||||
# Test mode (5 queries per complexity level)
|
||||
main(test_mode=True, run_epoch=3)
|
||||
|
||||
# Full mode (20 queries per complexity level)
|
||||
main(test_mode=False, run_epoch=3)
|
||||
```
|
||||
|
||||
## Key Components
|
||||
|
||||
### 1. Tool Selection: UseMockSearchOp
|
||||
|
||||
```python
|
||||
# Agent uses LLM to select appropriate tool
|
||||
tool_call = await self.select_tool(query, [SearchToolA(), SearchToolB(), SearchToolC()])
|
||||
|
||||
# Execute selected tool and record results
|
||||
result = ToolCallResult(
|
||||
create_time=timestamp,
|
||||
tool_name=tool_call.name,
|
||||
input={"query": query},
|
||||
output=content,
|
||||
token_cost=token_cost,
|
||||
success=success,
|
||||
time_cost=time_cost
|
||||
)
|
||||
```
|
||||
|
||||
### 2. Tool Call Result Evaluation
|
||||
|
||||
Results are automatically evaluated and scored:
|
||||
- `score`: 0.0 (failure/irrelevant) or 1.0 (complete success)
|
||||
- `success`: Tool execution status
|
||||
- `summary`: Brief description
|
||||
- `evaluation`: Detailed assessment
|
||||
|
||||
### 3. Tool Memory Schema
|
||||
|
||||
```python
|
||||
ToolMemory(
|
||||
workspace_id="workspace_id",
|
||||
memory_type="tool",
|
||||
when_to_use="Brief usage scenario description",
|
||||
content="Detailed performance analysis and recommendations",
|
||||
score=0.85,
|
||||
tool_call_results=[list of ToolCallResult],
|
||||
metadata={"tool_name": "SearchToolA"}
|
||||
)
|
||||
```
|
||||
|
||||
## Evaluation Metrics
|
||||
|
||||
### Per-Scenario Metrics
|
||||
- **Avg Score**: Average quality score (0.0-1.0)
|
||||
- **Total Calls**: Number of tool invocations
|
||||
- **Success Rate**: Percentage of successful executions
|
||||
|
||||
### Improvement Calculation
|
||||
```python
|
||||
improvement_percentage = ((with_memory_score - without_memory_score) / without_memory_score) * 100
|
||||
```
|
||||
|
||||
## Expected Results
|
||||
|
||||
### Hypothesis
|
||||
Tool Memory should enable the agent to:
|
||||
1. **Select optimal tools** based on query complexity
|
||||
2. **Improve average score** by 10-30% on test set
|
||||
3. **Increase consistency** across multiple epochs
|
||||
|
||||
### Sample Output
|
||||
|
||||
```
|
||||
==================================================================================================
|
||||
BENCHMARK RESULTS COMPARISON
|
||||
==================================================================================================
|
||||
Note: Avg Score = average quality score
|
||||
+---------------------------+--------------+-----------+
|
||||
| Scenario | Total Calls | Avg Score |
|
||||
+===========================+==============+===========+
|
||||
| Epoch1 - Train (No Memory)| 60 | 0.650 |
|
||||
+---------------------------+--------------+-----------+
|
||||
| Epoch1 - Test (No Memory) | 60 | 0.633 |
|
||||
+---------------------------+--------------+-----------+
|
||||
| Epoch1 - Test (With Memory)| 60 | 0.817 |
|
||||
+---------------------------+--------------+-----------+
|
||||
|
||||
==================================================================================================
|
||||
IMPROVEMENTS WITH TOOL MEMORY (Baseline: Test without memory)
|
||||
==================================================================================================
|
||||
Average Score : +29.07% ↑
|
||||
==================================================================================================
|
||||
```
|
||||
|
||||
## Running the Benchmark
|
||||
|
||||
### Prerequisites
|
||||
```bash
|
||||
pip install requests python-dotenv loguru tabulate
|
||||
```
|
||||
|
||||
### Start API Server
|
||||
```bash
|
||||
# Start ReMe API server
|
||||
python reme_ai/app.py --port 8002
|
||||
```
|
||||
|
||||
### Execute Benchmark
|
||||
```bash
|
||||
# Full benchmark (3 epochs, 60+60 queries per epoch)
|
||||
python cookbook/tool_memory/run_reme_tool_bench.py
|
||||
|
||||
# Quick test (3 epochs, 15+15 queries per epoch)
|
||||
# Modify main() call: main(test_mode=True, run_epoch=3)
|
||||
```
|
||||
|
||||
### Output Files
|
||||
- `tool_memory_benchmark_results.json`: Complete benchmark results
|
||||
- Console output: Real-time progress and comparison tables
|
||||
|
||||
## API Endpoints Used
|
||||
|
||||
1. **`/use_mock_search`**: Execute tool selection and search
|
||||
- Input: `workspace_id`, `query`
|
||||
- Output: `ToolCallResult` JSON
|
||||
|
||||
2. **`/add_tool_call_result`**: Add results to memory and get evaluation scores
|
||||
- Input: `workspace_id`, `tool_call_results` (list)
|
||||
- Output: `memory_list` with scored results
|
||||
|
||||
3. **`/summary_tool_memory`**: Summarize tool performance
|
||||
- Input: `workspace_id`, `tool_names` (comma-separated)
|
||||
- Output: Updated `ToolMemory` with content
|
||||
|
||||
4. **`/retrieve_tool_memory`**: Retrieve formatted tool memory
|
||||
- Input: `workspace_id`, `tool_names`
|
||||
- Output: Markdown-formatted memory content
|
||||
|
||||
5. **`/vector_store`**: Delete workspace
|
||||
- Input: `workspace_id`, `action: "delete"`
|
||||
|
||||
## Concurrency Control
|
||||
|
||||
- **Max workers**: 4 parallel queries
|
||||
- **Rate limiting**: 1 second delay between submissions
|
||||
- **Timeout**: 120 seconds per API call
|
||||
|
||||
## References
|
||||
|
||||
- Tool Memory Schema: `reme_ai/schema/memory.py`
|
||||
- Mock Tools Implementation: `reme_ai/agent/tools/mock_search_tools.py`
|
||||
- LLM-based Search Op: `reme_ai/agent/tools/llm_mock_search_op.py`
|
||||
- Tool Selection Op: `reme_ai/agent/tools/use_mock_search_op.py`
|
||||
821
docs/tool_memory/tool_memory.md
Normal file
821
docs/tool_memory/tool_memory.md
Normal file
|
|
@ -0,0 +1,821 @@
|
|||
# Tool Memory in ReMe
|
||||
|
||||
## 1. Background: Why Tool Memory?
|
||||
|
||||
### The MCP Tool Selection Challenge
|
||||
|
||||
In modern AI agent systems, LLMs face a rapidly expanding ecosystem of MCP (Model Context Protocol) tools. With hundreds or thousands of available tools, a critical problem emerges:
|
||||
|
||||
**The Core Problem: Tool Description is Not Enough**
|
||||
|
||||
When an LLM faces numerous MCP tools, it relies heavily on tool descriptions to decide which tool to use and how to use it. However:
|
||||
|
||||
- **Ambiguous Descriptions**: Many tools have similar descriptions but different performance characteristics
|
||||
- **Hidden Complexity**: Static descriptions can't capture runtime behaviors, edge cases, or failure patterns
|
||||
- **Parameter Confusion**: Tools may accept similar parameters with different optimal values
|
||||
- **No Quality Signal**: Descriptions don't tell you which tools are reliable, fast, or cost-effective
|
||||
|
||||
**Example: Web Search Tools**
|
||||
|
||||
Imagine an LLM choosing between three search tools:
|
||||
```
|
||||
Tool A: "Search the web for information"
|
||||
Tool B: "Perform web searches with customizable parameters"
|
||||
Tool C: "Query search engines and return results"
|
||||
```
|
||||
|
||||
The descriptions are nearly identical, but in reality:
|
||||
- Tool A: 95% success rate, avg 2.3s, best for technical queries
|
||||
- Tool B: 70% success rate, avg 5.8s, often times out with >20 results
|
||||
- Tool C: 85% success rate, avg 3.1s, good for general queries
|
||||
|
||||
**Without historical data, the LLM can't make informed decisions.**
|
||||
|
||||
### The Solution: Tool Memory as Context Enhancement
|
||||
|
||||
Tool Memory solves this by providing **learned context from historical usage**, transforming static tool descriptions into dynamic, data-driven guidance:
|
||||
|
||||
**1. Rule-Based Statistics** (Objective Metrics)
|
||||
- **Success Rate**: "This tool succeeds 92% of the time"
|
||||
- **Performance**: "Average execution time: 2.3s, token cost: 150"
|
||||
- **Usage Patterns**: "Most successful calls use max_results=10-20"
|
||||
|
||||
**2. LLM-as-Judge Evaluation** (Qualitative Insights)
|
||||
- **Quality Assessment**: LLM evaluates each call's effectiveness
|
||||
- **Pattern Recognition**: Identifies why some calls succeed and others fail
|
||||
- **Actionable Recommendations**: Synthesizes guidelines from patterns
|
||||
|
||||
**3. Enhanced Context for LLM Decision-Making**
|
||||
|
||||
Instead of just a tool description, the LLM now receives:
|
||||
|
||||
```
|
||||
Tool: web_search
|
||||
|
||||
Static Description:
|
||||
"Search the web for information"
|
||||
|
||||
+ Tool Memory Context:
|
||||
"Based on 150 historical calls:
|
||||
- Success rate: 92% (138 successful, 12 failed)
|
||||
- Avg time: 2.3s, Avg tokens: 150
|
||||
- Best for: Technical documentation, tutorials (95% success)
|
||||
- Optimal params: max_results=5-20, language='en'
|
||||
- Common failures: Generic queries timeout, max_results>50 unreliable
|
||||
- Recommendation: Use specific multi-word queries with filter_type='technical_docs'"
|
||||
```
|
||||
|
||||
This enriched context enables the LLM to:
|
||||
- **Choose the right tool** based on task requirements and reliability
|
||||
- **Use optimal parameters** learned from successful historical calls
|
||||
- **Avoid known pitfalls** that caused previous failures
|
||||
- **Estimate costs** (time and tokens) before execution
|
||||
|
||||
### The Impact: From Static Descriptions to Dynamic Intelligence
|
||||
|
||||
**Traditional Approach (Static Descriptions Only):**
|
||||
```
|
||||
LLM: "I have 50 search tools, all with similar descriptions"
|
||||
→ Random choice or first match
|
||||
→ Trial-and-error parameter selection
|
||||
→ 75% success rate, repeated failures
|
||||
```
|
||||
|
||||
**Tool Memory Approach (Description + Historical Context):**
|
||||
```
|
||||
LLM: "I have 50 search tools, but Tool A has 95% success for technical queries"
|
||||
→ Informed choice based on data
|
||||
→ Use proven parameter configurations
|
||||
→ 92% success rate, optimized performance
|
||||
```
|
||||
|
||||
**Real-World Impact:**
|
||||
|
||||
```
|
||||
Before Tool Memory:
|
||||
- Success rate: 75%
|
||||
- Average time cost: 5.2s
|
||||
- Token cost: 200+ per call
|
||||
- Repeated parameter errors
|
||||
- Random tool selection
|
||||
|
||||
After Tool Memory:
|
||||
- Success rate: 92% (+17%)
|
||||
- Average time cost: 2.8s (-46%)
|
||||
- Token cost: 150 per call (-25%)
|
||||
- Consistent best practices
|
||||
- Data-driven tool selection
|
||||
```
|
||||
|
||||
### Why This Matters for MCP Ecosystem
|
||||
|
||||
As the MCP ecosystem grows, Tool Memory becomes essential:
|
||||
|
||||
1. **Scalability**: LLMs can navigate thousands of tools with confidence
|
||||
2. **Quality Control**: Tools with poor performance get flagged automatically
|
||||
3. **Continuous Improvement**: Every call improves the knowledge base
|
||||
4. **Transfer Learning**: Insights from one agent benefit all agents in the workspace
|
||||
|
||||
**Tool Memory transforms tool descriptions from static documentation into living, learned manuals that improve with every use.**
|
||||
|
||||
## 2. What is Tool Memory?
|
||||
|
||||
Tool Memory is a structured knowledge base that captures insights from tool usage history. Each Tool Memory represents accumulated wisdom about a specific tool.
|
||||
|
||||
### Data Structure
|
||||
|
||||
#### ToolMemory
|
||||
|
||||
`ToolMemory` is the core data structure that stores comprehensive information about a tool's usage patterns:
|
||||
|
||||
```python
|
||||
class ToolMemory(BaseMemory):
|
||||
memory_type: str = "tool" # Type identifier
|
||||
workspace_id: str # Workspace identifier
|
||||
memory_id: str # Unique memory ID
|
||||
when_to_use: str # Tool name (serves as unique identifier)
|
||||
content: str # Synthesized usage guidelines
|
||||
score: float # Overall quality score
|
||||
time_created: str # Creation timestamp
|
||||
time_modified: str # Last modification timestamp
|
||||
author: str # Creator (typically LLM model name)
|
||||
tool_call_results: List[ToolCallResult] # Historical invocation records
|
||||
metadata: dict # Additional metadata
|
||||
```
|
||||
|
||||
**Key Fields:**
|
||||
- **`when_to_use`**: The tool name, used as the unique identifier for retrieval
|
||||
- **`content`**: Human-readable usage guidelines synthesized from historical data
|
||||
- **`tool_call_results`**: Complete history of tool invocations with evaluations
|
||||
- **`score`**: Overall quality metric for the tool's performance
|
||||
|
||||
#### ToolCallResult
|
||||
|
||||
Each tool invocation is captured as a `ToolCallResult`:
|
||||
|
||||
```python
|
||||
class ToolCallResult(BaseModel):
|
||||
create_time: str # Invocation timestamp
|
||||
tool_name: str # Name of the tool
|
||||
input: dict | str # Input parameters
|
||||
output: str # Tool output
|
||||
token_cost: int # Token consumption
|
||||
success: bool # Whether invocation succeeded
|
||||
time_cost: float # Time consumed (seconds)
|
||||
summary: str # Brief summary of the result
|
||||
evaluation: str # Detailed evaluation (generated by LLM)
|
||||
score: float # Evaluation score (0.0 for failure, 1.0 for success)
|
||||
metadata: dict # Additional metadata
|
||||
```
|
||||
|
||||
**Key Fields:**
|
||||
- **`input`/`output`**: The complete I/O data for analysis
|
||||
- **`summary`**: LLM-generated brief summary of what happened
|
||||
- **`evaluation`**: LLM-generated detailed analysis of the call quality
|
||||
- **`score`**: Binary evaluation (0.0 = failure, 1.0 = success)
|
||||
- **Performance metrics**: `time_cost`, `token_cost`, `success` for statistical analysis
|
||||
|
||||
### Tool Memory Lifecycle
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
A[Tool Call] --> B[Evaluate]
|
||||
B --> C[Store Memory]
|
||||
C --> D[(Vector Store)]
|
||||
D --> E[Agent Retrieves]
|
||||
E --> A
|
||||
C -.Periodic.-> F[Summarize]
|
||||
F --> C
|
||||
```
|
||||
|
||||
## 3. How Tool Memory Works: The Complete Flow
|
||||
|
||||
Tool Memory operates through three complementary operations that work together to create a learning loop:
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
A[Agent] -->|1 retrieve_tool_memory| B[(Vector Store)]
|
||||
B -->|Guidelines| A
|
||||
A -->|2 Execute Tool| C[Tool]
|
||||
C -->|Result| A
|
||||
A -->|3 add_tool_call_result| D[LLM Evaluate]
|
||||
D -->|Store| B
|
||||
B -->|Periodic| E[summary_tool_memory]
|
||||
E -->|4 Update Guidelines| B
|
||||
```
|
||||
|
||||
### Operation Flow
|
||||
|
||||
**1. retrieve_tool_memory** (Before Execution)
|
||||
- Agent queries: "How should I use `web_search` tool?"
|
||||
- Retrieves stored guidelines and historical patterns
|
||||
- Returns: Usage recommendations, parameter suggestions, common pitfalls
|
||||
|
||||
**2. Tool Execution**
|
||||
- Agent executes tool with informed parameters
|
||||
- Collects: input, output, time_cost, token_cost, success status
|
||||
|
||||
**3. add_tool_call_result** (After Execution)
|
||||
- Submits execution data for evaluation
|
||||
- LLM analyzes: Was it successful? What could be improved?
|
||||
- Generates: summary, evaluation, score (0.0 or 1.0)
|
||||
- Appends to tool's historical record in Vector Store
|
||||
|
||||
**4. summary_tool_memory** (Periodic)
|
||||
- Analyzes recent N tool calls (e.g., last 20-30)
|
||||
- Calculates statistics: success rate, avg costs, avg score
|
||||
- LLM synthesizes: Actionable usage guidelines
|
||||
- Updates the `content` field with comprehensive guidance
|
||||
|
||||
### Example Flow from Demo
|
||||
|
||||
Based on `use_tool_memory_demo.py`, here's a typical workflow:
|
||||
|
||||
```python
|
||||
# Step 1: Add tool call results (accumulate history)
|
||||
add_tool_call_results([
|
||||
{"tool_name": "web_search", "input": {...}, "output": "...", "success": True},
|
||||
{"tool_name": "web_search", "input": {...}, "output": "...", "success": False},
|
||||
# ... more results
|
||||
])
|
||||
|
||||
# Step 2: Generate usage guidelines (periodic)
|
||||
summarize_tool_memory("web_search")
|
||||
|
||||
# Step 3: Retrieve guidelines before next use
|
||||
memory = retrieve_tool_memory("web_search")
|
||||
# Returns:
|
||||
# "For web_search tool:
|
||||
# - Use max_results=5-20 for optimal performance
|
||||
# - Avoid generic queries, be specific
|
||||
# - Language parameter 'en' has 95% success rate
|
||||
# Statistics: 83% success, avg 2.3s, avg 150 tokens"
|
||||
|
||||
# Step 4: Agent uses guidelines for better execution
|
||||
execute_with_recommended_parameters()
|
||||
```
|
||||
|
||||
## 4. Operation Details: How to Use Each Component
|
||||
|
||||
### 4.1 `add_tool_call_result`
|
||||
|
||||
**Purpose**: Evaluate and store tool call results into Tool Memory.
|
||||
|
||||
**Flow**:
|
||||
```yaml
|
||||
add_tool_call_result:
|
||||
flow_content: parse_tool_call_result_op >> update_vector_store_op
|
||||
description: "Evaluates and adds tool call results to the tool memory database"
|
||||
```
|
||||
|
||||
**Process**:
|
||||
1. Receives raw tool call results
|
||||
2. Uses LLM to evaluate each call (generates summary, evaluation, score)
|
||||
3. Groups results by tool name
|
||||
4. Creates or updates ToolMemory objects
|
||||
5. Stores in Vector Store
|
||||
|
||||
**Configuration** (`default.yaml`):
|
||||
```yaml
|
||||
op:
|
||||
parse_tool_call_result_op:
|
||||
backend: parse_tool_call_result_op
|
||||
llm: default
|
||||
params:
|
||||
max_history_tool_call_cnt: 100 # Max calls to retain per tool
|
||||
evaluation_sleep_interval: 1.0 # Delay between evaluations (seconds)
|
||||
```
|
||||
|
||||
#### Usage with curl
|
||||
|
||||
```bash
|
||||
curl -X POST http://0.0.0.0:8002/add_tool_call_result \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "my_workspace",
|
||||
"tool_call_results": [
|
||||
{
|
||||
"create_time": "2025-10-21 10:30:00",
|
||||
"tool_name": "web_search",
|
||||
"input": {
|
||||
"query": "Python asyncio tutorial",
|
||||
"max_results": 10,
|
||||
"language": "en"
|
||||
},
|
||||
"output": "Found 10 relevant results including official docs and tutorials",
|
||||
"token_cost": 150,
|
||||
"success": true,
|
||||
"time_cost": 2.3
|
||||
},
|
||||
{
|
||||
"create_time": "2025-10-21 10:32:00",
|
||||
"tool_name": "web_search",
|
||||
"input": {
|
||||
"query": "test",
|
||||
"max_results": 100,
|
||||
"language": "unknown"
|
||||
},
|
||||
"output": "Error: Invalid language parameter",
|
||||
"token_cost": 50,
|
||||
"success": false,
|
||||
"time_cost": 0.5
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
**Response**:
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"answer": "Successfully evaluated and stored 2 tool call results",
|
||||
"metadata": {
|
||||
"memory_list": [
|
||||
{
|
||||
"when_to_use": "web_search",
|
||||
"memory_id": "abc123...",
|
||||
"tool_call_results": [
|
||||
{
|
||||
"tool_name": "web_search",
|
||||
"summary": "Successfully retrieved relevant Python asyncio documentation",
|
||||
"evaluation": "Good parameter choices with appropriate max_results and language settings",
|
||||
"score": 1.0,
|
||||
...
|
||||
},
|
||||
{
|
||||
"tool_name": "web_search",
|
||||
"summary": "Failed due to invalid language parameter",
|
||||
"evaluation": "Query too generic and language parameter not supported",
|
||||
"score": 0.0,
|
||||
...
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Usage with Python
|
||||
|
||||
```python
|
||||
import requests
|
||||
from datetime import datetime
|
||||
|
||||
def add_tool_call_results(tool_call_results: list) -> dict:
|
||||
"""Add tool call results to Tool Memory"""
|
||||
response = requests.post(
|
||||
url=f"{BASE_URL}add_tool_call_result",
|
||||
json={
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"tool_call_results": tool_call_results
|
||||
}
|
||||
)
|
||||
return response.json()
|
||||
|
||||
# Example: Record a tool invocation
|
||||
result = add_tool_call_results([{
|
||||
"create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": "web_search",
|
||||
"input": {"query": "Python asyncio", "max_results": 10},
|
||||
"output": "Found 10 relevant results...",
|
||||
"token_cost": 150,
|
||||
"success": True,
|
||||
"time_cost": 2.3
|
||||
}])
|
||||
```
|
||||
|
||||
**Complete examples**: See `cookbook/simple_demo/use_tool_memory_demo.py` for full working code.
|
||||
|
||||
---
|
||||
|
||||
### 4.2 `retrieve_tool_memory`
|
||||
|
||||
**Purpose**: Retrieve usage guidelines and historical data for specific tools.
|
||||
|
||||
**Flow**:
|
||||
```yaml
|
||||
retrieve_tool_memory:
|
||||
flow_content: retrieve_tool_memory_op
|
||||
description: "Retrieves tool memories from the vector database based on tool names"
|
||||
```
|
||||
|
||||
**Process**:
|
||||
1. Takes comma-separated tool names as input
|
||||
2. Searches Vector Store for exact matches (by `when_to_use` field)
|
||||
3. Returns complete ToolMemory objects with:
|
||||
- Usage guidelines (`content`)
|
||||
- Historical call records (`tool_call_results`)
|
||||
- Statistics and metadata
|
||||
|
||||
#### Usage with curl
|
||||
|
||||
```bash
|
||||
# Retrieve single tool
|
||||
curl -X POST http://0.0.0.0:8002/retrieve_tool_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "my_workspace",
|
||||
"tool_names": "web_search"
|
||||
}'
|
||||
|
||||
# Retrieve multiple tools (comma-separated)
|
||||
curl -X POST http://0.0.0.0:8002/retrieve_tool_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "my_workspace",
|
||||
"tool_names": "web_search,database_query,file_processor"
|
||||
}'
|
||||
```
|
||||
|
||||
**Response**:
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"answer": "Successfully retrieved 1 tool memories",
|
||||
"metadata": {
|
||||
"memory_list": [
|
||||
{
|
||||
"memory_type": "tool",
|
||||
"workspace_id": "my_workspace",
|
||||
"memory_id": "abc123...",
|
||||
"when_to_use": "web_search",
|
||||
"content": "## Usage Guidelines\n\n**Best Practices:**\n- Use max_results between 5-20 for optimal performance\n- Always specify language parameter (en has 95% success rate)\n- Avoid generic single-word queries\n\n**Common Pitfalls:**\n- max_results > 50 often causes timeouts\n- Unknown language values default to 'en' with warning\n\n## Statistics\n- **Success Rate**: 83.33%\n- **Average Score**: 0.833\n- **Average Time Cost**: 2.345s\n- **Average Token Cost**: 156.7",
|
||||
"score": 0.85,
|
||||
"time_created": "2025-10-20 10:00:00",
|
||||
"time_modified": "2025-10-21 10:35:00",
|
||||
"author": "gpt-4",
|
||||
"tool_call_results": [
|
||||
{
|
||||
"create_time": "2025-10-21 10:30:00",
|
||||
"tool_name": "web_search",
|
||||
"input": {"query": "Python asyncio", "max_results": 10},
|
||||
"output": "Found 10 results...",
|
||||
"summary": "Successfully retrieved relevant documentation",
|
||||
"evaluation": "Good parameter choices...",
|
||||
"score": 1.0,
|
||||
"token_cost": 150,
|
||||
"success": true,
|
||||
"time_cost": 2.3
|
||||
}
|
||||
// ... more historical calls
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Usage with Python
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
def retrieve_tool_memory(tool_names: str) -> dict:
|
||||
"""Retrieve tool memories by tool names"""
|
||||
response = requests.post(
|
||||
url=f"{BASE_URL}retrieve_tool_memory",
|
||||
json={
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"tool_names": tool_names
|
||||
}
|
||||
)
|
||||
return response.json()
|
||||
|
||||
# Example: Retrieve and use guidelines
|
||||
result = retrieve_tool_memory("web_search")
|
||||
if result['success']:
|
||||
memory = result['metadata']['memory_list'][0]
|
||||
print(f"Tool: {memory['when_to_use']}")
|
||||
print(f"Guidelines:\n{memory['content']}")
|
||||
```
|
||||
|
||||
**Complete examples**: See `cookbook/simple_demo/use_tool_memory_demo.py` for full working code.
|
||||
|
||||
---
|
||||
|
||||
### 4.3 `summary_tool_memory`
|
||||
|
||||
**Purpose**: Analyze historical tool calls and generate comprehensive usage guidelines.
|
||||
|
||||
**Flow**:
|
||||
```yaml
|
||||
summary_tool_memory:
|
||||
flow_content: summary_tool_memory_op >> update_vector_store_op
|
||||
description: "Analyzes tool call history and generates comprehensive usage patterns"
|
||||
```
|
||||
|
||||
**Process**:
|
||||
1. Retrieves existing ToolMemory by tool name
|
||||
2. Analyzes recent N tool calls (default: 30)
|
||||
3. Calculates statistics:
|
||||
- Success rate
|
||||
- Average score
|
||||
- Average time cost
|
||||
- Average token cost
|
||||
4. Uses LLM to synthesize actionable guidelines from call summaries
|
||||
5. Appends statistics to guidelines
|
||||
6. Updates ToolMemory content in Vector Store
|
||||
|
||||
**Configuration** (`default.yaml`):
|
||||
```yaml
|
||||
op:
|
||||
summary_tool_memory_op:
|
||||
backend: summary_tool_memory_op
|
||||
llm: default
|
||||
params:
|
||||
recent_call_count: 30 # Number of recent calls to analyze
|
||||
summary_sleep_interval: 1.0 # Delay between summaries (seconds)
|
||||
```
|
||||
|
||||
#### Usage with curl
|
||||
|
||||
```bash
|
||||
# Summarize single tool
|
||||
curl -X POST http://0.0.0.0:8002/summary_tool_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "my_workspace",
|
||||
"tool_names": "web_search"
|
||||
}'
|
||||
|
||||
# Summarize multiple tools (comma-separated)
|
||||
curl -X POST http://0.0.0.0:8002/summary_tool_memory \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "my_workspace",
|
||||
"tool_names": "web_search,database_query,file_processor"
|
||||
}'
|
||||
```
|
||||
|
||||
**Response**:
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"answer": "Successfully summarized 1 tool memories",
|
||||
"metadata": {
|
||||
"memory_list": [
|
||||
{
|
||||
"memory_type": "tool",
|
||||
"when_to_use": "web_search",
|
||||
"content": "## Usage Guidelines\n\n**Optimal Parameters:**\n- Set max_results between 5-20 for best balance of coverage and speed\n- Always specify language='en' for technical queries (95% success rate)\n- Use filter_type='technical_docs' for development-related searches\n\n**Success Patterns:**\n- Specific, multi-word queries perform significantly better than generic terms\n- Queries with clear intent (e.g., 'Python asyncio tutorial') return high-quality results\n- Technical terms and version numbers improve result relevance\n\n**Common Failures:**\n- Generic single-word queries (e.g., 'test') return poor results\n- max_results > 50 increases timeout risk (5 failures observed)\n- Invalid language codes cause fallback to default with warnings\n\n**Performance Insights:**\n- Typical response time: 1.5-3.5s for successful queries\n- Timeout threshold: 10s (consider simplifying complex queries)\n- Token cost scales with result count: ~150 tokens for 10 results\n\n**Recommendations:**\n1. Always validate language parameter before calling\n2. Start with max_results=10, adjust based on needs\n3. For time-sensitive operations, set timeout < 5s\n4. Monitor token costs for high-frequency usage\n\n## Statistics\n- **Success Rate**: 83.33%\n- **Average Score**: 0.833\n- **Average Time Cost**: 2.345s\n- **Average Token Cost**: 156.7",
|
||||
"memory_id": "abc123...",
|
||||
"time_modified": "2025-10-21 10:40:00",
|
||||
...
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Usage with Python
|
||||
|
||||
```python
|
||||
import requests
|
||||
|
||||
def summarize_tool_memory(tool_names: str) -> dict:
|
||||
"""Generate comprehensive usage guidelines for tools"""
|
||||
response = requests.post(
|
||||
url=f"{BASE_URL}summary_tool_memory",
|
||||
json={
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"tool_names": tool_names
|
||||
}
|
||||
)
|
||||
return response.json()
|
||||
|
||||
# Example: Generate guidelines
|
||||
result = summarize_tool_memory("web_search")
|
||||
if result['success']:
|
||||
memory = result['metadata']['memory_list'][0]
|
||||
print(f"Tool: {memory['when_to_use']}")
|
||||
print(f"Guidelines:\n{memory['content']}")
|
||||
```
|
||||
|
||||
**Complete examples**: See `cookbook/simple_demo/use_tool_memory_demo.py` for full working code.
|
||||
|
||||
---
|
||||
|
||||
## 5. Best Practices
|
||||
|
||||
### When to Record Tool Calls
|
||||
- **Always**: Record every tool invocation, including failures
|
||||
- **Include**: Complete input parameters, output, and performance metrics
|
||||
- **Timing**: Record immediately after tool execution completes
|
||||
|
||||
### When to Generate Summaries
|
||||
- **Initial**: After accumulating 20-30 tool calls for meaningful patterns
|
||||
- **Periodic**: Re-summarize every 50-100 new calls or weekly
|
||||
- **Trigger-based**: When success rate drops or patterns change significantly
|
||||
|
||||
### When to Retrieve Guidelines
|
||||
- **Before first use**: Always retrieve before using an unfamiliar tool
|
||||
- **Before critical operations**: Check latest guidelines for important tasks
|
||||
- **After updates**: Re-retrieve when tool memory has been updated
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
**For High-Volume Tools** (>100 calls/day):
|
||||
```yaml
|
||||
op:
|
||||
parse_tool_call_result_op:
|
||||
params:
|
||||
max_history_tool_call_cnt: 200 # Keep more history
|
||||
evaluation_sleep_interval: 0.5 # Faster evaluation
|
||||
|
||||
summary_tool_memory_op:
|
||||
params:
|
||||
recent_call_count: 50 # Analyze more calls
|
||||
```
|
||||
|
||||
**For Low-Volume Tools** (<20 calls/day):
|
||||
```yaml
|
||||
op:
|
||||
parse_tool_call_result_op:
|
||||
params:
|
||||
max_history_tool_call_cnt: 50 # Less history needed
|
||||
evaluation_sleep_interval: 1.0 # Standard rate
|
||||
|
||||
summary_tool_memory_op:
|
||||
params:
|
||||
recent_call_count: 20 # Analyze fewer calls
|
||||
```
|
||||
|
||||
### Quality Maintenance
|
||||
|
||||
1. **Monitor Metrics**:
|
||||
```python
|
||||
memory = retrieve_tool_memory("web_search")['metadata']['memory_list'][0]
|
||||
stats = ToolMemory(**memory).statistic(recent_frequency=30)
|
||||
|
||||
print(f"Success Rate: {stats['success_rate']:.2%}")
|
||||
print(f"Avg Score: {stats['avg_score']:.2f}")
|
||||
|
||||
if stats['success_rate'] < 0.7:
|
||||
print("⚠️ Low success rate - investigate tool issues")
|
||||
```
|
||||
|
||||
2. **Clean Old Memories**:
|
||||
- Delete tool memories for deprecated tools
|
||||
- Reset memories when tool behavior changes significantly
|
||||
|
||||
3. **Validate Guidelines**:
|
||||
- Periodically review generated guidelines for accuracy
|
||||
- Test recommended parameters in production scenarios
|
||||
|
||||
## 6. Memory Management
|
||||
|
||||
### Delete Workspace
|
||||
```bash
|
||||
curl -X POST http://0.0.0.0:8002/vector_store \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "my_workspace",
|
||||
"action": "delete"
|
||||
}'
|
||||
```
|
||||
|
||||
```python
|
||||
def delete_workspace(workspace_id: str):
|
||||
response = requests.post(
|
||||
url=f"{BASE_URL}vector_store",
|
||||
json={"workspace_id": workspace_id, "action": "delete"}
|
||||
)
|
||||
return response.json()
|
||||
```
|
||||
|
||||
### Dump Memories to Disk
|
||||
```bash
|
||||
curl -X POST http://0.0.0.0:8002/vector_store \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "my_workspace",
|
||||
"action": "dump",
|
||||
"path": "./memory_backup/"
|
||||
}'
|
||||
```
|
||||
|
||||
```python
|
||||
def dump_memory(workspace_id: str, path: str = "./"):
|
||||
response = requests.post(
|
||||
url=f"{BASE_URL}vector_store",
|
||||
json={"workspace_id": workspace_id, "action": "dump", "path": path}
|
||||
)
|
||||
return response.json()
|
||||
```
|
||||
|
||||
### Load Memories from Disk
|
||||
```bash
|
||||
curl -X POST http://0.0.0.0:8002/vector_store \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "my_workspace",
|
||||
"action": "load",
|
||||
"path": "./memory_backup/"
|
||||
}'
|
||||
```
|
||||
|
||||
```python
|
||||
def load_memory(workspace_id: str, path: str = "./"):
|
||||
response = requests.post(
|
||||
url=f"{BASE_URL}vector_store",
|
||||
json={"workspace_id": workspace_id, "action": "load", "path": path}
|
||||
)
|
||||
return response.json()
|
||||
```
|
||||
|
||||
## 7. Complete Working Example
|
||||
|
||||
For a complete, runnable example demonstrating the full Tool Memory lifecycle, see:
|
||||
|
||||
**`cookbook/simple_demo/use_tool_memory_demo.py`**
|
||||
|
||||
This demo includes:
|
||||
- **Workspace management**: Clean, delete, dump, and load operations
|
||||
- **Tool call recording**: Adding 30+ mock tool invocations with various scenarios
|
||||
- **Summarization**: Generating usage guidelines from historical data
|
||||
- **Retrieval**: Fetching and displaying tool memories
|
||||
- **Statistics**: Analyzing success rates, costs, and performance
|
||||
|
||||
Run the demo:
|
||||
```bash
|
||||
cd cookbook/simple_demo
|
||||
python use_tool_memory_demo.py
|
||||
```
|
||||
|
||||
**Key Workflow Steps:**
|
||||
1. **Clean workspace**: Remove existing data
|
||||
2. **Add tool calls**: Record 30+ invocations (success/failure scenarios)
|
||||
3. **Generate guidelines**: LLM analyzes patterns and creates recommendations
|
||||
4. **Retrieve memory**: Get usage guidelines for agent consumption
|
||||
5. **Persistence**: Test dump/load operations
|
||||
|
||||
## 8. Advanced Use Cases
|
||||
|
||||
### Use Case 1: Adaptive Parameter Tuning
|
||||
|
||||
Retrieve tool memory statistics and adapt parameters based on historical performance:
|
||||
- If `avg_time_cost > 5s`: Increase timeout
|
||||
- If `success_rate < 80%`: Enable retry logic
|
||||
- If `avg_token_cost` high: Reduce result limits
|
||||
|
||||
### Use Case 2: Multi-Tool Workflow Optimization
|
||||
|
||||
Retrieve memories for multiple tools at once and optimize workflow order based on:
|
||||
- Success rates: Execute reliable tools first
|
||||
- Time costs: Parallelize slow operations
|
||||
- Token costs: Budget-aware tool selection
|
||||
|
||||
### Use Case 3: Automated Quality Monitoring
|
||||
|
||||
Periodically check tool memory statistics and alert on:
|
||||
- Success rate degradation
|
||||
- Increasing time/token costs
|
||||
- Unusual failure patterns
|
||||
|
||||
**Implementation examples**: See `cookbook/simple_demo/use_tool_memory_demo.py` and the ToolBench evaluation scripts.
|
||||
|
||||
## 9. Benchmark Results
|
||||
|
||||
### Tool Memory Performance Evaluation
|
||||
|
||||
We evaluated Tool Memory effectiveness using a controlled benchmark with three mock search tools, each optimized for different query complexity levels (simple, moderate, complex). The benchmark compares agent performance with and without tool memory guidance across multiple epochs.
|
||||
|
||||
**Experimental Settings:**
|
||||
- **Model**: Qwen3-30B-Instruct with default parameters
|
||||
- **Task**: Single-turn tool selection and invocation
|
||||
- **Dataset**: 60 training queries + 60 test queries per epoch
|
||||
- **Tools**: 3 mock search tools with varying performance profiles
|
||||
- **Metrics**: Average quality score (0.0-1.0) based on LLM evaluation
|
||||
- **Baseline**: Test set performance without tool memory
|
||||
- **Replication**: Results averaged across 3 independent experimental runs
|
||||
|
||||
**Results (averaged across 3 epochs):**
|
||||
|
||||
| Scenario | Avg Score | Improvement |
|
||||
|----------|-----------|-------------|
|
||||
| Train (No Memory) | 0.650 | - |
|
||||
| Test (No Memory) | 0.672 | Baseline |
|
||||
| **Test (With Memory)** | **0.772** | **+14.88%** |
|
||||
|
||||
**Key Findings:**
|
||||
- **Consistent improvement**: Tool Memory boosted test performance by ~15% on average
|
||||
- **Knowledge transfer**: Training data successfully informed test-time tool selection
|
||||
- **Stability**: Improvement remained consistent across all 3 epochs (9.90% → 17.39% → 17.13%)
|
||||
|
||||
The benchmark demonstrates that Tool Memory enables agents to make data-driven tool selection decisions, significantly improving task success rates compared to relying solely on static tool descriptions.
|
||||
|
||||
**Benchmark Resources:**
|
||||
- **Design Documentation**: [`docs/tool_memory/tool_bench.md`](tool_bench.md) - Complete benchmark methodology and workflow
|
||||
- **Implementation**: [`cookbook/tool_memory/run_reme_tool_bench.py`](../../cookbook/tool_memory/run_reme_tool_bench.py) - Full benchmark script
|
||||
- **Query Dataset**: [`cookbook/tool_memory/query.json`](../../cookbook/tool_memory/query.json) - 60 train + 60 test queries across 3 complexity levels
|
||||
|
||||
---
|
||||
|
||||
## 10. References
|
||||
|
||||
- **Implementation**: See `reme_ai/summary/tool/` and `reme_ai/retrieve/tool/`
|
||||
- **Demo**: `cookbook/simple_demo/use_tool_memory_demo.py`
|
||||
- **Benchmark**: `cookbook/tool_memory/run_reme_tool_bench.py`
|
||||
- **Schema**: `reme_ai/schema/memory.py`
|
||||
- **Utilities**: `reme_ai/utils/tool_memory_utils.py`
|
||||
19
docs/tool_memory/tool_retrieve_ops.md
Normal file
19
docs/tool_memory/tool_retrieve_ops.md
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
# Tool Memory Retrieval Operations
|
||||
|
||||
## RetrieveToolMemoryOp
|
||||
|
||||
### Purpose
|
||||
|
||||
Retrieves tool memories from the vector database based on tool names, providing usage patterns, best practices, and historical call data.
|
||||
|
||||
### Functionality
|
||||
|
||||
- Accepts comma-separated tool names as input
|
||||
- Searches the vector store for exact tool name matches
|
||||
- Validates that retrieved memories are of type "tool"
|
||||
- Returns complete tool memories including usage guidelines and call history
|
||||
|
||||
### Parameters
|
||||
|
||||
This operation has no configurable parameters. It uses the default vector store configuration.
|
||||
|
||||
82
docs/tool_memory/tool_summary_ops.md
Normal file
82
docs/tool_memory/tool_summary_ops.md
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
# Tool Summary Operations
|
||||
|
||||
## ParseToolCallResultOp
|
||||
|
||||
### Purpose
|
||||
|
||||
Evaluates individual tool invocations and adds them to the tool memory database with comprehensive assessments.
|
||||
|
||||
### Functionality
|
||||
|
||||
- Receives tool call results with input parameters, output, and metadata
|
||||
- Uses LLM to evaluate each tool call based on success and parameter alignment
|
||||
- Generates summary, evaluation, and score (0.0 or 1.0) for each call
|
||||
- Appends evaluated results to existing tool memory or creates new memory
|
||||
- Maintains a sliding window of recent tool calls (configurable limit)
|
||||
|
||||
### Parameters
|
||||
|
||||
- `op.parse_tool_call_result_op.params.max_history_tool_call_cnt` (integer, default: `100`):
|
||||
- Maximum number of historical tool call results to retain per tool
|
||||
- When exceeded, oldest results are removed (FIFO)
|
||||
|
||||
- `op.parse_tool_call_result_op.params.evaluation_sleep_interval` (float, default: `1.0`):
|
||||
- Delay in seconds between concurrent evaluations
|
||||
- Prevents rate limiting when evaluating multiple calls
|
||||
|
||||
## SummaryToolMemoryOp
|
||||
|
||||
### Purpose
|
||||
|
||||
Analyzes accumulated tool call history and generates comprehensive usage patterns, best practices, and recommendations.
|
||||
|
||||
### Functionality
|
||||
|
||||
- Retrieves existing tool memories from the vector store
|
||||
- **Intelligently skips tools** where all recent calls have already been summarized (using `is_summarized` flag)
|
||||
- Analyzes the most recent N tool calls (configurable)
|
||||
- Calculates statistical metrics (success rate, average scores, costs)
|
||||
- Uses LLM to synthesize actionable usage guidelines
|
||||
- Updates tool memory content with generated insights
|
||||
- **Marks processed calls** as summarized to avoid redundant processing in future runs
|
||||
|
||||
### Smart Skip Logic
|
||||
|
||||
To optimize costs and performance, `SummaryToolMemoryOp` tracks which tool call results have been included in a summary:
|
||||
|
||||
- **Skip Condition**: If all recent N calls are already summarized (`is_summarized=True`), the tool is skipped entirely
|
||||
- **Trigger Condition**: If at least 1 recent call is new (`is_summarized=False`), re-summarization is triggered
|
||||
- **Automatic Marking**: After successful summarization, all processed calls are marked with `is_summarized=True`
|
||||
|
||||
**Example Behavior**:
|
||||
```
|
||||
Run 1: 30 new calls → Summarize all 30, mark as summarized
|
||||
Run 2: Same 30 calls → Skip (all already summarized) ✓ Cost savings
|
||||
Run 3: 30 old + 1 new → Re-summarize all 31, mark new call as summarized
|
||||
```
|
||||
|
||||
This ensures summaries stay fresh while avoiding unnecessary LLM calls.
|
||||
|
||||
### Parameters
|
||||
|
||||
- `op.summary_tool_memory_op.params.recent_call_count` (integer, default: `30`):
|
||||
- Number of most recent tool calls to analyze
|
||||
- Also determines the window for checking summarization status
|
||||
- Focuses on recent usage patterns
|
||||
|
||||
- `op.summary_tool_memory_op.params.summary_sleep_interval` (float, default: `1.0`):
|
||||
- Delay in seconds between concurrent summarizations
|
||||
- Prevents rate limiting when summarizing multiple tools
|
||||
|
||||
### Return Value
|
||||
|
||||
The operation returns a response message indicating:
|
||||
- Number of tools summarized (had new unsummarized calls)
|
||||
- Number of tools skipped (all recent calls already summarized)
|
||||
|
||||
Example: `"Successfully processed 5 tool memories: 2 summarized, 3 skipped (already up-to-date)"`
|
||||
|
||||
### See Also
|
||||
|
||||
For detailed implementation and examples, see [is_summarized Feature Documentation](is_summarized_feature.md).
|
||||
|
||||
13
mkdocs.yml
13
mkdocs.yml
|
|
@ -26,6 +26,12 @@ nav:
|
|||
- Retrieve Ops: task_memory/task_retrieve_ops.md
|
||||
- Summary Ops: task_memory/task_summary_ops.md
|
||||
|
||||
- Tool Memory:
|
||||
- Overview: tool_memory/tool_memory.md
|
||||
- Retrieve Ops: tool_memory/tool_retrieve_ops.md
|
||||
- Summary Ops: tool_memory/tool_summary_ops.md
|
||||
- Benchmark: tool_memory/tool_bench.md
|
||||
|
||||
- SOP Memory:
|
||||
- Making SOP Memories: sop_memory/making_sop_memories.md
|
||||
|
||||
|
|
@ -39,6 +45,7 @@ nav:
|
|||
- BFCL: cookbook/bfcl/quickstart.md
|
||||
- FrozenLake: cookbook/frozenlake/quickstart.md
|
||||
|
||||
- TODO: todo.md
|
||||
|
||||
- Contribution Guide: contribution.md
|
||||
|
||||
|
|
@ -64,3 +71,9 @@ markdown_extensions:
|
|||
shadcn.extensions.echarts.alpha:
|
||||
shadcn.extensions.codexec:
|
||||
shadcn.extensions.iconify:
|
||||
|
||||
|
||||
# pip install mkdocs-shadcn
|
||||
# mkdocs build
|
||||
# mkdocs serve
|
||||
# mkdocs gh-deploy --force
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||
|
||||
[project]
|
||||
name = "reme_ai"
|
||||
version = "0.1.9"
|
||||
version = "0.1.10"
|
||||
description = "Remember me"
|
||||
authors = [
|
||||
{ name = "jinli.yl", email = "jinli.yl@alibaba-inc.com" },
|
||||
|
|
@ -24,7 +24,7 @@ classifiers = [
|
|||
keywords = ["llm", "memory", "experience", "memoryscope", "ai", "mcp", "http"]
|
||||
|
||||
dependencies = [
|
||||
"flowllm==0.1.9",
|
||||
"flowllm>=0.1.11.1",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,14 @@
|
|||
from reme_ai import react
|
||||
from reme_ai import retrieve
|
||||
from reme_ai import summary
|
||||
from reme_ai import vector_store
|
||||
import warnings
|
||||
|
||||
__version__ = "0.1.9"
|
||||
from pydantic.warnings import PydanticDeprecatedSince20
|
||||
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="websockets")
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="uvicorn")
|
||||
warnings.filterwarnings("ignore", category=PydanticDeprecatedSince20)
|
||||
|
||||
from . import agent
|
||||
from . import retrieve
|
||||
from . import summary
|
||||
from . import vector_store
|
||||
|
||||
__version__ = "0.1.10"
|
||||
|
|
|
|||
2
reme_ai/agent/__init__.py
Normal file
2
reme_ai/agent/__init__.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
from . import react
|
||||
from . import tools
|
||||
3
reme_ai/agent/tools/__init__.py
Normal file
3
reme_ai/agent/tools/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from .llm_mock_search_op import LLMMockSearchOp
|
||||
from .mock_search_tools import SearchToolA, SearchToolB, SearchToolC
|
||||
from .use_mock_search_op import UseMockSearchOp
|
||||
307
reme_ai/agent/tools/llm_mock_search_op.py
Normal file
307
reme_ai/agent/tools/llm_mock_search_op.py
Normal file
|
|
@ -0,0 +1,307 @@
|
|||
import asyncio
|
||||
import json
|
||||
import random
|
||||
from typing import Dict, Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from flowllm.context import FlowContext, C
|
||||
from flowllm.enumeration.role import Role
|
||||
from flowllm.op.base_async_tool_op import BaseAsyncToolOp
|
||||
from flowllm.schema.message import Message
|
||||
from flowllm.schema.tool_call import ToolCall
|
||||
|
||||
|
||||
@C.register_op()
|
||||
class LLMMockSearchOp(BaseAsyncToolOp):
|
||||
"""
|
||||
Mock search operation that uses LLM to classify queries and simulate different scenarios.
|
||||
|
||||
Supports three query complexity levels:
|
||||
- simple: Simple factual queries with short, direct answers
|
||||
- medium: Medium complexity queries requiring balanced performance
|
||||
- complex: Complex research queries requiring comprehensive, in-depth results
|
||||
|
||||
Each scenario can be configured with:
|
||||
- success_rate: Probability of successful response (vs "Service busy" error)
|
||||
- extra_time: Extra sleep time in seconds to simulate latency
|
||||
- relevance_ratio: Probability of returning relevant results (vs random query results)
|
||||
"""
|
||||
file_path: str = __file__
|
||||
|
||||
def __init__(self,
|
||||
llm: str = "qwen3_30b_instruct",
|
||||
simple_config: Dict[str, Any] = None,
|
||||
medium_config: Dict[str, Any] = None,
|
||||
complex_config: Dict[str, Any] = None,
|
||||
seed: int = 0,
|
||||
**kwargs):
|
||||
"""
|
||||
Initialize the LLM Mock Search Op.
|
||||
|
||||
Args:
|
||||
llm: LLM model name to use for classification and content generation
|
||||
simple_config: Configuration for simple queries
|
||||
- success_rate: float (0-1), default 0.95
|
||||
- extra_time: float (seconds), default 0.5
|
||||
- relevance_ratio: float (0-1), default 0.98
|
||||
medium_config: Configuration for medium complexity queries
|
||||
- success_rate: float (0-1), default 0.85
|
||||
- extra_time: float (seconds), default 1.0
|
||||
- relevance_ratio: float (0-1), default 0.90
|
||||
complex_config: Configuration for complex queries
|
||||
- success_rate: float (0-1), default 0.70
|
||||
- extra_time: float (seconds), default 1.5
|
||||
- relevance_ratio: float (0-1), default 0.80
|
||||
seed: Random seed for deterministic behavior, default 0
|
||||
"""
|
||||
super().__init__(llm=llm, **kwargs)
|
||||
|
||||
# Set random seed for deterministic behavior
|
||||
self.seed = seed
|
||||
random.seed(self.seed)
|
||||
|
||||
# Default configurations for each scenario
|
||||
self.simple_config = {
|
||||
"success_rate": 0.95,
|
||||
"extra_time": 0.5,
|
||||
"relevance_ratio": 0.98,
|
||||
"content_length": "short"
|
||||
}
|
||||
if simple_config:
|
||||
self.simple_config.update(simple_config)
|
||||
|
||||
self.medium_config = {
|
||||
"success_rate": 0.85,
|
||||
"extra_time": 1.0,
|
||||
"relevance_ratio": 0.90,
|
||||
"content_length": "medium"
|
||||
}
|
||||
if medium_config:
|
||||
self.medium_config.update(medium_config)
|
||||
|
||||
self.complex_config = {
|
||||
"success_rate": 0.70,
|
||||
"extra_time": 1.5,
|
||||
"relevance_ratio": 0.80,
|
||||
"content_length": "long"
|
||||
}
|
||||
if complex_config:
|
||||
self.complex_config.update(complex_config)
|
||||
|
||||
def build_tool_call(self) -> ToolCall:
|
||||
return ToolCall(**{
|
||||
"description": "Use search keywords to retrieve relevant information from the internet.",
|
||||
"input_schema": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "search keyword or query",
|
||||
"required": True
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
async def classify_query(self, query: str) -> str:
|
||||
"""
|
||||
Classify the query into simple, medium, or complex using LLM.
|
||||
|
||||
Args:
|
||||
query: The search query to classify
|
||||
|
||||
Returns:
|
||||
Classification result: "simple", "medium", or "complex"
|
||||
"""
|
||||
classification_prompt = self.prompt_format(
|
||||
prompt_name="classification_prompt",
|
||||
query=query
|
||||
)
|
||||
|
||||
messages = [Message(role=Role.USER, content=classification_prompt)]
|
||||
|
||||
response = await self.llm.achat(messages=messages)
|
||||
classification = response.content.strip().lower()
|
||||
|
||||
# Extract classification from response
|
||||
if "simple" in classification:
|
||||
return "simple"
|
||||
elif "complex" in classification:
|
||||
return "complex"
|
||||
else:
|
||||
return "medium"
|
||||
|
||||
async def generate_search_result(self, query: str, complexity: str, config: Dict[str, Any]) -> str:
|
||||
"""
|
||||
Generate mock search results using LLM based on query complexity.
|
||||
|
||||
Args:
|
||||
query: The search query
|
||||
complexity: Query complexity level
|
||||
config: Configuration for this complexity level
|
||||
|
||||
Returns:
|
||||
Generated search result content
|
||||
"""
|
||||
content_length = config["content_length"]
|
||||
|
||||
generation_prompt = self.prompt_format(
|
||||
prompt_name="generation_prompt",
|
||||
query=query,
|
||||
complexity=complexity,
|
||||
content_length=content_length
|
||||
)
|
||||
|
||||
messages = [Message(role=Role.USER, content=generation_prompt)]
|
||||
|
||||
response = await self.llm.achat(messages=messages)
|
||||
return response.content
|
||||
|
||||
async def generate_random_result(self) -> str:
|
||||
"""
|
||||
Generate a random/irrelevant search result to simulate low relevance.
|
||||
|
||||
Returns:
|
||||
Random search result content
|
||||
"""
|
||||
random_topics = [
|
||||
"the history of ancient civilizations",
|
||||
"modern technology trends",
|
||||
"climate change impacts",
|
||||
"space exploration achievements",
|
||||
"culinary traditions around the world",
|
||||
"evolution of music genres",
|
||||
"breakthroughs in medical science",
|
||||
"architectural wonders",
|
||||
"wildlife conservation efforts",
|
||||
"developments in artificial intelligence"
|
||||
]
|
||||
|
||||
random_query = random.choice(random_topics)
|
||||
generation_prompt = self.prompt_format(
|
||||
prompt_name="generation_prompt",
|
||||
query=random_query,
|
||||
complexity="simple",
|
||||
content_length="short"
|
||||
)
|
||||
|
||||
messages = [Message(role=Role.USER, content=generation_prompt)]
|
||||
response = await self.llm.achat(messages=messages)
|
||||
|
||||
return f"[Low Relevance Result]\n{response.content}"
|
||||
|
||||
async def async_execute(self):
|
||||
query: str = self.input_dict["query"]
|
||||
logger.info(f"LLMMockSearchOp processing query: {query}")
|
||||
|
||||
# Step 1: Classify the query
|
||||
complexity = await self.classify_query(query)
|
||||
logger.info(f"Query classified as: {complexity}")
|
||||
|
||||
# Step 2: Get configuration for this complexity
|
||||
if complexity == "simple":
|
||||
config = self.simple_config
|
||||
elif complexity == "medium":
|
||||
config = self.medium_config
|
||||
else: # complex
|
||||
config = self.complex_config
|
||||
|
||||
logger.info(f"Using config: {config}")
|
||||
|
||||
# Step 3: Simulate extra time delay
|
||||
extra_time = config["extra_time"]
|
||||
await asyncio.sleep(extra_time)
|
||||
logger.info(f"Simulated extra delay: {extra_time:.2f}s")
|
||||
|
||||
# Step 4: Check success rate
|
||||
if random.random() > config["success_rate"]:
|
||||
error_message = "Search service is currently busy. Please try again later."
|
||||
logger.warning(f"Simulated failure: {error_message}")
|
||||
result_dict = {
|
||||
"success": False,
|
||||
"content": error_message,
|
||||
"query": query,
|
||||
"complexity": complexity
|
||||
}
|
||||
self.set_result(json.dumps(result_dict, ensure_ascii=False))
|
||||
return
|
||||
|
||||
# Step 5: Check relevance ratio
|
||||
if random.random() > config["relevance_ratio"]:
|
||||
# Generate random/irrelevant result
|
||||
# NOTE: success=True because technically the tool executed without errors,
|
||||
# but the content is irrelevant (low quality), which should result in score=0.0 during evaluation
|
||||
logger.info("Generating low relevance result (success=True but low quality)")
|
||||
content = await self.generate_random_result()
|
||||
result_dict = {
|
||||
"success": True, # Technical execution succeeded
|
||||
"content": content,
|
||||
"query": query,
|
||||
"complexity": complexity,
|
||||
"is_relevant": False # Mark as irrelevant for debugging
|
||||
}
|
||||
else:
|
||||
# Generate relevant result
|
||||
logger.info("Generating relevant result")
|
||||
content = await self.generate_search_result(query, complexity, config)
|
||||
result_dict = {
|
||||
"success": True,
|
||||
"content": content,
|
||||
"query": query,
|
||||
"complexity": complexity,
|
||||
"is_relevant": True # Mark as relevant for debugging
|
||||
}
|
||||
|
||||
self.set_result(json.dumps(result_dict, ensure_ascii=False))
|
||||
|
||||
|
||||
async def async_main():
|
||||
from flowllm.app import FlowLLMApp
|
||||
|
||||
async with FlowLLMApp(load_default_config=True):
|
||||
# Test with different query types
|
||||
test_queries = [
|
||||
"What is the capital of France?", # Simple
|
||||
"How does quantum computing work?", # Medium
|
||||
"Analyze the impact of artificial intelligence on global economy, employment, and society", # Complex
|
||||
]
|
||||
|
||||
# Custom configurations for testing
|
||||
custom_simple = {
|
||||
"success_rate": 1,
|
||||
"extra_time": 0,
|
||||
"relevance_ratio": 1
|
||||
}
|
||||
|
||||
custom_medium = {
|
||||
"success_rate": 1,
|
||||
"extra_time": 0,
|
||||
"relevance_ratio": 1
|
||||
}
|
||||
|
||||
custom_complex = {
|
||||
"success_rate": 1,
|
||||
"extra_time": 0,
|
||||
"relevance_ratio": 1
|
||||
}
|
||||
|
||||
op = LLMMockSearchOp(
|
||||
simple_config=custom_simple,
|
||||
medium_config=custom_medium,
|
||||
complex_config=custom_complex
|
||||
)
|
||||
|
||||
for query in test_queries:
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f"Testing query: {query}")
|
||||
print(f"{'=' * 80}")
|
||||
|
||||
context = FlowContext(query=query)
|
||||
await op.async_call(context=context)
|
||||
result = json.loads(context.llm_mock_search_result)
|
||||
print(f"Success: {result['success']}")
|
||||
print(f"Query: {result['query']}")
|
||||
print(f"Complexity: {result['complexity']}")
|
||||
print(f"Content:\n{result['content']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(async_main())
|
||||
76
reme_ai/agent/tools/llm_mock_search_prompt.yaml
Normal file
76
reme_ai/agent/tools/llm_mock_search_prompt.yaml
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
classification_prompt: |
|
||||
You are a query complexity classifier. Analyze the following search query and classify it into one of three categories:
|
||||
|
||||
1. **simple** - Simple factual queries that:
|
||||
- Ask for a single, direct fact
|
||||
- Have a clear, unambiguous answer
|
||||
- Require minimal context or explanation
|
||||
- Examples: "What is the capital of France?", "Who invented the telephone?", "When did World War 2 end?"
|
||||
|
||||
2. **medium** - Medium complexity queries that:
|
||||
- Require some explanation or context
|
||||
- May involve multiple related facts
|
||||
- Need balanced depth without being exhaustive
|
||||
- Examples: "How does photosynthesis work?", "What are the main causes of climate change?", "Explain blockchain technology"
|
||||
|
||||
3. **complex** - Complex research queries that:
|
||||
- Require comprehensive, multi-dimensional analysis
|
||||
- Involve multiple subtopics or perspectives
|
||||
- Need in-depth exploration and connections
|
||||
- Examples: "Analyze the impact of AI on the global economy", "Compare different renewable energy solutions", "What are the geopolitical implications of space exploration?"
|
||||
|
||||
Query to classify: {query}
|
||||
|
||||
Respond with ONLY one word: simple, medium, or complex.
|
||||
|
||||
classification_prompt_zh: |
|
||||
你是一个查询复杂度分类器。分析以下搜索查询并将其分类为以下三类之一：
|
||||
|
||||
1. **simple** - 简单事实查询：
|
||||
- 询问单一、直接的事实
|
||||
- 有明确、无歧义的答案
|
||||
- 需要最少的上下文或解释
|
||||
- 示例："法国的首都是什么？"、"谁发明了电话？"、"第二次世界大战何时结束？"
|
||||
|
||||
2. **medium** - 中等复杂度查询：
|
||||
- 需要一些解释或上下文
|
||||
- 可能涉及多个相关事实
|
||||
- 需要平衡的深度但不需要详尽无遗
|
||||
- 示例："光合作用如何工作？"、"气候变化的主要原因是什么？"、"解释区块链技术"
|
||||
|
||||
3. **complex** - 复杂研究查询：
|
||||
- 需要全面、多维度的分析
|
||||
- 涉及多个子主题或观点
|
||||
- 需要深入探索和联系
|
||||
- 示例："分析人工智能对全球经济的影响"、"比较不同的可再生能源解决方案"、"太空探索的地缘政治影响是什么？"
|
||||
|
||||
要分类的查询：{query}
|
||||
|
||||
只用一个词回答：simple、medium 或 complex。
|
||||
|
||||
generation_prompt: |
|
||||
You are a search engine generating mock search results. Generate a {content_length} response for the following query.
|
||||
|
||||
Query: {query}
|
||||
Complexity Level: {complexity}
|
||||
|
||||
Instructions based on content length:
|
||||
- **short**: Provide a concise answer in 1-3 sentences. Be direct and factual.
|
||||
- **medium**: Provide a balanced answer in 2-4 paragraphs. Include key details and some context.
|
||||
- **long**: Provide a comprehensive answer in 4-6 paragraphs. Include multiple perspectives, detailed explanations, and relevant context.
|
||||
|
||||
Generate the search result content now:
|
||||
|
||||
generation_prompt_zh: |
|
||||
你是一个搜索引擎，正在生成模拟搜索结果。为以下查询生成一个 {content_length} 的响应。
|
||||
|
||||
查询：{query}
|
||||
复杂度级别：{complexity}
|
||||
|
||||
根据内容长度的指示：
|
||||
- **short**（短）：提供 1-3 句话的简洁答案。要直接和事实性。
|
||||
- **medium**（中）：提供 2-4 段的平衡答案。包括关键细节和一些上下文。
|
||||
- **long**（长）：提供 4-6 段的全面答案。包括多个角度、详细解释和相关上下文。
|
||||
|
||||
现在生成搜索结果内容：
|
||||
|
||||
115
reme_ai/agent/tools/mock_search_tools.py
Normal file
115
reme_ai/agent/tools/mock_search_tools.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
from flowllm.context import C
|
||||
from flowllm.schema.tool_call import ToolCall
|
||||
|
||||
from reme_ai.agent.tools.llm_mock_search_op import LLMMockSearchOp
|
||||
|
||||
|
||||
@C.register_op()
|
||||
class SearchToolA(LLMMockSearchOp):
|
||||
def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs):
|
||||
# Configure for fast but shallow performance
|
||||
simple_config = {
|
||||
"success_rate": 0.9, # High success rate for simple queries
|
||||
"extra_time": 0, # Very fast (0.2-0.5s range)
|
||||
"relevance_ratio": 0.9, # High relevance
|
||||
"content_length": "short" # Concise answers
|
||||
}
|
||||
|
||||
medium_config = {
|
||||
"success_rate": 0.2, # Lower success for medium queries
|
||||
"extra_time": 0, # Still fast
|
||||
"relevance_ratio": 0.2, # Moderate relevance
|
||||
"content_length": "short" # Limited depth
|
||||
}
|
||||
|
||||
complex_config = {
|
||||
"success_rate": 0.5, # Poor success rate for complex queries
|
||||
"extra_time": 0, # Fast but insufficient
|
||||
"relevance_ratio": 0.5, # Low relevance (often misses key aspects)
|
||||
"content_length": "short" # Too shallow for complex topics
|
||||
}
|
||||
|
||||
super().__init__(llm=llm,
|
||||
simple_config=simple_config,
|
||||
medium_config=medium_config,
|
||||
complex_config=complex_config,
|
||||
**kwargs)
|
||||
|
||||
def build_tool_call(self) -> ToolCall:
|
||||
tool_call = super().build_tool_call()
|
||||
tool_call.description += " Best suited for simple queries."
|
||||
return tool_call
|
||||
|
||||
@C.register_op()
|
||||
class SearchToolB(LLMMockSearchOp):
|
||||
def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs):
|
||||
# Configure for balanced performance
|
||||
simple_config = {
|
||||
"success_rate": 0.3, # Very high success rate
|
||||
"extra_time": 0, # Moderate speed (1.0-1.5s range)
|
||||
"relevance_ratio": 0.3, # High relevance
|
||||
"content_length": "medium" # More detailed than needed for simple
|
||||
}
|
||||
|
||||
medium_config = {
|
||||
"success_rate": 0.9, # Excellent success rate
|
||||
"extra_time": 0, # Balanced speed
|
||||
"relevance_ratio": 0.9, # High relevance
|
||||
"content_length": "medium" # Perfect depth for medium queries
|
||||
}
|
||||
|
||||
complex_config = {
|
||||
"success_rate": 0.5, # Good success rate
|
||||
"extra_time": 0, # Still reasonable speed
|
||||
"relevance_ratio": 0.5, # Decent relevance but not exhaustive
|
||||
"content_length": "medium" # Covers main points but lacks depth
|
||||
}
|
||||
|
||||
super().__init__(llm=llm,
|
||||
simple_config=simple_config,
|
||||
medium_config=medium_config,
|
||||
complex_config=complex_config,
|
||||
**kwargs)
|
||||
|
||||
def build_tool_call(self) -> ToolCall:
|
||||
tool_call = super().build_tool_call()
|
||||
tool_call.description += " Best suited for medium complexity queries."
|
||||
return tool_call
|
||||
|
||||
|
||||
@C.register_op()
|
||||
class SearchToolC(LLMMockSearchOp):
|
||||
|
||||
def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs):
|
||||
# Configure for comprehensive but costly performance
|
||||
simple_config = {
|
||||
"success_rate": 0.3, # Good but not optimal (over-processing)
|
||||
"extra_time": 0, # Slow (3.0-4.0s range)
|
||||
"relevance_ratio": 0.3, # High relevance but unnecessary depth
|
||||
"content_length": "long" # Too detailed for simple queries
|
||||
}
|
||||
|
||||
medium_config = {
|
||||
"success_rate": 0.4, # High success rate
|
||||
"extra_time": 0, # Slow but thorough
|
||||
"relevance_ratio": 0.4, # High relevance with extra context
|
||||
"content_length": "long" # More depth than needed
|
||||
}
|
||||
|
||||
complex_config = {
|
||||
"success_rate": 0.9, # Excellent success rate
|
||||
"extra_time": 0, # Slow but comprehensive (3.5-5.0s range)
|
||||
"relevance_ratio": 0.9, # Very high relevance
|
||||
"content_length": "long" # Perfect depth for complex queries
|
||||
}
|
||||
|
||||
super().__init__(llm=llm,
|
||||
simple_config=simple_config,
|
||||
medium_config=medium_config,
|
||||
complex_config=complex_config,
|
||||
**kwargs)
|
||||
|
||||
def build_tool_call(self) -> ToolCall:
|
||||
tool_call = super().build_tool_call()
|
||||
tool_call.description += " Best suited for complex queries."
|
||||
return tool_call
|
||||
140
reme_ai/agent/tools/use_mock_search_op.py
Normal file
140
reme_ai/agent/tools/use_mock_search_op.py
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
|
||||
from flowllm.context import C
|
||||
from flowllm.enumeration.role import Role
|
||||
from flowllm.op.base_async_tool_op import BaseAsyncToolOp
|
||||
from flowllm.schema.message import Message
|
||||
from flowllm.schema.tool_call import ToolCall
|
||||
from flowllm.utils.timer import Timer
|
||||
from flowllm.utils.token_utils import TokenCounter
|
||||
from loguru import logger
|
||||
|
||||
from reme_ai.agent.tools.mock_search_tools import SearchToolA, SearchToolB, SearchToolC
|
||||
from reme_ai.schema.memory import ToolCallResult
|
||||
|
||||
|
||||
@C.register_op()
|
||||
class UseMockSearchOp(BaseAsyncToolOp):
|
||||
file_path: str = __file__
|
||||
|
||||
def __init__(self, llm: str = "qwen3_30b_instruct", **kwargs):
|
||||
super().__init__(llm=llm, save_answer=True, **kwargs)
|
||||
|
||||
def build_tool_call(self) -> ToolCall:
|
||||
return ToolCall(**{
|
||||
"description": "Intelligently selects and executes the most appropriate search tool based on query complexity. "
|
||||
"Automatically tracks performance metrics and records tool usage for optimization.",
|
||||
"input_schema": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "query",
|
||||
"required": True
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
async def select_tool(self, query: str, tool_ops: list[BaseAsyncToolOp]) -> ToolCall | None:
|
||||
assistant_message = await self.llm.achat(messages=[Message(role=Role.USER, content=query)],
|
||||
tools=[x.tool_call for x in tool_ops])
|
||||
logger.info(f"assistant_message={assistant_message.model_dump_json()}")
|
||||
if assistant_message.tool_calls:
|
||||
return assistant_message.tool_calls[0]
|
||||
|
||||
return None
|
||||
|
||||
async def async_execute(self):
|
||||
query: str = self.input_dict["query"]
|
||||
logger.info(f"query={query}")
|
||||
|
||||
tool_ops = [
|
||||
SearchToolA(),
|
||||
SearchToolB(),
|
||||
SearchToolC(),
|
||||
]
|
||||
|
||||
# Step 1: Select the appropriate tool using LLM
|
||||
tool_call = await self.select_tool(query, tool_ops)
|
||||
|
||||
if tool_call is None:
|
||||
# No tool selected
|
||||
error_result = ToolCallResult(
|
||||
create_time=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
tool_name="None",
|
||||
input={"query": query},
|
||||
output="No appropriate tool was selected for the query",
|
||||
token_cost=0,
|
||||
success=False,
|
||||
time_cost=0.0
|
||||
)
|
||||
self.set_result(error_result.model_dump_json())
|
||||
return
|
||||
|
||||
# Step 2: Execute the selected tool
|
||||
selected_op = None
|
||||
for op in tool_ops:
|
||||
if op.tool_call.name == tool_call.name:
|
||||
selected_op = op
|
||||
break
|
||||
|
||||
if selected_op is None:
|
||||
# Tool not found (should not happen)
|
||||
error_result = ToolCallResult(
|
||||
create_time=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
tool_name=tool_call.name,
|
||||
input=tool_call.arguments,
|
||||
output=f"Tool {tool_call.name} not found in available tools",
|
||||
token_cost=0,
|
||||
success=False,
|
||||
time_cost=0.0
|
||||
)
|
||||
self.set_result(error_result.model_dump_json())
|
||||
return
|
||||
|
||||
# Step 3: Execute the tool with timer
|
||||
timer = Timer("tool execute")
|
||||
with timer:
|
||||
await selected_op.async_call(query=query)
|
||||
selected_op_output = json.loads(selected_op.output)
|
||||
content = selected_op_output["content"]
|
||||
success = selected_op_output["success"]
|
||||
token_cost = TokenCounter().count(content)
|
||||
|
||||
time_cost = timer.time_cost
|
||||
|
||||
# Create ToolCallResult
|
||||
tool_call_result = ToolCallResult(
|
||||
create_time=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
tool_name=tool_call.name,
|
||||
input={"query": query},
|
||||
output=content,
|
||||
token_cost=token_cost,
|
||||
success=success,
|
||||
time_cost=round(time_cost, 3)
|
||||
)
|
||||
|
||||
self.set_result(tool_call_result.model_dump_json())
|
||||
|
||||
|
||||
async def async_main():
|
||||
from flowllm.app import FlowLLMApp
|
||||
|
||||
async with FlowLLMApp(load_default_config=True):
|
||||
test_queries = [
|
||||
"What is the capital of France?",
|
||||
"How does quantum computing work?",
|
||||
"Analyze the impact of artificial intelligence on global economy, employment, and society",
|
||||
"When was Python programming language created?",
|
||||
"Compare different types of renewable energy sources",
|
||||
]
|
||||
|
||||
for query in test_queries:
|
||||
op = UseMockSearchOp()
|
||||
await op.async_call(query=query)
|
||||
print(op.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(async_main())
|
||||
|
||||
|
|
@ -1,8 +1,4 @@
|
|||
import sys
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="websockets")
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="uvicorn")
|
||||
|
||||
from flowllm.app import FlowLLMApp
|
||||
|
||||
|
|
|
|||
|
|
@ -49,14 +49,41 @@ flow:
|
|||
description: "A list of conversation trajectory information, including message content and score. This field does not need to be filled in, the system will complete it automatically."
|
||||
required: false
|
||||
|
||||
retrieve_task_memory_simple:
|
||||
flow_content: build_query_op >> recall_vector_store_op >> merge_memory_op
|
||||
description: "Retrieves the most relevant top-k memory experiences from historical data based on the current query with simplified processing"
|
||||
retrieve_tool_memory:
|
||||
flow_content: retrieve_tool_memory_op
|
||||
description: "Retrieves tool memories from the vector database based on tool names to provide tool usage patterns and best practices"
|
||||
input_schema:
|
||||
tool_names:
|
||||
type: string
|
||||
description: "Comma-separated tool names (e.g., 'tool_name1,tool_name2')"
|
||||
required: true
|
||||
|
||||
add_tool_call_result:
|
||||
flow_content: parse_tool_call_result_op >> update_vector_store_op
|
||||
description: "Evaluates and adds tool call results to the tool memory database, creating new memory or updating existing memory for the specified tool"
|
||||
input_schema:
|
||||
tool_call_results:
|
||||
type: array
|
||||
description: "List of tool call result objects, each containing: tool_name, input, output, success, time_cost, token_cost, create_time"
|
||||
required: true
|
||||
|
||||
summary_tool_memory:
|
||||
flow_content: summary_tool_memory_op >> update_vector_store_op
|
||||
description: "Analyzes tool call history and generates comprehensive usage patterns, best practices, and recommendations for the specified tools"
|
||||
input_schema:
|
||||
tool_names:
|
||||
type: string
|
||||
description: "Comma-separated tool names to summarize (e.g., 'tool_name1,tool_name2')"
|
||||
required: true
|
||||
|
||||
use_mock_search:
|
||||
flow_content: use_mock_search_op
|
||||
description: "Simulates intelligent search tool selection and execution based on query complexity, with automatic tool memory recording"
|
||||
input_schema:
|
||||
query:
|
||||
type: string
|
||||
description: "current query"
|
||||
required: true
|
||||
description: "User search query to process"
|
||||
required: true
|
||||
|
||||
summary_task_memory_simple:
|
||||
flow_content: simple_summary_op >> update_vector_store_op
|
||||
|
|
@ -136,6 +163,21 @@ op:
|
|||
params:
|
||||
enable_llm_rewrite: true
|
||||
|
||||
# tool memory ops
|
||||
parse_tool_call_result_op:
|
||||
backend: parse_tool_call_result_op
|
||||
llm: default
|
||||
params:
|
||||
max_history_tool_call_cnt: 100
|
||||
evaluation_sleep_interval: 1.0
|
||||
|
||||
summary_tool_memory_op:
|
||||
backend: summary_tool_memory_op
|
||||
llm: default
|
||||
params:
|
||||
recent_call_count: 20
|
||||
summary_sleep_interval: 1.0
|
||||
|
||||
llm:
|
||||
default:
|
||||
backend: openai_compatible
|
||||
|
|
@ -143,6 +185,12 @@ llm:
|
|||
params:
|
||||
temperature: 0.6
|
||||
|
||||
wk1:
|
||||
backend: openai_compatible
|
||||
model_name: qwen3-30b-a3b-instruct-2507
|
||||
params:
|
||||
temperature: 0.6
|
||||
|
||||
qwen3_30b_instruct:
|
||||
backend: openai_compatible
|
||||
model_name: qwen3-30b-a3b-instruct-2507
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
from . import personal
|
||||
from . import task
|
||||
from . import tool
|
||||
|
|
|
|||
1
reme_ai/retrieve/tool/__init__.py
Normal file
1
reme_ai/retrieve/tool/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from .retrieve_tool_memory_op import RetrieveToolMemoryOp
|
||||
71
reme_ai/retrieve/tool/retrieve_tool_memory_op.py
Normal file
71
reme_ai/retrieve/tool/retrieve_tool_memory_op.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
from typing import List
|
||||
|
||||
from flowllm import C, BaseAsyncOp
|
||||
from flowllm.schema.vector_node import VectorNode
|
||||
from loguru import logger
|
||||
|
||||
from reme_ai.schema.memory import ToolMemory, vector_node_to_memory
|
||||
|
||||
|
||||
@C.register_op()
|
||||
class RetrieveToolMemoryOp(BaseAsyncOp):
|
||||
file_path: str = __file__
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
async def async_execute(self):
|
||||
tool_names: str = self.context.get("tool_names", "")
|
||||
workspace_id: str = self.context.workspace_id
|
||||
|
||||
if not tool_names:
|
||||
logger.warning("tool_names is empty, skipping processing")
|
||||
self.context.response.answer = "tool_names is required"
|
||||
self.context.response.success = False
|
||||
return
|
||||
|
||||
# Split tool names by comma
|
||||
tool_name_list = [name.strip() for name in tool_names.split(",") if name.strip()]
|
||||
logger.info(f"workspace_id={workspace_id} retrieving {len(tool_name_list)} tools: {tool_name_list}")
|
||||
|
||||
# Search for each tool in the vector store
|
||||
matched_tool_memories: List[ToolMemory] = []
|
||||
|
||||
for tool_name in tool_name_list:
|
||||
nodes: List[VectorNode] = await self.vector_store.async_search(
|
||||
query=tool_name,
|
||||
workspace_id=workspace_id,
|
||||
top_k=1
|
||||
)
|
||||
|
||||
if nodes:
|
||||
top_node = nodes[0]
|
||||
memory = vector_node_to_memory(top_node)
|
||||
|
||||
# Ensure it's a ToolMemory and when_to_use matches
|
||||
if isinstance(memory, ToolMemory) and memory.when_to_use == tool_name:
|
||||
matched_tool_memories.append(memory)
|
||||
logger.info(f"Found tool_memory for tool_name={tool_name}, "
|
||||
f"memory_id={memory.memory_id}, "
|
||||
f"total_calls={len(memory.tool_call_results)}")
|
||||
else:
|
||||
logger.warning(f"No exact match found for tool_name={tool_name}")
|
||||
else:
|
||||
logger.warning(f"No memory found for tool_name={tool_name}")
|
||||
|
||||
if not matched_tool_memories:
|
||||
logger.info("No matching tool memories found")
|
||||
self.context.response.answer = "No matching tool memories found"
|
||||
self.context.response.success = False
|
||||
return
|
||||
|
||||
# Set response
|
||||
self.context.response.answer = f"Successfully retrieved {len(matched_tool_memories)} tool memories"
|
||||
self.context.response.success = True
|
||||
self.context.response.metadata["memory_list"] = matched_tool_memories
|
||||
|
||||
# Log retrieval results
|
||||
for memory in matched_tool_memories:
|
||||
logger.info(f"Retrieved tool: {memory.when_to_use}, "
|
||||
f"total_calls={len(memory.tool_call_results)}, "
|
||||
f"content_length={len(memory.content)}")
|
||||
|
|
@ -1,8 +1,10 @@
|
|||
import datetime
|
||||
from abc import ABC
|
||||
from typing import List
|
||||
from uuid import uuid4
|
||||
|
||||
from flowllm.schema.vector_node import VectorNode
|
||||
from mcp.types import CallToolResult, TextContent
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
|
|
@ -105,7 +107,118 @@ class PersonalMemory(BaseMemory):
|
|||
metadata=metadata.pop("metadata", {}))
|
||||
|
||||
|
||||
def vector_node_to_memory(node: VectorNode) -> BaseMemory:
|
||||
class ToolCallResult(BaseModel):
|
||||
create_time: str = Field(default="", description="Time of tool invocation")
|
||||
tool_name: str = Field(default=..., description="Name of the tool")
|
||||
input: dict | str = Field(default="", description="Tool input")
|
||||
output: str = Field(default="", description="Tool output")
|
||||
token_cost: int = Field(default=-1, description="Token consumption of the tool")
|
||||
success: bool = Field(default=True, description="Whether the tool invocation was successful")
|
||||
time_cost: float = Field(default=0, description="Time consumed by the tool invocation, in seconds")
|
||||
summary: str = Field(default="", description="Brief summary of the tool call result")
|
||||
evaluation: str = Field(default="", description="Detailed evaluation for the tool invocation")
|
||||
score: float = Field(default=0, description="Score of the Evaluation (0.0 for failure, 1.0 for complete success)")
|
||||
is_summarized: bool = Field(default=False, description="Whether this tool call has been included in a summary")
|
||||
|
||||
metadata: dict = Field(default_factory=dict)
|
||||
|
||||
def from_mcp_tool_result(self, tool_result: CallToolResult, max_char_len: int = None):
|
||||
text_list = []
|
||||
for content in tool_result.content:
|
||||
if isinstance(content, TextContent):
|
||||
text_list.append(content.text)
|
||||
|
||||
else:
|
||||
raise NotImplementedError(f"content.type={type(content)} not supported")
|
||||
content = "\n".join(text_list)
|
||||
|
||||
if max_char_len:
|
||||
content = content[:max_char_len]
|
||||
self.output = content
|
||||
|
||||
self.success = not tool_result.is_error
|
||||
self.metadata.update(tool_result.meta)
|
||||
|
||||
|
||||
class ToolMemory(BaseMemory):
|
||||
memory_type: str = Field(default="tool")
|
||||
tool_call_results: List[ToolCallResult] = Field(default_factory=list)
|
||||
|
||||
def to_vector_node(self) -> VectorNode:
|
||||
return VectorNode(unique_id=self.memory_id,
|
||||
workspace_id=self.workspace_id,
|
||||
content=self.when_to_use,
|
||||
metadata={
|
||||
"memory_type": self.memory_type,
|
||||
"content": self.content,
|
||||
"score": self.score,
|
||||
"time_created": self.time_created,
|
||||
"time_modified": self.time_modified,
|
||||
"author": self.author,
|
||||
"tool_call_results": [x.model_dump() for x in self.tool_call_results],
|
||||
"metadata": self.metadata,
|
||||
})
|
||||
|
||||
def statistic(self, recent_frequency: int = 20) -> dict:
|
||||
"""
|
||||
Calculate statistical information for the most recent N tool calls.
|
||||
Returns avg token_cost, success rate, avg time_cost, and avg score.
|
||||
"""
|
||||
if not self.tool_call_results:
|
||||
return {
|
||||
"total_calls": 0,
|
||||
"recent_calls_analyzed": 0,
|
||||
"avg_token_cost": 0.0,
|
||||
"success_rate": 0.0,
|
||||
"avg_time_cost": 0.0,
|
||||
"avg_score": 0.0
|
||||
}
|
||||
|
||||
# Get the most recent N tool calls (or all if less than N)
|
||||
recent_calls = self.tool_call_results[-recent_frequency:]
|
||||
total_calls = len(self.tool_call_results)
|
||||
recent_calls_count = len(recent_calls)
|
||||
|
||||
# Calculate statistics
|
||||
total_token_cost = sum(call.token_cost for call in recent_calls if call.token_cost >= 0)
|
||||
valid_token_calls = [call for call in recent_calls if call.token_cost >= 0]
|
||||
avg_token_cost = total_token_cost / len(valid_token_calls) if valid_token_calls else 0.0
|
||||
|
||||
successful_calls = sum(1 for call in recent_calls if call.success)
|
||||
success_rate = successful_calls / recent_calls_count if recent_calls_count > 0 else 0.0
|
||||
|
||||
total_time_cost = sum(call.time_cost for call in recent_calls)
|
||||
avg_time_cost = total_time_cost / recent_calls_count if recent_calls_count > 0 else 0.0
|
||||
|
||||
total_score = sum(call.score for call in recent_calls)
|
||||
avg_score = total_score / recent_calls_count if recent_calls_count > 0 else 0.0
|
||||
|
||||
return {
|
||||
"avg_token_cost": round(avg_token_cost, 2),
|
||||
"avg_time_cost": round(avg_time_cost, 3),
|
||||
"success_rate": round(success_rate, 4),
|
||||
"avg_score": round(avg_score, 3)
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_vector_node(cls, node: VectorNode) -> "ToolMemory":
|
||||
metadata = node.metadata.copy()
|
||||
tool_call_results = [ToolCallResult(**result) for result in metadata.pop("tool_call_results", [])]
|
||||
return cls(workspace_id=node.workspace_id,
|
||||
memory_id=node.unique_id,
|
||||
when_to_use=node.content,
|
||||
memory_type=metadata.pop("memory_type"),
|
||||
content=metadata.pop("content"),
|
||||
score=metadata.pop("score"),
|
||||
time_created=metadata.pop("time_created"),
|
||||
time_modified=metadata.pop("time_modified"),
|
||||
author=metadata.pop("author"),
|
||||
tool_call_results=tool_call_results,
|
||||
metadata=metadata.pop("metadata", {}))
|
||||
|
||||
|
||||
|
||||
def vector_node_to_memory(node: VectorNode):
|
||||
memory_type = node.metadata.get("memory_type")
|
||||
if memory_type == "task":
|
||||
return TaskMemory.from_vector_node(node)
|
||||
|
|
@ -113,6 +226,9 @@ def vector_node_to_memory(node: VectorNode) -> BaseMemory:
|
|||
elif memory_type == "personal":
|
||||
return PersonalMemory.from_vector_node(node)
|
||||
|
||||
elif memory_type == "tool":
|
||||
return ToolMemory.from_vector_node(node)
|
||||
|
||||
else:
|
||||
raise RuntimeError(f"memory_type={memory_type} not supported!")
|
||||
|
||||
|
|
@ -125,11 +241,14 @@ def dict_to_memory(memory_dict: dict):
|
|||
elif memory_type == "personal":
|
||||
return PersonalMemory(**memory_dict)
|
||||
|
||||
elif memory_type == "tool":
|
||||
return ToolMemory(**memory_dict)
|
||||
|
||||
else:
|
||||
raise RuntimeError(f"memory_type={memory_type} not supported!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
def task_main():
|
||||
e1 = TaskMemory(
|
||||
workspace_id="w_1024",
|
||||
memory_id="123",
|
||||
|
|
@ -142,3 +261,78 @@ if __name__ == "__main__":
|
|||
print(v1.model_dump_json(indent=2))
|
||||
e2 = vector_node_to_memory(v1)
|
||||
print(e2.model_dump_json(indent=2))
|
||||
|
||||
|
||||
def personal_main():
|
||||
p1 = PersonalMemory(
|
||||
workspace_id="w_2048",
|
||||
memory_id="456",
|
||||
when_to_use="personal memory test case",
|
||||
content="personal test content",
|
||||
target="user_preferences",
|
||||
reflection_subject="learning_style",
|
||||
score=0.85,
|
||||
metadata={"category": "user_profile"})
|
||||
print("PersonalMemory test:")
|
||||
print(p1.model_dump_json(indent=2))
|
||||
v1 = p1.to_vector_node()
|
||||
print("VectorNode:")
|
||||
print(v1.model_dump_json(indent=2))
|
||||
p2 = vector_node_to_memory(v1)
|
||||
print("Reconstructed PersonalMemory:")
|
||||
print(p2.model_dump_json(indent=2))
|
||||
|
||||
|
||||
def tool_main():
|
||||
# Create sample tool call results
|
||||
tool_result1 = ToolCallResult(
|
||||
create_time="2025-10-15 10:30:00",
|
||||
tool_name="file_reader",
|
||||
input={"file_path": "/test/file.txt"},
|
||||
output="File content successfully read",
|
||||
token_cost=50,
|
||||
success=True,
|
||||
time_cost=0.5,
|
||||
evaluation="Successfully executed",
|
||||
score=0.95
|
||||
)
|
||||
|
||||
tool_result2 = ToolCallResult(
|
||||
create_time="2025-10-15 10:31:00",
|
||||
tool_name="data_processor",
|
||||
input={"data": "sample_data", "format": "json"},
|
||||
output="Data processed successfully",
|
||||
token_cost=75,
|
||||
success=True,
|
||||
time_cost=1.2,
|
||||
evaluation="Good performance",
|
||||
score=0.88
|
||||
)
|
||||
|
||||
t1 = ToolMemory(
|
||||
workspace_id="w_4096",
|
||||
memory_id="789",
|
||||
memory_type="tool",
|
||||
when_to_use="tool execution memory test",
|
||||
content="tool execution test content",
|
||||
score=0.92,
|
||||
tool_call_results=[tool_result1, tool_result2],
|
||||
metadata={"execution_context": "test_environment"})
|
||||
|
||||
print("ToolMemory test:")
|
||||
print(t1.model_dump_json(indent=2))
|
||||
v1 = t1.to_vector_node()
|
||||
print("VectorNode:")
|
||||
print(v1.model_dump_json(indent=2))
|
||||
t2 = ToolMemory.from_vector_node(v1)
|
||||
print("Reconstructed ToolMemory:")
|
||||
print(t2.model_dump_json(indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("=== Task Memory Test ===")
|
||||
# task_main()
|
||||
print("\n=== Personal Memory Test ===")
|
||||
# personal_main()
|
||||
print("\n=== Tool Memory Test ===")
|
||||
tool_main()
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
from . import personal
|
||||
from . import task
|
||||
from . import tool
|
||||
|
|
|
|||
2
reme_ai/summary/tool/__init__.py
Normal file
2
reme_ai/summary/tool/__init__.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
from .parse_tool_call_result_op import ParseToolCallResultOp
|
||||
from .summary_tool_memory_op import SummaryToolMemoryOp
|
||||
167
reme_ai/summary/tool/parse_tool_call_result_op.py
Normal file
167
reme_ai/summary/tool/parse_tool_call_result_op.py
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
import asyncio
|
||||
from collections import defaultdict
|
||||
from typing import List
|
||||
|
||||
from flowllm import C, BaseAsyncOp
|
||||
from flowllm.enumeration.role import Role
|
||||
from flowllm.schema.message import Message
|
||||
from flowllm.schema.vector_node import VectorNode
|
||||
from flowllm.utils.common_utils import extract_content
|
||||
from loguru import logger
|
||||
|
||||
from reme_ai.schema.memory import ToolMemory, ToolCallResult, vector_node_to_memory
|
||||
|
||||
|
||||
@C.register_op()
|
||||
class ParseToolCallResultOp(BaseAsyncOp):
|
||||
file_path: str = __file__
|
||||
|
||||
def __init__(self,
|
||||
max_history_tool_call_cnt: int = 100,
|
||||
evaluation_sleep_interval: float = 1.0,
|
||||
**kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.max_history_tool_call_cnt: int = max_history_tool_call_cnt
|
||||
self.evaluation_sleep_interval: float = evaluation_sleep_interval
|
||||
|
||||
async def _evaluate_single_tool_call(self, tool_call_result: ToolCallResult, index: int) -> ToolCallResult:
|
||||
await asyncio.sleep(self.evaluation_sleep_interval * index)
|
||||
|
||||
prompt = self.prompt_format(
|
||||
prompt_name="evaluate_tool_call_prompt",
|
||||
tool_name=tool_call_result.tool_name,
|
||||
input_params=str(tool_call_result.input),
|
||||
output=tool_call_result.output,
|
||||
success_flag=str(tool_call_result.success),
|
||||
time_cost=tool_call_result.time_cost,
|
||||
token_cost=tool_call_result.token_cost)
|
||||
|
||||
def parse_evaluation(message: Message) -> ToolCallResult:
|
||||
content = message.content.strip()
|
||||
eval_data = extract_content(content, "json")
|
||||
|
||||
# 更新 tool_call_result - 包含 summary, evaluation 和 score
|
||||
tool_call_result.summary = eval_data.get("summary", "")
|
||||
tool_call_result.evaluation = eval_data.get("evaluation", "")
|
||||
tool_call_result.score = float(eval_data.get("score", 0.0))
|
||||
|
||||
# 验证 score 是否符合 2 档要求 (0.0, 1.0)
|
||||
if tool_call_result.score not in [0.0, 1.0]:
|
||||
if tool_call_result.score < 0.5:
|
||||
tool_call_result.score = 0.0
|
||||
else:
|
||||
tool_call_result.score = 1.0
|
||||
|
||||
# 打印完整的prompt和result
|
||||
logger.info(f"\n{'='*80}\nLLM Evaluation [Index {index}]\n{'='*80}\n"
|
||||
f"PROMPT:\n{prompt}\n\n"
|
||||
f"RESULT:\n{content}\n"
|
||||
f"{'='*80}\n")
|
||||
|
||||
return tool_call_result
|
||||
|
||||
# 调用 LLM 进行评估
|
||||
result = await self.llm.achat(messages=[Message(role=Role.USER, content=prompt)], callback_fn=parse_evaluation)
|
||||
|
||||
return result
|
||||
|
||||
async def async_execute(self):
|
||||
tool_call_results: list = self.context.get("tool_call_results", [])
|
||||
tool_call_results = [ToolCallResult(**x) if isinstance(x, dict) else x for x in tool_call_results]
|
||||
workspace_id: str = self.context.workspace_id
|
||||
|
||||
if not tool_call_results:
|
||||
self.context.response.answer = "No valid tool_call_results"
|
||||
self.context.response.success = False
|
||||
return
|
||||
|
||||
# 使用基类的 submit_async_task 提交所有评估任务
|
||||
for index, tool_call_result in enumerate(tool_call_results):
|
||||
self.submit_async_task(self._evaluate_single_tool_call, tool_call_result, index)
|
||||
|
||||
# 使用基类的 join_async_task 等待所有任务完成
|
||||
# 注意: 基类已经过滤掉异常,返回的只包含成功的结果
|
||||
evaluated_results = await self.join_async_task(return_exceptions=True)
|
||||
|
||||
tool_results_by_name = defaultdict(list)
|
||||
for result in evaluated_results:
|
||||
tool_results_by_name[result.tool_name].append(result)
|
||||
|
||||
# 处理每个 tool_name 的结果
|
||||
all_memory_list = []
|
||||
all_deleted_memory_ids = []
|
||||
|
||||
for tool_name, tool_call_results in tool_results_by_name.items():
|
||||
nodes: List[VectorNode] = await self.vector_store.async_search(query=tool_name,
|
||||
workspace_id=workspace_id,
|
||||
top_k=1)
|
||||
|
||||
tool_memory: ToolMemory | None = None
|
||||
exist_node: bool = False
|
||||
|
||||
if nodes:
|
||||
top_node = nodes[0]
|
||||
memory: ToolMemory = vector_node_to_memory(top_node)
|
||||
|
||||
# 确保是 ToolMemory 类型且 when_to_use 与 tool_name 匹配
|
||||
if isinstance(memory, ToolMemory) and memory.when_to_use == tool_name:
|
||||
tool_memory = memory
|
||||
exist_node = True
|
||||
|
||||
# 如果没有找到匹配的 memory，创建新的
|
||||
if tool_memory is None:
|
||||
tool_memory = ToolMemory(workspace_id=workspace_id, when_to_use=tool_name)
|
||||
|
||||
tool_memory.tool_call_results.extend(tool_call_results)
|
||||
|
||||
# 保留最近的 n 个
|
||||
if len(tool_memory.tool_call_results) > self.max_history_tool_call_cnt:
|
||||
tool_memory.tool_call_results = tool_memory.tool_call_results[-self.max_history_tool_call_cnt:]
|
||||
|
||||
# 更新修改时间
|
||||
tool_memory.update_modified_time()
|
||||
|
||||
# 如果是更新现有的 memory，需要先删除旧的
|
||||
if exist_node:
|
||||
all_deleted_memory_ids.append(tool_memory.memory_id)
|
||||
|
||||
all_memory_list.append(tool_memory)
|
||||
|
||||
# 设置返回结果
|
||||
self.context.response.metadata["deleted_memory_ids"] = all_deleted_memory_ids
|
||||
self.context.response.metadata["memory_list"] = all_memory_list
|
||||
|
||||
|
||||
async def main():
|
||||
"""Simple test for ParseToolCallResultOp"""
|
||||
from flowllm.app import FlowLLMApp
|
||||
from datetime import datetime
|
||||
|
||||
async with FlowLLMApp(load_default_config=True):
|
||||
op = ParseToolCallResultOp()
|
||||
|
||||
# Create simple test data
|
||||
tool_call_results = [
|
||||
{
|
||||
"create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": "test_tool",
|
||||
"input": {
|
||||
"query": "search for python asyncio documentation",
|
||||
"max_results": 10,
|
||||
"filter_type": "official_docs",
|
||||
"language": "en"
|
||||
},
|
||||
"output": "Found 10 relevant documentation pages for Python asyncio. Top results include: 1) Official Python docs for asyncio module, 2) Real Python asyncio tutorial, 3) Stack Overflow asyncio examples. All results are from official sources as requested.",
|
||||
"token_cost": 150,
|
||||
"success": True,
|
||||
"time_cost": 2.3
|
||||
}
|
||||
]
|
||||
workspace_id = "test_workspace1"
|
||||
|
||||
await op.async_call(tool_call_results=tool_call_results, workspace_id=workspace_id)
|
||||
logger.info(f"Response: {op.context.response.model_dump_json()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
57
reme_ai/summary/tool/parse_tool_call_result_prompt.yaml
Normal file
57
reme_ai/summary/tool/parse_tool_call_result_prompt.yaml
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
evaluate_tool_call_prompt: |
|
||||
You are an expert in evaluating tool invocation results. Please evaluate the following tool call based on two criteria:
|
||||
|
||||
1. **Success**: Whether the tool invocation was successful
|
||||
2. **Parameter Alignment**: Whether the result aligns with the expected behavior given the input parameters
|
||||
|
||||
## Tool Call Information:
|
||||
- Tool Name: {tool_name}
|
||||
- Success Flag: {success_flag}
|
||||
- Time Cost: {time_cost}s
|
||||
- Token Cost: {token_cost} tokens
|
||||
- Input Parameters: {input_params}
|
||||
- Output: {output}
|
||||
|
||||
## Evaluation Criteria:
|
||||
|
||||
### Important: Score independently from the success flag
|
||||
The `success_flag` indicates whether the tool executed without technical errors.
|
||||
The `score` should evaluate the QUALITY and RELEVANCE of the result.
|
||||
|
||||
A tool can execute successfully (success=True) but still produce low-quality or irrelevant results (score=0.0).
|
||||
|
||||
### Evaluation Dimensions:
|
||||
|
||||
1. **Technical Execution** (reflected in success_flag):
|
||||
- Did the tool run without errors?
|
||||
- Are there error messages or exceptions in the output?
|
||||
|
||||
2. **Result Quality** (what you should score):
|
||||
- Is the output relevant to the input query/parameters?
|
||||
- Does the output provide meaningful and useful information?
|
||||
- Is the result appropriate given the input parameters?
|
||||
- Does the output match the tool's intended purpose?
|
||||
|
||||
## Response Format:
|
||||
Please provide your evaluation in the following JSON format:
|
||||
|
||||
```json
|
||||
{{
|
||||
"summary": "A brief one-sentence summary of the tool call result",
|
||||
"evaluation": "A brief evaluation (2-3 sentences) explaining the result quality and relevance, NOT just repeating the success flag.",
|
||||
"score": 1.0
|
||||
}}
|
||||
```
|
||||
|
||||
## Scoring Guidelines (Focus on Result Quality):
|
||||
- **1.0**: High Quality - The output is relevant, useful, and appropriate for the given input parameters
|
||||
- **0.0**: Low Quality - The output is irrelevant, incorrect, unhelpful, or inappropriate for the input
|
||||
|
||||
## Examples:
|
||||
- success=True, but output is "No results found" for a reasonable query → score=0.0 (technically succeeded but unhelpful)
|
||||
- success=True, but output contains generic/irrelevant information → score=0.0 (poor quality)
|
||||
- success=True, and output provides relevant, useful information → score=1.0 (good quality)
|
||||
- success=False, with error messages → score=0.0 (failed execution)
|
||||
|
||||
Please evaluate carefully and provide a score of exactly 0.0 or 1.0 based on OUTPUT QUALITY, not just the success flag.
|
||||
|
||||
462
reme_ai/summary/tool/summary_tool_memory_op.py
Normal file
462
reme_ai/summary/tool/summary_tool_memory_op.py
Normal file
|
|
@ -0,0 +1,462 @@
|
|||
import asyncio
|
||||
from typing import List
|
||||
|
||||
from flowllm import C, BaseAsyncOp
|
||||
from flowllm.enumeration.role import Role
|
||||
from flowllm.schema.message import Message
|
||||
from flowllm.schema.vector_node import VectorNode
|
||||
from flowllm.utils.common_utils import extract_content
|
||||
from loguru import logger
|
||||
|
||||
from reme_ai.schema.memory import ToolMemory, vector_node_to_memory
|
||||
|
||||
|
||||
@C.register_op()
|
||||
class SummaryToolMemoryOp(BaseAsyncOp):
|
||||
file_path: str = __file__
|
||||
|
||||
def __init__(self,
|
||||
recent_call_count: int = 30,
|
||||
summary_sleep_interval: float = 1.0,
|
||||
**kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.recent_call_count: int = recent_call_count
|
||||
self.summary_sleep_interval: float = summary_sleep_interval
|
||||
|
||||
@staticmethod
|
||||
def _format_call_summaries_markdown(recent_calls: List) -> str:
|
||||
"""Format tool call summaries as markdown."""
|
||||
if not recent_calls:
|
||||
return "No recent calls available."
|
||||
|
||||
lines = []
|
||||
for i, call in enumerate(recent_calls, 1):
|
||||
lines.append(f"### Call #{i}")
|
||||
lines.append(f"- **Summary**: {call.summary}")
|
||||
lines.append(f"- **Evaluation**: {call.evaluation}")
|
||||
lines.append(f"- **Score**: {call.score}")
|
||||
lines.append(f"- **Success**: {call.success}")
|
||||
lines.append(f"- **Time Cost**: {call.time_cost}s")
|
||||
lines.append(f"- **Token Cost**: {call.token_cost}")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@staticmethod
|
||||
def _format_statistics_markdown(statistics: dict) -> str:
|
||||
"""Format statistics as markdown."""
|
||||
lines = [f"- **Success Rate**: {statistics.get('success_rate', 0):.2%}",
|
||||
f"- **Average Score**: {statistics.get('avg_score', 0):.3f}",
|
||||
f"- **Average Time Cost**: {statistics.get('avg_time_cost', 0):.3f}s",
|
||||
f"- **Average Token Cost**: {statistics.get('avg_token_cost', 0):.1f}"]
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
async def _summarize_single_tool(self, tool_memory: ToolMemory, index: int) -> ToolMemory:
|
||||
await asyncio.sleep(self.summary_sleep_interval * index)
|
||||
|
||||
# Get the most recent N tool calls
|
||||
recent_calls = tool_memory.tool_call_results[-self.recent_call_count:]
|
||||
|
||||
if not recent_calls:
|
||||
logger.warning(f"No tool call results found for tool: {tool_memory.when_to_use}")
|
||||
return tool_memory
|
||||
|
||||
# Log how many unsummarized calls we're processing
|
||||
unsummarized_count = sum(1 for call in recent_calls if not call.is_summarized)
|
||||
logger.info(f"Summarizing tool {tool_memory.when_to_use}: "
|
||||
f"{unsummarized_count}/{len(recent_calls)} unsummarized calls")
|
||||
|
||||
# Get statistics
|
||||
statistics = tool_memory.statistic(recent_frequency=self.recent_call_count)
|
||||
|
||||
# Format data as markdown
|
||||
call_summaries_md = self._format_call_summaries_markdown(recent_calls)
|
||||
statistics_md = self._format_statistics_markdown(statistics)
|
||||
|
||||
# Don't include statistics in prompt - only call summaries
|
||||
prompt = self.prompt_format(prompt_name="summarize_tool_usage_prompt",
|
||||
tool_name=tool_memory.when_to_use,
|
||||
call_summaries=call_summaries_md)
|
||||
|
||||
def parse_summary(message: Message) -> ToolMemory:
|
||||
content = message.content.strip()
|
||||
# Extract content from txt code block
|
||||
llm_summary = extract_content(content, "txt")
|
||||
|
||||
# Append statistics markdown to LLM result
|
||||
tool_memory.content = f"{llm_summary}\n\n## Statistics\n{statistics_md}"
|
||||
|
||||
# Mark all recent calls as summarized
|
||||
for call in recent_calls:
|
||||
call.is_summarized = True
|
||||
|
||||
# Update modified time
|
||||
tool_memory.update_modified_time()
|
||||
|
||||
logger.info(f"Summarized tool {index}: tool_name={tool_memory.when_to_use}, "
|
||||
f"content_length={len(tool_memory.content)}, "
|
||||
f"marked {len(recent_calls)} calls as summarized")
|
||||
return tool_memory
|
||||
|
||||
# Call LLM to generate summary
|
||||
result = await self.llm.achat(messages=[Message(role=Role.USER, content=prompt)], callback_fn=parse_summary)
|
||||
|
||||
return result
|
||||
|
||||
async def async_execute(self):
|
||||
tool_names: str = self.context.get("tool_names", "")
|
||||
workspace_id: str = self.context.workspace_id
|
||||
|
||||
if not tool_names:
|
||||
logger.warning("tool_names is empty, skipping processing")
|
||||
self.context.response.answer = "tool_names is required"
|
||||
self.context.response.success = False
|
||||
return
|
||||
|
||||
# Split tool names by comma
|
||||
tool_name_list = [name.strip() for name in tool_names.split(",") if name.strip()]
|
||||
logger.info(f"workspace_id={workspace_id} processing {len(tool_name_list)} tools: {tool_name_list}")
|
||||
|
||||
# Search for each tool in the vector store
|
||||
matched_tool_memories: List[ToolMemory] = []
|
||||
|
||||
for tool_name in tool_name_list:
|
||||
nodes: List[VectorNode] = await self.vector_store.async_search(query=tool_name,
|
||||
workspace_id=workspace_id,
|
||||
top_k=1)
|
||||
|
||||
if nodes:
|
||||
top_node = nodes[0]
|
||||
memory = vector_node_to_memory(top_node)
|
||||
|
||||
# Ensure it's a ToolMemory and when_to_use matches
|
||||
if isinstance(memory, ToolMemory) and memory.when_to_use == tool_name:
|
||||
matched_tool_memories.append(memory)
|
||||
logger.info(f"Found tool_memory for tool_name={tool_name}, "
|
||||
f"memory_id={memory.memory_id}, "
|
||||
f"total_calls={len(memory.tool_call_results)}")
|
||||
else:
|
||||
logger.warning(f"No exact match found for tool_name={tool_name}")
|
||||
else:
|
||||
logger.warning(f"No memory found for tool_name={tool_name}")
|
||||
|
||||
if not matched_tool_memories:
|
||||
logger.info("No matching tool memories found")
|
||||
self.context.response.answer = "No matching tool memories found"
|
||||
self.context.response.success = False
|
||||
return
|
||||
|
||||
# Check which tools need summarization
|
||||
tools_need_summary = []
|
||||
tools_skipped = []
|
||||
|
||||
for tool_memory in matched_tool_memories:
|
||||
recent_calls = tool_memory.tool_call_results[-self.recent_call_count:]
|
||||
unsummarized_count = sum(1 for call in recent_calls if not call.is_summarized)
|
||||
|
||||
if unsummarized_count > 0:
|
||||
tools_need_summary.append(tool_memory)
|
||||
else:
|
||||
tools_skipped.append(tool_memory)
|
||||
logger.info(f"Skipping tool {tool_memory.when_to_use}: all recent {len(recent_calls)} calls already summarized")
|
||||
|
||||
# Concurrently summarize tool memories that need it
|
||||
if tools_need_summary:
|
||||
logger.info(f"Starting concurrent summarization of {len(tools_need_summary)} tool memories")
|
||||
|
||||
# 使用基类的 submit_async_task 提交所有总结任务
|
||||
for index, tool_memory in enumerate(tools_need_summary):
|
||||
self.submit_async_task(self._summarize_single_tool, tool_memory, index)
|
||||
|
||||
# 使用基类的 join_async_task 等待所有任务完成
|
||||
# 注意: 基类已经过滤掉异常,返回的只包含成功的结果
|
||||
valid_summarized_memories = await self.join_async_task(return_exceptions=True)
|
||||
logger.info(f"Completed summarization of {len(valid_summarized_memories)} tool memories")
|
||||
else:
|
||||
valid_summarized_memories = []
|
||||
logger.info("All tool memories are up-to-date, no summarization needed")
|
||||
|
||||
# Combine summarized and skipped memories
|
||||
all_memories = valid_summarized_memories + tools_skipped
|
||||
|
||||
# Set response
|
||||
self.context.response.answer = (f"Successfully processed {len(all_memories)} tool memories: "
|
||||
f"{len(valid_summarized_memories)} summarized, "
|
||||
f"{len(tools_skipped)} skipped (already up-to-date)")
|
||||
self.context.response.success = True
|
||||
self.context.response.metadata["memory_list"] = all_memories
|
||||
self.context.response.metadata["deleted_memory_ids"] = [m.memory_id for m in all_memories]
|
||||
|
||||
# Log summary for each tool
|
||||
for memory in valid_summarized_memories:
|
||||
logger.info(f"Tool: {memory.when_to_use}, "
|
||||
f"Content: {memory.content[:100]}...")
|
||||
|
||||
|
||||
async def main():
|
||||
from flowllm.app import FlowLLMApp
|
||||
from reme_ai.summary.tool.parse_tool_call_result_op import ParseToolCallResultOp
|
||||
from reme_ai.vector_store.update_vector_store_op import UpdateVectorStoreOp
|
||||
from datetime import datetime, timedelta
|
||||
import random
|
||||
|
||||
async with FlowLLMApp(load_default_config=True):
|
||||
workspace_id = "test_workspace_complex"
|
||||
tool_name = "web_search_tool"
|
||||
|
||||
# ===== 第一步: 准备 30 条模拟的工具调用记录 =====
|
||||
# 模拟不同场景的调用记录:成功、参数错误、超时、返回空结果等
|
||||
logger.info("=" * 80)
|
||||
logger.info("步骤1: 准备 30 条工具调用记录,模拟真实使用场景")
|
||||
logger.info("=" * 80)
|
||||
|
||||
base_time = datetime.now() - timedelta(days=7)
|
||||
tool_call_results = []
|
||||
|
||||
# 场景1: 成功的搜索 (15条)
|
||||
success_queries = [
|
||||
"Python asyncio tutorial", "machine learning basics", "React hooks guide",
|
||||
"Docker best practices", "SQL optimization tips", "Git workflow strategies",
|
||||
"RESTful API design", "microservices architecture", "Redis caching patterns",
|
||||
"Kubernetes deployment", "GraphQL advantages", "MongoDB schema design",
|
||||
"JWT authentication", "OAuth2 flow", "WebSocket real-time"
|
||||
]
|
||||
|
||||
for i, query in enumerate(success_queries):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=i * 2)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"query": query,
|
||||
"max_results": random.randint(5, 20),
|
||||
"language": "en",
|
||||
"filter_type": "technical_docs"
|
||||
},
|
||||
"output": f"Found {random.randint(8, 20)} relevant results for '{query}'. Top results include official documentation, tutorials, and best practice guides.",
|
||||
"token_cost": random.randint(100, 300),
|
||||
"success": True,
|
||||
"time_cost": round(random.uniform(1.5, 3.5), 2)
|
||||
})
|
||||
|
||||
# 场景2: 参数不合理导致的部分成功 (8条)
|
||||
for i in range(8):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=30 + i * 3)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"query": f"test query {i}", # 查询词过于简单
|
||||
"max_results": 100, # 请求过多结果
|
||||
"language": "unknown", # 语言参数错误
|
||||
},
|
||||
"output": f"Warning: language 'unknown' not supported, using default. Query too generic, returning limited results. Found {random.randint(2, 5)} results.",
|
||||
"token_cost": random.randint(50, 150),
|
||||
"success": True,
|
||||
"time_cost": round(random.uniform(2.0, 4.0), 2)
|
||||
})
|
||||
|
||||
# 场景3: 超时或失败 (5条)
|
||||
for i in range(5):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=54 + i * 4)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"query": f"extremely complex query with many conditions {i}",
|
||||
"max_results": 50,
|
||||
"language": "en",
|
||||
"filter_type": "all"
|
||||
},
|
||||
"output": "Error: Request timeout after 10 seconds. Try simplifying the query or reducing max_results.",
|
||||
"token_cost": 20,
|
||||
"success": False,
|
||||
"time_cost": 10.0
|
||||
})
|
||||
|
||||
# 场景4: 空结果 (2条)
|
||||
for i in range(2):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=74 + i * 5)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"query": f"xyzabc123nonexistent{i}", # 不存在的内容
|
||||
"max_results": 10,
|
||||
"language": "en",
|
||||
},
|
||||
"output": "No results found for the given query. Please try different keywords.",
|
||||
"token_cost": 30,
|
||||
"success": True,
|
||||
"time_cost": 1.2
|
||||
})
|
||||
|
||||
logger.info(f"准备了 {len(tool_call_results)} 条工具调用记录")
|
||||
logger.info(f"- 成功调用: 15 条")
|
||||
logger.info(f"- 参数不合理: 8 条")
|
||||
logger.info(f"- 超时失败: 5 条")
|
||||
logger.info(f"- 空结果: 2 条")
|
||||
|
||||
# ===== 第二步: 使用 ParseToolCallResultOp >> UpdateVectorStoreOp 串联运行 =====
|
||||
logger.info("\n" + "=" * 80)
|
||||
logger.info("步骤2: ParseToolCallResultOp >> UpdateVectorStoreOp 串联评估并保存")
|
||||
logger.info("=" * 80)
|
||||
|
||||
# 使用 >> 操作符串联两个Op,自动传递context和metadata
|
||||
pipeline = ParseToolCallResultOp(evaluation_sleep_interval=0.1) >> UpdateVectorStoreOp()
|
||||
|
||||
await pipeline.async_call(
|
||||
tool_call_results=tool_call_results,
|
||||
tool_name=tool_name,
|
||||
workspace_id=workspace_id
|
||||
)
|
||||
|
||||
if not pipeline.context.response.success:
|
||||
logger.error(f"Pipeline failed: {pipeline.context.response.answer}")
|
||||
return
|
||||
|
||||
logger.info(f"✓ Pipeline 完成")
|
||||
logger.info(f" 评估了 {len(tool_call_results)} 条记录")
|
||||
logger.info(f" 每条记录包含: summary, evaluation, score (0.0/0.5/1.0)")
|
||||
|
||||
# 显示一些评估结果示例
|
||||
memory_list = pipeline.context.response.metadata.get("memory_list", [])
|
||||
if memory_list:
|
||||
tool_memory = memory_list[0]
|
||||
logger.info(f"\n评估结果示例 (前3条):")
|
||||
for i, result in enumerate(tool_memory.tool_call_results[:3], 1):
|
||||
logger.info(f" 调用 #{i}:")
|
||||
logger.info(f" 查询: {result.input.get('query', 'N/A')}")
|
||||
logger.info(f" 评分: {result.score}")
|
||||
logger.info(f" 总结: {result.summary[:80]}...")
|
||||
logger.info(f" 评价: {result.evaluation[:80]}...")
|
||||
|
||||
# 显示向量数据库更新结果
|
||||
update_result = pipeline.context.response.metadata.get("update_result", {})
|
||||
logger.info(f"\n✓ 向量数据库更新完成:")
|
||||
logger.info(f" 删除记录数: {update_result.get('deleted_count', 0)}")
|
||||
logger.info(f" 插入记录数: {update_result.get('inserted_count', 0)}")
|
||||
|
||||
# ===== 第三步: 使用 SummaryToolMemoryOp 总结工具使用模式 =====
|
||||
logger.info("\n" + "=" * 80)
|
||||
logger.info("步骤3: 使用 SummaryToolMemoryOp 从 30 条记录中提取使用模式和建议")
|
||||
logger.info("=" * 80)
|
||||
|
||||
summary_op = SummaryToolMemoryOp(
|
||||
recent_call_count=30, # 分析最近30条记录
|
||||
summary_sleep_interval=0.5
|
||||
)
|
||||
await summary_op.async_call(
|
||||
tool_names=tool_name,
|
||||
workspace_id=workspace_id
|
||||
)
|
||||
|
||||
if not summary_op.context.response.success:
|
||||
logger.error(f"SummaryToolMemoryOp failed: {summary_op.context.response.answer}")
|
||||
return
|
||||
|
||||
logger.info(f"✓ SummaryToolMemoryOp 完成")
|
||||
|
||||
# ===== 第四步: 展示 summary 的价值 =====
|
||||
logger.info("\n" + "=" * 80)
|
||||
logger.info("步骤4: 展示 Summary 如何将分散的调用记录转化为有价值的使用指南")
|
||||
logger.info("=" * 80)
|
||||
|
||||
summarized_memories = summary_op.context.response.metadata.get("memory_list", [])
|
||||
if summarized_memories:
|
||||
summarized_memory = summarized_memories[0]
|
||||
|
||||
logger.info(f"\n工具名称: {summarized_memory.when_to_use}")
|
||||
logger.info(f"\n统计信息:")
|
||||
stats = summarized_memory.statistic(recent_frequency=30)
|
||||
logger.info(f" 总调用次数: {len(summarized_memory.tool_call_results)}")
|
||||
logger.info(f" 成功率: {stats['success_rate']:.1%}")
|
||||
logger.info(f" 平均评分: {stats['avg_score']:.2f}")
|
||||
logger.info(f" 平均耗时: {stats['avg_time_cost']:.2f}s")
|
||||
logger.info(f" 平均Token消耗: {stats['avg_token_cost']:.1f}")
|
||||
|
||||
logger.info(f"\n" + "=" * 60)
|
||||
logger.info("Summary 生成的使用指南 (从30条分散记录中提取):")
|
||||
logger.info("=" * 60)
|
||||
logger.info(summarized_memory.content)
|
||||
logger.info("=" * 60)
|
||||
|
||||
# ===== 第五步: 验证跳过逻辑 - 再次运行应该跳过 =====
|
||||
logger.info("\n" + "=" * 80)
|
||||
logger.info("步骤5: 验证跳过逻辑 - 再次运行 SummaryToolMemoryOp 应该跳过已总结的记录")
|
||||
logger.info("=" * 80)
|
||||
|
||||
# 需要先保存更新后的 memory (带有 is_summarized=True 标记)
|
||||
from reme_ai.vector_store.update_vector_store_op import UpdateVectorStoreOp
|
||||
update_op = UpdateVectorStoreOp()
|
||||
await update_op.async_call(
|
||||
memory_list=summarized_memories,
|
||||
workspace_id=workspace_id
|
||||
)
|
||||
logger.info("✓ 已保存带有 is_summarized 标记的 tool memory")
|
||||
|
||||
# 再次运行总结
|
||||
summary_op2 = SummaryToolMemoryOp(
|
||||
recent_call_count=30,
|
||||
summary_sleep_interval=0.5
|
||||
)
|
||||
await summary_op2.async_call(
|
||||
tool_names=tool_name,
|
||||
workspace_id=workspace_id
|
||||
)
|
||||
|
||||
if summary_op2.context.response.success:
|
||||
logger.info("✓ 第二次总结完成 (预期应该跳过)")
|
||||
else:
|
||||
logger.info(f"第二次总结失败: {summary_op2.context.response.answer}")
|
||||
|
||||
# ===== 第六步: 添加新记录并验证增量总结 =====
|
||||
logger.info("\n" + "=" * 80)
|
||||
logger.info("步骤6: 添加 1 条新记录,验证会触发重新总结")
|
||||
logger.info("=" * 80)
|
||||
|
||||
# 添加一条新的工具调用记录
|
||||
new_tool_call = {
|
||||
"create_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"query": "FastAPI async best practices",
|
||||
"max_results": 15,
|
||||
"language": "en",
|
||||
"filter_type": "technical_docs"
|
||||
},
|
||||
"output": "Found 12 excellent resources on FastAPI async patterns including official docs, real-world examples, and performance tips.",
|
||||
"token_cost": 180,
|
||||
"success": True,
|
||||
"time_cost": 2.1
|
||||
}
|
||||
|
||||
# 使用 ParseToolCallResultOp 评估新记录
|
||||
new_pipeline = ParseToolCallResultOp(evaluation_sleep_interval=0.1) >> UpdateVectorStoreOp()
|
||||
await new_pipeline.async_call(
|
||||
tool_call_results=[new_tool_call],
|
||||
tool_name=tool_name,
|
||||
workspace_id=workspace_id
|
||||
)
|
||||
logger.info("✓ 添加并评估了 1 条新记录")
|
||||
|
||||
# 第三次运行总结 - 这次应该会执行
|
||||
summary_op3 = SummaryToolMemoryOp(
|
||||
recent_call_count=30,
|
||||
summary_sleep_interval=0.5
|
||||
)
|
||||
await summary_op3.async_call(
|
||||
tool_names=tool_name,
|
||||
workspace_id=workspace_id
|
||||
)
|
||||
|
||||
if summary_op3.context.response.success:
|
||||
logger.info("✓ 第三次总结完成 (因为有新记录)")
|
||||
summarized_memories3 = summary_op3.context.response.metadata.get("memory_list", [])
|
||||
if summarized_memories3:
|
||||
summarized_memory3 = summarized_memories3[0]
|
||||
unsummarized = sum(1 for call in summarized_memory3.tool_call_results[-30:] if not call.is_summarized)
|
||||
logger.info(f" 最近30条中未总结的记录数: {unsummarized} (应该为 0)")
|
||||
else:
|
||||
logger.info(f"第三次总结失败: {summary_op3.context.response.answer}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
24
reme_ai/summary/tool/summary_tool_memory_prompt.yaml
Normal file
24
reme_ai/summary/tool/summary_tool_memory_prompt.yaml
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
summarize_tool_usage_prompt: |
|
||||
You are an expert in analyzing tool usage patterns and generating practical usage guidelines.
|
||||
|
||||
## Tool Information:
|
||||
- Tool Name: {tool_name}
|
||||
|
||||
## Recent Tool Call History:
|
||||
{call_summaries}
|
||||
|
||||
## Your Task:
|
||||
Based on the tool call history, generate a concise and logical tool usage description following this structure:
|
||||
|
||||
1. **Core Function**: What this tool does and when to use it
|
||||
2. **Success Patterns**: Parameter patterns and usage scenarios that work well
|
||||
3. **Common Issues**: Main pitfalls to avoid and why they fail
|
||||
4. **Best Practices**: 2-3 actionable recommendations
|
||||
|
||||
## Response Format:
|
||||
Provide a structured, concise description (max 200 words). Focus on actionable insights derived from actual usage data. Avoid generic advice.
|
||||
|
||||
```txt
|
||||
[Your concise, data-driven tool usage description]
|
||||
```
|
||||
|
||||
195
reme_ai/utils/tool_memory_utils.py
Normal file
195
reme_ai/utils/tool_memory_utils.py
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
import random
|
||||
from datetime import datetime, timedelta
|
||||
from typing import List, Dict, Any
|
||||
|
||||
|
||||
def create_mock_tool_call_results(tool_name: str, count: int = 30) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Create mock tool call results for testing
|
||||
|
||||
Args:
|
||||
tool_name: Name of the tool to create results for
|
||||
count: Number of mock results to create
|
||||
|
||||
Returns:
|
||||
List of tool call result dictionaries
|
||||
"""
|
||||
base_time = datetime.now() - timedelta(days=7)
|
||||
tool_call_results = []
|
||||
|
||||
if tool_name == "web_search":
|
||||
# Scenario 1: Successful searches (15 calls)
|
||||
success_queries = [
|
||||
"Python asyncio tutorial", "machine learning basics", "React hooks guide",
|
||||
"Docker best practices", "SQL optimization tips", "Git workflow strategies",
|
||||
"RESTful API design", "microservices architecture", "Redis caching patterns",
|
||||
"Kubernetes deployment", "GraphQL advantages", "MongoDB schema design",
|
||||
"JWT authentication", "OAuth2 flow", "WebSocket real-time"
|
||||
]
|
||||
|
||||
for i, query in enumerate(success_queries):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=i * 2)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"query": query,
|
||||
"max_results": random.randint(5, 20),
|
||||
"language": "en",
|
||||
"filter_type": "technical_docs"
|
||||
},
|
||||
"output": f"Found {random.randint(8, 20)} relevant results for '{query}'. Top results include official documentation, tutorials, and best practice guides.",
|
||||
"token_cost": random.randint(100, 300),
|
||||
"success": True,
|
||||
"time_cost": round(random.uniform(1.5, 3.5), 2)
|
||||
})
|
||||
|
||||
# Scenario 2: Poor parameters (8 calls)
|
||||
for i in range(8):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=30 + i * 3)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"query": f"test query {i}",
|
||||
"max_results": 100,
|
||||
"language": "unknown",
|
||||
},
|
||||
"output": f"Warning: language 'unknown' not supported, using default. Query too generic, returning limited results. Found {random.randint(2, 5)} results.",
|
||||
"token_cost": random.randint(50, 150),
|
||||
"success": True,
|
||||
"time_cost": round(random.uniform(2.0, 4.0), 2)
|
||||
})
|
||||
|
||||
# Scenario 3: Timeouts or failures (5 calls)
|
||||
for i in range(5):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=54 + i * 4)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"query": f"extremely complex query with many conditions {i}",
|
||||
"max_results": 50,
|
||||
"language": "en",
|
||||
"filter_type": "all"
|
||||
},
|
||||
"output": "Error: Request timeout after 10 seconds. Try simplifying the query or reducing max_results.",
|
||||
"token_cost": 20,
|
||||
"success": False,
|
||||
"time_cost": 10.0
|
||||
})
|
||||
|
||||
# Scenario 4: Empty results (2 calls)
|
||||
for i in range(2):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=74 + i * 5)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"query": f"xyzabc123nonexistent{i}",
|
||||
"max_results": 10,
|
||||
"language": "en",
|
||||
},
|
||||
"output": "No results found for the given query. Please try different keywords.",
|
||||
"token_cost": 30,
|
||||
"success": True,
|
||||
"time_cost": 1.2
|
||||
})
|
||||
|
||||
elif tool_name == "database_query":
|
||||
# Scenario 1: Successful queries (12 calls)
|
||||
for i in range(12):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=i * 3)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"table": f"users_{i % 3}",
|
||||
"query": f"SELECT * FROM table WHERE id > {i * 10}",
|
||||
"limit": random.randint(10, 100)
|
||||
},
|
||||
"output": f"Query executed successfully. Returned {random.randint(5, 50)} rows in {round(random.uniform(0.1, 0.5), 3)}s.",
|
||||
"token_cost": random.randint(20, 80),
|
||||
"success": True,
|
||||
"time_cost": round(random.uniform(0.1, 0.5), 3)
|
||||
})
|
||||
|
||||
# Scenario 2: Slow queries (6 calls)
|
||||
for i in range(6):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=36 + i * 4)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"table": "large_table",
|
||||
"query": f"SELECT * FROM large_table WHERE name LIKE '%pattern%' ORDER BY created_at",
|
||||
"limit": 1000
|
||||
},
|
||||
"output": f"Query executed but took longer than expected. Returned {random.randint(100, 1000)} rows. Consider adding indexes.",
|
||||
"token_cost": random.randint(50, 150),
|
||||
"success": True,
|
||||
"time_cost": round(random.uniform(5.0, 10.0), 2)
|
||||
})
|
||||
|
||||
# Scenario 3: Query errors (4 calls)
|
||||
for i in range(4):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=60 + i * 5)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"table": "invalid_table",
|
||||
"query": f"SELECT * FROM invalid_table WHERE bad_column = {i}",
|
||||
"limit": 10
|
||||
},
|
||||
"output": f"Error: Table 'invalid_table' does not exist or column 'bad_column' not found.",
|
||||
"token_cost": 10,
|
||||
"success": False,
|
||||
"time_cost": 0.05
|
||||
})
|
||||
|
||||
elif tool_name == "file_processor":
|
||||
# Scenario 1: Successful file processing (10 calls)
|
||||
file_types = ["csv", "json", "xml", "txt", "pdf"]
|
||||
for i in range(10):
|
||||
file_type = file_types[i % len(file_types)]
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=i * 4)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"file_path": f"/data/file_{i}.{file_type}",
|
||||
"operation": "read",
|
||||
"encoding": "utf-8"
|
||||
},
|
||||
"output": f"Successfully processed {file_type.upper()} file. Size: {random.randint(100, 5000)}KB, Records: {random.randint(100, 10000)}",
|
||||
"token_cost": random.randint(50, 200),
|
||||
"success": True,
|
||||
"time_cost": round(random.uniform(1.0, 3.0), 2)
|
||||
})
|
||||
|
||||
# Scenario 2: Large file warnings (5 calls)
|
||||
for i in range(5):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=40 + i * 6)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"file_path": f"/data/large_file_{i}.csv",
|
||||
"operation": "read",
|
||||
"encoding": "utf-8"
|
||||
},
|
||||
"output": f"Warning: Large file detected ({random.randint(50, 200)}MB). Processing may take longer. Consider using streaming mode.",
|
||||
"token_cost": random.randint(200, 500),
|
||||
"success": True,
|
||||
"time_cost": round(random.uniform(10.0, 30.0), 2)
|
||||
})
|
||||
|
||||
# Scenario 3: File not found (3 calls)
|
||||
for i in range(3):
|
||||
tool_call_results.append({
|
||||
"create_time": (base_time + timedelta(hours=70 + i * 8)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"tool_name": tool_name,
|
||||
"input": {
|
||||
"file_path": f"/invalid/path/file_{i}.txt",
|
||||
"operation": "read",
|
||||
"encoding": "utf-8"
|
||||
},
|
||||
"output": "Error: File not found. Please check the file path and ensure the file exists.",
|
||||
"token_cost": 10,
|
||||
"success": False,
|
||||
"time_cost": 0.01
|
||||
})
|
||||
|
||||
return tool_call_results[:count]
|
||||
|
|
@ -110,23 +110,34 @@ async def run2(session):
|
|||
result = await response.json()
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
|
||||
|
||||
async def run3(session):
|
||||
workspace_id = "default2"
|
||||
|
||||
async with session.post(
|
||||
f"{base_url}/add_tool_call_result",
|
||||
json={
|
||||
"tool_call_results": [
|
||||
{"a": 1},
|
||||
{"a": 2},
|
||||
],
|
||||
"workspace_id": workspace_id,
|
||||
},
|
||||
headers={"Content-Type": "application/json"}
|
||||
) as response:
|
||||
result = await response.json()
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
|
||||
|
||||
async def main():
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# 获取工具列表
|
||||
print("获取工具列表...")
|
||||
async with session.get(f"{base_url}/list") as response:
|
||||
if response.status == 200:
|
||||
tools = await response.json()
|
||||
print("可用工具:")
|
||||
for tool in tools:
|
||||
print(json.dumps(tool, ensure_ascii=False))
|
||||
else:
|
||||
print(f"获取工具列表失败: {response.status}")
|
||||
return
|
||||
|
||||
# await run1(session)
|
||||
await run2(session)
|
||||
# await run2(session)
|
||||
await run3(session)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
|
|
|||
6
test/test5.py
Normal file
6
test/test5.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import tiktoken
|
||||
enc = tiktoken.get_encoding("o200k_base")
|
||||
|
||||
# r = enc.encode("我爱吃西瓜，你说啥")
|
||||
r = enc.encode("hello world aaaaaaaaaaaa")
|
||||
print(len(r))
|
||||
13
test/test6.py
Normal file
13
test/test6.py
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
import tiktoken
|
||||
|
||||
def count_tokens(text: str) -> int:
|
||||
"""计算给定文本在指定模型下的 token 数量"""
|
||||
encoding = tiktoken.get_encoding("o200k_base")
|
||||
tokens = encoding.encode(text)
|
||||
return len(tokens)
|
||||
|
||||
# 示例使用
|
||||
text = "你好，世界！Hello, world!"
|
||||
token_count = count_tokens(text)
|
||||
print(f"Token 数量: {token_count}")
|
||||
print(len(text) / 4)
|
||||
Loading…
Add table
Reference in a new issue