mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-08-28 05:25:04 +00:00
refactor(reme_ai): formatting and minor code improvements
- Removed unnecessary comments - Improved code formatting across multiple files - Updated import statements and removed unused imports - Refactored some function definitions for better readability
This commit is contained in:
jinli.yl
parent
72db149964
commit
0c2fb9a522
40 changed files with 854 additions and 516 deletions
|
|
@ -1,26 +0,0 @@
|
|||
[
|
||||
{
|
||||
"workspace_id": "test_workspace4",
|
||||
"memory_id": "922a556d988f47bcadd8228d9f2235dc",
|
||||
"memory_type": "task",
|
||||
"when_to_use": "When conducting a company analysis requiring multi-dimensional insights (e.g., financials, market position, strategic shifts)",
|
||||
"content": "Split the analysis into 3-4 distinct subtopics (e.g., financial performance, market challenges, strategic initiatives) and perform targeted web searches for each. Use language matching the user's query (Chinese in this case) to ensure relevant results. Avoid broad single searches that may miss critical angles.",
|
||||
"score": 0.0,
|
||||
"time_created": "2025-08-29 01:44:37",
|
||||
"time_modified": "2025-08-29 01:44:37",
|
||||
"author": "qwen3-30b-a3b-thinking-2507",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"workspace_id": "test_workspace4",
|
||||
"memory_id": "5cac948c7e8141dfb7723ab482d3aa8c",
|
||||
"memory_type": "task",
|
||||
"when_to_use": "When verifying if retrieved information is sufficient to answer the user's query",
|
||||
"content": "Always cross-check that the retrieved data covers all core aspects of the query before terminating. In this case, the financial metrics, market challenges, and strategic pivots were all covered in the search results, making further searches unnecessary. Never assume coverage—explicitly validate completeness.",
|
||||
"score": 0.0,
|
||||
"time_created": "2025-08-29 01:44:37",
|
||||
"time_modified": "2025-08-29 01:44:37",
|
||||
"author": "qwen3-30b-a3b-thinking-2507",
|
||||
"metadata": {}
|
||||
}
|
||||
]
|
||||
File diff suppressed because one or more lines are too long
123
cookbook/simple_demo/personal_memory.jsonl
Normal file
123
cookbook/simple_demo/personal_memory.jsonl
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
{
|
||||
"answer": "",
|
||||
"messages": [],
|
||||
"success": true,
|
||||
"metadata": {
|
||||
"memory_list": [
|
||||
{
|
||||
"workspace_id": "personal_memory_demo",
|
||||
"memory_id": "42f9d6e7e8c34310b284d36cfb73a68d",
|
||||
"memory_type": "personal",
|
||||
"when_to_use": "John Smith, 28, San Francisco, tech company",
|
||||
"content": "user's name is John Smith, he is 28 years old, and works at a tech company in San Francisco",
|
||||
"score": 0.0,
|
||||
"time_created": "2025-08-31 15:45:33",
|
||||
"time_modified": "2025-08-31 15:45:33",
|
||||
"author": "qwen3-30b-a3b-instruct-2507",
|
||||
"metadata": {
|
||||
"keywords": "John Smith, 28, San Francisco, tech company",
|
||||
"source_message": "My name is John Smith, I'm 28 years old, and I work at a tech company in San Francisco",
|
||||
"observation_type": "personal_info"
|
||||
},
|
||||
"target": "user",
|
||||
"reflection_subject": ""
|
||||
},
|
||||
{
|
||||
"workspace_id": "personal_memory_demo",
|
||||
"memory_id": "9be49b1479e540c7abf0257595e49972",
|
||||
"memory_type": "personal",
|
||||
"when_to_use": "software engineer, backend, Python, Go",
|
||||
"content": "user is a software engineer specializing in backend development using Python and Go",
|
||||
"score": 0.0,
|
||||
"time_created": "2025-08-31 15:45:33",
|
||||
"time_modified": "2025-08-31 15:45:33",
|
||||
"author": "qwen3-30b-a3b-instruct-2507",
|
||||
"metadata": {
|
||||
"keywords": "software engineer, backend, Python, Go",
|
||||
"source_message": "I'm a software engineer, mainly doing backend development using Python and Go",
|
||||
"observation_type": "personal_info"
|
||||
},
|
||||
"target": "user",
|
||||
"reflection_subject": ""
|
||||
},
|
||||
{
|
||||
"workspace_id": "personal_memory_demo",
|
||||
"memory_id": "5287138b20ff432a920cd07d0e034579",
|
||||
"memory_type": "personal",
|
||||
"when_to_use": "basketball, sci-fi movies, Dune Part 2",
|
||||
"content": "user enjoys playing basketball and watching sci-fi movies, recently watched Dune Part 2",
|
||||
"score": 0.0,
|
||||
"time_created": "2025-08-31 15:45:33",
|
||||
"time_modified": "2025-08-31 15:45:33",
|
||||
"author": "qwen3-30b-a3b-instruct-2507",
|
||||
"metadata": {
|
||||
"keywords": "basketball, sci-fi movies, Dune Part 2",
|
||||
"source_message": "I enjoy playing basketball and watching sci-fi movies. I recently watched Dune Part 2",
|
||||
"observation_type": "personal_info"
|
||||
},
|
||||
"target": "user",
|
||||
"reflection_subject": ""
|
||||
},
|
||||
{
|
||||
"workspace_id": "personal_memory_demo",
|
||||
"memory_id": "c8f329903b82404c9d36e8a61bb30d67",
|
||||
"memory_type": "personal",
|
||||
"when_to_use": "cat, Shadow, pet",
|
||||
"content": "user has a 3-year-old cat named Shadow",
|
||||
"score": 0.0,
|
||||
"time_created": "2025-08-31 15:45:33",
|
||||
"time_modified": "2025-08-31 15:45:33",
|
||||
"author": "qwen3-30b-a3b-instruct-2507",
|
||||
"metadata": {
|
||||
"keywords": "cat, Shadow, pet",
|
||||
"source_message": "I have a cat named Shadow who is 3 years old",
|
||||
"observation_type": "personal_info"
|
||||
},
|
||||
"target": "user",
|
||||
"reflection_subject": ""
|
||||
},
|
||||
{
|
||||
"workspace_id": "personal_memory_demo",
|
||||
"memory_id": "7166549431bf4f019acb8331377f7def",
|
||||
"memory_type": "personal",
|
||||
"when_to_use": "Japanese cuisine, sushi, ramen",
|
||||
"content": "user is interested in Japanese cuisine, especially sushi and ramen",
|
||||
"score": 0.0,
|
||||
"time_created": "2025-08-31 15:45:33",
|
||||
"time_modified": "2025-08-31 15:45:33",
|
||||
"author": "qwen3-30b-a3b-instruct-2507",
|
||||
"metadata": {
|
||||
"keywords": "Japanese cuisine, sushi, ramen",
|
||||
"source_message": "I'm really interested in Japanese cuisine, especially sushi and ramen",
|
||||
"observation_type": "personal_info"
|
||||
},
|
||||
"target": "user",
|
||||
"reflection_subject": ""
|
||||
},
|
||||
{
|
||||
"workspace_id": "personal_memory_demo",
|
||||
"memory_id": "4ab4fe610ce845b6b174d03449df2881",
|
||||
"memory_type": "personal",
|
||||
"when_to_use": "Japan, Tokyo, Kyoto, trip",
|
||||
"content": "user is planning a trip to Japan in September 2025, mainly to Tokyo and Kyoto",
|
||||
"score": 0.0,
|
||||
"time_created": "2025-08-31 15:45:32",
|
||||
"time_modified": "2025-08-31 15:45:32",
|
||||
"author": "qwen3-30b-a3b-instruct-2507",
|
||||
"metadata": {
|
||||
"keywords": "Japan, Tokyo, Kyoto, trip",
|
||||
"time_info": "September 2025",
|
||||
"source_message": "I'm planning a trip to Japan next month, mainly to Tokyo and Kyoto",
|
||||
"observation_type": "personal_info_with_time"
|
||||
},
|
||||
"target": "user",
|
||||
"reflection_subject": ""
|
||||
}
|
||||
],
|
||||
"deleted_memory_ids": [],
|
||||
"update_result": {
|
||||
"deleted_count": 0,
|
||||
"inserted_count": 6
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
import json
|
||||
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
base_url = "http://0.0.0.0:8002/"
|
||||
workspace_id = "test_workspace4"
|
||||
|
||||
|
||||
def run_agent(query: str, dump_messages: bool = False):
|
||||
response = requests.post(url=base_url + "react", json={"query": query})
|
||||
if response.status_code != 200:
|
||||
print(response.text)
|
||||
return []
|
||||
|
||||
response = response.json()
|
||||
|
||||
answer = response["answer"]
|
||||
print(answer)
|
||||
|
||||
messages = response["messages"]
|
||||
if dump_messages:
|
||||
with open("messages.jsonl", "w") as f:
|
||||
f.write(json.dumps(messages, indent=2, ensure_ascii=False))
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
def run_summary(messages: list, enable_dump_memory: bool = True):
|
||||
response = requests.post(url=base_url + "summary_task_memory_simple", json={
|
||||
"workspace_id": workspace_id,
|
||||
"trajectories": [
|
||||
{"messages": messages, "score": 1.0}
|
||||
]
|
||||
})
|
||||
|
||||
if response.status_code != 200:
|
||||
print(response.text)
|
||||
return
|
||||
|
||||
response = response.json()
|
||||
memory_list = response["metadata"]["memory_list"]
|
||||
if enable_dump_memory:
|
||||
with open("memory.jsonl", "w") as f:
|
||||
f.write(json.dumps(memory_list, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
def run_retrieve(query: str):
|
||||
response = requests.post(url=base_url + "retrieve_task_memory_simple", json={
|
||||
"workspace_id": workspace_id,
|
||||
"query": query,
|
||||
})
|
||||
|
||||
if response.status_code != 200:
|
||||
print(response.text)
|
||||
return ""
|
||||
|
||||
response = response.json()
|
||||
answer: str = response["answer"]
|
||||
print(f"answer={answer}")
|
||||
return answer
|
||||
|
||||
|
||||
def run_agent_with_memory(query_first: str, query_second: str, enable_dump_memory: bool = True):
|
||||
messages = run_agent(query=query_second)
|
||||
run_summary(messages, enable_dump_memory)
|
||||
retrieved_memory = run_retrieve(query_first)
|
||||
messages = run_agent(query=f"{retrieved_memory}\n\nUser Question:\n{query_first}")
|
||||
return messages
|
||||
|
||||
|
||||
def dump_memory():
|
||||
response = requests.post(url=base_url + "vector_store", json={
|
||||
"workspace_id": workspace_id,
|
||||
"action": "dump",
|
||||
"path": "./",
|
||||
})
|
||||
|
||||
if response.status_code != 200:
|
||||
print(response.text)
|
||||
return
|
||||
|
||||
print(response.json())
|
||||
|
||||
|
||||
def load_memory():
|
||||
response = requests.post(url=base_url + "vector_store", json={
|
||||
"workspace_id": "test_workspace2",
|
||||
"action": "load",
|
||||
"path": "./",
|
||||
})
|
||||
|
||||
if response.status_code != 200:
|
||||
print(response.text)
|
||||
return
|
||||
|
||||
print(response.json())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
query1 = "Analyze Xiaomi Corporation"
|
||||
query2 = "Analyze the company Tesla."
|
||||
|
||||
run_agent(query=query1, dump_messages=True)
|
||||
run_agent_with_memory(query_first=query1, query_second=query2)
|
||||
dump_memory()
|
||||
load_memory()
|
||||
78
cookbook/simple_demo/task_memory.jsonl
Normal file
78
cookbook/simple_demo/task_memory.jsonl
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
[
|
||||
{
|
||||
"workspace_id": "test_workspace",
|
||||
"memory_id": "4ff6a7a783a84940a046e557de7542f0",
|
||||
"memory_type": "task",
|
||||
"when_to_use": "When analyzing a complex organization with multiple dimensions (history, products, finances, leadership), especially when the query is open-ended or broad like 'analyze [company]'",
|
||||
"content": "The agent successfully decomposed the task into four targeted web searches: company overview/history, products/services, financial performance, and leadership impact. This multi-angle approach ensured comprehensive coverage of key domains. Each search was designed to extract distinct, high-value information layers, enabling synthesis into a structured, well-rounded analysis. The use of varied queries prevented redundancy and maximized information diversity.",
|
||||
"score": 0.92,
|
||||
"time_created": "2025-08-31 15:42:44",
|
||||
"time_modified": "2025-08-31 15:42:44",
|
||||
"author": "qwen3-30b-a3b-instruct-2507",
|
||||
"metadata": {
|
||||
"when_to_use": "When analyzing a complex organization with multiple dimensions (history, products, finances, leadership), especially when the query is open-ended or broad like 'analyze [company]'",
|
||||
"experience": "The agent successfully decomposed the task into four targeted web searches: company overview/history, products/services, financial performance, and leadership impact. This multi-angle approach ensured comprehensive coverage of key domains. Each search was designed to extract distinct, high-value information layers, enabling synthesis into a structured, well-rounded analysis. The use of varied queries prevented redundancy and maximized information diversity.",
|
||||
"tags": [
|
||||
"multi-faceted analysis",
|
||||
"information decomposition",
|
||||
"web_search strategy",
|
||||
"comprehensive coverage"
|
||||
],
|
||||
"confidence": 0.9,
|
||||
"step_type": "reasoning",
|
||||
"tools_used": [
|
||||
"web_search"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"workspace_id": "test_workspace",
|
||||
"memory_id": "2ad5b6aae8724e87944e69ea4476a97c",
|
||||
"memory_type": "task",
|
||||
"when_to_use": "When initial results from a general search are insufficient or too broad, and deeper insights are needed on specific subtopics such as financial trends, product portfolios, or leadership dynamics",
|
||||
"content": "The agent proactively called the same tool (web_search) multiple times with different, focused queries to gather information from various perspectives. This iterative technique allowed for layered understanding—e.g., separating historical milestones from current financial challenges and leadership influence. It demonstrated strategic use of repetition not as redundancy but as a method to triangulate data across domains, significantly improving depth and accuracy.",
|
||||
"score": 0.85,
|
||||
"time_created": "2025-08-31 15:42:44",
|
||||
"time_modified": "2025-08-31 15:42:44",
|
||||
"author": "qwen3-30b-a3b-instruct-2507",
|
||||
"metadata": {
|
||||
"when_to_use": "When initial results from a general search are insufficient or too broad, and deeper insights are needed on specific subtopics such as financial trends, product portfolios, or leadership dynamics",
|
||||
"experience": "The agent proactively called the same tool (web_search) multiple times with different, focused queries to gather information from various perspectives. This iterative technique allowed for layered understanding—e.g., separating historical milestones from current financial challenges and leadership influence. It demonstrated strategic use of repetition not as redundancy but as a method to triangulate data across domains, significantly improving depth and accuracy.",
|
||||
"tags": [
|
||||
"iterative querying",
|
||||
"perspective diversification",
|
||||
"information triangulation",
|
||||
"search optimization"
|
||||
],
|
||||
"confidence": 0.85,
|
||||
"step_type": "action",
|
||||
"tools_used": [
|
||||
"web_search"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"workspace_id": "test_workspace",
|
||||
"memory_id": "87afc1367c6c4b739bbe191e1f47c875",
|
||||
"memory_type": "task",
|
||||
"when_to_use": "After gathering raw data from external sources, when synthesizing findings into a coherent, structured response that balances factual reporting with critical insight",
|
||||
"content": "The agent effectively integrated results from multiple searches into a unified summary with clear sectioning: history, products, finance, and leadership. It highlighted both strengths (innovation, mission-driven vision) and risks (financial decline, leadership dependency). This synthesis transformed fragmented data into actionable intelligence, demonstrating how structured summarization enhances usability and decision-making value in agent outputs.",
|
||||
"score": 0.85,
|
||||
"time_created": "2025-08-31 15:42:44",
|
||||
"time_modified": "2025-08-31 15:42:44",
|
||||
"author": "qwen3-30b-a3b-instruct-2507",
|
||||
"metadata": {
|
||||
"when_to_use": "After gathering raw data from external sources, when synthesizing findings into a coherent, structured response that balances factual reporting with critical insight",
|
||||
"experience": "The agent effectively integrated results from multiple searches into a unified summary with clear sectioning: history, products, finance, and leadership. It highlighted both strengths (innovation, mission-driven vision) and risks (financial decline, leadership dependency). This synthesis transformed fragmented data into actionable intelligence, demonstrating how structured summarization enhances usability and decision-making value in agent outputs.",
|
||||
"tags": [
|
||||
"synthesis",
|
||||
"structured summarization",
|
||||
"critical insight",
|
||||
"data integration"
|
||||
],
|
||||
"confidence": 0.9,
|
||||
"step_type": "observation",
|
||||
"tools_used": []
|
||||
}
|
||||
}
|
||||
]
|
||||
85
cookbook/simple_demo/use_personal_memory_demo.py
Normal file
85
cookbook/simple_demo/use_personal_memory_demo.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
import asyncio
|
||||
import json
|
||||
import aiohttp
|
||||
|
||||
# API base URL
|
||||
base_url = "http://0.0.0.0:8002"
|
||||
|
||||
async def main():
|
||||
# Create a unique workspace ID
|
||||
workspace_id = "personal_memory_demo"
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# Step 1: Clear existing memories in the workspace
|
||||
print("Clearing existing memories...")
|
||||
async with session.post(
|
||||
f"{base_url}/vector_store",
|
||||
json={
|
||||
"action": "delete",
|
||||
"workspace_id": workspace_id,
|
||||
},
|
||||
headers={"Content-Type": "application/json"}
|
||||
) as response:
|
||||
result = await response.json()
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
|
||||
# Step 2: Create a conversation with rich personal information
|
||||
print("\nCreating conversation with personal information...")
|
||||
messages = [
|
||||
{"role": "user", "content": "My name is John Smith, I'm 28 years old, and I work at a tech company in San Francisco"},
|
||||
{"role": "assistant", "content": "Nice to meet you, John!"},
|
||||
{"role": "user", "content": "I'm a software engineer, mainly doing backend development using Python and Go"},
|
||||
{"role": "assistant", "content": "I see, you're a backend engineer working with Python and Go."},
|
||||
{"role": "user", "content": "I enjoy playing basketball and watching sci-fi movies. I recently watched Dune Part 2"},
|
||||
{"role": "assistant", "content": "Basketball and sci-fi movies are great hobbies! Dune Part 2 was indeed amazing."},
|
||||
{"role": "user", "content": "I have a cat named Shadow who is 3 years old"},
|
||||
{"role": "assistant", "content": "Shadow sounds adorable! 3-year-old cats are quite playful."},
|
||||
{"role": "user", "content": "I'm planning a trip to Japan next month, mainly to Tokyo and Kyoto"},
|
||||
{"role": "assistant", "content": "Your Japan trip sounds exciting! Tokyo and Kyoto are both wonderful destinations with their own unique charm."},
|
||||
{"role": "user", "content": "I'm really interested in Japanese cuisine, especially sushi and ramen"},
|
||||
{"role": "assistant", "content": "Japanese cuisine is delicious! Sushi and ramen are very popular choices."},
|
||||
]
|
||||
|
||||
# Step 3: Summarize personal memories from the conversation
|
||||
print("\nSummarizing personal memories...")
|
||||
async with session.post(
|
||||
f"{base_url}/summary_personal_memory",
|
||||
json={
|
||||
"messages": messages,
|
||||
"workspace_id": workspace_id,
|
||||
},
|
||||
headers={"Content-Type": "application/json"}
|
||||
) as response:
|
||||
result = await response.json()
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
# Wait for the memories to be processed and stored
|
||||
print("\nWaiting for memories to be processed...")
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# Step 4: Retrieve personal memories with different queries
|
||||
queries = [
|
||||
"What's my name and age?",
|
||||
"What do I do for work?",
|
||||
"What are my hobbies?",
|
||||
"Do I have any pets?",
|
||||
"What are my travel plans?",
|
||||
"What foods do I like?"
|
||||
]
|
||||
|
||||
print("\nRetrieving personal memories...")
|
||||
for query in queries:
|
||||
print(f"\nQuery: {query}")
|
||||
async with session.post(
|
||||
f"{base_url}/retrieve_personal_memory",
|
||||
json={
|
||||
"query": query,
|
||||
"workspace_id": workspace_id,
|
||||
},
|
||||
headers={"Content-Type": "application/json"}
|
||||
) as response:
|
||||
result = await response.json()
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
291
cookbook/simple_demo/use_task_memory_demo.py
Normal file
291
cookbook/simple_demo/use_task_memory_demo.py
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Task Memory Demo for MemoryScope
|
||||
|
||||
This script demonstrates how to use the task memory capabilities of MemoryScope.
|
||||
It shows how to run an agent, summarize conversations, retrieve memories, and
|
||||
manage the memory workspace.
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
# API configuration
|
||||
BASE_URL = "http://0.0.0.0:8002/"
|
||||
WORKSPACE_ID = "test_workspace"
|
||||
|
||||
|
||||
def handle_api_response(response: requests.Response) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Handle API response with proper error checking
|
||||
|
||||
Args:
|
||||
response: Response object from requests
|
||||
|
||||
Returns:
|
||||
Response JSON if successful, None otherwise
|
||||
"""
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.text)
|
||||
return None
|
||||
|
||||
return response.json()
|
||||
|
||||
|
||||
def delete_workspace() -> None:
|
||||
"""
|
||||
Delete the current workspace from the vector store
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
response = requests.post(
|
||||
url=f"{BASE_URL}vector_store",
|
||||
json={
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"action": "delete",
|
||||
}
|
||||
)
|
||||
|
||||
result = handle_api_response(response)
|
||||
if result:
|
||||
print(f"Workspace '{WORKSPACE_ID}' deleted successfully")
|
||||
|
||||
|
||||
def run_agent(query: str, dump_messages: bool = False) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Run the agent with a specific query
|
||||
|
||||
Args:
|
||||
query: The query to send to the agent
|
||||
dump_messages: Whether to save messages to a file
|
||||
|
||||
Returns:
|
||||
List of message objects from the conversation
|
||||
"""
|
||||
response = requests.post(
|
||||
url=f"{BASE_URL}react",
|
||||
json={"query": query}
|
||||
)
|
||||
|
||||
result = handle_api_response(response)
|
||||
if not result:
|
||||
return []
|
||||
|
||||
# Extract and display the answer
|
||||
answer = result.get("answer", "")
|
||||
print(f"Agent response: {answer}")
|
||||
|
||||
# Get the conversation messages
|
||||
messages = result.get("messages", [])
|
||||
|
||||
# Optionally save messages to file
|
||||
if dump_messages and messages:
|
||||
with open("messages.jsonl", "w") as f:
|
||||
f.write(json.dumps(messages, indent=2, ensure_ascii=False))
|
||||
print(f"Messages saved to messages.jsonl")
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
def run_summary(messages: List[Dict[str, Any]], enable_dump_memory: bool = True) -> None:
|
||||
"""
|
||||
Generate a summary of conversation messages and create task memories
|
||||
|
||||
Args:
|
||||
messages: List of message objects from a conversation
|
||||
enable_dump_memory: Whether to save memory list to a file
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
if not messages:
|
||||
print("No messages to summarize")
|
||||
return
|
||||
|
||||
response = requests.post(
|
||||
# url=f"{BASE_URL}summary_task_memory_simple",
|
||||
url=f"{BASE_URL}summary_task_memory",
|
||||
json={
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"trajectories": [
|
||||
{"messages": messages, "score": 1.0}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
result = handle_api_response(response)
|
||||
if not result:
|
||||
return
|
||||
|
||||
# Extract memory list from response
|
||||
memory_list = result.get("metadata", {}).get("memory_list", [])
|
||||
print(f"Memory list: {memory_list}")
|
||||
|
||||
# Optionally save memory list to file
|
||||
if enable_dump_memory and memory_list:
|
||||
with open("memory.jsonl", "w") as f:
|
||||
f.write(json.dumps(memory_list, indent=2, ensure_ascii=False))
|
||||
print(f"Memory saved to memory.jsonl")
|
||||
|
||||
|
||||
def run_retrieve(query: str) -> str:
|
||||
"""
|
||||
Retrieve relevant task memories based on a query
|
||||
|
||||
Args:
|
||||
query: The query to retrieve relevant memories
|
||||
|
||||
Returns:
|
||||
String containing the retrieved memory answer
|
||||
"""
|
||||
response = requests.post(
|
||||
# url=f"{BASE_URL}retrieve_task_memory_simple",
|
||||
url=f"{BASE_URL}retrieve_task_memory",
|
||||
json={
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"query": query,
|
||||
}
|
||||
)
|
||||
|
||||
result = handle_api_response(response)
|
||||
if not result:
|
||||
return ""
|
||||
|
||||
# Extract and return the answer
|
||||
answer = result.get("answer", "")
|
||||
print(f"Retrieved memory: {answer}")
|
||||
return answer
|
||||
|
||||
|
||||
def run_agent_with_memory(query_first: str, query_second: str, enable_dump_memory: bool = True) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Run the agent with memory augmentation
|
||||
|
||||
This function demonstrates how to use task memory to enhance agent responses:
|
||||
1. First run the agent with the second query to build memory
|
||||
2. Then summarize the conversation to create memories
|
||||
3. Retrieve relevant memories for the first query
|
||||
4. Run the agent with the first query augmented with retrieved memories
|
||||
|
||||
Args:
|
||||
query_first: The query to run with memory augmentation
|
||||
query_second: The query to build initial memories
|
||||
enable_dump_memory: Whether to save memory list to a file
|
||||
|
||||
Returns:
|
||||
List of message objects from the final conversation
|
||||
"""
|
||||
# Run agent with second query to build initial memories
|
||||
print(f"\n--- Building memories with query: '{query_second}' ---")
|
||||
messages = run_agent(query=query_second)
|
||||
|
||||
# Summarize conversation to create memories
|
||||
print("\n--- Summarizing conversation to create memories ---")
|
||||
run_summary(messages, enable_dump_memory)
|
||||
time.sleep(1)
|
||||
|
||||
# Retrieve relevant memories for the first query
|
||||
print(f"\n--- Retrieving memories for query: '{query_first}' ---")
|
||||
retrieved_memory = run_retrieve(query_first)
|
||||
|
||||
# Run agent with first query augmented with retrieved memories
|
||||
print(f"\n--- Running agent with memory-augmented query ---")
|
||||
augmented_query = f"{retrieved_memory}\n\nUser Question:\n{query_first}"
|
||||
print(f"Augmented query: {augmented_query}")
|
||||
messages = run_agent(query=augmented_query)
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
def dump_memory(path: str = "./") -> None:
|
||||
"""
|
||||
Dump the vector store memories to disk
|
||||
|
||||
Args:
|
||||
path: Directory path to save the memories
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
response = requests.post(
|
||||
url=f"{BASE_URL}vector_store",
|
||||
json={
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"action": "dump",
|
||||
"path": path,
|
||||
}
|
||||
)
|
||||
|
||||
result = handle_api_response(response)
|
||||
if result:
|
||||
print(f"Memory dumped to {path}")
|
||||
|
||||
|
||||
def load_memory(path: str = "./") -> None:
|
||||
"""
|
||||
Load memories from disk into the vector store
|
||||
|
||||
Args:
|
||||
path: Directory path to load the memories from
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
response = requests.post(
|
||||
url=f"{BASE_URL}vector_store",
|
||||
json={
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"action": "load",
|
||||
"path": path,
|
||||
}
|
||||
)
|
||||
|
||||
result = handle_api_response(response)
|
||||
if result:
|
||||
print(f"Memory loaded from {path}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""
|
||||
Main function to demonstrate task memory workflow
|
||||
"""
|
||||
# Define example queries
|
||||
query1 = "Analyze Xiaomi Corporation"
|
||||
query2 = "Analyze the company Tesla."
|
||||
|
||||
print("=== Task Memory Demo ===")
|
||||
|
||||
# Step 1: Clean up workspace
|
||||
print("\n1. Deleting workspace...")
|
||||
delete_workspace()
|
||||
|
||||
# Step 2: Run agent with first query and save messages
|
||||
print("\n2. Running agent with first query...")
|
||||
run_agent(query=query1, dump_messages=True)
|
||||
|
||||
# Step 3: Demonstrate memory-augmented agent
|
||||
print("\n3. Running memory-augmented agent workflow...")
|
||||
run_agent_with_memory(query_first=query1, query_second=query2)
|
||||
|
||||
# Step 4: Demonstrate memory persistence
|
||||
print("\n4. Dumping memory to disk...")
|
||||
dump_memory()
|
||||
|
||||
print("\n5. Loading memory from disk...")
|
||||
load_memory()
|
||||
|
||||
print("\n=== Demo Complete ===")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
14
example.env
14
example.env
|
|
@ -1,11 +1,9 @@
|
|||
OPENAI_API_KEY=sk-xxxx
|
||||
OPENAI_BASE_URL=https://xxxx/v1
|
||||
FLOW_EMBEDDING_API_KEY=sk-xxxx
|
||||
FLOW_EMBEDDING_BASE_URL=https://xxxx/v1
|
||||
|
||||
EMBEDDING_API_KEY=sk-xxxx
|
||||
EMBEDDING_BASE_URL=https://xxxx/v1
|
||||
FLOW_LLM_API_KEY=sk-xxxx
|
||||
FLOW_LLM_BASE_URL=https://xxxx/v1
|
||||
|
||||
LLM_API_KEY=sk-xxxx
|
||||
LLM_BASE_URL=https://xxxx/v1
|
||||
FLOW_ES_HOSTS=http://0.0.0.0:9200
|
||||
|
||||
ES_HOSTS=http://0.0.0.0:9200
|
||||
DASHSCOPE_API_KEY=sk-xxxx
|
||||
FLOW_USE_FRAMEWORK=true
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|||
name = "reme_ai"
|
||||
version = "0.1.0"
|
||||
description = "Remember me of memory and experience"
|
||||
authors = [{ name = "reme_ai team", email = "reme_ai@alibaba-inc.com" }]
|
||||
authors = [{ name = "reme_ai_team", email = "reme_ai_team@alibaba-inc.com" }]
|
||||
license = { file = "LICENSE" }
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
|
|
|
|||
|
|
@ -97,32 +97,36 @@ flow:
|
|||
description: "Whether to update the utility attribute of the retrieved task memory."
|
||||
required: true
|
||||
|
||||
delete_task_memory:
|
||||
flow_content: delete_memory_op >> update_vector_store_op
|
||||
description: "Delete task memories when utility/freq < utility_threshold and freq >= freq_threshold"
|
||||
input_schema:
|
||||
workspace_id:
|
||||
type: "str"
|
||||
description: "workspace id"
|
||||
required: true
|
||||
freq_threshold:
|
||||
type: "int"
|
||||
description: "The retrieved frequency threshold for deleting task memory."
|
||||
required: true
|
||||
utility_threshold:
|
||||
type: "float"
|
||||
description: "The utility/freq threshold for deleting task memory."
|
||||
required: true
|
||||
|
||||
# reconsolidate_personal_memory:
|
||||
# flow_content: load_not_reflected_memory_op >> get_reflection_subject_op >> update_insight_op >> long_contra_repeat_op >> update_vector_store_op
|
||||
# description: "Consolidate personal memories by generating topic insights, updating values, resolving conflicts, and updating vector store"
|
||||
delete_task_memory:
|
||||
flow_content: delete_memory_op >> update_vector_store_op
|
||||
description: "Delete task memories when utility/freq < utility_threshold and freq >= freq_threshold"
|
||||
input_schema:
|
||||
workspace_id:
|
||||
type: "str"
|
||||
description: "workspace id"
|
||||
required: true
|
||||
freq_threshold:
|
||||
type: "int"
|
||||
description: "The retrieved frequency threshold for deleting task memory."
|
||||
required: true
|
||||
utility_threshold:
|
||||
type: "float"
|
||||
description: "The utility/freq threshold for deleting task memory."
|
||||
required: true
|
||||
|
||||
react:
|
||||
flow_content: simple_react_op
|
||||
description: "React to the current task with an agent"
|
||||
input_schema:
|
||||
query:
|
||||
type: "str"
|
||||
description: "user query"
|
||||
required: true
|
||||
|
||||
llm:
|
||||
default:
|
||||
backend: openai_compatible
|
||||
# model_name: qwen3-30b-a3b-thinking-2507
|
||||
# model_name: qwen3-30b-a3b-thinking-2507
|
||||
model_name: qwen3-30b-a3b-instruct-2507
|
||||
params:
|
||||
temperature: 0.6
|
||||
|
|
@ -138,6 +142,6 @@ vector_store:
|
|||
default:
|
||||
backend: elasticsearch
|
||||
embedding_model: default
|
||||
params:
|
||||
# hosts: "http://localhost:9200"
|
||||
hosts: "http://11.160.132.46:8200"
|
||||
# params:
|
||||
# hosts: "http://localhost:9200"
|
||||
# hosts: "http://11.160.132.46:8200"
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
from .simple_react_op import SimpleReactOp
|
||||
from .simple_react_op import SimpleReactOp
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from flowllm import C
|
||||
from flowllm.context.flow_context import FlowContext
|
||||
from flowllm.op.agent.react_v1_op import ReactV1Op
|
||||
from flowllm.op.agent.react_v2_op import ReactV2Op
|
||||
|
||||
|
||||
@C.register_op()
|
||||
class SimpleReactOp(ReactV1Op):
|
||||
class SimpleReactOp(ReactV2Op):
|
||||
...
|
||||
|
||||
|
||||
|
|
@ -18,4 +18,4 @@ if __name__ == "__main__":
|
|||
op(context=context)
|
||||
# from reme_ai.schema import Message
|
||||
# result = op.llm.chat(messages=[Message(**{"role": "user", "content": "你叫什么名字？"})])
|
||||
# print("!!!", result)
|
||||
# print("!!!", result)
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
from . import task
|
||||
from . import personal
|
||||
from . import task
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
query_build: |
|
||||
# Execution Process
|
||||
{execution_process}
|
||||
|
||||
Read through the entire execution process to understand which part is currently being executed.
|
||||
Generate a `query` that reflects the current state, which will later be used to search for similar problems in the database and help resolve the issue at hand.
|
||||
# Execution Process
|
||||
{execution_process}
|
||||
|
||||
Read through the entire execution process to understand which part is currently being executed.
|
||||
Generate a `query` that reflects the current state, which will later be used to search for similar problems in the database and help resolve the issue at hand.
|
||||
|
|
|
|||
|
|
@ -20,8 +20,7 @@ class MergeMemoryOp(BaseOp):
|
|||
if not memory.content:
|
||||
continue
|
||||
|
||||
content_collector.append(f"- when_to_use: {memory.when_to_use}\n"
|
||||
f"content: {memory.content}\n")
|
||||
content_collector.append(f"- {memory.when_to_use} {memory.content}\n")
|
||||
content_collector.append("Please consider the helpful parts from these in answering the question, "
|
||||
"to make the response more comprehensive and substantial.")
|
||||
self.context.response.answer = "\n".join(content_collector)
|
||||
|
|
|
|||
|
|
@ -146,4 +146,4 @@ class RerankMemoryOp(BaseLLMOp):
|
|||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing rerank response: {e}")
|
||||
return []
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ class RewriteMemoryOp(BaseLLMOp):
|
|||
|
||||
# Generate initial context message
|
||||
rewritten_memory = self._generate_context_message(query, messages, memory_list)
|
||||
|
||||
|
||||
# Store results in context
|
||||
self.context.response.answer = rewritten_memory
|
||||
self.context.response.metadata["memory_list"] = [memory.model_dump() for memory in memory_list]
|
||||
|
|
@ -41,7 +41,6 @@ class RewriteMemoryOp(BaseLLMOp):
|
|||
def _generate_context_message(self, query: str, messages: List[Message], memories: List[BaseMemory]) -> str:
|
||||
"""Generate context message from retrieved memories"""
|
||||
if not memories:
|
||||
|
||||
return ""
|
||||
|
||||
try:
|
||||
|
|
@ -147,4 +146,4 @@ class RewriteMemoryOp(BaseLLMOp):
|
|||
# If JSON parsing fails, return the response as-is for fallback
|
||||
return response.strip()
|
||||
|
||||
return ""
|
||||
return ""
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
from flowllm.schema.message import Message, Trajectory, ToolCall, Role
|
||||
from flowllm.schema.message import Message, Role, Trajectory # noqa
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ class BaseMemory(BaseModel, ABC):
|
|||
metadata: dict = Field(default_factory=dict)
|
||||
|
||||
def update_modified_time(self):
|
||||
self.modified_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
self.time_modified = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
def update_metadata(self, new_metadata):
|
||||
self.metadata = new_metadata
|
||||
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
from . import personal
|
||||
from . import task
|
||||
from . import personal
|
||||
|
|
@ -5,4 +5,4 @@ from .get_reflection_subject_op import GetReflectionSubjectOp
|
|||
from .info_filter_op import InfoFilterOp
|
||||
from .load_today_memory_op import LoadTodayMemoryOp
|
||||
from .long_contra_repeat_op import LongContraRepeatOp
|
||||
from .update_insight_op import UpdateInsightOp
|
||||
from .update_insight_op import UpdateInsightOp
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ class GetReflectionSubjectOp(BaseLLMOp):
|
|||
Args:
|
||||
memory_contents: List of memory content strings
|
||||
existing_subjects: List of already existing subject strings
|
||||
user_name: Target user name
|
||||
user_name: Target username
|
||||
num_questions: Maximum number of new subjects to generate
|
||||
|
||||
Returns:
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
info_filter_system_zh: |
|
||||
任务：对所给{batch_size}个句子中所含有的关于{user_name}的信息打分，分数为0,1,2或3。
|
||||
注意：其中0表示不包含用户信息，1表示句子中只包含用户假设的信息或者用户虚构的内容比如用户创作的小说或剧本，2表示包含用户的一般信息，时效性信息或者需要猜测才能得到的用户信息，3表示明确含有或者可以确定推断出关于用户的重要信息，或者用户要求记录。
|
||||
{user_name}的重要信息可以包含用户基本信息，用户画像信息，用户兴趣偏好信息，用户性格，用户价值观，用户人际关系，用户重大事件转折点等等重要信息。
|
||||
对每个句子都做一次信息打分，一共输出{batch_size}个分数，不需要写最终结果。
|
||||
请一定要按如下格式依次输出，最后的结果一定要加<>:
|
||||
思考：思考的依据和过程，30字以内。
|
||||
结果：<句子序号> <分数:0或1或2或3>
|
||||
任务：对所给{batch_size}个句子中所含有的关于{user_name}的信息打分，分数为0,1,2或3。
|
||||
注意：其中0表示不包含用户信息，1表示句子中只包含用户假设的信息或者用户虚构的内容比如用户创作的小说或剧本，2表示包含用户的一般信息，时效性信息或者需要猜测才能得到的用户信息，3表示明确含有或者可以确定推断出关于用户的重要信息，或者用户要求记录。
|
||||
{user_name}的重要信息可以包含用户基本信息，用户画像信息，用户兴趣偏好信息，用户性格，用户价值观，用户人际关系，用户重大事件转折点等等重要信息。
|
||||
对每个句子都做一次信息打分，一共输出{batch_size}个分数，不需要写最终结果。
|
||||
请一定要按如下格式依次输出，最后的结果一定要加<>:
|
||||
思考：思考的依据和过程，30字以内。
|
||||
结果：<句子序号> <分数:0或1或2或3>
|
||||
|
||||
info_filter_system: |
|
||||
Task: Score the information about {user_name} contained in the given batch of {batch_size} sentences, with scores of 0, 1, 2, or 3.
|
||||
|
|
@ -118,7 +118,7 @@ info_filter_few_shot: |
|
|||
Sentences:
|
||||
1 {user_name}: I've just joined Google.
|
||||
2 {user_name}: There are too many mosquitoes when sleeping outdoors. What should I do?
|
||||
3 {user_name}: Is creativity related to extraversion?
|
||||
3 {user_name}: Is creativity related to extroversion?
|
||||
4 {user_name}: Should the review, modification, and standardization of all personnel files in a district or county be handled by the cadre archive center or by the county human resources and social security bureau?
|
||||
5 {user_name}: If I am planning to have a child with a woman, as a man, how can I protect the woman and the baby and how can I prepare to ensure high sperm quality for the benefit of the offspring?
|
||||
6 {user_name}: If we go out to play together, would you feel happy?
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ class UpdateInsightOp(BaseLLMOp):
|
|||
logger.info(f"Successfully updated {len(updated_insights)} insight memories")
|
||||
|
||||
def _score_insights_by_relevance(self, insight_memories: List[PersonalMemory],
|
||||
observation_memories: List[PersonalMemory],
|
||||
observation_memories: List[PersonalMemory],
|
||||
threshold: float) -> List[tuple]:
|
||||
"""
|
||||
Score insight memories based on relevance to observation memories.
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ class ComparativeExtractionOp(BaseLLMOp):
|
|||
|
||||
# Hard comparison: success vs failure (if similarity search is enabled)
|
||||
if (success_trajectories and failure_trajectories and
|
||||
self.op_params.get("enable_similarity_comparison", False)):
|
||||
self.op_params.get("enable_similarity_comparison", False)):
|
||||
|
||||
similar_pairs = self._find_similar_step_sequences(success_trajectories, failure_trajectories)
|
||||
logger.info(f"Found {len(similar_pairs)} similar pairs for hard comparison")
|
||||
|
|
@ -47,7 +47,8 @@ class ComparativeExtractionOp(BaseLLMOp):
|
|||
self.context.comparative_task_memories = comparative_task_memories
|
||||
|
||||
@staticmethod
|
||||
def _find_highest_lowest_scoring_trajectories(trajectories: List[Trajectory]) -> Tuple[Optional[Trajectory], Optional[Trajectory]]:
|
||||
def _find_highest_lowest_scoring_trajectories(trajectories: List[Trajectory]) -> Tuple[
|
||||
Optional[Trajectory], Optional[Trajectory]]:
|
||||
"""Find the highest and lowest scoring trajectories"""
|
||||
if len(trajectories) < 2:
|
||||
return None, None
|
||||
|
|
@ -107,7 +108,8 @@ class ComparativeExtractionOp(BaseLLMOp):
|
|||
return self.llm.chat(messages=[Message(content=prompt)], callback_fn=parse_task_memories)
|
||||
|
||||
def _extract_hard_comparative_task_memory(self, success_steps: List[Message],
|
||||
failure_steps: List[Message], similarity_score: float) -> List[BaseMemory]:
|
||||
failure_steps: List[Message], similarity_score: float) -> List[
|
||||
BaseMemory]:
|
||||
"""Extract hard comparative task memory (success vs failure)"""
|
||||
prompt = self.prompt_format(
|
||||
prompt_name="hard_comparative_step_task_memory_prompt",
|
||||
|
|
@ -134,7 +136,6 @@ class ComparativeExtractionOp(BaseLLMOp):
|
|||
|
||||
return self.llm.chat(messages=[Message(content=prompt)], callback_fn=parse_task_memories)
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _get_trajectory_steps(trajectory: Trajectory) -> List[Message]:
|
||||
"""Get trajectory steps, prioritizing segmented steps"""
|
||||
|
|
@ -148,7 +149,8 @@ class ComparativeExtractionOp(BaseLLMOp):
|
|||
return trajectory.messages
|
||||
|
||||
def _find_similar_step_sequences(self, success_trajectories: List[Trajectory],
|
||||
failure_trajectories: List[Trajectory]) -> List[Tuple[List[Message], List[Message], float]]:
|
||||
failure_trajectories: List[Trajectory]) -> List[
|
||||
Tuple[List[Message], List[Message], float]]:
|
||||
"""Find similar step sequences for comparison"""
|
||||
if not self.op_params.get("enable_similarity_comparison", False):
|
||||
return []
|
||||
|
|
@ -184,7 +186,8 @@ class ComparativeExtractionOp(BaseLLMOp):
|
|||
failure_texts = [merge_messages_content(seq) for seq in failure_step_sequences]
|
||||
|
||||
# Get embedding vectors
|
||||
if hasattr(self.context, 'vector_store') and self.context.vector_store and hasattr(self.context.vector_store, 'embedding_model'):
|
||||
if hasattr(self.context, 'vector_store') and self.context.vector_store and hasattr(
|
||||
self.context.vector_store, 'embedding_model'):
|
||||
success_embeddings = self.context.vector_store.embedding_model.get_embeddings(success_texts)
|
||||
failure_embeddings = self.context.vector_store.embedding_model.get_embeddings(failure_texts)
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class FailureExtractionOp(BaseLLMOp):
|
|||
def execute(self):
|
||||
"""Extract task memories from failed trajectories"""
|
||||
failure_trajectories: List[Trajectory] = self.context.get("failure_trajectories", [])
|
||||
|
||||
|
||||
if not failure_trajectories:
|
||||
logger.info("No failure trajectories found for extraction")
|
||||
return
|
||||
|
|
@ -23,7 +23,7 @@ class FailureExtractionOp(BaseLLMOp):
|
|||
logger.info(f"Extracting task memories from {len(failure_trajectories)} failed trajectories")
|
||||
|
||||
failure_task_memories = []
|
||||
|
||||
|
||||
# Process trajectories
|
||||
for trajectory in failure_trajectories:
|
||||
if hasattr(trajectory, 'segments') and trajectory.segments:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
from typing import List
|
||||
from loguru import logger
|
||||
|
||||
from flowllm import C, BaseOp
|
||||
from loguru import logger
|
||||
|
||||
from reme_ai.schema.memory import BaseMemory
|
||||
|
||||
|
||||
|
|
@ -23,8 +24,9 @@ class MemoryDeduplicationOp(BaseOp):
|
|||
# Perform deduplication
|
||||
deduplicated_task_memories = self._deduplicate_task_memories(task_memories)
|
||||
|
||||
logger.info(f"Deduplication complete: {len(deduplicated_task_memories)} deduplicated task memories out of {len(task_memories)}")
|
||||
|
||||
logger.info(
|
||||
f"Deduplication complete: {len(deduplicated_task_memories)} deduplicated task memories out of {len(task_memories)}")
|
||||
|
||||
# Update context
|
||||
self.context.memory_list = deduplicated_task_memories
|
||||
|
||||
|
|
@ -84,7 +86,8 @@ class MemoryDeduplicationOp(BaseOp):
|
|||
if hasattr(node, 'embedding') and node.embedding:
|
||||
existing_embeddings.append(node.embedding)
|
||||
|
||||
logger.debug(f"Retrieved {len(existing_embeddings)} existing task memory embeddings from workspace {workspace_id}")
|
||||
logger.debug(
|
||||
f"Retrieved {len(existing_embeddings)} existing task memory embeddings from workspace {workspace_id}")
|
||||
return existing_embeddings
|
||||
|
||||
except Exception as e:
|
||||
|
|
@ -111,10 +114,9 @@ class MemoryDeduplicationOp(BaseOp):
|
|||
logger.error(f"Error generating embedding for task memory: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _is_similar_to_existing_task_memories(self, current_embedding: List[float],
|
||||
existing_embeddings: List[List[float]],
|
||||
threshold: float) -> bool:
|
||||
existing_embeddings: List[List[float]],
|
||||
threshold: float) -> bool:
|
||||
"""Check if current embedding is similar to existing embeddings"""
|
||||
for existing_embedding in existing_embeddings:
|
||||
similarity = self._calculate_cosine_similarity(current_embedding, existing_embedding)
|
||||
|
|
@ -124,8 +126,8 @@ class MemoryDeduplicationOp(BaseOp):
|
|||
return False
|
||||
|
||||
def _is_similar_to_current_task_memories(self, current_embedding: List[float],
|
||||
current_task_memories: List[BaseMemory],
|
||||
threshold: float) -> bool:
|
||||
current_task_memories: List[BaseMemory],
|
||||
threshold: float) -> bool:
|
||||
for existing_task_memory in current_task_memories:
|
||||
existing_embedding = self._get_task_memory_embedding(existing_task_memory)
|
||||
if existing_embedding is None:
|
||||
|
|
@ -158,4 +160,4 @@ class MemoryDeduplicationOp(BaseOp):
|
|||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating cosine similarity: {e}")
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ class MemoryValidationOp(BaseLLMOp):
|
|||
logger.warning(f"Task memory validation failed: {reason}")
|
||||
|
||||
logger.info(f"Validated {len(validated_task_memories)} out of {len(task_memories)} task memories")
|
||||
|
||||
|
||||
# Update context
|
||||
self.context.response.answer = json.dumps([x.model_dump() for x in validated_task_memories])
|
||||
self.context.response.metadata["memory_list"] = validated_task_memories
|
||||
|
|
@ -62,7 +62,7 @@ class MemoryValidationOp(BaseLLMOp):
|
|||
def parse_validation(message: Message) -> Dict[str, Any]:
|
||||
try:
|
||||
response_content = message.content
|
||||
|
||||
|
||||
# Parse validation result
|
||||
# Extract JSON blocks
|
||||
json_pattern = r'```json\s*([\s\S]*?)\s*```'
|
||||
|
|
@ -73,19 +73,20 @@ class MemoryValidationOp(BaseLLMOp):
|
|||
else:
|
||||
parsed = {}
|
||||
|
||||
is_valid = parsed.get("is_valid",True)
|
||||
score = parsed.get("score",0.5)
|
||||
is_valid = parsed.get("is_valid", True)
|
||||
score = parsed.get("score", 0.5)
|
||||
|
||||
# Set validation threshold
|
||||
validation_threshold = self.op_params.get("validation_threshold", 0.5)
|
||||
|
||||
|
||||
return {
|
||||
"is_valid": is_valid and score >= validation_threshold,
|
||||
"score": score,
|
||||
"feedback": response_content,
|
||||
"reason": "" if (is_valid and score >= validation_threshold) else f"Low validation score ({score:.2f}) or marked as invalid"
|
||||
"reason": "" if (
|
||||
is_valid and score >= validation_threshold) else f"Low validation score ({score:.2f}) or marked as invalid"
|
||||
}
|
||||
|
||||
|
||||
except Exception as e_inner:
|
||||
logger.exception(f"Error parsing validation response: {e_inner}")
|
||||
return {
|
||||
|
|
@ -104,4 +105,4 @@ class MemoryValidationOp(BaseLLMOp):
|
|||
"score": 0.0,
|
||||
"feedback": "",
|
||||
"reason": f"LLM validation error: {str(e)}"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,14 +12,14 @@ class PDFPreprocessOp(BaseOp):
|
|||
"""Process PDF files using MinerU and chunk content"""
|
||||
pdf_path = self.context.get("pdf_path")
|
||||
output_dir = self.context.get("output_dir")
|
||||
|
||||
|
||||
if not pdf_path:
|
||||
logger.error("No PDF path provided in context")
|
||||
return
|
||||
|
||||
# Process PDF
|
||||
processor = MinerUPDFProcessor(log_level="INFO")
|
||||
|
||||
|
||||
try:
|
||||
content_list, markdown_content = processor.process_pdf(
|
||||
pdf_path=pdf_path,
|
||||
|
|
@ -28,21 +28,21 @@ class PDFPreprocessOp(BaseOp):
|
|||
lang=self.op_params.get("lang"),
|
||||
backend=self.op_params.get("backend", "pipeline")
|
||||
)
|
||||
|
||||
|
||||
# Create chunks if requested
|
||||
chunks = []
|
||||
if self.op_params.get("create_chunks", True):
|
||||
max_length = self.op_params.get("max_chunk_length", 4000)
|
||||
chunks = chunk_pdf_content(content_list, max_length=max_length)
|
||||
|
||||
|
||||
# Store results in context
|
||||
self.context.pdf_content_list = content_list
|
||||
self.context.pdf_markdown_content = markdown_content
|
||||
self.context.pdf_chunks = chunks
|
||||
|
||||
|
||||
logger.info(f"PDF processing completed: {len(content_list)} content blocks, "
|
||||
f"{len(chunks)} chunks, {len(markdown_content)} characters of markdown")
|
||||
|
||||
f"{len(chunks)} chunks, {len(markdown_content)} characters of markdown")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"PDF processing failed: {e}")
|
||||
self.context.pdf_content_list = []
|
||||
|
|
|
|||
|
|
@ -32,9 +32,9 @@ class SimpleComparativeSummaryOp(BaseLLMOp):
|
|||
task_memory_content = tm_dict.get("experience", "").strip()
|
||||
if when_to_use and task_memory_content:
|
||||
task_memory_list.append(TaskMemory(workspace_id=self.context.get("workspace_id", ""),
|
||||
when_to_use=when_to_use,
|
||||
when_to_use=when_to_use,
|
||||
content=task_memory_content,
|
||||
author=getattr(self.llm, 'model_name', 'system')))
|
||||
author=getattr(self.llm, 'model_name', 'system')))
|
||||
|
||||
return task_memory_list
|
||||
|
||||
|
|
@ -62,7 +62,7 @@ class SimpleComparativeSummaryOp(BaseLLMOp):
|
|||
|
||||
if task_trajectories[0].score > task_trajectories[-1].score:
|
||||
task_memories = self.compare_summary_trajectory(trajectory_a=task_trajectories[0],
|
||||
trajectory_b=task_trajectories[-1])
|
||||
trajectory_b=task_trajectories[-1])
|
||||
memory_list.extend(task_memories)
|
||||
|
||||
self.context.response.answer = json.dumps([x.model_dump() for x in memory_list])
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class SuccessExtractionOp(BaseLLMOp):
|
|||
def execute(self):
|
||||
"""Extract task memories from successful trajectories"""
|
||||
success_trajectories: List[Trajectory] = self.context.success_trajectories
|
||||
|
||||
|
||||
if not success_trajectories:
|
||||
logger.info("No success trajectories found for extraction")
|
||||
return
|
||||
|
|
@ -23,7 +23,7 @@ class SuccessExtractionOp(BaseLLMOp):
|
|||
logger.info(f"Extracting task memories from {len(success_trajectories)} successful trajectories")
|
||||
|
||||
success_task_memories = []
|
||||
|
||||
|
||||
# Process trajectories
|
||||
for trajectory in success_trajectories:
|
||||
if "segments" in trajectory.metadata:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import json
|
||||
from typing import List, Dict
|
||||
|
||||
from flowllm import C, BaseOp
|
||||
|
|
@ -57,7 +56,7 @@ class TrajectoryPreprocessOp(BaseOp):
|
|||
|
||||
def _modify_tool_calls(self, messages: List[Dict]) -> List[Dict]:
|
||||
new_messages = []
|
||||
|
||||
|
||||
for msg in messages:
|
||||
if 'tool_calls' in msg:
|
||||
processed_tool_calls = []
|
||||
|
|
@ -72,5 +71,5 @@ class TrajectoryPreprocessOp(BaseOp):
|
|||
processed_tool_calls.append(tool_call)
|
||||
msg['tool_calls'] = processed_tool_calls
|
||||
new_messages.append(msg)
|
||||
|
||||
|
||||
return new_messages
|
||||
|
|
@ -35,9 +35,8 @@ class TrajectorySegmentationOp(BaseLLMOp):
|
|||
segmented_count += 1
|
||||
|
||||
logger.info(f"Segmented {segmented_count} trajectories")
|
||||
|
||||
# Update context with segmented trajectories
|
||||
|
||||
# Update context with segmented trajectories
|
||||
|
||||
def _get_target_trajectories(self, all_trajectories: List[Trajectory],
|
||||
success_trajectories: List[Trajectory],
|
||||
|
|
@ -116,4 +115,4 @@ class TrajectorySegmentationOp(BaseLLMOp):
|
|||
numbers = re.findall(r'\b\d+\b', response)
|
||||
segment_points = [int(num) for num in numbers if int(num) > 0]
|
||||
|
||||
return sorted(list(set(segment_points)))
|
||||
return sorted(list(set(segment_points)))
|
||||
|
|
|
|||
|
|
@ -231,7 +231,7 @@ class MinerUPDFProcessor:
|
|||
Returns:
|
||||
Tuple[List[Dict[str, Any]], str]: A tuple containing:
|
||||
- content_list: Structured content list with metadata
|
||||
- markdown_content: Raw markdown text content
|
||||
- markdown_content: Raw Markdown text content
|
||||
"""
|
||||
# Locate generated output files - handle both flat and nested directory structures
|
||||
md_file = output_dir / f"{file_stem}.md"
|
||||
|
|
@ -277,7 +277,8 @@ class MinerUPDFProcessor:
|
|||
item[field_name] = str(absolute_img_path)
|
||||
self.logger.debug(f"Updated {field_name}: {img_path} -> {item[field_name]}")
|
||||
|
||||
self.logger.info(f"Successfully read JSON file: {json_file}, containing {len(content_list)} content blocks")
|
||||
self.logger.info(
|
||||
f"Successfully read JSON file: {json_file}, containing {len(content_list)} content blocks")
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to read JSON file {json_file}: {e}")
|
||||
|
|
@ -466,32 +467,32 @@ class MinerUPDFProcessor:
|
|||
"table_count": 0,
|
||||
"has_formulas": False
|
||||
}
|
||||
|
||||
|
||||
for item in content_list:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
|
||||
content_type = item.get("type", "unknown")
|
||||
stats["content_types"][content_type] = stats["content_types"].get(content_type, 0) + 1
|
||||
|
||||
|
||||
if content_type == "text":
|
||||
text = item.get("text", "")
|
||||
stats["text_stats"]["total_characters"] += len(text)
|
||||
stats["text_stats"]["total_words"] += len(text.split())
|
||||
|
||||
|
||||
level = item.get("text_level", 0)
|
||||
if level > 0:
|
||||
stats["text_stats"]["title_levels"][level] = stats["text_stats"]["title_levels"].get(level, 0) + 1
|
||||
|
||||
|
||||
elif content_type == "image":
|
||||
stats["image_count"] += 1
|
||||
|
||||
|
||||
elif content_type == "table":
|
||||
stats["table_count"] += 1
|
||||
|
||||
|
||||
elif content_type == "formula":
|
||||
stats["has_formulas"] = True
|
||||
|
||||
|
||||
return stats
|
||||
|
||||
def validate_output_quality(self, content_list: List[Dict[str, Any]], markdown_content: str) -> Dict[str, Any]:
|
||||
|
|
@ -518,20 +519,20 @@ class MinerUPDFProcessor:
|
|||
"suggestions": [],
|
||||
"quality_score": 100
|
||||
}
|
||||
|
||||
|
||||
# Check if content was extracted
|
||||
if not content_list and not markdown_content.strip():
|
||||
validation["is_valid"] = False
|
||||
validation["warnings"].append("No content was extracted from the PDF")
|
||||
validation["quality_score"] = 0
|
||||
return validation
|
||||
|
||||
|
||||
# Check content diversity
|
||||
stats = self.get_content_statistics(content_list)
|
||||
if stats["total_blocks"] < 5:
|
||||
validation["warnings"].append("Very few content blocks extracted - document may be complex or image-heavy")
|
||||
validation["quality_score"] -= 20
|
||||
|
||||
|
||||
# Check text content ratio
|
||||
text_blocks = stats["content_types"].get("text", 0)
|
||||
if text_blocks == 0:
|
||||
|
|
@ -540,16 +541,17 @@ class MinerUPDFProcessor:
|
|||
elif text_blocks / stats["total_blocks"] < 0.3:
|
||||
validation["suggestions"].append("Low text content ratio - document may benefit from OCR processing")
|
||||
validation["quality_score"] -= 10
|
||||
|
||||
|
||||
# Check for images without processing
|
||||
if stats["image_count"] > 0 and stats["content_types"].get("text", 0) == 0:
|
||||
validation["suggestions"].append("Images detected but no text extracted - consider using VLM backend for image analysis")
|
||||
|
||||
validation["suggestions"].append(
|
||||
"Images detected but no text extracted - consider using VLM backend for image analysis")
|
||||
|
||||
# Check markdown length vs content blocks
|
||||
if len(markdown_content.strip()) < 100 and stats["total_blocks"] > 10:
|
||||
validation["warnings"].append("Markdown content seems unusually short for the number of content blocks")
|
||||
validation["quality_score"] -= 15
|
||||
|
||||
|
||||
return validation
|
||||
|
||||
|
||||
|
|
@ -656,22 +658,23 @@ if __name__ == "__main__":
|
|||
and working with the extracted content.
|
||||
"""
|
||||
import sys
|
||||
|
||||
|
||||
|
||||
# Example usage
|
||||
def example_usage():
|
||||
"""Demonstrate basic PDF processing workflow."""
|
||||
try:
|
||||
# Initialize processor
|
||||
processor = MinerUPDFProcessor.create_with_defaults(log_level="INFO")
|
||||
|
||||
|
||||
# Example PDF path (replace with actual PDF file)
|
||||
pdf_path = "example_document.pdf"
|
||||
|
||||
|
||||
if not Path(pdf_path).exists():
|
||||
print(f"Example PDF file not found: {pdf_path}")
|
||||
print("Please provide a valid PDF file path to test the processor.")
|
||||
return
|
||||
|
||||
|
||||
# Process PDF with different methods
|
||||
print("Processing PDF with auto method...")
|
||||
content_list, markdown_content = processor.process_pdf(
|
||||
|
|
@ -679,7 +682,7 @@ if __name__ == "__main__":
|
|||
method="auto",
|
||||
lang="en" # Specify language for better OCR results
|
||||
)
|
||||
|
||||
|
||||
# Generate statistics
|
||||
stats = processor.get_content_statistics(content_list)
|
||||
print(f"Processing Statistics:")
|
||||
|
|
@ -687,7 +690,7 @@ if __name__ == "__main__":
|
|||
print(f" Content types: {stats['content_types']}")
|
||||
print(f" Text characters: {stats['text_stats']['total_characters']}")
|
||||
print(f" Text words: {stats['text_stats']['total_words']}")
|
||||
|
||||
|
||||
# Validate output quality
|
||||
validation = processor.validate_output_quality(content_list, markdown_content)
|
||||
print(f"Quality Score: {validation['quality_score']}/100")
|
||||
|
|
@ -695,7 +698,7 @@ if __name__ == "__main__":
|
|||
print("Warnings:", validation['warnings'])
|
||||
if validation['suggestions']:
|
||||
print("Suggestions:", validation['suggestions'])
|
||||
|
||||
|
||||
# Save results
|
||||
output_path = Path(pdf_path).stem + "_processed"
|
||||
saved_files = processor.save_results(
|
||||
|
|
@ -704,19 +707,20 @@ if __name__ == "__main__":
|
|||
output_path=output_path
|
||||
)
|
||||
print(f"Results saved to: {saved_files}")
|
||||
|
||||
|
||||
# Create text chunks for downstream processing
|
||||
chunks = chunk_pdf_content(content_list, max_length=2000)
|
||||
print(f"Created {len(chunks)} text chunks")
|
||||
|
||||
|
||||
# Display first chunk as example
|
||||
if chunks:
|
||||
print("First chunk preview:")
|
||||
print(chunks[0][:200] + "..." if len(chunks[0]) > 200 else chunks[0])
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error during processing: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
||||
# Run example if script is executed directly
|
||||
example_usage()
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from loguru import logger
|
|||
def merge_messages_content(messages: List[Message | dict]) -> str:
|
||||
return merge_messages_content_flowllm(messages)
|
||||
|
||||
|
||||
def parse_json_experience_response(response: str) -> List[dict]:
|
||||
"""Parse JSON formatted experience response"""
|
||||
try:
|
||||
|
|
@ -82,155 +83,6 @@ def get_trajectory_context(trajectory: Trajectory, step_sequence: List[Message])
|
|||
logger.error(f"Error getting trajectory context: {e}")
|
||||
return f"Query: {trajectory.metadata.get('query', 'N/A')}"
|
||||
|
||||
|
||||
def parse_observation_response(response_text: str) -> List[dict]:
|
||||
"""Parse observation response to extract structured data"""
|
||||
# Pattern to match both Chinese and English observation formats
|
||||
pattern = r"信息：<(\d+)>\s*<>\s*<([^<>]+)>\s*<([^<>]*)>|Information:\s*<(\d+)>\s*<>\s*<([^<>]+)>\s*<([^<>]*)>"
|
||||
matches = re.findall(pattern, response_text, re.IGNORECASE | re.MULTILINE)
|
||||
|
||||
observations = []
|
||||
for match in matches:
|
||||
# Handle both Chinese and English patterns
|
||||
if match[0]: # Chinese pattern
|
||||
idx_str, content, keywords = match[0], match[1], match[2]
|
||||
else: # English pattern
|
||||
idx_str, content, keywords = match[3], match[4], match[5]
|
||||
|
||||
try:
|
||||
idx = int(idx_str)
|
||||
# Skip if content indicates no meaningful observation
|
||||
content_lower = content.lower().strip()
|
||||
if content_lower not in ['无', 'none', '', 'repeat']:
|
||||
observations.append({
|
||||
"index": idx,
|
||||
"content": content.strip(),
|
||||
"keywords": keywords.strip() if keywords else ""
|
||||
})
|
||||
except ValueError:
|
||||
logger.warning(f"Invalid index format: {idx_str}")
|
||||
continue
|
||||
|
||||
return observations
|
||||
|
||||
|
||||
def parse_observation_with_time_response(response_text: str) -> List[dict]:
|
||||
"""Parse observation with time response to extract structured data"""
|
||||
# Pattern to match both Chinese and English observation formats with time information
|
||||
# Chinese: 信息：<1> <时间信息或不输出> <明确的重要信息或"无"> <关键词>
|
||||
# English: Information: <1> <Time information or do not output> <Clear important information or "None"> <Keywords>
|
||||
pattern = r"信息：<(\d+)>\s*<([^<>]*)>\s*<([^<>]+)>\s*<([^<>]*)>|Information:\s*<(\d+)>\s*<([^<>]*)>\s*<([^<>]+)>\s*<([^<>]*)>"
|
||||
matches = re.findall(pattern, response_text, re.IGNORECASE | re.MULTILINE)
|
||||
|
||||
observations = []
|
||||
for match in matches:
|
||||
# Handle both Chinese and English patterns
|
||||
if match[0]: # Chinese pattern
|
||||
idx_str, time_info, content, keywords = match[0], match[1], match[2], match[3]
|
||||
else: # English pattern
|
||||
idx_str, time_info, content, keywords = match[4], match[5], match[6], match[7]
|
||||
|
||||
try:
|
||||
idx = int(idx_str)
|
||||
# Skip if content indicates no meaningful observation
|
||||
content_lower = content.lower().strip()
|
||||
if content_lower not in ['无', 'none', '', 'repeat']:
|
||||
observations.append({
|
||||
"index": idx,
|
||||
"time_info": time_info.strip() if time_info else "",
|
||||
"content": content.strip(),
|
||||
"keywords": keywords.strip() if keywords else ""
|
||||
})
|
||||
except ValueError:
|
||||
logger.warning(f"Invalid index format: {idx_str}")
|
||||
continue
|
||||
|
||||
return observations
|
||||
|
||||
|
||||
def parse_reflection_subjects_response(response_text: str, existing_subjects: List[str] = None) -> List[str]:
|
||||
"""Parse reflection subjects response to extract new subject attributes"""
|
||||
if existing_subjects is None:
|
||||
existing_subjects = []
|
||||
|
||||
# Split response into lines and clean up
|
||||
lines = response_text.strip().split('\n')
|
||||
subjects = []
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
# Skip empty lines, "None" responses, and existing subjects
|
||||
if (line and
|
||||
line not in ['无', 'None', ''] and
|
||||
line not in existing_subjects and
|
||||
not line.startswith('新增') and # Skip Chinese header
|
||||
not line.startswith('New ') and # Skip English header
|
||||
len(line) > 1): # Skip single character responses
|
||||
subjects.append(line)
|
||||
|
||||
logger.info(f"Parsed {len(subjects)} new reflection subjects from response")
|
||||
return subjects
|
||||
|
||||
|
||||
def parse_info_filter_response(response_text: str) -> List[tuple]:
|
||||
"""Parse info filter response to extract message scores"""
|
||||
import re
|
||||
|
||||
# Pattern to match both Chinese and English result formats
|
||||
# Chinese: 结果：<序号> <分数>
|
||||
# English: Result: <Index> <Score>
|
||||
pattern = r"结果：<(\d+)>\s*<([0-3])>|Result:\s*<(\d+)>\s*<([0-3])>"
|
||||
matches = re.findall(pattern, response_text, re.IGNORECASE | re.MULTILINE)
|
||||
|
||||
scores = []
|
||||
for match in matches:
|
||||
# Handle both Chinese and English patterns
|
||||
if match[0]: # Chinese pattern
|
||||
idx_str, score_str = match[0], match[1]
|
||||
else: # English pattern
|
||||
idx_str, score_str = match[2], match[3]
|
||||
|
||||
try:
|
||||
idx = int(idx_str)
|
||||
score = score_str
|
||||
scores.append((idx, score))
|
||||
except ValueError:
|
||||
logger.warning(f"Invalid index or score format: {idx_str}, {score_str}")
|
||||
continue
|
||||
|
||||
logger.info(f"Parsed {len(scores)} info filter scores from response")
|
||||
return scores
|
||||
|
||||
|
||||
def parse_long_contra_repeat_response(response_text: str) -> List[tuple]:
|
||||
"""Parse long contra repeat response to extract judgments"""
|
||||
import re
|
||||
|
||||
# Pattern to match both Chinese and English judgment formats
|
||||
# Chinese: 判断：<序号> <矛盾|被包含|无> <修改后的内容>
|
||||
# English: Judgment: <Index> <Contradiction|Contained|None> <Modified content>
|
||||
pattern = r"判断：<(\d+)>\s*<(矛盾|被包含|无)>\s*<([^<>]*)>|Judgment:\s*<(\d+)>\s*<(Contradiction|Contained|None)>\s*<([^<>]*)>"
|
||||
matches = re.findall(pattern, response_text, re.IGNORECASE | re.MULTILINE)
|
||||
|
||||
judgments = []
|
||||
for match in matches:
|
||||
# Handle both Chinese and English patterns
|
||||
if match[0]: # Chinese pattern
|
||||
idx_str, judgment, modified_content = match[0], match[1], match[2]
|
||||
else: # English pattern
|
||||
idx_str, judgment, modified_content = match[3], match[4], match[5]
|
||||
|
||||
try:
|
||||
idx = int(idx_str)
|
||||
judgments.append((idx, judgment, modified_content))
|
||||
except ValueError:
|
||||
logger.warning(f"Invalid index format: {idx_str}")
|
||||
continue
|
||||
|
||||
logger.info(f"Parsed {len(judgments)} long contra repeat judgments from response")
|
||||
return judgments
|
||||
|
||||
|
||||
def parse_update_insight_response(response_text: str, language: str = "en") -> str:
|
||||
"""Parse update insight response to extract updated insight content"""
|
||||
import re
|
||||
|
|
@ -261,61 +113,3 @@ def parse_update_insight_response(response_text: str, language: str = "en") -> s
|
|||
|
||||
logger.warning("No insight content found in response")
|
||||
return ""
|
||||
|
||||
|
||||
def load_memories_from_vector_store(workspace_id: str, filter_criteria: dict, top_k: int,
|
||||
memory_category: str = ""):
|
||||
"""
|
||||
Load memories from vector store based on filter criteria.
|
||||
|
||||
Args:
|
||||
workspace_id: The workspace identifier
|
||||
filter_criteria: Dictionary containing filter criteria for memory retrieval
|
||||
top_k: Maximum number of memories to retrieve
|
||||
memory_category: Category label to add to memory metadata
|
||||
|
||||
Returns:
|
||||
List of PersonalMemory objects loaded from vector store
|
||||
"""
|
||||
from reme_ai.schema.memory import PersonalMemory
|
||||
|
||||
try:
|
||||
# This is a placeholder implementation - in a real scenario, you would
|
||||
# integrate with your actual vector store (e.g., Chroma, Pinecone, etc.)
|
||||
logger.info(f"Loading memories from vector store for workspace: {workspace_id}")
|
||||
logger.info(f"Filter criteria: {filter_criteria}")
|
||||
logger.info(f"Top K: {top_k}")
|
||||
|
||||
# For now, return empty list as placeholder
|
||||
# In real implementation, this would:
|
||||
# 1. Connect to vector store
|
||||
# 2. Apply filter criteria
|
||||
# 3. Retrieve top_k memories
|
||||
# 4. Convert vector nodes to PersonalMemory objects
|
||||
# 5. Add memory_category to metadata
|
||||
|
||||
memories = []
|
||||
|
||||
# Placeholder: Create some example memories for testing
|
||||
if memory_category == "insight":
|
||||
for i in range(min(top_k, 2)):
|
||||
memory = PersonalMemory(
|
||||
workspace_id=workspace_id,
|
||||
memory_type="personal_insight",
|
||||
content=f"Sample insight memory {i + 1} for {filter_criteria.get('target', 'user')}",
|
||||
target=filter_criteria.get('target', 'user'),
|
||||
when_to_use=f"When analyzing user behavior patterns {i + 1}",
|
||||
author="system",
|
||||
metadata={
|
||||
"memory_category": memory_category,
|
||||
"filter_criteria": filter_criteria
|
||||
}
|
||||
)
|
||||
memories.append(memory)
|
||||
|
||||
logger.info(f"Loaded {len(memories)} memories from vector store")
|
||||
return memories
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error loading memories from vector store: {e}")
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
from typing import List, Iterable
|
||||
from typing import Iterable
|
||||
|
||||
from flowllm import C, BaseLLMOp
|
||||
from flowllm.schema.vector_node import VectorNode
|
||||
from loguru import logger
|
||||
|
||||
from reme_ai.schema.memory import BaseMemory, dict_to_memory
|
||||
|
||||
|
||||
@C.register_op()
|
||||
|
|
@ -14,14 +12,14 @@ class DeleteMemoryOp(BaseLLMOp):
|
|||
workspace_id: str = self.context.workspace_id
|
||||
freq_threshold: int = self.context.freq_threshold
|
||||
utility_threshold: float = self.context.utility_threshold
|
||||
nodes: Iterable[VectorNode] = self.vector_store._iter_workspace_nodes(workspace_id=workspace_id)
|
||||
|
||||
nodes: Iterable[VectorNode] = self.vector_store.iter_workspace_nodes(workspace_id=workspace_id)
|
||||
|
||||
deleted_memory_ids = []
|
||||
for node in nodes:
|
||||
freq = node["metadata"]["metadata"]["freq"]
|
||||
utility = node["metadata"]["metadata"]["utility"]
|
||||
if freq >= freq_threshold:
|
||||
if utility*1.0 / freq < utility_threshold:
|
||||
if utility * 1.0 / freq < utility_threshold:
|
||||
deleted_memory_ids.append(node["unique_id"])
|
||||
|
||||
|
||||
self.context.deleted_memory_ids = deleted_memory_ids
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from typing import List
|
||||
from flowllm import C, BaseLLMOp, BaseOp
|
||||
|
||||
from flowllm import C, BaseOp
|
||||
from loguru import logger
|
||||
|
||||
from reme_ai.schema.memory import BaseMemory, dict_to_memory
|
||||
|
|
@ -11,11 +12,11 @@ class UpdateMemoryFreqOp(BaseOp):
|
|||
|
||||
def execute(self):
|
||||
memory_dicts: List[dict] = self.context.memory_dicts
|
||||
|
||||
|
||||
if not memory_dicts:
|
||||
logger.info("No memories to update freq")
|
||||
return
|
||||
|
||||
|
||||
memory_list: List[BaseMemory] = [dict_to_memory(memory_dict) for memory_dict in memory_dicts]
|
||||
new_memory_list = []
|
||||
deleted_memory_ids = []
|
||||
|
|
@ -24,10 +25,9 @@ class UpdateMemoryFreqOp(BaseOp):
|
|||
metadata = memory.metadata
|
||||
metadata["freq"] = metadata.get("freq", 0) + 1
|
||||
memory.update_metadata(metadata)
|
||||
|
||||
|
||||
deleted_memory_ids.append(memory.memory_id)
|
||||
new_memory_list.append(memory)
|
||||
|
||||
self.context.deleted_memory_ids = deleted_memory_ids
|
||||
self.context.memory_list = new_memory_list
|
||||
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
from typing import List
|
||||
from flowllm import C, BaseLLMOp, BaseOp
|
||||
|
||||
from flowllm import C, BaseOp
|
||||
from loguru import logger
|
||||
|
||||
from reme_ai.schema.memory import BaseMemory
|
||||
|
|
@ -12,7 +13,7 @@ class UpdateMemoryUtilityOp(BaseOp):
|
|||
def execute(self):
|
||||
memory_dicts: List[dict] = self.context.memory_dicts
|
||||
update_utility = self.context.update_utility
|
||||
|
||||
|
||||
if not memory_dicts or not update_utility:
|
||||
logger.info("No memories to update utility")
|
||||
return
|
||||
|
|
@ -24,9 +25,8 @@ class UpdateMemoryUtilityOp(BaseOp):
|
|||
metadata = memory.metadata
|
||||
metadata["utility"] = metadata.get("utility", 0) + 1
|
||||
memory.update_metadata(metadata)
|
||||
|
||||
|
||||
new_memory_list.append(memory)
|
||||
|
||||
self.context.response.metadata["memory_list"] = new_memory_list
|
||||
self.context.response.metadata["deleted_memory_ids"] = self.context.deleted_memory_ids
|
||||
|
||||
|
|
@ -17,7 +17,8 @@ class VectorStoreActionOp(BaseLLMOp):
|
|||
dest_workspace_id=workspace_id)
|
||||
|
||||
elif action == "delete":
|
||||
result = self.vector_store.delete_workspace(workspace_id=workspace_id)
|
||||
if self.vector_store.exist_workspace(workspace_id):
|
||||
result = self.vector_store.delete_workspace(workspace_id=workspace_id)
|
||||
|
||||
elif action == "delete_ids":
|
||||
memory_ids: list = self.context.memory_ids
|
||||
|
|
@ -25,6 +26,7 @@ class VectorStoreActionOp(BaseLLMOp):
|
|||
|
||||
elif action == "dump":
|
||||
path: str = self.context.path
|
||||
|
||||
def node_to_memory(node: VectorNode) -> dict:
|
||||
return vector_node_to_memory(node).model_dump()
|
||||
|
||||
|
|
@ -34,6 +36,7 @@ class VectorStoreActionOp(BaseLLMOp):
|
|||
|
||||
elif action == "load":
|
||||
path: str = self.context.path
|
||||
|
||||
def memory_dict_to_node(memory_dict: dict) -> VectorNode:
|
||||
memory: BaseMemory = dict_to_memory(memory_dict=memory_dict)
|
||||
return memory.to_vector_node()
|
||||
|
|
@ -41,7 +44,7 @@ class VectorStoreActionOp(BaseLLMOp):
|
|||
result = self.vector_store.load_workspace(workspace_id=workspace_id,
|
||||
path=path,
|
||||
callback_fn=memory_dict_to_node)
|
||||
|
||||
|
||||
else:
|
||||
raise ValueError(f"invalid action={action}")
|
||||
|
||||
|
|
|
|||
3
test_workspace.jsonl
Normal file
3
test_workspace.jsonl
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
{"workspace_id": "test_workspace", "memory_id": "4ff6a7a783a84940a046e557de7542f0", "memory_type": "task", "when_to_use": "When analyzing a complex organization with multiple dimensions (history, products, finances, leadership), especially when the query is open-ended or broad like 'analyze [company]'", "content": "The agent successfully decomposed the task into four targeted web searches: company overview/history, products/services, financial performance, and leadership impact. This multi-angle approach ensured comprehensive coverage of key domains. Each search was designed to extract distinct, high-value information layers, enabling synthesis into a structured, well-rounded analysis. The use of varied queries prevented redundancy and maximized information diversity.", "score": 0.0, "time_created": "2025-08-31 15:42:44", "time_modified": "2025-08-31 15:42:44", "author": "qwen3-30b-a3b-instruct-2507", "metadata": {"when_to_use": "When analyzing a complex organization with multiple dimensions (history, products, finances, leadership), especially when the query is open-ended or broad like 'analyze [company]'", "experience": "The agent successfully decomposed the task into four targeted web searches: company overview/history, products/services, financial performance, and leadership impact. This multi-angle approach ensured comprehensive coverage of key domains. Each search was designed to extract distinct, high-value information layers, enabling synthesis into a structured, well-rounded analysis. The use of varied queries prevented redundancy and maximized information diversity.", "tags": ["multi-faceted analysis", "information decomposition", "web_search strategy", "comprehensive coverage"], "confidence": 0.9, "step_type": "reasoning", "tools_used": ["web_search"]}}
|
||||
{"workspace_id": "test_workspace", "memory_id": "2ad5b6aae8724e87944e69ea4476a97c", "memory_type": "task", "when_to_use": "When initial results from a general search are insufficient or too broad, and deeper insights are needed on specific subtopics such as financial trends, product portfolios, or leadership dynamics", "content": "The agent proactively called the same tool (web_search) multiple times with different, focused queries to gather information from various perspectives. This iterative technique allowed for layered understanding—e.g., separating historical milestones from current financial challenges and leadership influence. It demonstrated strategic use of repetition not as redundancy but as a method to triangulate data across domains, significantly improving depth and accuracy.", "score": 0.0, "time_created": "2025-08-31 15:42:44", "time_modified": "2025-08-31 15:42:44", "author": "qwen3-30b-a3b-instruct-2507", "metadata": {"when_to_use": "When initial results from a general search are insufficient or too broad, and deeper insights are needed on specific subtopics such as financial trends, product portfolios, or leadership dynamics", "experience": "The agent proactively called the same tool (web_search) multiple times with different, focused queries to gather information from various perspectives. This iterative technique allowed for layered understanding—e.g., separating historical milestones from current financial challenges and leadership influence. It demonstrated strategic use of repetition not as redundancy but as a method to triangulate data across domains, significantly improving depth and accuracy.", "tags": ["iterative querying", "perspective diversification", "information triangulation", "search optimization"], "confidence": 0.85, "step_type": "action", "tools_used": ["web_search"]}}
|
||||
{"workspace_id": "test_workspace", "memory_id": "87afc1367c6c4b739bbe191e1f47c875", "memory_type": "task", "when_to_use": "After gathering raw data from external sources, when synthesizing findings into a coherent, structured response that balances factual reporting with critical insight", "content": "The agent effectively integrated results from multiple searches into a unified summary with clear sectioning: history, products, finance, and leadership. It highlighted both strengths (innovation, mission-driven vision) and risks (financial decline, leadership dependency). This synthesis transformed fragmented data into actionable intelligence, demonstrating how structured summarization enhances usability and decision-making value in agent outputs.", "score": 0.0, "time_created": "2025-08-31 15:42:44", "time_modified": "2025-08-31 15:42:44", "author": "qwen3-30b-a3b-instruct-2507", "metadata": {"when_to_use": "After gathering raw data from external sources, when synthesizing findings into a coherent, structured response that balances factual reporting with critical insight", "experience": "The agent effectively integrated results from multiple searches into a unified summary with clear sectioning: history, products, finance, and leadership. It highlighted both strengths (innovation, mission-driven vision) and risks (financial decline, leadership dependency). This synthesis transformed fragmented data into actionable intelligence, demonstrating how structured summarization enhances usability and decision-making value in agent outputs.", "tags": ["synthesis", "structured summarization", "critical insight", "data integration"], "confidence": 0.9, "step_type": "observation", "tools_used": []}}
|
||||
Loading…
Add table
Reference in a new issue