mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-08-28 05:25:04 +00:00
feat(core): add simple request methods and improve memory management
This commit is contained in:
jinli.yl
parent
013fba8538
commit
e63a3fa632
19 changed files with 1116 additions and 160 deletions
|
|
@ -8,7 +8,7 @@ A modular evaluation pipeline that:
|
|||
4. Generates comprehensive metrics
|
||||
|
||||
Usage:
|
||||
python bench/halumem/eval_reme_simple_v4.py \
|
||||
python benchmark/halumem/eval_reme.py \
|
||||
--data_path /Users/yuli/workspace/HaluMem/data/HaluMem-Medium.jsonl \
|
||||
--top_k 20 --user_num 100 --max_concurrency 20
|
||||
"""
|
||||
|
|
@ -26,14 +26,13 @@ from typing import Any
|
|||
|
||||
from loguru import logger
|
||||
|
||||
from eval_tools import evaluation_for_question2, answer_question_with_memories
|
||||
from reme_ai.core.enumeration import MemoryType
|
||||
from reme_ai.core.schema import MemoryNode
|
||||
from reme_ai.reme import ReMe
|
||||
from reme.core.schema import MemoryNode, Message
|
||||
from reme.reme import ReMe
|
||||
|
||||
|
||||
# ==================== Configuration ====================
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalConfig:
|
||||
"""Evaluation configuration parameters."""
|
||||
|
|
@ -42,7 +41,7 @@ class EvalConfig:
|
|||
user_num: int = 1
|
||||
max_concurrency: int = 2
|
||||
batch_size: int = 20
|
||||
output_dir: str = "bench_results/reme_simple_v4"
|
||||
output_dir: str = "bench_results/reme"
|
||||
|
||||
|
||||
# ==================== Utilities ====================
|
||||
|
|
@ -138,7 +137,7 @@ class FileManager:
|
|||
"""Check if user has cached results."""
|
||||
user_dir = self.get_user_dir(user_name)
|
||||
return any(f.name.startswith("session_") and f.suffix == ".json"
|
||||
for f in user_dir.iterdir())
|
||||
for f in user_dir.iterdir())
|
||||
|
||||
def combine_results(self, output_file: str):
|
||||
"""Combine all user session files into a single JSONL file."""
|
||||
|
|
@ -177,6 +176,94 @@ class FileManager:
|
|||
f_out.write(json.dumps(user_data, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
# ==================== Evaluation Functions ====================
|
||||
|
||||
async def answer_question_with_memories(
|
||||
reme: ReMe,
|
||||
question: str,
|
||||
memories: str,
|
||||
user_id: str = None,
|
||||
model_name: str = "qwen3-30b-a3b-instruct-2507"
|
||||
):
|
||||
"""
|
||||
Answer a question using retrieved memories with PROMPT_MEMZERO_JSON template.
|
||||
|
||||
Args:
|
||||
reme: ReMe instance with default_llm and prompt_handler
|
||||
question: The question to answer
|
||||
memories: The retrieved memories (formatted as context)
|
||||
user_id: Optional user ID for context formatting
|
||||
model_name: Model name to use for LLM request
|
||||
|
||||
Returns:
|
||||
dict with 'reasoning' and 'answer' fields
|
||||
"""
|
||||
# Format context with memories
|
||||
if user_id:
|
||||
context = reme.prompt_handler.prompt_format(
|
||||
"TEMPLATE_MEMOS",
|
||||
user_id=user_id,
|
||||
memories=memories
|
||||
)
|
||||
else:
|
||||
context = f"Memories:\n{memories}"
|
||||
|
||||
# Use PROMPT_MEMZERO_JSON template for structured JSON response
|
||||
prompt = reme.prompt_handler.prompt_format(
|
||||
"PROMPT_MEMZERO_JSON",
|
||||
context=context,
|
||||
question=question
|
||||
)
|
||||
|
||||
result = await reme.llm.simple_request_for_json(
|
||||
prompt=prompt,
|
||||
model_name=model_name
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def evaluation_for_question(
|
||||
reme: ReMe,
|
||||
question: str,
|
||||
reference_answer: str,
|
||||
key_memory_points: str,
|
||||
response: str,
|
||||
dialogue: str = None,
|
||||
model_name: str = "qwen3-max"
|
||||
):
|
||||
"""
|
||||
Question-Answering Evaluation with optional Dialogue Context.
|
||||
|
||||
Args:
|
||||
reme: ReMe instance with default_llm and prompt_handler
|
||||
question: The question string to be evaluated.
|
||||
reference_answer: The reference (gold-standard) answer.
|
||||
key_memory_points: The memory points used to derive the reference answer.
|
||||
response: The answer produced by the memory system.
|
||||
dialogue: Optional formatted dialogue history (role, content, time_created).
|
||||
model_name: Model name to use for LLM request
|
||||
|
||||
Returns:
|
||||
dict with 'reasoning' and 'evaluation_result' fields
|
||||
"""
|
||||
prompt = reme.prompt_handler.prompt_format(
|
||||
"EVALUATION_PROMPT_FOR_QUESTION",
|
||||
question=question,
|
||||
reference_answer=reference_answer,
|
||||
key_memory_points=key_memory_points,
|
||||
response=response,
|
||||
dialogue=dialogue if dialogue else ""
|
||||
)
|
||||
|
||||
result = await reme.llm.simple_request_for_json(
|
||||
prompt=prompt,
|
||||
model_name=model_name
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ==================== Memory Operations ====================
|
||||
|
||||
class MemoryProcessor:
|
||||
|
|
@ -186,59 +273,48 @@ class MemoryProcessor:
|
|||
self.reme = reme
|
||||
|
||||
async def add_memories(
|
||||
self,
|
||||
user_id: str,
|
||||
messages: list[dict],
|
||||
batch_size: int = 10000
|
||||
) -> tuple[list[str], list[list[dict]], float]:
|
||||
self,
|
||||
user_id: str,
|
||||
messages: list[dict],
|
||||
batch_size: int = 10000
|
||||
) -> tuple[list[str], list, float]:
|
||||
"""
|
||||
Add memories in batches using ReMe and return extracted memory contents.
|
||||
|
||||
Returns:
|
||||
tuple: (extracted_memories, agent_messages, total_duration_ms)
|
||||
"""
|
||||
added_memories: list[MemoryNode] = []
|
||||
deleted_memories: list[str] = []
|
||||
all_agent_messages: list = []
|
||||
extracted_memories = []
|
||||
summary_messages = []
|
||||
total_duration_ms = 0
|
||||
|
||||
for i in range(0, len(messages), batch_size):
|
||||
batch = messages[i:i + batch_size]
|
||||
start = time.time()
|
||||
|
||||
memory_nodes, agent_messages, success = await self.reme.summary_v4(
|
||||
# Use new summary API
|
||||
result = await self.reme.summary_memory(
|
||||
messages=batch,
|
||||
user_id=user_id
|
||||
user_name=user_id,
|
||||
version="default",
|
||||
return_dict=True,
|
||||
)
|
||||
|
||||
duration_ms = (time.time() - start) * 1000
|
||||
total_duration_ms += duration_ms
|
||||
|
||||
# Save agent messages for this batch
|
||||
if agent_messages:
|
||||
all_agent_messages.extend(agent_messages)
|
||||
memory_nodes: list[MemoryNode] = result["answer"]
|
||||
messages: list[Message] = result["messages"]
|
||||
extracted_memories.extend([m.model_dump_json(exclude_none=True) for m in memory_nodes])
|
||||
summary_messages.extend([m.simple_dump() for m in messages])
|
||||
|
||||
if memory_nodes:
|
||||
for node in memory_nodes:
|
||||
if isinstance(node, MemoryNode) and node.memory_type == MemoryType.HISTORY:
|
||||
continue
|
||||
|
||||
if isinstance(node, MemoryNode):
|
||||
added_memories.append(node)
|
||||
|
||||
if isinstance(node, str):
|
||||
deleted_memories.append(node)
|
||||
|
||||
extracted_memories = deleted_memories
|
||||
extracted_memories += ["[delete]" + n.format_memory() for n in added_memories if n.memory_id in deleted_memories]
|
||||
extracted_memories += ["[add]" + n.format_memory() for n in added_memories if n.memory_id not in deleted_memories]
|
||||
return extracted_memories, all_agent_messages, total_duration_ms
|
||||
return extracted_memories, messages, total_duration_ms
|
||||
|
||||
async def search_memory(
|
||||
self,
|
||||
query: str,
|
||||
user_id: str,
|
||||
top_k: int = 20
|
||||
self,
|
||||
query: str,
|
||||
user_id: str,
|
||||
top_k: int = 20
|
||||
) -> tuple[dict, list, float]:
|
||||
"""
|
||||
Search memory using ReMe and return structured answer with reasoning.
|
||||
|
|
@ -249,25 +325,34 @@ class MemoryProcessor:
|
|||
"""
|
||||
start = time.time()
|
||||
|
||||
# Retrieve memories from ReMe
|
||||
memories_response, agent_messages, success = await self.reme.retrieve_v4(
|
||||
# Retrieve memories from ReMe using new API
|
||||
result = await self.reme.retrieve_memory(
|
||||
query=query,
|
||||
user_id=user_id,
|
||||
top_k=top_k
|
||||
top_k=top_k,
|
||||
user_name=user_id,
|
||||
version="default",
|
||||
return_dict=True,
|
||||
)
|
||||
|
||||
# Extract memories from response
|
||||
memories = result["answer"]
|
||||
messages = [x.model_dump_json(exclude_none=True) for x in result["messages"]]
|
||||
retrieved_nodes = [x.model_dump_json(exclude_none=True) for x in result["retrieved_nodes"]]
|
||||
|
||||
# Use LLM to generate structured answer from memories
|
||||
answer_result = await answer_question_with_memories(
|
||||
reme=self.reme,
|
||||
question=query,
|
||||
memories=memories_response,
|
||||
memories=memories,
|
||||
user_id=user_id
|
||||
)
|
||||
|
||||
# Add original memories to the result
|
||||
answer_result["memories"] = memories_response
|
||||
answer_result["memories"] = memories
|
||||
answer_result["retrieved_nodes"] = retrieved_nodes
|
||||
|
||||
duration_ms = (time.time() - start) * 1000
|
||||
return answer_result, agent_messages, duration_ms
|
||||
return answer_result, messages, duration_ms
|
||||
|
||||
|
||||
# ==================== Evaluation ====================
|
||||
|
|
@ -275,17 +360,18 @@ class MemoryProcessor:
|
|||
class QuestionAnsweringEvaluator:
|
||||
"""Evaluates question answering performance."""
|
||||
|
||||
def __init__(self, memory_processor: MemoryProcessor, top_k: int):
|
||||
def __init__(self, memory_processor: MemoryProcessor, reme: ReMe, top_k: int):
|
||||
self.memory_processor = memory_processor
|
||||
self.reme = reme
|
||||
self.top_k = top_k
|
||||
|
||||
async def evaluate_questions(
|
||||
self,
|
||||
questions: list[dict],
|
||||
user_name: str,
|
||||
uuid: str,
|
||||
session_id: int,
|
||||
formatted_dialogue: str
|
||||
self,
|
||||
questions: list[dict],
|
||||
user_name: str,
|
||||
uuid: str,
|
||||
session_id: int,
|
||||
formatted_dialogue: str
|
||||
) -> list[dict]:
|
||||
"""Evaluate all questions for a session."""
|
||||
results = []
|
||||
|
|
@ -301,15 +387,17 @@ class QuestionAnsweringEvaluator:
|
|||
system_answer = answer_dict.get("answer", "")
|
||||
system_reasoning = answer_dict.get("reasoning", "")
|
||||
retrieved_memories = answer_dict.get("memories", "")
|
||||
retrieved_nodes = answer_dict.get("retrieved_nodes", "")
|
||||
|
||||
# Evaluate response
|
||||
evidence_text = "\n".join([e["memory_content"] for e in qa["evidence"]])
|
||||
eval_result = await evaluation_for_question2(
|
||||
qa["question"],
|
||||
qa["answer"],
|
||||
evidence_text,
|
||||
system_answer,
|
||||
formatted_dialogue
|
||||
eval_result = await evaluation_for_question(
|
||||
reme=self.reme,
|
||||
question=qa["question"],
|
||||
reference_answer=qa["answer"],
|
||||
key_memory_points=evidence_text,
|
||||
response=system_answer,
|
||||
dialogue=formatted_dialogue
|
||||
)
|
||||
|
||||
# Build result record
|
||||
|
|
@ -320,7 +408,8 @@ class QuestionAnsweringEvaluator:
|
|||
"system_response": system_answer,
|
||||
"system_reasoning": system_reasoning,
|
||||
"retrieved_memories": retrieved_memories,
|
||||
"retrieve_messages": [m.model_dump() for m in agent_messages],
|
||||
"retrieved_nodes": retrieved_nodes,
|
||||
"retrieve_messages": agent_messages,
|
||||
"search_duration_ms": duration_ms,
|
||||
"result_type": eval_result.get("evaluation_result"),
|
||||
"question_answering_reasoning": eval_result.get("reasoning", "")
|
||||
|
|
@ -418,25 +507,50 @@ class MetricsAggregator:
|
|||
|
||||
# ==================== Main Pipeline ====================
|
||||
|
||||
class HaluMemEvaluatorV4:
|
||||
class HaluMemEvaluator:
|
||||
"""HaluMem evaluator with proper resource management."""
|
||||
|
||||
def __init__(self, config: EvalConfig):
|
||||
self.config = config
|
||||
self.reme = ReMe()
|
||||
|
||||
# Load evaluation prompts into ReMe's prompt handler
|
||||
prompts_yaml_path = Path(__file__).parent / "eval_reme.yaml"
|
||||
self.reme.prompt_handler.load_prompt_by_file(prompts_yaml_path)
|
||||
|
||||
self.file_manager = FileManager(config.output_dir)
|
||||
self.memory_processor = MemoryProcessor(self.reme)
|
||||
self.qa_evaluator = QuestionAnsweringEvaluator(
|
||||
self.memory_processor,
|
||||
self.reme,
|
||||
config.top_k
|
||||
)
|
||||
self.data_loader = DataLoader()
|
||||
|
||||
async def __aenter__(self):
|
||||
"""Async context manager entry."""
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Async context manager exit with cleanup."""
|
||||
await self.reme.close()
|
||||
return False
|
||||
|
||||
def __enter__(self):
|
||||
"""Sync context manager entry."""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Sync context manager exit with cleanup."""
|
||||
self.reme.close_sync()
|
||||
return False
|
||||
|
||||
async def process_session(
|
||||
self,
|
||||
session: dict,
|
||||
session_id: int,
|
||||
user_name: str,
|
||||
uuid: str
|
||||
self,
|
||||
session: dict,
|
||||
session_id: int,
|
||||
user_name: str,
|
||||
uuid: str
|
||||
) -> dict:
|
||||
"""Process a single session using ReMe."""
|
||||
session_data = {
|
||||
|
|
@ -463,7 +577,7 @@ class HaluMemEvaluatorV4:
|
|||
session_data.update({
|
||||
"dialogue": dialogue,
|
||||
"extracted_memories": extracted_memories,
|
||||
"summary_messages": [m.model_dump() for m in agent_messages],
|
||||
"summary_messages": agent_messages,
|
||||
"add_dialogue_duration_ms": duration_ms
|
||||
})
|
||||
|
||||
|
|
@ -509,6 +623,19 @@ class HaluMemEvaluatorV4:
|
|||
"""Run the complete evaluation pipeline using ReMe."""
|
||||
start_time = time.time()
|
||||
|
||||
# Load user data first to get user names
|
||||
all_users = self.data_loader.load_jsonl(self.config.data_path)
|
||||
users_to_process = all_users[:self.config.user_num]
|
||||
|
||||
# Extract all user names and delete all profiles
|
||||
all_user_names = [
|
||||
self.data_loader.extract_user_name(user_data["persona_info"])
|
||||
for user_data in all_users
|
||||
]
|
||||
if all_user_names:
|
||||
await self.reme.delete_all_profiles(all_user_names)
|
||||
logger.info(f"Deleted all profiles for {len(all_user_names)} users")
|
||||
|
||||
# Clear existing data
|
||||
await self.reme.vector_store.delete_all()
|
||||
|
||||
|
|
@ -519,10 +646,6 @@ class HaluMemEvaluatorV4:
|
|||
logger.info(f"Cleared meta_memory directory: {meta_memory_path}")
|
||||
meta_memory_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load user data
|
||||
all_users = self.data_loader.load_jsonl(self.config.data_path)
|
||||
users_to_process = all_users[:self.config.user_num]
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("HALUMEM EVALUATION - REME - QUESTION ANSWERING")
|
||||
print(f"Users: {len(users_to_process)} | Concurrency: {self.config.max_concurrency}")
|
||||
|
|
@ -631,13 +754,13 @@ class HaluMemEvaluatorV4:
|
|||
|
||||
# ==================== Entry Point ====================
|
||||
|
||||
def main(
|
||||
data_path: str,
|
||||
top_k: int = 20,
|
||||
user_num: int = 1,
|
||||
max_concurrency: int = 2
|
||||
async def main_async(
|
||||
data_path: str,
|
||||
top_k: int,
|
||||
user_num: int,
|
||||
max_concurrency: int
|
||||
):
|
||||
"""Main entry point for ReMe evaluation."""
|
||||
"""Main async entry point for ReMe evaluation with proper resource cleanup."""
|
||||
config = EvalConfig(
|
||||
data_path=data_path,
|
||||
top_k=top_k,
|
||||
|
|
@ -645,8 +768,24 @@ def main(
|
|||
max_concurrency=max_concurrency
|
||||
)
|
||||
|
||||
evaluator = HaluMemEvaluatorV4(config)
|
||||
asyncio.run(evaluator.run_evaluation())
|
||||
# Use async context manager for automatic cleanup
|
||||
async with HaluMemEvaluator(config) as evaluator:
|
||||
await evaluator.run_evaluation()
|
||||
|
||||
|
||||
def main(
|
||||
data_path: str,
|
||||
top_k: int,
|
||||
user_num: int,
|
||||
max_concurrency: int
|
||||
):
|
||||
"""Main entry point for ReMe evaluation."""
|
||||
asyncio.run(main_async(
|
||||
data_path=data_path,
|
||||
top_k=top_k,
|
||||
user_num=user_num,
|
||||
max_concurrency=max_concurrency
|
||||
))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
@ -676,7 +815,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument(
|
||||
"--max_concurrency",
|
||||
type=int,
|
||||
default=2,
|
||||
default=100,
|
||||
help="Maximum concurrent user processing (default: 2)"
|
||||
)
|
||||
|
||||
548
benchmark/halumem/eval_reme.yaml
Normal file
548
benchmark/halumem/eval_reme.yaml
Normal file
|
|
@ -0,0 +1,548 @@
|
|||
TEMPLATE_MEMOS: |
|
||||
Memories for user {user_id}:
|
||||
{memories}
|
||||
|
||||
PROMPT_MEMZERO_JSON: |
|
||||
# CONTEXT:
|
||||
{context}
|
||||
|
||||
# CONTEXT PRIORITY:
|
||||
When the context contains information from multiple sources, follow this strict priority order:
|
||||
1. **Historical Dialogue** (highest priority) - Direct conversation content
|
||||
2. **Extracted Memories** (medium priority) - Summarized memory points
|
||||
3. **User Profile** (lowest priority) - General user information
|
||||
|
||||
# Question:
|
||||
{question}
|
||||
|
||||
# OUTPUT FORMAT:
|
||||
Do not hallucinate; strictly answer the user's question based on the content of the CONTEXT.
|
||||
Please provide your response in the following JSON format:
|
||||
|
||||
```json
|
||||
{{
|
||||
"reasoning": "reasoning content",
|
||||
"answer": "Provide a detailed answer"
|
||||
}}
|
||||
```
|
||||
|
||||
PROMPT_MEMZERO_JSON2: |
|
||||
# CONTEXT:
|
||||
{context}
|
||||
|
||||
# CONTEXT PRIORITY:
|
||||
When the context contains information from multiple sources, follow this strict priority order:
|
||||
1. **Historical Dialogue** (highest priority) - Direct conversation content
|
||||
2. **Extracted Memories** (medium priority) - Summarized memory points
|
||||
3. **User Profile** (lowest priority) - General user information
|
||||
|
||||
# Question:
|
||||
{question}
|
||||
|
||||
# OUTPUT FORMAT:
|
||||
Do not hallucinate; strictly answer the user's question based on the content of the CONTEXT.
|
||||
Please provide your response in the following JSON format:
|
||||
|
||||
```json
|
||||
{{
|
||||
"reasoning": "reasoning content",
|
||||
"answer": "Provide a detailed answer"
|
||||
}}
|
||||
```
|
||||
|
||||
PROMPT_MEMZERO: |
|
||||
You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
|
||||
|
||||
# CONTEXT:
|
||||
You have access to memories from two speakers in a conversation. These memories contain
|
||||
timestamped information that may be relevant to answering the question.
|
||||
|
||||
# INSTRUCTIONS:
|
||||
1. Carefully analyze all provided memories from both speakers
|
||||
2. Pay special attention to the timestamps to determine the answer
|
||||
3. If the question asks about a specific event or fact, look for direct evidence in the memories
|
||||
4. If the memories contain contradictory information, prioritize the most recent memory
|
||||
5. If there is a question about time references (like "last year", "two months ago", etc.),
|
||||
calculate the actual date based on the memory timestamp. For example, if a memory from
|
||||
4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
|
||||
6. Always convert relative time references to specific dates, months, or years. For example,
|
||||
convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory
|
||||
timestamp. Ignore the reference while answering the question.
|
||||
7. Focus only on the content of the memories from both speakers. Do not confuse character
|
||||
names mentioned in memories with the actual users who created those memories.
|
||||
8. The answer should be less than 5-6 words.
|
||||
|
||||
# APPROACH (Think step by step):
|
||||
1. First, examine all memories that contain information related to the question
|
||||
2. Examine the timestamps and content of these memories carefully
|
||||
3. Look for explicit mentions of dates, times, locations, or events that answer the question
|
||||
4. If the answer requires calculation (e.g., converting relative time references), show your work
|
||||
5. Formulate a precise, concise answer based solely on the evidence in the memories
|
||||
6. Double-check that your answer directly addresses the question asked
|
||||
7. Ensure your final answer is specific and avoids vague time references
|
||||
|
||||
{context}
|
||||
|
||||
Question: {question}
|
||||
|
||||
Answer:
|
||||
|
||||
PROMPT_ZEP: |
|
||||
You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
|
||||
|
||||
# CONTEXT:
|
||||
You have access to memories from a conversation. These memories contain
|
||||
timestamped information that may be relevant to answering the question.
|
||||
|
||||
# INSTRUCTIONS:
|
||||
1. Carefully analyze all provided memories
|
||||
2. Pay special attention to the timestamps to determine the answer
|
||||
3. If the question asks about a specific event or fact, look for direct evidence in the memories
|
||||
4. If the memories contain contradictory information, prioritize the most recent memory
|
||||
5. If there is a question about time references (like "last year", "two months ago", etc.),
|
||||
calculate the actual date based on the memory timestamp. For example, if a memory from
|
||||
4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
|
||||
6. Always convert relative time references to specific dates, months, or years. For example,
|
||||
convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory
|
||||
timestamp. Ignore the reference while answering the question.
|
||||
7. Focus only on the content of the memories. Do not confuse character
|
||||
names mentioned in memories with the actual users who created those memories.
|
||||
8. The answer should be less than 5-6 words.
|
||||
|
||||
# APPROACH (Think step by step):
|
||||
1. First, examine all memories that contain information related to the question
|
||||
2. Examine the timestamps and content of these memories carefully
|
||||
3. Look for explicit mentions of dates, times, locations, or events that answer the question
|
||||
4. If the answer requires calculation (e.g., converting relative time references), show your work
|
||||
5. Formulate a precise, concise answer based solely on the evidence in the memories
|
||||
6. Double-check that your answer directly addresses the question asked
|
||||
7. Ensure your final answer is specific and avoids vague time references
|
||||
|
||||
Context:
|
||||
|
||||
{context}
|
||||
|
||||
Question: {question}
|
||||
Answer:
|
||||
|
||||
PROMPT_MEMOS: |
|
||||
You are a knowledgeable and helpful AI assistant.
|
||||
|
||||
# CONTEXT:
|
||||
You have access to memories from two speakers in a conversation. These memories contain
|
||||
timestamped information that may be relevant to answering the question.
|
||||
|
||||
# INSTRUCTIONS:
|
||||
1. Carefully analyze all provided memories. Synthesize information across different entries if needed to form a complete answer.
|
||||
2. Pay close attention to the timestamps to determine the answer. If memories contain contradictory information, the **most recent memory** is the source of truth.
|
||||
3. If the question asks about a specific event or fact, look for direct evidence in the memories.
|
||||
4. Your answer must be grounded in the memories. However, you may use general world knowledge to interpret or complete information found within a memory (e.g., identifying a landmark mentioned by description).
|
||||
5. If the question involves time references (like "last year", "two months ago", etc.), you **must** calculate the actual date based on the memory's timestamp. For example, if a memory from 4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
|
||||
6. Always convert relative time references to specific dates, months, or years in your final answer.
|
||||
7. Do not confuse character names mentioned in memories with the actual users who created them.
|
||||
8. The answer must be brief (under 5-6 words) and direct, with no extra description.
|
||||
|
||||
# APPROACH (Think step by step):
|
||||
1. First, examine all memories that contain information related to the question.
|
||||
2. Synthesize findings from multiple memories if a single entry is insufficient.
|
||||
3. Examine timestamps and content carefully, looking for explicit dates, times, locations, or events.
|
||||
4. If the answer requires calculation (e.g., converting relative time references), perform the calculation.
|
||||
5. Formulate a precise, concise answer based on the evidence from the memories (and allowed world knowledge).
|
||||
6. Double-check that your answer directly addresses the question asked and adheres to all instructions.
|
||||
7. Ensure your final answer is specific and avoids vague time references.
|
||||
|
||||
{context}
|
||||
|
||||
Question: {question}
|
||||
|
||||
Answer:
|
||||
|
||||
PROMPT_MEMOBASE: |
|
||||
You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
|
||||
|
||||
# CONTEXT:
|
||||
You have access to memories from two speakers in a conversation. These memories contain
|
||||
timestamped information that may be relevant to answering the question.
|
||||
|
||||
# INSTRUCTIONS:
|
||||
1. Carefully analyze all provided memories from both speakers
|
||||
2. Pay special attention to the timestamps to determine the answer
|
||||
3. If the question asks about a specific event or fact, look for direct evidence in the memories
|
||||
4. If the memories contain contradictory information, prioritize the most recent memory
|
||||
5. If there is a question about time references (like "last year", "two months ago", etc.), calculate the actual date based on the memory timestamp. For example, if a memory from 4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
|
||||
6. Always convert relative time references to specific dates, months, or years. For example, convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory timestamp. Ignore the reference while answering the question.
|
||||
7. Focus only on the content of the memories from both speakers. Do not confuse character names mentioned in memories with the actual users who created those memories.
|
||||
8. The answer should be less than 5-6 words.
|
||||
|
||||
# APPROACH (Think step by step):
|
||||
1. First, examine all memories that contain information related to the question
|
||||
2. Examine the timestamps and content of these memories carefully
|
||||
3. Look for explicit mentions of dates, times, locations, or events that answer the question
|
||||
4. If the answer requires calculation (e.g., converting relative time references), show your work
|
||||
5. Formulate a precise, concise answer based solely on the evidence in the memories
|
||||
6. Double-check that your answer directly addresses the question asked
|
||||
7. Ensure your final answer is specific and avoids vague time references
|
||||
|
||||
{context}
|
||||
|
||||
Question: {question}
|
||||
|
||||
Answer:
|
||||
|
||||
|
||||
EVALUATION_PROMPT_FOR_MEMORY_INTEGRITY: |
|
||||
You are a strict **"Memory Integrity" evaluator**.
|
||||
Your core task is to assess whether an AI memory system has **missed any key memory points** after processing a conversation. This evaluation measures the system’s **memory integrity**, i.e., its ability to resist **amnesia** or **omission**.
|
||||
|
||||
# Evaluation Context & Data:
|
||||
|
||||
1. **Extracted Memories:**
|
||||
These are all the memory items actually extracted by the memory system.
|
||||
{memories}
|
||||
|
||||
2. **Expected Memory Point:**
|
||||
The key memory point that *should* have been extracted.
|
||||
{expected_memory_point}
|
||||
|
||||
# Evaluation Instructions:
|
||||
|
||||
1. For each **Expected Memory Point**, search within the **Extracted Memories** list for corresponding or related information. Ignore unrelated items.
|
||||
2. Based on the following scoring rubric, rate how well the memory system captured the **Expected Memory Point** and provide a detailed explanation.
|
||||
|
||||
# Scoring Rubric:
|
||||
|
||||
* **2:** Fully covered or implied.
|
||||
One or more items in “Extracted Memories” fully cover or logically imply all information in the “Expected Memory Point.”
|
||||
|
||||
* **1:** Partially covered or mentioned.
|
||||
Some information in “Extracted Memories” mentions part of the “Expected Memory Point,” but key information is missing, inaccurate, or slightly incorrect.
|
||||
|
||||
* **0:** Not mentioned or incorrect.
|
||||
“Extracted Memories” contains no mention of the “Expected Memory Point,” or the corresponding information is entirely wrong.
|
||||
|
||||
# Scoring Notes:
|
||||
|
||||
* For **compound Expected Memory Points** (with multiple elements such as person/event/time/location/preference, etc.):
|
||||
|
||||
* All elements correct → **2 points**
|
||||
* Some elements correct / uncertain → **1 point**
|
||||
* Key elements missing or wrong → **0 points**
|
||||
|
||||
* Semantic matching is acceptable; exact wording is **not** required.
|
||||
|
||||
* If “Extracted Memories” contains **conflicting information**, assign the **best possible coverage score** and mention the conflict in your reasoning.
|
||||
|
||||
* Extra or stylistically different memories do **not** reduce the score; only the coverage of the **Expected Memory Point** matters.
|
||||
|
||||
* For uncertain wording (“might,” “probably,” “tends to,” etc.):
|
||||
|
||||
* If the Expected Memory Point is a definite statement, usually assign **1 point**.
|
||||
|
||||
* If critical fields (e.g., time, entity name, relationship) are partly wrong but others match → **1 point**.
|
||||
|
||||
* If all key fields are wrong or missing → **0 points**.
|
||||
|
||||
# Output Format:
|
||||
|
||||
Please output your result in the following JSON format:
|
||||
|
||||
```json
|
||||
{{
|
||||
"reasoning": "Provide a concise justification for the score",
|
||||
"score": "2|1|0"
|
||||
}}
|
||||
```
|
||||
|
||||
EVALUATION_PROMPT_FOR_MEMORY_ACCURACY: |
|
||||
You are a **Dialogue Memory Accuracy Evaluator.** Your task is to evaluate the **accuracy** of a memory extracted by an AI memory system, based on three given inputs: the dialogue content, the *target (gold)* memory points (the correct annotated memories), and the *candidate* memory to be evaluated. The goal is to output a **structured evaluation result**.
|
||||
|
||||
# Input Content
|
||||
|
||||
* **Dialogue:**
|
||||
{dialogue}
|
||||
|
||||
* **Golden Memories (Target Memory Points):**
|
||||
The correct memory points pre-annotated for this dialogue in the evaluation dataset.
|
||||
{golden_memories}
|
||||
|
||||
* **Candidate Memory:**
|
||||
The memory extracted by the system to be evaluated.
|
||||
{candidate_memory}
|
||||
|
||||
# Evaluation Principles and Definitions
|
||||
|
||||
### 1) Support / Entailment
|
||||
|
||||
* An **information point** (atomic fact) in the candidate memory is considered *supported* if it can be directly stated or semantically entailed (via synonym, paraphrase, or equivalent expression) by the *Dialogue* or *Golden Memories*.
|
||||
* Only the given dialogue and golden memories can be used for judgment — **no external knowledge** or assumptions are allowed.
|
||||
Any information not appearing in or inferable from these two sources is considered *unsupported*.
|
||||
* Pay careful attention to **negation**, **quantities**, **time**, and **subjects**.
|
||||
If the candidate statement contradicts the dialogue or golden memories, it is considered a **conflict**.
|
||||
|
||||
### 2) Memory Accuracy Score (integer: 0 / 1 / 2)
|
||||
|
||||
* **2 points:** Every information point in the candidate memory is supported by the dialogue or golden memories, with **no contradictions or hallucinations**.
|
||||
* **1 point:** The candidate memory is *partially correct* (at least one supported information point) but also includes *unsupported* or *contradictory* content.
|
||||
* **0 points:** The candidate memory is **entirely unsupported or contradictory** to the sources (i.e., a “hallucinated memory”).
|
||||
|
||||
> Note:
|
||||
>
|
||||
> * If a candidate memory contains multiple information points, **any unsupported or contradictory element** prevents a full score (2).
|
||||
> * If both supported and unsupported/conflicting content appear, assign a score of **1**.
|
||||
|
||||
### 3) Inclusion in Golden Memories (Boolean field-level judgment)
|
||||
|
||||
**Definition:**
|
||||
|
||||
* **Atomic information point:** the smallest factual unit in the candidate memory (e.g., *name = Li Si*, *age = 25*, *location = Beijing*, *preference = coffee*, *budget ≤ 2000*, *meeting_time = Wednesday 10:00*, *tool = Zoom*, etc.).
|
||||
* **Field / Slot:** the semantic dimension of an information point (e.g., *name*, *age*, *residence*, *food preference*, *budget*, *meeting time*, *meeting tool*, etc.).
|
||||
|
||||
**Judgment Rules (independent of correctness):**
|
||||
|
||||
* **true:**
|
||||
Every atomic information point in the candidate memory has a corresponding **field** in the golden memories (allowing for synonyms, paraphrases, or equivalent expressions; ignore value, polarity, or quantity differences).
|
||||
|
||||
* Note: A single field in the gold list may match multiple candidate points (e.g., multiple “drink preference” facts can be covered by one “drink preference” field in gold).
|
||||
* **false:**
|
||||
If **any** atomic information point’s field in the candidate memory cannot be found in the golden memories, mark as *false*.
|
||||
|
||||
**Important Notes:**
|
||||
|
||||
* Field matching is restricted to fields that are **explicitly present or semantically recognizable** in the golden memories — no external knowledge may be used to expand the field set.
|
||||
* Differences in **values** (e.g., “Zhang San” vs. “Li Si”), **polarity** (like/dislike), or **exact number/time** do **not** affect this Boolean judgment.
|
||||
|
||||
# Evaluation Procedure
|
||||
|
||||
For each candidate memory:
|
||||
|
||||
1. **Decompose** it into atomic information points (e.g., name, number, location, preference).
|
||||
2. For each information point, **search** the dialogue and golden memories for supporting or contradictory evidence.
|
||||
3. Assign the **accuracy_score** (0 / 1 / 2) according to the rules above.
|
||||
4. Determine **is_included_in_golden_memories (true/false)**:
|
||||
|
||||
* Identify each information point’s field;
|
||||
* If *all* fields exist in the golden memories, mark as *true*; otherwise, *false*.
|
||||
5. Provide a **concise Chinese explanation** in `"reason"`, citing key evidence (short excerpts allowed), and clearly state any unsupported or contradictory parts if applicable.
|
||||
|
||||
# Output Format (strictly required)
|
||||
|
||||
Output **only one JSON object**, with the following three fields:
|
||||
|
||||
* `"accuracy_score"`: `"0"` or `"1"` or `"2"`
|
||||
* `"is_included_in_golden_memories"`: `"true"` or `"false"`
|
||||
* `"reason"`: `"brief explanation in Chinese"`
|
||||
|
||||
Do **not** include any other text, explanation, or fields.
|
||||
Do **not** include the candidate memory text inside the JSON.
|
||||
|
||||
Please output **only** the following JSON (in a code block):
|
||||
|
||||
```json
|
||||
{{
|
||||
"accuracy_score": "2 | 1 | 0",
|
||||
"is_included_in_golden_memories": "true | false",
|
||||
"reason": "Brief explanation in Chinese"
|
||||
}}
|
||||
```
|
||||
|
||||
EVALUATION_PROMPT_FOR_UPDATE_MEMORY: |
|
||||
Your task is to **evaluate the update accuracy** of an AI memory system.
|
||||
Based on the information provided below, determine whether the system-generated **“Generated Memories”** correctly **includes** the **Target Memory for Update**.
|
||||
|
||||
# Background Information
|
||||
|
||||
The following information is provided for evaluation:
|
||||
|
||||
1. **Generated Memories:**
|
||||
This is the list of memory points generated by the system after the current dialogue.
|
||||
{memories}
|
||||
|
||||
2. **Target Memory for Update:**
|
||||
This is the correct, updated version of the memory point that should have been produced — the one we focus on in this evaluation.
|
||||
{updated_memory}
|
||||
|
||||
3. **Original Memory Content:**
|
||||
This is the original version of the target memory before the update.
|
||||
{original_memory}
|
||||
|
||||
# Evaluation Criteria
|
||||
|
||||
Please make your judgment **strictly based on the content update of the “Target Memory for Update.”**
|
||||
Use the following categories:
|
||||
|
||||
### Correct Update
|
||||
|
||||
* **Generated Memories** **contains all information points** from the “Target Memory for Update,” accurately and completely reflecting the intended update.
|
||||
* **Key fields** (e.g., date, time, values, proper nouns, etc.) must match exactly.
|
||||
* The **original memory** is effectively replaced or marked as outdated.
|
||||
* Synonymous or slightly rephrased expressions are acceptable.
|
||||
|
||||
### Hallucinated Update
|
||||
|
||||
* **Factual error:** The **Generated Memories** includes a new memory related to the “Target Memory for Update,” but its content contains factual mistakes or contradictions compared to the correct update.
|
||||
|
||||
### Omitted Update
|
||||
|
||||
* **Completely omitted:** The **Generated Memories** contains no new memory related to the “Target Memory for Update.”
|
||||
* **Partially omitted:** A related new memory was generated in **Generated Memories**, but it **misses key information** that should have been included.
|
||||
|
||||
### Other
|
||||
|
||||
Used for update failures that do **not clearly fall** into the above categories of “Hallucination” or “Omission.”
|
||||
|
||||
# Output Requirements
|
||||
|
||||
Please return your evaluation strictly in the following JSON format and provide a concise explanation.
|
||||
|
||||
```json
|
||||
{{
|
||||
"reason": "Briefly explain your reasoning here and why it fits this category.",
|
||||
"evaluation_result": "Correct | Hallucination | Omission | Other"
|
||||
}}
|
||||
```
|
||||
|
||||
EVALUATION_PROMPT_FOR_QUESTION: |
|
||||
You are an **evaluation expert for AI memory system question answering**.
|
||||
Based **only** on the provided **“Question”**, **“Reference Answer”**, and **“Key Memory Points”** (the essential facts needed to derive the reference answer), strictly evaluate the **accuracy** of the **“Memory System Response.”** Classify it as one of **“Correct”**, **“Hallucination”**, or **“Omission.”** Do **not** use any external knowledge or subjective inference. Finally, output your judgment **strictly** in the specified JSON format.
|
||||
|
||||
# Evaluation Criteria
|
||||
|
||||
## Answer Type Classification
|
||||
|
||||
### 1. Correct
|
||||
|
||||
* The “Memory System Response” accurately answers the “Question,” and its content is **semantically equivalent** to the “Reference Answer.”
|
||||
* It contains **no contradictions** with the “Key Memory Points” or “Reference Answer.”
|
||||
* It introduces **no unsupported details** beyond the “Key Memory Points” that could alter the conclusion.
|
||||
* Synonyms, paraphrasing, and reasonable summarization are acceptable.
|
||||
|
||||
### 2. Hallucination
|
||||
|
||||
* The “Memory System Response” includes information or facts that **contradict or are inconsistent** with the “Reference Answer” or the “Key Memory Points.”
|
||||
* When the “Reference Answer” is labeled as *unknown/uncertain*, yet the response provides a specific verifiable fact or conclusion.
|
||||
* Extra irrelevant information that does **not change** the conclusion is **not** considered hallucination by itself; however, if it **changes or misleads** the conclusion, or **contradicts** the “Key Memory Points,” it should be judged as a **Hallucination**.
|
||||
|
||||
### 3. Omission
|
||||
|
||||
* The response is **incomplete** compared to the “Reference Answer.”
|
||||
* It explicitly states “don’t know,” “can’t remember,” or “no related memory,” even though relevant information exists in the “Key Memory Points.”
|
||||
* For multi-element questions, **all elements must be correct and present**; omission of **any** element is considered an **Omission**.
|
||||
|
||||
## Priority Rules (Conflict Handling)
|
||||
|
||||
* If the response contains **both missing necessary information** and **fabricated/contradictory information**, classify it as **Hallucination**.
|
||||
* If there is **no fabrication/contradiction** but some necessary information is missing, classify it as **Omission**.
|
||||
* Only when the meaning is **fully equivalent** to the reference answer should it be classified as **Correct**.
|
||||
|
||||
## Detailed Guidelines and Tolerance
|
||||
|
||||
* Equivalent expressions of numbers, times, and units are acceptable, but the **numerical values themselves must not differ**.
|
||||
* For multi-element questions, **all elements must be complete and accurate**; missing any element counts as **Omission**.
|
||||
* If the reference answer is *“unknown / cannot be determined”* and the system provides a definite fact, that is a **Hallucination**.
|
||||
If the system also answers *“unknown”* (without guessing), it may be **Correct**.
|
||||
* The evaluation must rely **only** on the *Reference Answer*, *Key Memory Points*, and *System Response* — no external context, world knowledge, or speculative reasoning is allowed.
|
||||
|
||||
# Information for Evaluation
|
||||
|
||||
* **Question:**
|
||||
{question}
|
||||
|
||||
* **Reference Answer:**
|
||||
{reference_answer}
|
||||
|
||||
* **Key Memory Points:**
|
||||
{key_memory_points}
|
||||
|
||||
* **Memory System Response:**
|
||||
{response}
|
||||
|
||||
# Output Requirements
|
||||
|
||||
Please provide your evaluation result **strictly** in the JSON format below.
|
||||
Do **not** add any extra explanation or comments outside the JSON block.
|
||||
|
||||
```json
|
||||
{{
|
||||
"reasoning": "Provide a concise and traceable evaluation rationale: first compare the system’s response with the Key Memory Points (which were correctly used, which were missing, and whether there was any fabrication/contradiction), then assess its consistency with the Reference Answer, and finally state the classification basis.",
|
||||
"evaluation_result": "Correct | Hallucination | Omission"
|
||||
}}
|
||||
```
|
||||
|
||||
|
||||
EVALUATION_PROMPT_FOR_QUESTION2: |
|
||||
You are an **evaluation expert for AI memory system question answering**.
|
||||
|
||||
Based **only** on the provided **"Question"**, **"Reference Answer"**, and **"Key Memory Points"** (the essential facts needed to derive the reference answer), strictly evaluate the **accuracy** of the **"Memory System Response."** Classify it as one of **"Correct"**, **"Hallucination"**, or **"Omission."** Do **not** use any external knowledge or subjective inference. Finally, output your judgment **strictly** in the specified JSON format.
|
||||
|
||||
# Evaluation Criteria
|
||||
|
||||
## Answer Type Classification
|
||||
|
||||
### 1. Correct
|
||||
|
||||
* The "Memory System Response" accurately answers the "Question," and its content is **semantically equivalent** to the "Reference Answer."
|
||||
* It contains **no contradictions** with the "Key Memory Points" or "Reference Answer."
|
||||
* **Extra details not present in the Key Memory Points are allowed and should not be penalized**, as long as they:
|
||||
- Do not contradict the Key Memory Points or Reference Answer
|
||||
- Do not change or mislead the core conclusion
|
||||
- Are reasonable additional context that the memory system may have retained from the conversation
|
||||
* The memory system may have stored additional information beyond the Key Memory Points. Such extra information should be treated as **supplementary context** rather than hallucination, provided it does not conflict with the core answer.
|
||||
* Synonyms, paraphrasing, and reasonable summarization are acceptable.
|
||||
|
||||
### 2. Hallucination
|
||||
|
||||
* The "Memory System Response" includes information or facts that **contradict or are inconsistent** with the "Reference Answer" or the "Key Memory Points."
|
||||
* The response provides information that **directly contradicts** known facts from the Key Memory Points.
|
||||
* When the "Reference Answer" is labeled as *unknown/uncertain*, yet the response provides a specific verifiable fact or conclusion.
|
||||
* **Important:** Extra information that is NOT in Key Memory Points is **NOT automatically a hallucination**. Only classify as hallucination if the extra information:
|
||||
- Directly contradicts the Key Memory Points or Reference Answer
|
||||
- Changes or misleads the core conclusion in a way that makes the answer incorrect
|
||||
- Provides a definitive answer when the Reference Answer indicates uncertainty
|
||||
|
||||
### 3. Omission
|
||||
|
||||
* The response is **incomplete** compared to the "Reference Answer."
|
||||
* It explicitly states "don't know," "can't remember," or "no related memory," even though relevant information exists in the "Key Memory Points."
|
||||
* For multi-element questions, **all elements must be correct and present**; omission of **any** element is considered an **Omission**.
|
||||
|
||||
## Priority Rules (Conflict Handling)
|
||||
|
||||
* If the response contains **both missing necessary information** and **fabricated/contradictory information**, classify it as **Hallucination**.
|
||||
* If there is **no fabrication/contradiction** but some necessary information is missing, classify it as **Omission**.
|
||||
* If the core answer is correct and complete, classify as **Correct** even if there are extra details not in Key Memory Points (as long as they don't contradict or mislead).
|
||||
|
||||
## Detailed Guidelines and Tolerance
|
||||
|
||||
* Equivalent expressions of numbers, times, and units are acceptable, but the **numerical values themselves must not differ**.
|
||||
* For multi-element questions, **all elements must be complete and accurate**; missing any element counts as **Omission**.
|
||||
* If the reference answer is *"unknown / cannot be determined"* and the system provides a definite fact, that is a **Hallucination**.
|
||||
If the system also answers *"unknown"* (without guessing), it may be **Correct**.
|
||||
* **Focus on evaluating whether the core answer to the question is correct**, not whether the response is limited to only the Key Memory Points.
|
||||
* Extra contextual information (e.g., additional preferences, related details) should be viewed as enrichment, not as errors, unless they contradict or mislead.
|
||||
|
||||
# Information for Evaluation
|
||||
|
||||
* **Question:**
|
||||
{question}
|
||||
|
||||
* **Reference Answer:**
|
||||
{reference_answer}
|
||||
|
||||
* **Key Memory Points:**
|
||||
{key_memory_points}
|
||||
|
||||
* **Memory System Response:**
|
||||
{response}
|
||||
|
||||
# Output Requirements
|
||||
|
||||
Please provide your evaluation result **strictly** in the JSON format below.
|
||||
Do **not** add any extra explanation or comments outside the JSON block.
|
||||
|
||||
```json
|
||||
{{
|
||||
"reasoning": "Provide a concise and traceable evaluation rationale: first verify that the system's response correctly includes all required elements from the Reference Answer, then check if any information contradicts the Key Memory Points or Reference Answer. Extra details not in Key Memory Points should be noted but not penalized unless they contradict or mislead. Finally state the classification basis.",
|
||||
"evaluation_result": "Correct | Hallucination | Omission"
|
||||
}}
|
||||
```
|
||||
"""
|
||||
|
|
@ -73,3 +73,10 @@ class BaseMemoryAgent(BaseReact, metaclass=ABCMeta):
|
|||
def author(self) -> str:
|
||||
"""Returns the LLM model name as the author identifier."""
|
||||
return self.llm.model_name
|
||||
|
||||
@property
|
||||
def retrieved_nodes(self) -> list[MemoryNode]:
|
||||
"""Returns the retrieved nodes."""
|
||||
if "retrieved_nodes" not in self.context:
|
||||
self.context.retrieved_nodes = []
|
||||
return self.context.retrieved_nodes
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from ..base_memory_agent import BaseMemoryAgent
|
||||
from ....core.enumeration import Role, MemoryType
|
||||
from ....core.op import BaseTool
|
||||
from ....core.schema import Message, MemoryNode
|
||||
from ....core.schema import Message
|
||||
from ....core.utils import format_messages
|
||||
|
||||
|
||||
|
|
@ -12,10 +12,6 @@ class PersonalRetriever(BaseMemoryAgent):
|
|||
|
||||
memory_type: MemoryType = MemoryType.PERSONAL
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.retrieved_nodes: list[MemoryNode] = []
|
||||
|
||||
async def build_messages(self) -> list[Message]:
|
||||
if self.context.get("query"):
|
||||
context = self.context.query
|
||||
|
|
@ -59,3 +55,9 @@ class PersonalRetriever(BaseMemoryAgent):
|
|||
retrieved_nodes=self.retrieved_nodes,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def execute(self):
|
||||
result = await super().execute()
|
||||
result["retrieved_nodes"] = self.retrieved_nodes
|
||||
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -85,19 +85,26 @@ class PersonalSummarizer(BaseMemoryAgent):
|
|||
messages_phase1 = await self._build_phase1_messages()
|
||||
for i, message in enumerate(messages_phase1):
|
||||
role = message.name or message.role
|
||||
logger.info(f"[{self.__class__.__name__}-S1] role={role} {message.simple_dump(as_dict=False)}")
|
||||
logger.info(f"[{self.__class__.__name__} S1] role={role} {message.simple_dump(as_dict=False)}")
|
||||
tools_phase1, messages_phase1, success_phase1 = await self.react(messages_phase1, tools[:-1], stage="S1")
|
||||
|
||||
messages_phase2 = await self._build_phase2_messages()
|
||||
for i, message in enumerate(messages_phase2):
|
||||
role = message.name or message.role
|
||||
logger.info(f"[{self.__class__.__name__}-S2] role={role} {message.simple_dump(as_dict=False)}")
|
||||
logger.info(f"[{self.__class__.__name__} S2] role={role} {message.simple_dump(as_dict=False)}")
|
||||
tools_phase2, messages_phase2, success_phase2 = await self.react(messages_phase2, tools[-1:], stage="S2")
|
||||
|
||||
success = success_phase1 and success_phase2
|
||||
messages = messages_phase1 + messages_phase2
|
||||
tools = tools_phase1 + tools_phase2
|
||||
memory_nodes = []
|
||||
for tool in tools:
|
||||
if tool.memory_nodes:
|
||||
memory_nodes.extend(tool.memory_nodes)
|
||||
|
||||
return {
|
||||
"answer": (messages_phase1[-1].content if success_phase1 else "")
|
||||
+ (messages_phase2[-1].content if success_phase2 else ""),
|
||||
"success": success_phase1 and success_phase2,
|
||||
"messages": messages_phase1 + messages_phase2,
|
||||
"tools": tools_phase1 + tools_phase2,
|
||||
"answer": memory_nodes,
|
||||
"success": success,
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ system_prompt_phase1: |
|
|||
### Step 2: Add New Memories
|
||||
Use `add_memory` to add new memories:
|
||||
- Extract and summarize important information about **{memory_target}**
|
||||
- Set `conversation_time` (format: 2020-01-01 00:00:00; use 0000-00-00 00:00:00 if unavailable)
|
||||
- Set `update_time` (format: 2020-01-01 00:00:00; use 0000-00-00 00:00:00 if unavailable)
|
||||
- If the information is completely identical to existing memory, skip adding
|
||||
|
||||
user_message_phase1: |
|
||||
|
|
@ -30,15 +30,15 @@ system_prompt_phase2: |
|
|||
{context}
|
||||
|
||||
## Current User Profile:
|
||||
UserProfile format: `profile_id=<id> conversation_time=<timestamp> <content>`.
|
||||
UserProfile format: `profile_id=<id> update_time=<timestamp> <content>`.
|
||||
{user_profile}
|
||||
|
||||
## Task: Update Profile with `UpdateUserProfile`
|
||||
**CRITICAL**: Extract ONLY explicitly stated information. DO NOT infer, assume, or fabricate.
|
||||
|
||||
Synchronize profile/memories with new information from the conversation, including **{memory_target}**' current status:
|
||||
- `profile_ids_to_delete`: Remove outdated, conflicting, or redundant entries.
|
||||
- `profiles_to_add`: Add new profiles/memories with `conversation_time`, e.g. `YYYY-MM-DD HH:MM:SS`, {memory_target} did something.
|
||||
- `profile_ids_to_delete`: Remove conflicting, or redundant entries.
|
||||
- `profiles_to_add`: Add new profiles/memories with `update_time`, e.g. `YYYY-MM-DD HH:MM:SS`, {memory_target} did something.
|
||||
- Maintain profiles that are concise, mutually exclusive, and collectively comprehensive with no information loss.
|
||||
|
||||
user_message_phase2: |
|
||||
|
|
|
|||
|
|
@ -62,19 +62,23 @@ class ReMeRetriever(BaseMemoryAgent):
|
|||
hands_off_tool = tools[0]
|
||||
agents: list[BaseMemoryAgent] = hands_off_tool.response.metadata["agents"]
|
||||
|
||||
answer = ""
|
||||
answer = []
|
||||
success = True
|
||||
messages = []
|
||||
tools = []
|
||||
retrieved_nodes = []
|
||||
|
||||
for agent in agents:
|
||||
answer += "\n" + agent.response.answer
|
||||
answer.append(agent.response.answer)
|
||||
success = success and agent.response.success
|
||||
messages += agent.response.metadata["messages"]
|
||||
tools += agent.response.metadata["tools"]
|
||||
messages.extend(agent.response.metadata["messages"])
|
||||
tools.extend(agent.response.metadata["tools"])
|
||||
retrieved_nodes.extend(agent.response.metadata["retrieved_nodes"])
|
||||
|
||||
return {
|
||||
"answer": answer.strip(),
|
||||
"answer": "\n".join(answer),
|
||||
"success": True,
|
||||
"messages": self.messages,
|
||||
"tools": tools,
|
||||
"retrieved_nodes": retrieved_nodes,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,19 +58,19 @@ class ReMeSummarizer(BaseMemoryAgent):
|
|||
hands_off_tool = tools[0]
|
||||
agents: list[BaseMemoryAgent] = hands_off_tool.response.metadata["agents"]
|
||||
|
||||
answer = ""
|
||||
success = True
|
||||
messages = []
|
||||
tools = []
|
||||
memory_nodes = []
|
||||
for agent in agents:
|
||||
answer += "\n" + agent.response.answer
|
||||
success = success and agent.response.success
|
||||
messages += agent.response.metadata["messages"]
|
||||
tools += agent.response.metadata["tools"]
|
||||
messages.extend(agent.response.metadata["messages"])
|
||||
tools.extend(agent.response.metadata["tools"])
|
||||
memory_nodes.extend(agent.response.answer)
|
||||
|
||||
return {
|
||||
"answer": answer.strip(),
|
||||
"answer": memory_nodes,
|
||||
"success": True,
|
||||
"messages": self.messages,
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from ..enumeration import ChunkEnum, Role
|
|||
from ..schema import Message
|
||||
from ..schema import StreamChunk
|
||||
from ..schema import ToolCall
|
||||
from ..utils import extract_content
|
||||
|
||||
|
||||
class BaseLLM(ABC):
|
||||
|
|
@ -458,6 +459,43 @@ class BaseLLM(ABC):
|
|||
time.sleep(1 + i)
|
||||
return default_value
|
||||
|
||||
async def simple_request(
|
||||
self,
|
||||
prompt: str,
|
||||
model_name: str,
|
||||
callback_fn: Callable[[Message], Any] | None = None,
|
||||
default_value: Any = None,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""Make a simple request using the LLM."""
|
||||
assistant_message = await self.chat(
|
||||
messages=[Message(role=Role.USER, content=prompt)],
|
||||
model_name=model_name,
|
||||
callback_fn=callback_fn,
|
||||
default_value=default_value,
|
||||
**kwargs,
|
||||
)
|
||||
return assistant_message.content
|
||||
|
||||
async def simple_request_for_json(
|
||||
self,
|
||||
prompt: str,
|
||||
model_name: str,
|
||||
**kwargs,
|
||||
) -> dict:
|
||||
"""Make a simple request using the LLM and extract JSON."""
|
||||
|
||||
def extract_fn(message: Message) -> dict:
|
||||
return extract_content(message.content)
|
||||
|
||||
return await self.chat(
|
||||
messages=[Message(role=Role.USER, content=prompt)],
|
||||
model_name=model_name,
|
||||
callback_fn=extract_fn,
|
||||
default_value={},
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def close(self):
|
||||
"""Release async resources."""
|
||||
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ class BaseReact(BaseOp):
|
|||
logger.warning(f"{prefix} unknown tool_call={tool_call.name}")
|
||||
continue
|
||||
|
||||
logger.info(f"{prefix} submit tool_calls={tool_call.simple_output_dump(as_dict=False)}")
|
||||
logger.info(f"{prefix} submit tool_call[{tool_call.name}] arguments={tool_call.arguments}")
|
||||
|
||||
# Create independent tool copy with unique ID
|
||||
tool_copy: BaseTool = tool_dict[tool_call.name].copy()
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
import json
|
||||
import re
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from ..enumeration import Role
|
||||
|
|
@ -170,18 +171,18 @@ def extract_content(text: str, language_tag: str = "json", greedy: bool = False)
|
|||
pattern = rf"```\s*{re.escape(language_tag)}\s*({quantifier})\s*```"
|
||||
match = re.search(pattern, text, re.DOTALL)
|
||||
|
||||
if match:
|
||||
result = match.group(1).strip()
|
||||
else:
|
||||
result = text
|
||||
if not match:
|
||||
return None
|
||||
|
||||
content = match.group(1).strip()
|
||||
|
||||
if language_tag == "json":
|
||||
try:
|
||||
result = json.loads(result)
|
||||
return json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
result = None
|
||||
|
||||
return result
|
||||
return None
|
||||
else:
|
||||
return content
|
||||
|
||||
|
||||
def deduplicate_memories(memories: list[MemoryNode]) -> list[MemoryNode]:
|
||||
|
|
|
|||
240
reme/reme.py
240
reme/reme.py
|
|
@ -1,23 +1,24 @@
|
|||
"""ReMe application classes for simplified configuration and execution."""
|
||||
"""ReMe classes for simplified configuration and execution."""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from .agent.memory.default import ReMeSummarizer, PersonalSummarizer, PersonalRetriever, ReMeRetriever
|
||||
from .config import ReMeConfigParser
|
||||
from .core.context import ServiceContext
|
||||
from .core.context import PromptHandler, ServiceContext
|
||||
from .core.embedding import BaseEmbeddingModel
|
||||
from .core.enumeration import MemoryType
|
||||
from .core.flow import BaseFlow
|
||||
from .core.llm import BaseLLM
|
||||
from .core.schema import Response, Message
|
||||
from .core.schema import Response, Message, MemoryNode, VectorNode
|
||||
from .core.token_counter import BaseTokenCounter
|
||||
from .core.utils import execute_stream_task
|
||||
from .core.utils import execute_stream_task, get_now_time
|
||||
from .core.vector_store import BaseVectorStore
|
||||
from .tool.memory import UpdateUserProfile, RetrieveMemory, AddMemory, HandsOff, ReadHistory
|
||||
from .tool.memory import UpdateUserProfile, RetrieveMemory, AddMemory, HandsOff, ReadHistory, ReadUserProfile
|
||||
|
||||
|
||||
class ReMe:
|
||||
"""ReMe application with config file support and flow execution methods."""
|
||||
"""ReMe with config file support and flow execution methods."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -50,6 +51,8 @@ class ReMe:
|
|||
**kwargs,
|
||||
)
|
||||
|
||||
self.prompt_handler = PromptHandler(language=self.service_context.language)
|
||||
|
||||
async def __aenter__(self):
|
||||
"""Async context manager entry."""
|
||||
return self
|
||||
|
|
@ -59,11 +62,11 @@ class ReMe:
|
|||
return self
|
||||
|
||||
async def close(self):
|
||||
"""Close the application."""
|
||||
"""Close"""
|
||||
return await self.service_context.close()
|
||||
|
||||
def close_sync(self):
|
||||
"""Close the application synchronously."""
|
||||
"""Close synchronously"""
|
||||
self.service_context.close_sync()
|
||||
|
||||
async def __aexit__(self, exc_type=None, exc_val=None, exc_tb=None):
|
||||
|
|
@ -77,26 +80,26 @@ class ReMe:
|
|||
return False
|
||||
|
||||
@property
|
||||
def default_llm(self) -> BaseLLM:
|
||||
def llm(self) -> BaseLLM:
|
||||
"""Return the default LLM instance from the service context."""
|
||||
return self.service_context.llms["default"]
|
||||
|
||||
@property
|
||||
def default_embedding_model(self) -> BaseEmbeddingModel:
|
||||
def embedding_model(self) -> BaseEmbeddingModel:
|
||||
"""Return the default embedding model instance from the service context."""
|
||||
return self.service_context.embedding_models["default"]
|
||||
|
||||
@property
|
||||
def default_vector_store(self) -> BaseVectorStore:
|
||||
def vector_store(self) -> BaseVectorStore:
|
||||
"""Return the default vector store instance from the service context."""
|
||||
return self.service_context.vector_stores["default"]
|
||||
|
||||
@property
|
||||
def default_token_counter(self) -> BaseTokenCounter:
|
||||
def token_counter(self) -> BaseTokenCounter:
|
||||
"""Return the default token counter instance from the service context."""
|
||||
return self.service_context.token_counters["default"]
|
||||
|
||||
async def summary(
|
||||
async def summary_memory(
|
||||
self,
|
||||
messages: list[Message | dict],
|
||||
description: str = "",
|
||||
|
|
@ -104,11 +107,11 @@ class ReMe:
|
|||
enable_thinking_params: bool = False,
|
||||
meta_memories: list[dict] = None,
|
||||
version: str = "default",
|
||||
return_dict: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
) -> str | dict:
|
||||
"""Summarize messages and store them in memory for the specified user(s)."""
|
||||
if user_name:
|
||||
|
||||
if isinstance(user_name, str):
|
||||
for message in messages:
|
||||
if isinstance(message, dict) and not message.get("name"):
|
||||
|
|
@ -147,17 +150,22 @@ class ReMe:
|
|||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
return await reme_summarizer.call(
|
||||
result = await reme_summarizer.call(
|
||||
messages=messages,
|
||||
description=description,
|
||||
service_context=self.service_context,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if return_dict:
|
||||
return result
|
||||
else:
|
||||
return result["answer"]
|
||||
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
async def retrieve(
|
||||
async def retrieve_memory(
|
||||
self,
|
||||
query: str = "",
|
||||
top_k: int = 20,
|
||||
|
|
@ -167,8 +175,9 @@ class ReMe:
|
|||
enable_thinking_params: bool = False,
|
||||
meta_memories: list[dict] = None,
|
||||
version: str = "default",
|
||||
return_dict: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
) -> str | dict:
|
||||
"""Retrieve relevant memories for the specified user(s) based on query or messages."""
|
||||
if user_name:
|
||||
if isinstance(user_name, str):
|
||||
|
|
@ -210,7 +219,7 @@ class ReMe:
|
|||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
return await reme_retriever.call(
|
||||
result = await reme_retriever.call(
|
||||
query=query,
|
||||
messages=messages,
|
||||
description=description,
|
||||
|
|
@ -218,9 +227,200 @@ class ReMe:
|
|||
**kwargs,
|
||||
)
|
||||
|
||||
if return_dict:
|
||||
return result
|
||||
else:
|
||||
return result["answer"]
|
||||
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
async def add_memory(
|
||||
self,
|
||||
memory_content: str,
|
||||
user_name: str,
|
||||
memory_type: str | MemoryType | None = None,
|
||||
memory_target: str = "",
|
||||
when_to_use: str = "",
|
||||
ref_memory_id: str = "",
|
||||
author: str = "",
|
||||
score: float = 0,
|
||||
conversation_time: str = "",
|
||||
**kwargs,
|
||||
) -> MemoryNode:
|
||||
"""Add a new memory to the vector store for the specified user."""
|
||||
|
||||
if user_name:
|
||||
memory_type = MemoryType.PERSONAL
|
||||
memory_target = user_name
|
||||
else:
|
||||
memory_type = MemoryType(memory_type)
|
||||
assert memory_target, "memory_target is required"
|
||||
|
||||
metadata = kwargs.copy()
|
||||
if conversation_time:
|
||||
metadata["conversation_time"] = conversation_time
|
||||
|
||||
memory_node = MemoryNode(
|
||||
memory_type=memory_type,
|
||||
memory_target=memory_target,
|
||||
when_to_use=when_to_use,
|
||||
content=memory_content,
|
||||
ref_memory_id=ref_memory_id,
|
||||
author=author,
|
||||
score=score,
|
||||
metadata=metadata,
|
||||
)
|
||||
vector_node = memory_node.to_vector_node()
|
||||
await self.vector_store.delete([vector_node.vector_id])
|
||||
await self.vector_store.insert([vector_node])
|
||||
|
||||
return memory_node
|
||||
|
||||
async def update_memory(
|
||||
self,
|
||||
memory_id: str,
|
||||
memory_content: str,
|
||||
user_name: str,
|
||||
memory_type: str | MemoryType | None = None,
|
||||
memory_target: str = "",
|
||||
when_to_use: str = "",
|
||||
ref_memory_id: str = "",
|
||||
author: str = "",
|
||||
score: float = 0,
|
||||
conversation_time: str = "",
|
||||
**kwargs,
|
||||
) -> MemoryNode:
|
||||
"""Update an existing memory in the vector store by its ID."""
|
||||
|
||||
if user_name:
|
||||
memory_type = MemoryType.PERSONAL
|
||||
memory_target = user_name
|
||||
else:
|
||||
memory_type = MemoryType(memory_type)
|
||||
assert memory_target, "memory_target is required"
|
||||
|
||||
metadata = kwargs.copy()
|
||||
if conversation_time:
|
||||
metadata["conversation_time"] = conversation_time
|
||||
|
||||
memory_node = MemoryNode(
|
||||
memory_type=memory_type,
|
||||
memory_target=memory_target,
|
||||
when_to_use=when_to_use,
|
||||
content=memory_content,
|
||||
ref_memory_id=ref_memory_id,
|
||||
author=author,
|
||||
score=score,
|
||||
metadata=metadata,
|
||||
)
|
||||
vector_node = memory_node.to_vector_node()
|
||||
await self.vector_store.delete([memory_id, vector_node.vector_id])
|
||||
await self.vector_store.insert([vector_node])
|
||||
|
||||
return memory_node
|
||||
|
||||
async def delete_memory(self, memory_id: str | list[str]):
|
||||
"""Delete one or more memories from the vector store by their IDs."""
|
||||
vector_ids = [memory_id] if isinstance(memory_id, str) else memory_id
|
||||
await self.vector_store.delete(vector_ids)
|
||||
|
||||
async def delete_all_memories(self):
|
||||
"""Delete all memories from the vector store."""
|
||||
await self.vector_store.delete_all()
|
||||
|
||||
async def get_memory(self, memory_id: str | list[str]) -> MemoryNode | list[MemoryNode]:
|
||||
"""Retrieve one or more memories from the vector store by their IDs."""
|
||||
vector_ids = [memory_id] if isinstance(memory_id, str) else memory_id
|
||||
vector_nodes = await self.vector_store.get(vector_ids)
|
||||
if isinstance(vector_nodes, VectorNode):
|
||||
return vector_nodes.to_memory_node()
|
||||
else:
|
||||
return [node.to_memory_node() for node in vector_nodes]
|
||||
|
||||
async def get_all_memories(self) -> list[MemoryNode]:
|
||||
"""Retrieve all memories from the vector store."""
|
||||
return [node.to_memory_node() for node in await self.vector_store.list()]
|
||||
|
||||
@staticmethod
|
||||
async def get_profiles(user_name: str | list[str]) -> str | list[str]:
|
||||
"""Retrieve user profile(s) from the system for the specified user(s)."""
|
||||
read_profile = ReadUserProfile(show_id="profile")
|
||||
if isinstance(user_name, str):
|
||||
return await read_profile.call(memory_target=user_name)
|
||||
else:
|
||||
return [await read_profile.call(memory_target=name) for name in user_name]
|
||||
|
||||
@staticmethod
|
||||
async def add_profile(
|
||||
profile_key: str,
|
||||
profile_value: str,
|
||||
user_name: str,
|
||||
update_time: str | None = None,
|
||||
) -> MemoryNode:
|
||||
"""Add user profile to ReMe system."""
|
||||
update_user_profile = UpdateUserProfile()
|
||||
if update_time is None:
|
||||
update_time = get_now_time()
|
||||
|
||||
await update_user_profile.call(
|
||||
profile_ids_to_delete=[],
|
||||
profiles_to_add=[
|
||||
{
|
||||
"update_time": update_time,
|
||||
"profile_key": profile_key,
|
||||
"profile_value": profile_value,
|
||||
},
|
||||
],
|
||||
memory_target=user_name,
|
||||
)
|
||||
return update_user_profile.memory_nodes[0]
|
||||
|
||||
@staticmethod
|
||||
async def update_profile(
|
||||
profile_id: str,
|
||||
profile_key: str,
|
||||
profile_value: str,
|
||||
user_name: str,
|
||||
update_time: str | None = None,
|
||||
) -> MemoryNode:
|
||||
"""Add user profile to ReMe system."""
|
||||
update_user_profile = UpdateUserProfile()
|
||||
if update_time is None:
|
||||
update_time = get_now_time()
|
||||
|
||||
await update_user_profile.call(
|
||||
profile_ids_to_delete=[profile_id],
|
||||
profiles_to_add=[
|
||||
{
|
||||
"update_time": update_time,
|
||||
"profile_key": profile_key,
|
||||
"profile_value": profile_value,
|
||||
},
|
||||
],
|
||||
memory_target=user_name,
|
||||
)
|
||||
return update_user_profile.memory_nodes[0]
|
||||
|
||||
@staticmethod
|
||||
async def delete_all_profiles(user_name: str | list[str]):
|
||||
"""Delete all user profiles from ReMe system."""
|
||||
if isinstance(user_name, str):
|
||||
user_name = [user_name]
|
||||
|
||||
read_profile = ReadUserProfile(show_id="profile")
|
||||
update_profile = UpdateUserProfile()
|
||||
for memory_target in user_name:
|
||||
await read_profile.call(memory_target=memory_target)
|
||||
profile_ids = [profile.memory_id for profile in read_profile.memory_nodes]
|
||||
await update_profile.call(profile_ids_to_delete=profile_ids, memory_target=memory_target)
|
||||
|
||||
async def context_offload(self):
|
||||
"""working memory summary"""
|
||||
|
||||
async def context_reload(self):
|
||||
"""working memory retrieve"""
|
||||
|
||||
async def execute_flow(self, name: str, **kwargs) -> Response:
|
||||
"""Execute a flow with the given name and parameters."""
|
||||
assert name in self.service_context.flows, f"Flow {name} not found"
|
||||
|
|
@ -248,7 +448,7 @@ class ReMe:
|
|||
|
||||
|
||||
def main():
|
||||
"""Main entry point for running ReMe application from command line."""
|
||||
"""Main entry point for running ReMe from command line."""
|
||||
with ReMe(*sys.argv[1:]) as app:
|
||||
app.run_service()
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ class BaseMemoryTool(BaseTool, metaclass=ABCMeta):
|
|||
self.enable_multiple: bool = enable_multiple
|
||||
self.enable_thinking_params: bool = enable_thinking_params
|
||||
self.local_memory_path: str = local_memory_path
|
||||
self.memory_nodes: list[MemoryNode | str] = []
|
||||
|
||||
def _build_tool_call(self) -> ToolCall:
|
||||
"""Build and return the tool call schema"""
|
||||
|
|
@ -88,9 +87,16 @@ class BaseMemoryTool(BaseTool, metaclass=ABCMeta):
|
|||
@property
|
||||
def retrieved_nodes(self) -> list[MemoryNode]:
|
||||
"""Get the retrieved nodes from context."""
|
||||
return self.context.get("retrieved_nodes")
|
||||
return self.context["retrieved_nodes"]
|
||||
|
||||
@property
|
||||
def author(self) -> str:
|
||||
"""Get the author from context."""
|
||||
return self.context.get("author", "")
|
||||
|
||||
@property
|
||||
def memory_nodes(self) -> list[MemoryNode | str]:
|
||||
"""Get the memory nodes from context."""
|
||||
if "memory_nodes" not in self.context:
|
||||
self.context.memory_nodes = []
|
||||
return self.context["memory_nodes"]
|
||||
|
|
|
|||
|
|
@ -40,7 +40,8 @@ class ReadHistory(BaseMemoryTool):
|
|||
logger.warning(output)
|
||||
return output
|
||||
|
||||
memory = MemoryNode.from_vector_node(nodes[0])
|
||||
output = f"Historical Dialogue[{history_id}]\n{memory.content}"
|
||||
logger.info(f"Successfully read history memory: {history_id}")
|
||||
memory_node: MemoryNode = MemoryNode.from_vector_node(nodes[0])
|
||||
self.retrieved_nodes.append(memory_node)
|
||||
output = f"Historical Dialogue[{history_id}]\n{memory_node.content}"
|
||||
logger.info(f"Successfully read history memory_node: {history_id}")
|
||||
return output
|
||||
|
|
|
|||
|
|
@ -38,7 +38,8 @@ class ReadUserProfile(BaseMemoryTool):
|
|||
return ""
|
||||
|
||||
nodes = [MemoryNode(**data) for data in cached_data]
|
||||
nodes.sort(key=lambda n: n.metadata.get("conversation_time", ""))
|
||||
self.memory_nodes = nodes
|
||||
nodes.sort(key=lambda n: n.metadata.get("update_time", ""))
|
||||
|
||||
formatted_profiles = []
|
||||
for node in nodes:
|
||||
|
|
@ -46,8 +47,8 @@ class ReadUserProfile(BaseMemoryTool):
|
|||
if self.show_id == "profile":
|
||||
parts.append(f"profile_id={node.memory_id}")
|
||||
|
||||
if conv_time := node.metadata.get("conversation_time"):
|
||||
parts.append(f"conversation_time={conv_time}")
|
||||
if update_time := node.metadata.get("update_time"):
|
||||
parts.append(f"update_time={update_time}")
|
||||
|
||||
parts.append(f"{node.when_to_use}: {node.content}")
|
||||
|
||||
|
|
@ -58,4 +59,4 @@ class ReadUserProfile(BaseMemoryTool):
|
|||
|
||||
logger.info(f"Read {len(formatted_profiles)} profiles from cache key: {self.memory_cache_key}")
|
||||
|
||||
return "### User Profile\n" + "\n".join(formatted_profiles).strip()
|
||||
return "\n".join(formatted_profiles).strip()
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
from loguru import logger
|
||||
|
||||
from ..base_memory_tool import BaseMemoryTool
|
||||
from ....core.schema import ToolCall
|
||||
from ....core.schema.memory_node import MemoryNode
|
||||
from ....core.enumeration import MemoryType
|
||||
from ....core.schema import ToolCall, MemoryNode
|
||||
from ....core.utils import deduplicate_memories
|
||||
|
||||
|
||||
|
|
@ -34,9 +34,9 @@ class UpdateUserProfile(BaseMemoryTool):
|
|||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"conversation_time": {
|
||||
"update_time": {
|
||||
"type": "string",
|
||||
"description": "Conversation time, e.g. '2020-01-01 00:00:00'",
|
||||
"description": "Update time, e.g. '2020-01-01 00:00:00'",
|
||||
},
|
||||
"profile_key": {
|
||||
"type": "string",
|
||||
|
|
@ -47,7 +47,7 @@ class UpdateUserProfile(BaseMemoryTool):
|
|||
"description": "Profile value or content, e.g. 'John Smith'",
|
||||
},
|
||||
},
|
||||
"required": ["conversation_time", "profile_key", "profile_value"],
|
||||
"required": ["update_time", "profile_key", "profile_value"],
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -58,6 +58,8 @@ class UpdateUserProfile(BaseMemoryTool):
|
|||
|
||||
async def execute(self):
|
||||
# Get and deduplicate profile IDs to delete
|
||||
self.context.memory_type = MemoryType.PERSONAL
|
||||
|
||||
profile_ids_to_delete = self.context.get("profile_ids_to_delete", [])
|
||||
profile_ids_to_delete = list(dict.fromkeys([pid for pid in profile_ids_to_delete if pid]))
|
||||
profiles_to_add = self.context.get("profiles_to_add", [])
|
||||
|
|
@ -88,12 +90,13 @@ class UpdateUserProfile(BaseMemoryTool):
|
|||
content=profile.get("profile_value", ""),
|
||||
ref_memory_id=self.history_node.memory_id,
|
||||
author=self.author,
|
||||
metadata={"conversation_time": profile.get("conversation_time", "")},
|
||||
metadata={"update_time": profile.get("update_time", "")},
|
||||
)
|
||||
new_nodes.append(node)
|
||||
logger.info(f"Added {len(new_nodes)} new profiles.")
|
||||
|
||||
# Deduplicate and save updated profiles
|
||||
self.memory_nodes.extend(new_nodes)
|
||||
updated_nodes = deduplicate_memories(existing_nodes + new_nodes)
|
||||
nodes_data = [node.model_dump(exclude_none=True) for node in updated_nodes]
|
||||
self.local_memory.save(self.memory_cache_key, nodes_data)
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ class AddMemory(BaseMemoryTool):
|
|||
|
||||
await self.vector_store.delete(vector_ids=vector_ids)
|
||||
await self.vector_store.insert(nodes=vector_nodes)
|
||||
self.memory_nodes = memory_nodes
|
||||
self.memory_nodes.extend(memory_nodes)
|
||||
|
||||
output = f"Successfully added {len(memory_nodes)} memories to vector_store."
|
||||
logger.info(output)
|
||||
|
|
|
|||
|
|
@ -117,7 +117,6 @@ class RetrieveMemory(BaseMemoryTool):
|
|||
retrieved_memory_ids = {node.memory_id for node in self.retrieved_nodes if node.memory_id}
|
||||
new_memory_nodes = [node for node in memory_nodes if node.memory_id not in retrieved_memory_ids]
|
||||
self.retrieved_nodes.extend(new_memory_nodes)
|
||||
self.memory_nodes = new_memory_nodes
|
||||
|
||||
if not new_memory_nodes:
|
||||
output = "No new memory_nodes found matching the query (duplicates removed)."
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ reme = ReMe(vector_store={"collection_name": "reme"})
|
|||
async def test_reme():
|
||||
"""Tests ReMe memory system with personal information storage and retrieval."""
|
||||
# 构建一段包含个人信息的对话
|
||||
await reme.default_vector_store.delete_all()
|
||||
await reme.vector_store.delete_all()
|
||||
|
||||
messages = [
|
||||
{
|
||||
|
|
@ -53,7 +53,7 @@ async def test_reme():
|
|||
print("=" * 60)
|
||||
|
||||
# 对对话进行总结，生成记忆
|
||||
await reme.summary(
|
||||
await reme.summary_memory(
|
||||
messages=messages,
|
||||
user_name="zhangwei",
|
||||
description="用户自我介绍和技术兴趣分享",
|
||||
|
|
@ -66,7 +66,7 @@ async def test_reme():
|
|||
print("=" * 60)
|
||||
|
||||
# 列出所有存储的记忆节点
|
||||
nodes: list[VectorNode] = await reme.default_vector_store.list()
|
||||
nodes: list[VectorNode] = await reme.vector_store.list()
|
||||
for i, node in enumerate(nodes, 1):
|
||||
memory_node = MemoryNode.from_vector_node(node)
|
||||
print(f"{i} {memory_node.model_dump_json()}")
|
||||
|
|
@ -78,25 +78,25 @@ async def test_reme():
|
|||
# 测试问题1: 检索用户姓名
|
||||
query1 = "用户叫什么名字？"
|
||||
print(f"\n问题1: {query1}")
|
||||
result1 = await reme.retrieve(query=query1, user_name="zhangwei")
|
||||
result1 = await reme.retrieve_memory(query=query1, user_name="zhangwei")
|
||||
print(f"检索结果:\n{result1}")
|
||||
|
||||
# 测试问题2: 检索技术背景
|
||||
query2 = "用户擅长什么编程语言和技术方向？"
|
||||
print(f"\n问题2: {query2}")
|
||||
result2 = await reme.retrieve(query=query2, user_name="zhangwei")
|
||||
result2 = await reme.retrieve_memory(query=query2, user_name="zhangwei")
|
||||
print(f"检索结果:\n{result2}")
|
||||
|
||||
# 测试问题3: 检索个人信息
|
||||
query3 = "用户的工作地点和联系方式是什么？"
|
||||
print(f"\n问题3: {query3}")
|
||||
result3 = await reme.retrieve(query=query3, user_name="zhangwei")
|
||||
result3 = await reme.retrieve_memory(query=query3, user_name="zhangwei")
|
||||
print(f"检索结果:\n{result3}")
|
||||
|
||||
# 测试问题4: 检索兴趣爱好
|
||||
query4 = "用户平时有什么爱好或活动？"
|
||||
print(f"\n问题4: {query4}")
|
||||
result4 = await reme.retrieve(query=query4, user_name="zhangwei")
|
||||
result4 = await reme.retrieve_memory(query=query4, user_name="zhangwei")
|
||||
print(f"检索结果:\n{result4}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue