mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-08-28 05:25:04 +00:00
feat(benchmark): add HaluMem evaluation suite with ReMe integration
This commit is contained in:
jinli.yl
parent
5054cf22df
commit
9f114cb9b4
26 changed files with 1715 additions and 131 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -34,5 +34,6 @@ test_compact_storage/*
|
|||
test_working_memory/*
|
||||
*.code-workspace
|
||||
local_vector_store/*
|
||||
chroma_vector_store/*
|
||||
bench_results/*
|
||||
meta_memory/*
|
||||
|
|
@ -158,10 +158,11 @@ async def process_user_async(
|
|||
tmp_file = os.path.join(tmp_dir, f"{user_data['uuid']}.json")
|
||||
|
||||
# Clear existing memories for this user
|
||||
await reme.vector_store.delete_collection(f"reme_eval_{user_name}")
|
||||
collection_name = f"reme_eval_{user_name}".replace(" ", "_").lower()
|
||||
await reme.vector_store.delete_collection(collection_name)
|
||||
|
||||
# Update collection name for this user
|
||||
reme.vector_store.set_collection_name(f"reme_eval_{user_name}")
|
||||
reme.vector_store.set_collection_name(collection_name)
|
||||
|
||||
new_user_data = {
|
||||
"uuid": user_data["uuid"],
|
||||
|
|
@ -177,35 +178,40 @@ async def process_user_async(
|
|||
|
||||
# Add messages to ReMe
|
||||
dialogue = session["dialogue"]
|
||||
# Parse timestamp and format as "YYYY-MM-DD HH:MM:SS"
|
||||
date_format = "%b %d, %Y, %H:%M:%S"
|
||||
# dt = datetime.strptime(session["start_time"], date_format).replace(tzinfo=timezone.utc)
|
||||
# time_created = dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
formatted_dialogue = [
|
||||
{
|
||||
"role": turn["role"],
|
||||
"content": turn["content"],
|
||||
"time_created": datetime.strptime(turn["timestamp"], date_format)
|
||||
"time_created": datetime.strptime(turn["timestamp"], "%b %d, %Y, %H:%M:%S")
|
||||
.replace(tzinfo=timezone.utc)
|
||||
.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
for turn in dialogue
|
||||
]
|
||||
|
||||
# Add memory
|
||||
result, duration_ms = await add_memory_async(
|
||||
reme=reme,
|
||||
user_id=user_name,
|
||||
messages=formatted_dialogue,
|
||||
)
|
||||
memories = []
|
||||
for memory_modes in result:
|
||||
for memory_mode in memory_modes:
|
||||
if not isinstance(memory_mode, MemoryNode):
|
||||
continue
|
||||
# Add memory - process every 2 messages
|
||||
result = []
|
||||
total_duration_ms = 0
|
||||
batch_size = 4
|
||||
|
||||
memories.append(memory_mode.content)
|
||||
for i in range(0, len(formatted_dialogue), batch_size):
|
||||
batch = formatted_dialogue[i : i + batch_size]
|
||||
batch_result, duration_ms = await add_memory_async(
|
||||
reme=reme,
|
||||
user_id=user_name,
|
||||
messages=batch,
|
||||
)
|
||||
result.extend(batch_result)
|
||||
total_duration_ms += duration_ms
|
||||
|
||||
duration_ms = total_duration_ms
|
||||
|
||||
memories = []
|
||||
for memory_mode in result:
|
||||
if not isinstance(memory_mode, MemoryNode):
|
||||
continue
|
||||
|
||||
memories.append(memory_mode.content)
|
||||
|
||||
print(memories)
|
||||
|
||||
887
bench/halumem/eval_reme.py
Normal file
887
bench/halumem/eval_reme.py
Normal file
|
|
@ -0,0 +1,887 @@
|
|||
"""
|
||||
Complete evaluation script for ReMe on HaluMem benchmark.
|
||||
|
||||
This script performs the full evaluation pipeline:
|
||||
1. Load HaluMem data
|
||||
2. Process each user's sessions with ReMe (summary + retrieve)
|
||||
3. Evaluate memory integrity, accuracy, updates, and question answering
|
||||
4. Generate metrics and statistics
|
||||
|
||||
Usage:
|
||||
python bench/halumem/eval_reme.py --data_path /Users/yuli/workspace/HaluMem/data/HaluMem-Long.jsonl --version v1 \
|
||||
--top_k 20 --user_num 1
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from eval_tools import (
|
||||
_PROMPTS,
|
||||
evaluation_for_memory_accuracy,
|
||||
evaluation_for_memory_integrity,
|
||||
evaluation_for_question,
|
||||
evaluation_for_update_memory,
|
||||
)
|
||||
from llms import llm_request
|
||||
from reme_ai.core.enumeration import MemoryType
|
||||
from reme_ai.core.schema import MemoryNode
|
||||
from reme_ai.reme import ReMe
|
||||
|
||||
# Template for formatting memories (from shared YAML config)
|
||||
TEMPLATE_MEMOS = _PROMPTS["TEMPLATE_MEMOS"]
|
||||
|
||||
# Prompt for question answering (using optimized PROMPT_MEMOS)
|
||||
PROMPT_MEMOS = _PROMPTS["PROMPT_MEMOS"]
|
||||
|
||||
reme: ReMe = ReMe()
|
||||
|
||||
|
||||
def extract_user_name(persona_info: str):
|
||||
"""Extract user name from persona info."""
|
||||
match = re.search(r"Name:\s*(.*?); Gender:", persona_info)
|
||||
if match:
|
||||
username = match.group(1).strip()
|
||||
return username
|
||||
else:
|
||||
raise ValueError("No name found.")
|
||||
|
||||
|
||||
def iter_jsonl(file_path: str):
|
||||
"""Iterate over lines in a JSONL file."""
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
yield json.loads(line)
|
||||
|
||||
|
||||
def compute_f1(precision: float, recall: float) -> float:
|
||||
"""Compute F1-score from precision and recall."""
|
||||
if precision + recall == 0:
|
||||
return 0.0
|
||||
return 2 * (precision * recall) / (precision + recall)
|
||||
|
||||
|
||||
# ==================== Stage 1: Data Processing ====================
|
||||
|
||||
|
||||
async def add_memory_async(user_id: str, messages: list[dict]) -> tuple[list[MemoryNode], float]:
|
||||
"""Add memory to ReMe system asynchronously."""
|
||||
start = time.time()
|
||||
result = await reme.summary(messages=messages, user_id=user_id)
|
||||
duration_ms = (time.time() - start) * 1000
|
||||
return result, duration_ms
|
||||
|
||||
|
||||
async def search_memory_async(query: str, user_id: str, top_k: int = 20):
|
||||
"""Search memory from ReMe system asynchronously."""
|
||||
start = time.time()
|
||||
memories = await reme.retrieve(query=query, user_id=user_id, top_k=top_k)
|
||||
|
||||
# Format the context
|
||||
context = TEMPLATE_MEMOS.format(user_id=user_id, memories=memories)
|
||||
duration_ms = (time.time() - start) * 1000
|
||||
return context, memories, duration_ms
|
||||
|
||||
|
||||
async def process_user_stage1(
|
||||
user_data: dict,
|
||||
top_k_value: int,
|
||||
save_path: str,
|
||||
version: str,
|
||||
):
|
||||
"""Stage 1: Process user data through ReMe (summary + retrieve)."""
|
||||
user_name = extract_user_name(user_data["persona_info"]) + f"_{version}"
|
||||
sessions = user_data["sessions"]
|
||||
|
||||
tmp_dir = os.path.join(save_path, "tmp")
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
tmp_file = os.path.join(tmp_dir, f"{user_data['uuid']}.json")
|
||||
|
||||
new_user_data = {
|
||||
"uuid": user_data["uuid"],
|
||||
"user_name": user_name,
|
||||
"sessions": [],
|
||||
}
|
||||
|
||||
for idx, session in enumerate(sessions):
|
||||
logger.info(f"Processing user {user_name}: session {idx}/{len(sessions)}")
|
||||
new_session = {
|
||||
"memory_points": session["memory_points"],
|
||||
"dialogue": session["dialogue"],
|
||||
}
|
||||
|
||||
# Format dialogue
|
||||
dialogue = session["dialogue"]
|
||||
formatted_dialogue = [
|
||||
{
|
||||
"role": turn["role"],
|
||||
"content": turn["content"],
|
||||
"time_created": datetime.strptime(turn["timestamp"], "%b %d, %Y, %H:%M:%S")
|
||||
.replace(tzinfo=timezone.utc)
|
||||
.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
for turn in dialogue
|
||||
]
|
||||
|
||||
# Process in batches
|
||||
result = []
|
||||
total_duration_ms = 0
|
||||
batch_size = 20
|
||||
|
||||
for i in range(0, len(formatted_dialogue), batch_size):
|
||||
batch = formatted_dialogue[i : i + batch_size]
|
||||
batch_result, duration_ms = await add_memory_async(
|
||||
user_id=user_name,
|
||||
messages=batch,
|
||||
)
|
||||
if batch_result:
|
||||
result.extend(batch_result)
|
||||
total_duration_ms += duration_ms
|
||||
|
||||
duration_ms = total_duration_ms
|
||||
|
||||
# Extract memory content
|
||||
memories = []
|
||||
for memory_node in result:
|
||||
if isinstance(memory_node, MemoryNode) and memory_node.memory_type is not MemoryType.HISTORY:
|
||||
memories.append(memory_node.content)
|
||||
|
||||
if session.get("is_generated_qa_session", False):
|
||||
new_session["add_dialogue_duration_ms"] = duration_ms
|
||||
new_session["is_generated_qa_session"] = True
|
||||
del new_session["dialogue"]
|
||||
del new_session["memory_points"]
|
||||
new_user_data["sessions"].append(new_session)
|
||||
continue
|
||||
|
||||
# Store extracted memories
|
||||
new_session["extracted_memories"] = memories
|
||||
new_session["add_dialogue_duration_ms"] = duration_ms
|
||||
|
||||
# Search updated memories for memory points
|
||||
for memory in new_session["memory_points"]:
|
||||
if memory["is_update"] == "False" or not memory.get("original_memories"):
|
||||
continue
|
||||
|
||||
_, memories_from_system, duration_ms = await search_memory_async(
|
||||
query=memory["memory_content"],
|
||||
user_id=user_name,
|
||||
top_k=10,
|
||||
)
|
||||
|
||||
memory["memories_from_system"] = memories_from_system
|
||||
|
||||
# Process questions
|
||||
if "questions" not in session:
|
||||
new_user_data["sessions"].append(new_session)
|
||||
continue
|
||||
|
||||
new_session["questions"] = []
|
||||
|
||||
for qa in session["questions"]:
|
||||
context, _, duration_ms = await search_memory_async(
|
||||
query=qa["question"],
|
||||
user_id=user_name,
|
||||
top_k=top_k_value,
|
||||
)
|
||||
|
||||
new_qa = copy.deepcopy(qa)
|
||||
new_qa["context"] = context
|
||||
new_qa["search_duration_ms"] = duration_ms
|
||||
|
||||
prompt = PROMPT_MEMOS.format(
|
||||
context=context,
|
||||
question=qa["question"],
|
||||
)
|
||||
|
||||
start_time = time.time()
|
||||
response = await llm_request(prompt)
|
||||
new_qa["system_response"] = response
|
||||
new_qa["response_duration_ms"] = (time.time() - start_time) * 1000
|
||||
|
||||
new_session["questions"].append(new_qa)
|
||||
|
||||
# ==================== Evaluation for this session ====================
|
||||
session_eval_results = {
|
||||
"memory_integrity_records": [],
|
||||
"memory_accuracy_records": [],
|
||||
"memory_update_records": [],
|
||||
"question_answering_records": [],
|
||||
}
|
||||
|
||||
uuid = user_data["uuid"]
|
||||
golden_memories = session["memory_points"]
|
||||
extract_memories = new_session["extracted_memories"]
|
||||
extract_memories_str = "\n".join(extract_memories)
|
||||
|
||||
# Evaluate Memory Integrity
|
||||
logger.info(f"Evaluating Memory Integrity for session {idx}...")
|
||||
for memory in golden_memories:
|
||||
if memory["is_update"] == "True" and memory.get("memories_from_system", []):
|
||||
# Skip update memories for integrity check
|
||||
continue
|
||||
|
||||
new_memory = copy.deepcopy(memory)
|
||||
new_memory["uuid"] = uuid
|
||||
new_memory["session_id"] = idx
|
||||
|
||||
if extract_memories_str.strip() == "":
|
||||
new_memory["memory_integrity_score"] = 0
|
||||
session_eval_results["memory_integrity_records"].append(new_memory)
|
||||
continue
|
||||
|
||||
result = await evaluation_for_memory_integrity(extract_memories_str, memory["memory_content"])
|
||||
score = int(result.get("score"))
|
||||
new_memory["memory_integrity_score"] = score
|
||||
session_eval_results["memory_integrity_records"].append(new_memory)
|
||||
|
||||
# Evaluate Memory Accuracy
|
||||
logger.info(f"Evaluating Memory Accuracy for session {idx}...")
|
||||
dialogue = session["dialogue"]
|
||||
dialogue_str = []
|
||||
for turn in dialogue:
|
||||
dialogue_str.append(f'[{turn["timestamp"]}]{turn["role"]}: {turn["content"]}')
|
||||
if turn["role"] == "assistant":
|
||||
dialogue_str.append("")
|
||||
dialogue_str = "\n".join(dialogue_str)
|
||||
|
||||
golden_memories_str = "\n".join(
|
||||
[m["memory_content"] for m in golden_memories if m["memory_source"] != "interference"],
|
||||
)
|
||||
|
||||
for memory in extract_memories:
|
||||
new_memory = {
|
||||
"uuid": uuid,
|
||||
"session_id": idx,
|
||||
"memory_content": memory,
|
||||
}
|
||||
result = await evaluation_for_memory_accuracy(dialogue_str, golden_memories_str, memory)
|
||||
score = int(result.get("accuracy_score"))
|
||||
is_included_in_golden_memories = result.get("is_included_in_golden_memories", "false")
|
||||
new_memory["memory_accuracy_score"] = score
|
||||
new_memory["is_included_in_golden_memories"] = is_included_in_golden_memories
|
||||
session_eval_results["memory_accuracy_records"].append(new_memory)
|
||||
|
||||
# Evaluate Memory Update
|
||||
logger.info(f"Evaluating Memory Update for session {idx}...")
|
||||
for memory in golden_memories:
|
||||
if memory["is_update"] == "False" or not memory.get("original_memories"):
|
||||
continue
|
||||
|
||||
if not memory.get("memories_from_system", []):
|
||||
continue
|
||||
|
||||
update_memory = copy.deepcopy(memory)
|
||||
update_memory["uuid"] = uuid
|
||||
update_memory["session_id"] = idx
|
||||
|
||||
result = await evaluation_for_update_memory(
|
||||
"\n".join(update_memory["memories_from_system"]),
|
||||
update_memory["memory_content"],
|
||||
"\n".join(update_memory["original_memories"]),
|
||||
)
|
||||
update_type = result.get("evaluation_result")
|
||||
update_memory["memory_update_type"] = update_type
|
||||
session_eval_results["memory_update_records"].append(update_memory)
|
||||
|
||||
# Evaluate Question Answering
|
||||
if "questions" in new_session:
|
||||
logger.info(f"Evaluating Question Answering for session {idx}...")
|
||||
for qa in new_session["questions"]:
|
||||
new_qa = copy.deepcopy(qa)
|
||||
new_qa["uuid"] = uuid
|
||||
new_qa["session_id"] = idx
|
||||
|
||||
result = await evaluation_for_question(
|
||||
qa["question"],
|
||||
qa["answer"],
|
||||
"\n".join([i["memory_content"] for i in qa["evidence"]]),
|
||||
qa["system_response"],
|
||||
)
|
||||
result_type = result.get("evaluation_result")
|
||||
new_qa["result_type"] = result_type
|
||||
session_eval_results["question_answering_records"].append(new_qa)
|
||||
|
||||
# Store evaluation results in session
|
||||
new_session["evaluation_results"] = session_eval_results
|
||||
|
||||
new_user_data["sessions"].append(new_session)
|
||||
|
||||
# Save results
|
||||
with open(tmp_file, "w", encoding="utf-8") as f:
|
||||
json.dump(new_user_data, f, ensure_ascii=False, indent=2)
|
||||
session_size = len(new_user_data["sessions"])
|
||||
logger.info(f"✅ Saved user {user_name} to {tmp_file} session_size={session_size}")
|
||||
|
||||
logger.info(f"✅ Saved user {user_name} to {tmp_file} all!")
|
||||
return {"uuid": user_data["uuid"], "status": "ok", "path": tmp_file}
|
||||
|
||||
|
||||
# ==================== Stage 2: Evaluation ====================
|
||||
|
||||
|
||||
async def process_user_stage2(idx: int, user_data: dict):
|
||||
"""Stage 2: Extract evaluation results from user's sessions (already computed in Stage 1)."""
|
||||
user_name = user_data["user_name"]
|
||||
|
||||
eval_results = {
|
||||
"memory_integrity_records": [],
|
||||
"memory_accuracy_records": [],
|
||||
"memory_update_records": [],
|
||||
"question_answering_records": [],
|
||||
}
|
||||
|
||||
logger.info(f"[{idx}]{user_name}: Extracting evaluation results from sessions...")
|
||||
|
||||
# Extract evaluation results from each session
|
||||
for session in user_data["sessions"]:
|
||||
if session.get("is_generated_qa_session", False):
|
||||
continue
|
||||
|
||||
if "evaluation_results" not in session:
|
||||
logger.warning(f"[{idx}]{user_name}: Session missing evaluation_results, skipping...")
|
||||
continue
|
||||
|
||||
session_eval = session["evaluation_results"]
|
||||
eval_results["memory_integrity_records"].extend(session_eval.get("memory_integrity_records", []))
|
||||
eval_results["memory_accuracy_records"].extend(session_eval.get("memory_accuracy_records", []))
|
||||
eval_results["memory_update_records"].extend(session_eval.get("memory_update_records", []))
|
||||
eval_results["question_answering_records"].extend(session_eval.get("question_answering_records", []))
|
||||
|
||||
logger.info(
|
||||
f"[{idx}]{user_name}: Extracted {len(eval_results['memory_integrity_records'])} integrity, "
|
||||
f"{len(eval_results['memory_accuracy_records'])} accuracy, "
|
||||
f"{len(eval_results['memory_update_records'])} update, "
|
||||
f"{len(eval_results['question_answering_records'])} QA records",
|
||||
)
|
||||
|
||||
return eval_results
|
||||
|
||||
|
||||
def aggregate_eval_results(eval_results):
|
||||
"""Aggregate evaluation results and compute metrics."""
|
||||
|
||||
# Memory Integrity Evaluation
|
||||
memory_integrity_scores = 0
|
||||
memory_integrity_weighted_scores = 0
|
||||
memory_integrity_valid_num = 0
|
||||
memory_integrity_num = 0
|
||||
memory_integrity_weighted_valid_num = 0
|
||||
memory_integrity_weighted_num = 0
|
||||
interference_memory_scores = 0
|
||||
interference_memory_valid_num = 0
|
||||
interference_memory_num = 0
|
||||
|
||||
for item in eval_results["memory_integrity_records"]:
|
||||
item["is_valid"] = True
|
||||
|
||||
if item["memory_source"] != "interference":
|
||||
memory_integrity_num += 1
|
||||
memory_integrity_weighted_num += item["importance"]
|
||||
else:
|
||||
interference_memory_num += 1
|
||||
|
||||
if item["memory_integrity_score"] is None:
|
||||
item["is_valid"] = False
|
||||
continue
|
||||
|
||||
if item["memory_source"] != "interference":
|
||||
if item["memory_integrity_score"] == 2:
|
||||
memory_integrity_scores += 1
|
||||
memory_integrity_weighted_scores += 0.5 * item["memory_integrity_score"] * item["importance"]
|
||||
memory_integrity_valid_num += 1
|
||||
memory_integrity_weighted_valid_num += item["importance"]
|
||||
else:
|
||||
if item["memory_integrity_score"] == 0:
|
||||
interference_memory_scores += 1
|
||||
interference_memory_valid_num += 1
|
||||
|
||||
eval_results["overall_score"]["memory_integrity"]["recall(all)"] = (
|
||||
memory_integrity_scores / memory_integrity_num if memory_integrity_num > 0 else 0
|
||||
)
|
||||
eval_results["overall_score"]["memory_integrity"]["recall(valid)"] = (
|
||||
memory_integrity_scores / memory_integrity_valid_num if memory_integrity_valid_num > 0 else 0
|
||||
)
|
||||
eval_results["overall_score"]["memory_integrity"]["weighted_recall(all)"] = (
|
||||
memory_integrity_weighted_scores / memory_integrity_weighted_num if memory_integrity_weighted_num > 0 else 0
|
||||
)
|
||||
eval_results["overall_score"]["memory_integrity"]["weighted_recall(valid)"] = (
|
||||
memory_integrity_weighted_scores / memory_integrity_weighted_valid_num
|
||||
if memory_integrity_weighted_valid_num > 0
|
||||
else 0
|
||||
)
|
||||
eval_results["overall_score"]["memory_integrity"][
|
||||
"memory_valid_importance_sum"
|
||||
] = memory_integrity_weighted_valid_num
|
||||
eval_results["overall_score"]["memory_integrity"]["memory_importance_sum"] = memory_integrity_weighted_num
|
||||
eval_results["overall_score"]["memory_integrity"]["memory_valid_num"] = memory_integrity_valid_num
|
||||
eval_results["overall_score"]["memory_integrity"]["memory_num"] = memory_integrity_num
|
||||
eval_results["overall_score"]["memory_accuracy"]["interference_accuracy(all)"] = (
|
||||
interference_memory_scores / interference_memory_num if interference_memory_num > 0 else 0
|
||||
)
|
||||
eval_results["overall_score"]["memory_accuracy"]["interference_accuracy(valid)"] = (
|
||||
interference_memory_scores / interference_memory_valid_num if interference_memory_valid_num > 0 else 0
|
||||
)
|
||||
eval_results["overall_score"]["memory_accuracy"]["interference_memory_valid_num"] = interference_memory_valid_num
|
||||
eval_results["overall_score"]["memory_accuracy"]["interference_memory_num"] = interference_memory_num
|
||||
|
||||
# Memory Accuracy Evaluation
|
||||
target_memory_accuracy_scores = 0
|
||||
memory_accuracy_weighted_scores = 0
|
||||
target_memory_accuracy_valid_num = 0
|
||||
target_memory_accuracy_num = 0
|
||||
memory_accuracy_valid_num = 0
|
||||
memory_accuracy_num = 0
|
||||
|
||||
for item in eval_results["memory_accuracy_records"]:
|
||||
item["is_valid"] = True
|
||||
memory_accuracy_num += 1
|
||||
|
||||
if item["is_included_in_golden_memories"] in ["true", "True"]:
|
||||
target_memory_accuracy_num += 1
|
||||
|
||||
if item["memory_accuracy_score"] is None:
|
||||
item["is_valid"] = False
|
||||
continue
|
||||
|
||||
if item["is_included_in_golden_memories"] in ["true", "True"]:
|
||||
target_memory_accuracy_scores += 0.5 * item["memory_accuracy_score"]
|
||||
target_memory_accuracy_valid_num += 1
|
||||
|
||||
memory_accuracy_weighted_scores += 0.5 * item["memory_accuracy_score"]
|
||||
memory_accuracy_valid_num += 1
|
||||
|
||||
eval_results["overall_score"]["memory_accuracy"]["target_accuracy(all)"] = (
|
||||
target_memory_accuracy_scores / target_memory_accuracy_num if target_memory_accuracy_num > 0 else 0
|
||||
)
|
||||
eval_results["overall_score"]["memory_accuracy"]["target_accuracy(valid)"] = (
|
||||
target_memory_accuracy_scores / target_memory_accuracy_valid_num if target_memory_accuracy_valid_num > 0 else 0
|
||||
)
|
||||
eval_results["overall_score"]["memory_accuracy"]["target_memory_valid_num"] = target_memory_accuracy_valid_num
|
||||
eval_results["overall_score"]["memory_accuracy"]["target_memory_num"] = target_memory_accuracy_num
|
||||
eval_results["overall_score"]["memory_accuracy"]["weighted_accuracy(all)"] = (
|
||||
memory_accuracy_weighted_scores / memory_accuracy_num if memory_accuracy_num > 0 else 0
|
||||
)
|
||||
eval_results["overall_score"]["memory_accuracy"]["weighted_accuracy(valid)"] = (
|
||||
memory_accuracy_weighted_scores / memory_accuracy_valid_num if memory_accuracy_valid_num > 0 else 0
|
||||
)
|
||||
eval_results["overall_score"]["memory_accuracy"]["memory_valid_num"] = memory_accuracy_valid_num
|
||||
eval_results["overall_score"]["memory_accuracy"]["memory_num"] = memory_accuracy_num
|
||||
|
||||
# Memory Extraction F1-score
|
||||
eval_results["overall_score"]["memory_extraction_f1"] = compute_f1(
|
||||
precision=eval_results["overall_score"]["memory_accuracy"]["target_accuracy(all)"],
|
||||
recall=eval_results["overall_score"]["memory_integrity"]["recall(all)"],
|
||||
)
|
||||
|
||||
# Memory Update Evaluation
|
||||
correct_update_memory_num = 0
|
||||
hallucination_update_memory_num = 0
|
||||
omission_update_memory_num = 0
|
||||
other_update_memory_num = 0
|
||||
update_memory_num = 0
|
||||
update_memory_valid_num = 0
|
||||
|
||||
for item in eval_results["memory_update_records"]:
|
||||
item["is_valid"] = True
|
||||
update_memory_num += 1
|
||||
|
||||
if item["memory_update_type"] not in ["Correct", "Hallucination", "Omission", "Other"]:
|
||||
item["is_valid"] = False
|
||||
continue
|
||||
|
||||
if item["memory_update_type"] == "Correct":
|
||||
correct_update_memory_num += 1
|
||||
elif item["memory_update_type"] == "Hallucination":
|
||||
hallucination_update_memory_num += 1
|
||||
elif item["memory_update_type"] == "Omission":
|
||||
omission_update_memory_num += 1
|
||||
elif item["memory_update_type"] == "Other":
|
||||
other_update_memory_num += 1
|
||||
|
||||
update_memory_valid_num += 1
|
||||
|
||||
if update_memory_num > 0:
|
||||
eval_results["overall_score"]["memory_update"]["correct_update_memory_ratio(all)"] = (
|
||||
correct_update_memory_num / update_memory_num
|
||||
)
|
||||
eval_results["overall_score"]["memory_update"]["hallucination_update_memory_ratio(all)"] = (
|
||||
hallucination_update_memory_num / update_memory_num
|
||||
)
|
||||
eval_results["overall_score"]["memory_update"]["omission_update_memory_ratio(all)"] = (
|
||||
omission_update_memory_num / update_memory_num
|
||||
)
|
||||
eval_results["overall_score"]["memory_update"]["other_update_memory_ratio(all)"] = (
|
||||
other_update_memory_num / update_memory_num
|
||||
)
|
||||
else:
|
||||
eval_results["overall_score"]["memory_update"]["correct_update_memory_ratio(all)"] = 0
|
||||
eval_results["overall_score"]["memory_update"]["hallucination_update_memory_ratio(all)"] = 0
|
||||
eval_results["overall_score"]["memory_update"]["omission_update_memory_ratio(all)"] = 0
|
||||
eval_results["overall_score"]["memory_update"]["other_update_memory_ratio(all)"] = 0
|
||||
|
||||
if update_memory_valid_num > 0:
|
||||
eval_results["overall_score"]["memory_update"]["correct_update_memory_ratio(valid)"] = (
|
||||
correct_update_memory_num / update_memory_valid_num
|
||||
)
|
||||
eval_results["overall_score"]["memory_update"]["hallucination_update_memory_ratio(valid)"] = (
|
||||
hallucination_update_memory_num / update_memory_valid_num
|
||||
)
|
||||
eval_results["overall_score"]["memory_update"]["omission_update_memory_ratio(valid)"] = (
|
||||
omission_update_memory_num / update_memory_valid_num
|
||||
)
|
||||
eval_results["overall_score"]["memory_update"]["other_update_memory_ratio(valid)"] = (
|
||||
other_update_memory_num / update_memory_valid_num
|
||||
)
|
||||
else:
|
||||
eval_results["overall_score"]["memory_update"]["correct_update_memory_ratio(valid)"] = 0
|
||||
eval_results["overall_score"]["memory_update"]["hallucination_update_memory_ratio(valid)"] = 0
|
||||
eval_results["overall_score"]["memory_update"]["omission_update_memory_ratio(valid)"] = 0
|
||||
eval_results["overall_score"]["memory_update"]["other_update_memory_ratio(valid)"] = 0
|
||||
|
||||
eval_results["overall_score"]["memory_update"]["update_memory_valid_num"] = update_memory_valid_num
|
||||
eval_results["overall_score"]["memory_update"]["update_memory_num"] = update_memory_num
|
||||
|
||||
# Question-Answering Evaluation
|
||||
correct_qa_num = 0
|
||||
hallucination_qa_num = 0
|
||||
omission_qa_num = 0
|
||||
qa_num = 0
|
||||
qa_valid_num = 0
|
||||
|
||||
for item in eval_results["question_answering_records"]:
|
||||
item["is_valid"] = True
|
||||
qa_num += 1
|
||||
|
||||
if item["result_type"] not in ["Correct", "Hallucination", "Omission"]:
|
||||
item["is_valid"] = False
|
||||
continue
|
||||
|
||||
if item["result_type"] == "Correct":
|
||||
correct_qa_num += 1
|
||||
elif item["result_type"] == "Hallucination":
|
||||
hallucination_qa_num += 1
|
||||
elif item["result_type"] == "Omission":
|
||||
omission_qa_num += 1
|
||||
|
||||
qa_valid_num += 1
|
||||
|
||||
if qa_num > 0:
|
||||
eval_results["overall_score"]["question_answering"]["correct_qa_ratio(all)"] = correct_qa_num / qa_num
|
||||
eval_results["overall_score"]["question_answering"]["hallucination_qa_ratio(all)"] = (
|
||||
hallucination_qa_num / qa_num
|
||||
)
|
||||
eval_results["overall_score"]["question_answering"]["omission_qa_ratio(all)"] = omission_qa_num / qa_num
|
||||
else:
|
||||
eval_results["overall_score"]["question_answering"]["correct_qa_ratio(all)"] = 0
|
||||
eval_results["overall_score"]["question_answering"]["hallucination_qa_ratio(all)"] = 0
|
||||
eval_results["overall_score"]["question_answering"]["omission_qa_ratio(all)"] = 0
|
||||
|
||||
if qa_valid_num > 0:
|
||||
eval_results["overall_score"]["question_answering"]["correct_qa_ratio(valid)"] = correct_qa_num / qa_valid_num
|
||||
eval_results["overall_score"]["question_answering"]["hallucination_qa_ratio(valid)"] = (
|
||||
hallucination_qa_num / qa_valid_num
|
||||
)
|
||||
eval_results["overall_score"]["question_answering"]["omission_qa_ratio(valid)"] = omission_qa_num / qa_valid_num
|
||||
else:
|
||||
eval_results["overall_score"]["question_answering"]["correct_qa_ratio(valid)"] = 0
|
||||
eval_results["overall_score"]["question_answering"]["hallucination_qa_ratio(valid)"] = 0
|
||||
eval_results["overall_score"]["question_answering"]["omission_qa_ratio(valid)"] = 0
|
||||
|
||||
eval_results["overall_score"]["question_answering"]["qa_valid_num"] = qa_valid_num
|
||||
eval_results["overall_score"]["question_answering"]["qa_num"] = qa_num
|
||||
|
||||
# Memory Type Accuracy
|
||||
for item in eval_results["memory_integrity_records"]:
|
||||
if "memory_integrity_score" not in item or "importance" not in item:
|
||||
continue
|
||||
score = 1 if item["memory_integrity_score"] == 2 else 0
|
||||
eval_results["overall_score"]["memory_type_accuracy"][item["memory_type"]]["memory_integrity_acc"] += score
|
||||
eval_results["overall_score"]["memory_type_accuracy"][item["memory_type"]]["total_num"] += 1
|
||||
|
||||
for item in eval_results["memory_update_records"]:
|
||||
if "memory_update_type" not in item or "importance" not in item:
|
||||
continue
|
||||
score = 1 if item["memory_update_type"] == "Correct" else 0
|
||||
eval_results["overall_score"]["memory_type_accuracy"][item["memory_type"]]["memory_update_acc"] += score
|
||||
eval_results["overall_score"]["memory_type_accuracy"][item["memory_type"]]["total_num"] += 1
|
||||
|
||||
for key in eval_results["overall_score"]["memory_type_accuracy"]:
|
||||
if eval_results["overall_score"]["memory_type_accuracy"][key]["total_num"] > 0:
|
||||
total = eval_results["overall_score"]["memory_type_accuracy"][key]["total_num"]
|
||||
eval_results["overall_score"]["memory_type_accuracy"][key]["memory_integrity_acc"] = (
|
||||
eval_results["overall_score"]["memory_type_accuracy"][key]["memory_integrity_acc"] / total
|
||||
)
|
||||
eval_results["overall_score"]["memory_type_accuracy"][key]["memory_update_acc"] = (
|
||||
eval_results["overall_score"]["memory_type_accuracy"][key]["memory_update_acc"] / total
|
||||
)
|
||||
eval_results["overall_score"]["memory_type_accuracy"][key]["memory_acc"] = (
|
||||
eval_results["overall_score"]["memory_type_accuracy"][key]["memory_integrity_acc"]
|
||||
+ eval_results["overall_score"]["memory_type_accuracy"][key]["memory_update_acc"]
|
||||
)
|
||||
else:
|
||||
eval_results["overall_score"]["memory_type_accuracy"][key]["memory_integrity_acc"] = 0
|
||||
eval_results["overall_score"]["memory_type_accuracy"][key]["memory_update_acc"] = 0
|
||||
eval_results["overall_score"]["memory_type_accuracy"][key]["memory_acc"] = 0
|
||||
|
||||
return eval_results
|
||||
|
||||
|
||||
# ==================== Main Pipeline ====================
|
||||
|
||||
|
||||
async def main_async(
|
||||
data_path: str,
|
||||
version: str = "default",
|
||||
top_k: int = 20,
|
||||
user_num: int = 1,
|
||||
):
|
||||
"""Main evaluation pipeline."""
|
||||
frame = "reme"
|
||||
save_path = f"bench_results/{frame}-{version}/"
|
||||
os.makedirs(save_path, exist_ok=True)
|
||||
|
||||
output_file_stage1 = os.path.join(save_path, f"{frame}_eval_results.jsonl")
|
||||
output_file_stage2 = os.path.join(save_path, f"{frame}_eval_stat_result.json")
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# ==================== Stage 1: Data Processing ====================
|
||||
print("\n" + "=" * 80)
|
||||
print("STAGE 1: PROCESSING DATA WITH ReMe")
|
||||
print("=" * 80)
|
||||
|
||||
tmp_dir = os.path.join(save_path, "tmp")
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
|
||||
# Load all user data
|
||||
user_data_list = list(iter_jsonl(data_path))
|
||||
total_users = min(len(user_data_list), user_num)
|
||||
user_data_list = user_data_list[:total_users]
|
||||
|
||||
print(f"Processing {total_users} users sequentially...")
|
||||
|
||||
# Process users sequentially
|
||||
for idx, user_data in enumerate(user_data_list, 1):
|
||||
result = await process_user_stage1(user_data, top_k, save_path, version)
|
||||
print(f"[{idx}/{total_users}] ✅ Finished {user_data['uuid']} ({result['status']})")
|
||||
|
||||
# Combine all results into final output
|
||||
with open(output_file_stage1, "w", encoding="utf-8") as f_out:
|
||||
for file in os.listdir(tmp_dir):
|
||||
if file.endswith(".json"):
|
||||
file_path = os.path.join(tmp_dir, file)
|
||||
with open(file_path, "r", encoding="utf-8") as f_in:
|
||||
data = json.load(f_in)
|
||||
f_out.write(json.dumps(data, ensure_ascii=False) + "\n")
|
||||
|
||||
elapsed_stage1 = time.time() - start_time
|
||||
print(f"\n✅ Stage 1 completed in {elapsed_stage1:.2f}s")
|
||||
print(f"✅ Results saved to: {output_file_stage1}")
|
||||
|
||||
# ==================== Stage 2: Evaluation ====================
|
||||
print("\n" + "=" * 80)
|
||||
print("STAGE 2: EVALUATING MEMORY PERFORMANCE")
|
||||
print("=" * 80)
|
||||
|
||||
tmp_dir2 = os.path.join(save_path, "tmp2")
|
||||
os.makedirs(tmp_dir2, exist_ok=True)
|
||||
|
||||
start_stage2 = time.time()
|
||||
|
||||
for idx, user_data in enumerate(iter_jsonl(output_file_stage1), 1):
|
||||
uuid = user_data["uuid"]
|
||||
tmp_file = os.path.join(tmp_dir2, f"{uuid}.json")
|
||||
|
||||
if os.path.exists(tmp_file):
|
||||
print(f"⚡ Skipping user {uuid} ({idx}) — cached result found.")
|
||||
else:
|
||||
print(f"Processing user {uuid} ({idx})...")
|
||||
user_result = await process_user_stage2(idx, user_data)
|
||||
|
||||
with open(tmp_file, "w", encoding="utf-8") as f:
|
||||
json.dump(user_result, f, ensure_ascii=False, indent=4)
|
||||
|
||||
elapsed = time.time() - start_stage2
|
||||
print(f"✅ Finished user {uuid} ({idx}), elapsed {elapsed:.2f}s.")
|
||||
|
||||
# Calculate time consuming
|
||||
add_dialogue_duration_time = 0
|
||||
search_memory_duration_time = 0
|
||||
|
||||
for user_data in iter_jsonl(output_file_stage1):
|
||||
sessions = user_data["sessions"]
|
||||
|
||||
for session in sessions:
|
||||
if "add_dialogue_duration_ms" in session:
|
||||
add_dialogue_duration_time += session["add_dialogue_duration_ms"]
|
||||
|
||||
if "questions" in session:
|
||||
for question in session["questions"]:
|
||||
if "search_duration_ms" in question:
|
||||
search_memory_duration_time += question["search_duration_ms"]
|
||||
|
||||
add_dialogue_duration_time = add_dialogue_duration_time / 1000 / 60
|
||||
search_memory_duration_time = search_memory_duration_time / 1000 / 60
|
||||
|
||||
print("\n🔄 Aggregating all user results...")
|
||||
|
||||
eval_results = {
|
||||
"overall_score": {
|
||||
"memory_integrity": {},
|
||||
"memory_accuracy": {},
|
||||
"memory_extraction_f1": 0,
|
||||
"memory_update": {},
|
||||
"question_answering": {},
|
||||
"memory_type_accuracy": {
|
||||
"Event Memory": {
|
||||
"memory_integrity_acc": 0,
|
||||
"memory_update_acc": 0,
|
||||
"total_num": 0,
|
||||
},
|
||||
"Persona Memory": {
|
||||
"memory_integrity_acc": 0,
|
||||
"memory_update_acc": 0,
|
||||
"total_num": 0,
|
||||
},
|
||||
"Relationship Memory": {
|
||||
"memory_integrity_acc": 0,
|
||||
"memory_update_acc": 0,
|
||||
"total_num": 0,
|
||||
},
|
||||
},
|
||||
"time_consuming": {
|
||||
"add_dialogue_duration_time": add_dialogue_duration_time,
|
||||
"search_memory_duration_time": search_memory_duration_time,
|
||||
"total_duration_time": add_dialogue_duration_time + search_memory_duration_time,
|
||||
},
|
||||
},
|
||||
"memory_integrity_records": [],
|
||||
"memory_accuracy_records": [],
|
||||
"memory_update_records": [],
|
||||
"question_answering_records": [],
|
||||
}
|
||||
|
||||
for file_name in os.listdir(tmp_dir2):
|
||||
if not file_name.endswith(".json"):
|
||||
continue
|
||||
user_file = os.path.join(tmp_dir2, file_name)
|
||||
with open(user_file, "r", encoding="utf-8") as f:
|
||||
user_result = json.load(f)
|
||||
|
||||
eval_results["memory_accuracy_records"].extend(user_result.get("memory_accuracy_records", []))
|
||||
eval_results["memory_integrity_records"].extend(user_result.get("memory_integrity_records", []))
|
||||
eval_results["memory_update_records"].extend(user_result.get("memory_update_records", []))
|
||||
eval_results["question_answering_records"].extend(user_result.get("question_answering_records", []))
|
||||
|
||||
eval_results = aggregate_eval_results(eval_results)
|
||||
|
||||
with open(output_file_stage2, "w", encoding="utf-8") as f:
|
||||
json.dump(eval_results, f, ensure_ascii=False, indent=4)
|
||||
|
||||
elapsed_total = time.time() - start_time
|
||||
print(f"\n✅ All done in {elapsed_total:.2f}s. Results saved to {output_file_stage2}")
|
||||
|
||||
# Print summary
|
||||
print("\n" + "=" * 80)
|
||||
print("EVALUATION SUMMARY")
|
||||
print("=" * 80)
|
||||
print("\n📊 Memory Integrity:")
|
||||
print(f" - Recall (all): {eval_results['overall_score']['memory_integrity'].get('recall(all)', 0):.4f}")
|
||||
print(f" - Recall (valid): {eval_results['overall_score']['memory_integrity'].get('recall(valid)', 0):.4f}")
|
||||
print(f" - Weighted Recall (all): "
|
||||
f"{eval_results['overall_score']['memory_integrity'].get('weighted_recall(all)', 0):.4f}")
|
||||
|
||||
print(f"\n📊 Memory Accuracy:")
|
||||
print(f" - Target Accuracy (all): {eval_results['overall_score']['memory_accuracy'].get('target_accuracy(all)', 0):.4f}")
|
||||
print(f" - Target Accuracy (valid): {eval_results['overall_score']['memory_accuracy'].get('target_accuracy(valid)', 0):.4f}",
|
||||
)
|
||||
print(
|
||||
f" - Weighted Accuracy (all): {eval_results['overall_score']['memory_accuracy'].get('weighted_accuracy(all)', 0):.4f}",
|
||||
)
|
||||
|
||||
print(f"\n📊 Memory Extraction F1: {eval_results['overall_score']['memory_extraction_f1']:.4f}")
|
||||
|
||||
print(f"\n📊 Memory Update:")
|
||||
print(
|
||||
f" - Correct (all): {eval_results['overall_score']['memory_update'].get('correct_update_memory_ratio(all)', 0):.4f}",
|
||||
)
|
||||
print(
|
||||
f" - Hallucination (all): {eval_results['overall_score']['memory_update'].get('hallucination_update_memory_ratio(all)', 0):.4f}",
|
||||
)
|
||||
print(
|
||||
f" - Omission (all): {eval_results['overall_score']['memory_update'].get('omission_update_memory_ratio(all)', 0):.4f}",
|
||||
)
|
||||
|
||||
print(f"\n📊 Question Answering:")
|
||||
print(
|
||||
f" - Correct (all): {eval_results['overall_score']['question_answering'].get('correct_qa_ratio(all)', 0):.4f}",
|
||||
)
|
||||
print(
|
||||
f" - Hallucination (all): {eval_results['overall_score']['question_answering'].get('hallucination_qa_ratio(all)', 0):.4f}",
|
||||
)
|
||||
print(
|
||||
f" - Omission (all): {eval_results['overall_score']['question_answering'].get('omission_qa_ratio(all)', 0):.4f}",
|
||||
)
|
||||
|
||||
print(f"\n⏱️ Time Consuming:")
|
||||
print(f" - Add Dialogue: {add_dialogue_duration_time:.2f} min")
|
||||
print(f" - Search Memory: {search_memory_duration_time:.2f} min")
|
||||
print(f" - Total: {add_dialogue_duration_time + search_memory_duration_time:.2f} min")
|
||||
print("=" * 80)
|
||||
|
||||
|
||||
def main(
|
||||
data_path: str,
|
||||
version: str = "default",
|
||||
top_k: int = 20,
|
||||
user_num: int = 1,
|
||||
):
|
||||
"""Synchronous entry point."""
|
||||
asyncio.run(main_async(data_path, version, top_k, user_num))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Complete evaluation for ReMe on HaluMem benchmark")
|
||||
parser.add_argument(
|
||||
"--data_path",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Path to HaluMem data file (e.g., HaluMem-medium.jsonl)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
type=str,
|
||||
default="test",
|
||||
help="Version identifier for this evaluation run (default: test)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--top_k",
|
||||
type=int,
|
||||
default=20,
|
||||
help="Number of top memories to retrieve (default: 20)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--user_num",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of users to evaluate (default: 1)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
main(
|
||||
data_path=args.data_path,
|
||||
version=args.version,
|
||||
top_k=args.top_k,
|
||||
user_num=args.user_num,
|
||||
)
|
||||
104
bench/halumem/eval_tools.py
Normal file
104
bench/halumem/eval_tools.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
"""Evaluation tools for ReMe HaluMem benchmark."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from llms import llm_request_for_json
|
||||
|
||||
# Load prompts from YAML file
|
||||
_YAML_PATH = Path(__file__).parent / "halumem.yaml"
|
||||
with open(_YAML_PATH, "r", encoding="utf-8") as f:
|
||||
_PROMPTS = yaml.safe_load(f)
|
||||
|
||||
|
||||
async def evaluation_for_memory_integrity(
|
||||
extract_memories: str,
|
||||
target_memory: str,
|
||||
):
|
||||
"""
|
||||
Memory Integrity Evaluation
|
||||
extract_memories: A formatted string concatenating all memory points extracted by the memory system under evaluation.
|
||||
target_memory: The target key memory point.
|
||||
"""
|
||||
|
||||
prompt = _PROMPTS["EVALUATION_PROMPT_FOR_MEMORY_INTEGRITY"].format(
|
||||
memories=extract_memories,
|
||||
expected_memory_point=target_memory,
|
||||
)
|
||||
|
||||
result = await llm_request_for_json(prompt)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def evaluation_for_memory_accuracy(
|
||||
dialogue: str,
|
||||
golden_memories: str,
|
||||
candidate_memory: str,
|
||||
):
|
||||
"""
|
||||
Memory Accuracy Evaluation
|
||||
dialogue: The complete human-machine dialogue record.
|
||||
golden_memories: The core memory points for this dialogue segment in the evaluation set (the correct reference memories).
|
||||
candidate_memory: A specific memory point extracted by the memory system being evaluated.
|
||||
"""
|
||||
|
||||
prompt = _PROMPTS["EVALUATION_PROMPT_FOR_MEMORY_ACCURACY"].format(
|
||||
dialogue=dialogue,
|
||||
golden_memories=golden_memories,
|
||||
candidate_memory=candidate_memory,
|
||||
)
|
||||
|
||||
result = await llm_request_for_json(prompt)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def evaluation_for_update_memory(
|
||||
extract_memories: str,
|
||||
target_update_memory: str,
|
||||
original_memory: str,
|
||||
):
|
||||
"""
|
||||
Memory Update Evaluation
|
||||
extract_memories: A formatted string concatenating all memory points extracted by the memory system under evaluation.
|
||||
target_update_memory: The target updated memory point.
|
||||
original_memory: str: A formatted string concatenating all original memory points corresponding to the target updated memory point (i.e., all memories before the update).
|
||||
"""
|
||||
|
||||
prompt = _PROMPTS["EVALUATION_PROMPT_FOR_UPDATE_MEMORY"].format(
|
||||
memories=extract_memories,
|
||||
updated_memory=target_update_memory,
|
||||
original_memory=original_memory,
|
||||
)
|
||||
|
||||
result = await llm_request_for_json(prompt)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def evaluation_for_question(
|
||||
question: str,
|
||||
reference_answer: str,
|
||||
key_memory_points: str,
|
||||
response: str,
|
||||
):
|
||||
"""
|
||||
Question-Answering Evaluation
|
||||
question: The question string to be evaluated.
|
||||
reference_answer: The reference (gold-standard) answer.
|
||||
key_memory_points: The memory points used to derive the reference answer.
|
||||
response: The answer produced by the memory system.
|
||||
"""
|
||||
|
||||
prompt = _PROMPTS["EVALUATION_PROMPT_FOR_QUESTION"].format(
|
||||
question=question,
|
||||
reference_answer=reference_answer,
|
||||
key_memory_points=key_memory_points,
|
||||
response=response,
|
||||
)
|
||||
|
||||
result = await llm_request_for_json(prompt)
|
||||
|
||||
return result
|
||||
421
bench/halumem/halumem.yaml
Normal file
421
bench/halumem/halumem.yaml
Normal file
|
|
@ -0,0 +1,421 @@
|
|||
TEMPLATE_MEMOS: |
|
||||
Memories for user {user_id}:
|
||||
{memories}
|
||||
|
||||
PROMPT_MEMZERO: |
|
||||
You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
|
||||
|
||||
# CONTEXT:
|
||||
You have access to memories from two speakers in a conversation. These memories contain
|
||||
timestamped information that may be relevant to answering the question.
|
||||
|
||||
# INSTRUCTIONS:
|
||||
1. Carefully analyze all provided memories from both speakers
|
||||
2. Pay special attention to the timestamps to determine the answer
|
||||
3. If the question asks about a specific event or fact, look for direct evidence in the memories
|
||||
4. If the memories contain contradictory information, prioritize the most recent memory
|
||||
5. If there is a question about time references (like "last year", "two months ago", etc.),
|
||||
calculate the actual date based on the memory timestamp. For example, if a memory from
|
||||
4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
|
||||
6. Always convert relative time references to specific dates, months, or years. For example,
|
||||
convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory
|
||||
timestamp. Ignore the reference while answering the question.
|
||||
7. Focus only on the content of the memories from both speakers. Do not confuse character
|
||||
names mentioned in memories with the actual users who created those memories.
|
||||
8. The answer should be less than 5-6 words.
|
||||
|
||||
# APPROACH (Think step by step):
|
||||
1. First, examine all memories that contain information related to the question
|
||||
2. Examine the timestamps and content of these memories carefully
|
||||
3. Look for explicit mentions of dates, times, locations, or events that answer the question
|
||||
4. If the answer requires calculation (e.g., converting relative time references), show your work
|
||||
5. Formulate a precise, concise answer based solely on the evidence in the memories
|
||||
6. Double-check that your answer directly addresses the question asked
|
||||
7. Ensure your final answer is specific and avoids vague time references
|
||||
|
||||
{context}
|
||||
|
||||
Question: {question}
|
||||
|
||||
Answer:
|
||||
|
||||
PROMPT_ZEP: |
|
||||
You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
|
||||
|
||||
# CONTEXT:
|
||||
You have access to memories from a conversation. These memories contain
|
||||
timestamped information that may be relevant to answering the question.
|
||||
|
||||
# INSTRUCTIONS:
|
||||
1. Carefully analyze all provided memories
|
||||
2. Pay special attention to the timestamps to determine the answer
|
||||
3. If the question asks about a specific event or fact, look for direct evidence in the memories
|
||||
4. If the memories contain contradictory information, prioritize the most recent memory
|
||||
5. If there is a question about time references (like "last year", "two months ago", etc.),
|
||||
calculate the actual date based on the memory timestamp. For example, if a memory from
|
||||
4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
|
||||
6. Always convert relative time references to specific dates, months, or years. For example,
|
||||
convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory
|
||||
timestamp. Ignore the reference while answering the question.
|
||||
7. Focus only on the content of the memories. Do not confuse character
|
||||
names mentioned in memories with the actual users who created those memories.
|
||||
8. The answer should be less than 5-6 words.
|
||||
|
||||
# APPROACH (Think step by step):
|
||||
1. First, examine all memories that contain information related to the question
|
||||
2. Examine the timestamps and content of these memories carefully
|
||||
3. Look for explicit mentions of dates, times, locations, or events that answer the question
|
||||
4. If the answer requires calculation (e.g., converting relative time references), show your work
|
||||
5. Formulate a precise, concise answer based solely on the evidence in the memories
|
||||
6. Double-check that your answer directly addresses the question asked
|
||||
7. Ensure your final answer is specific and avoids vague time references
|
||||
|
||||
Context:
|
||||
|
||||
{context}
|
||||
|
||||
Question: {question}
|
||||
Answer:
|
||||
|
||||
PROMPT_MEMOS: |
|
||||
You are a knowledgeable and helpful AI assistant.
|
||||
|
||||
# CONTEXT:
|
||||
You have access to memories from two speakers in a conversation. These memories contain
|
||||
timestamped information that may be relevant to answering the question.
|
||||
|
||||
# INSTRUCTIONS:
|
||||
1. Carefully analyze all provided memories. Synthesize information across different entries if needed to form a complete answer.
|
||||
2. Pay close attention to the timestamps to determine the answer. If memories contain contradictory information, the **most recent memory** is the source of truth.
|
||||
3. If the question asks about a specific event or fact, look for direct evidence in the memories.
|
||||
4. Your answer must be grounded in the memories. However, you may use general world knowledge to interpret or complete information found within a memory (e.g., identifying a landmark mentioned by description).
|
||||
5. If the question involves time references (like "last year", "two months ago", etc.), you **must** calculate the actual date based on the memory's timestamp. For example, if a memory from 4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
|
||||
6. Always convert relative time references to specific dates, months, or years in your final answer.
|
||||
7. Do not confuse character names mentioned in memories with the actual users who created them.
|
||||
8. The answer must be brief (under 5-6 words) and direct, with no extra description.
|
||||
|
||||
# APPROACH (Think step by step):
|
||||
1. First, examine all memories that contain information related to the question.
|
||||
2. Synthesize findings from multiple memories if a single entry is insufficient.
|
||||
3. Examine timestamps and content carefully, looking for explicit dates, times, locations, or events.
|
||||
4. If the answer requires calculation (e.g., converting relative time references), perform the calculation.
|
||||
5. Formulate a precise, concise answer based on the evidence from the memories (and allowed world knowledge).
|
||||
6. Double-check that your answer directly addresses the question asked and adheres to all instructions.
|
||||
7. Ensure your final answer is specific and avoids vague time references.
|
||||
|
||||
{context}
|
||||
|
||||
Question: {question}
|
||||
|
||||
Answer:
|
||||
|
||||
PROMPT_MEMOBASE: |
|
||||
You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
|
||||
|
||||
# CONTEXT:
|
||||
You have access to memories from two speakers in a conversation. These memories contain
|
||||
timestamped information that may be relevant to answering the question.
|
||||
|
||||
# INSTRUCTIONS:
|
||||
1. Carefully analyze all provided memories from both speakers
|
||||
2. Pay special attention to the timestamps to determine the answer
|
||||
3. If the question asks about a specific event or fact, look for direct evidence in the memories
|
||||
4. If the memories contain contradictory information, prioritize the most recent memory
|
||||
5. If there is a question about time references (like "last year", "two months ago", etc.), calculate the actual date based on the memory timestamp. For example, if a memory from 4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
|
||||
6. Always convert relative time references to specific dates, months, or years. For example, convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory timestamp. Ignore the reference while answering the question.
|
||||
7. Focus only on the content of the memories from both speakers. Do not confuse character names mentioned in memories with the actual users who created those memories.
|
||||
8. The answer should be less than 5-6 words.
|
||||
|
||||
# APPROACH (Think step by step):
|
||||
1. First, examine all memories that contain information related to the question
|
||||
2. Examine the timestamps and content of these memories carefully
|
||||
3. Look for explicit mentions of dates, times, locations, or events that answer the question
|
||||
4. If the answer requires calculation (e.g., converting relative time references), show your work
|
||||
5. Formulate a precise, concise answer based solely on the evidence in the memories
|
||||
6. Double-check that your answer directly addresses the question asked
|
||||
7. Ensure your final answer is specific and avoids vague time references
|
||||
|
||||
{context}
|
||||
|
||||
Question: {question}
|
||||
|
||||
Answer:
|
||||
|
||||
|
||||
EVALUATION_PROMPT_FOR_MEMORY_INTEGRITY: |
|
||||
You are a strict **"Memory Integrity" evaluator**.
|
||||
Your core task is to assess whether an AI memory system has **missed any key memory points** after processing a conversation. This evaluation measures the system’s **memory integrity**, i.e., its ability to resist **amnesia** or **omission**.
|
||||
|
||||
# Evaluation Context & Data:
|
||||
|
||||
1. **Extracted Memories:**
|
||||
These are all the memory items actually extracted by the memory system.
|
||||
{memories}
|
||||
|
||||
2. **Expected Memory Point:**
|
||||
The key memory point that *should* have been extracted.
|
||||
{expected_memory_point}
|
||||
|
||||
# Evaluation Instructions:
|
||||
|
||||
1. For each **Expected Memory Point**, search within the **Extracted Memories** list for corresponding or related information. Ignore unrelated items.
|
||||
2. Based on the following scoring rubric, rate how well the memory system captured the **Expected Memory Point** and provide a detailed explanation.
|
||||
|
||||
# Scoring Rubric:
|
||||
|
||||
* **2:** Fully covered or implied.
|
||||
One or more items in “Extracted Memories” fully cover or logically imply all information in the “Expected Memory Point.”
|
||||
|
||||
* **1:** Partially covered or mentioned.
|
||||
Some information in “Extracted Memories” mentions part of the “Expected Memory Point,” but key information is missing, inaccurate, or slightly incorrect.
|
||||
|
||||
* **0:** Not mentioned or incorrect.
|
||||
“Extracted Memories” contains no mention of the “Expected Memory Point,” or the corresponding information is entirely wrong.
|
||||
|
||||
# Scoring Notes:
|
||||
|
||||
* For **compound Expected Memory Points** (with multiple elements such as person/event/time/location/preference, etc.):
|
||||
|
||||
* All elements correct → **2 points**
|
||||
* Some elements correct / uncertain → **1 point**
|
||||
* Key elements missing or wrong → **0 points**
|
||||
|
||||
* Semantic matching is acceptable; exact wording is **not** required.
|
||||
|
||||
* If “Extracted Memories” contains **conflicting information**, assign the **best possible coverage score** and mention the conflict in your reasoning.
|
||||
|
||||
* Extra or stylistically different memories do **not** reduce the score; only the coverage of the **Expected Memory Point** matters.
|
||||
|
||||
* For uncertain wording (“might,” “probably,” “tends to,” etc.):
|
||||
|
||||
* If the Expected Memory Point is a definite statement, usually assign **1 point**.
|
||||
|
||||
* If critical fields (e.g., time, entity name, relationship) are partly wrong but others match → **1 point**.
|
||||
|
||||
* If all key fields are wrong or missing → **0 points**.
|
||||
|
||||
# Output Format:
|
||||
|
||||
Please output your result in the following JSON format:
|
||||
|
||||
```json
|
||||
{{
|
||||
"reasoning": "Provide a concise justification for the score",
|
||||
"score": "2|1|0"
|
||||
}}
|
||||
```
|
||||
|
||||
EVALUATION_PROMPT_FOR_MEMORY_ACCURACY: |
|
||||
You are a **Dialogue Memory Accuracy Evaluator.** Your task is to evaluate the **accuracy** of a memory extracted by an AI memory system, based on three given inputs: the dialogue content, the *target (gold)* memory points (the correct annotated memories), and the *candidate* memory to be evaluated. The goal is to output a **structured evaluation result**.
|
||||
|
||||
# Input Content
|
||||
|
||||
* **Dialogue:**
|
||||
{dialogue}
|
||||
|
||||
* **Golden Memories (Target Memory Points):**
|
||||
The correct memory points pre-annotated for this dialogue in the evaluation dataset.
|
||||
{golden_memories}
|
||||
|
||||
* **Candidate Memory:**
|
||||
The memory extracted by the system to be evaluated.
|
||||
{candidate_memory}
|
||||
|
||||
# Evaluation Principles and Definitions
|
||||
|
||||
### 1) Support / Entailment
|
||||
|
||||
* An **information point** (atomic fact) in the candidate memory is considered *supported* if it can be directly stated or semantically entailed (via synonym, paraphrase, or equivalent expression) by the *Dialogue* or *Golden Memories*.
|
||||
* Only the given dialogue and golden memories can be used for judgment — **no external knowledge** or assumptions are allowed.
|
||||
Any information not appearing in or inferable from these two sources is considered *unsupported*.
|
||||
* Pay careful attention to **negation**, **quantities**, **time**, and **subjects**.
|
||||
If the candidate statement contradicts the dialogue or golden memories, it is considered a **conflict**.
|
||||
|
||||
### 2) Memory Accuracy Score (integer: 0 / 1 / 2)
|
||||
|
||||
* **2 points:** Every information point in the candidate memory is supported by the dialogue or golden memories, with **no contradictions or hallucinations**.
|
||||
* **1 point:** The candidate memory is *partially correct* (at least one supported information point) but also includes *unsupported* or *contradictory* content.
|
||||
* **0 points:** The candidate memory is **entirely unsupported or contradictory** to the sources (i.e., a “hallucinated memory”).
|
||||
|
||||
> Note:
|
||||
>
|
||||
> * If a candidate memory contains multiple information points, **any unsupported or contradictory element** prevents a full score (2).
|
||||
> * If both supported and unsupported/conflicting content appear, assign a score of **1**.
|
||||
|
||||
### 3) Inclusion in Golden Memories (Boolean field-level judgment)
|
||||
|
||||
**Definition:**
|
||||
|
||||
* **Atomic information point:** the smallest factual unit in the candidate memory (e.g., *name = Li Si*, *age = 25*, *location = Beijing*, *preference = coffee*, *budget ≤ 2000*, *meeting_time = Wednesday 10:00*, *tool = Zoom*, etc.).
|
||||
* **Field / Slot:** the semantic dimension of an information point (e.g., *name*, *age*, *residence*, *food preference*, *budget*, *meeting time*, *meeting tool*, etc.).
|
||||
|
||||
**Judgment Rules (independent of correctness):**
|
||||
|
||||
* **true:**
|
||||
Every atomic information point in the candidate memory has a corresponding **field** in the golden memories (allowing for synonyms, paraphrases, or equivalent expressions; ignore value, polarity, or quantity differences).
|
||||
|
||||
* Note: A single field in the gold list may match multiple candidate points (e.g., multiple “drink preference” facts can be covered by one “drink preference” field in gold).
|
||||
* **false:**
|
||||
If **any** atomic information point’s field in the candidate memory cannot be found in the golden memories, mark as *false*.
|
||||
|
||||
**Important Notes:**
|
||||
|
||||
* Field matching is restricted to fields that are **explicitly present or semantically recognizable** in the golden memories — no external knowledge may be used to expand the field set.
|
||||
* Differences in **values** (e.g., “Zhang San” vs. “Li Si”), **polarity** (like/dislike), or **exact number/time** do **not** affect this Boolean judgment.
|
||||
|
||||
# Evaluation Procedure
|
||||
|
||||
For each candidate memory:
|
||||
|
||||
1. **Decompose** it into atomic information points (e.g., name, number, location, preference).
|
||||
2. For each information point, **search** the dialogue and golden memories for supporting or contradictory evidence.
|
||||
3. Assign the **accuracy_score** (0 / 1 / 2) according to the rules above.
|
||||
4. Determine **is_included_in_golden_memories (true/false)**:
|
||||
|
||||
* Identify each information point’s field;
|
||||
* If *all* fields exist in the golden memories, mark as *true*; otherwise, *false*.
|
||||
5. Provide a **concise Chinese explanation** in `"reason"`, citing key evidence (short excerpts allowed), and clearly state any unsupported or contradictory parts if applicable.
|
||||
|
||||
# Output Format (strictly required)
|
||||
|
||||
Output **only one JSON object**, with the following three fields:
|
||||
|
||||
* `"accuracy_score"`: `"0"` or `"1"` or `"2"`
|
||||
* `"is_included_in_golden_memories"`: `"true"` or `"false"`
|
||||
* `"reason"`: `"brief explanation in Chinese"`
|
||||
|
||||
Do **not** include any other text, explanation, or fields.
|
||||
Do **not** include the candidate memory text inside the JSON.
|
||||
|
||||
Please output **only** the following JSON (in a code block):
|
||||
|
||||
```json
|
||||
{{
|
||||
"accuracy_score": "2 | 1 | 0",
|
||||
"is_included_in_golden_memories": "true | false",
|
||||
"reason": "Brief explanation in Chinese"
|
||||
}}
|
||||
```
|
||||
|
||||
EVALUATION_PROMPT_FOR_UPDATE_MEMORY: |
|
||||
Your task is to **evaluate the update accuracy** of an AI memory system.
|
||||
Based on the information provided below, determine whether the system-generated **“Generated Memories”** correctly **includes** the **Target Memory for Update**.
|
||||
|
||||
# Background Information
|
||||
|
||||
The following information is provided for evaluation:
|
||||
|
||||
1. **Generated Memories:**
|
||||
This is the list of memory points generated by the system after the current dialogue.
|
||||
{memories}
|
||||
|
||||
2. **Target Memory for Update:**
|
||||
This is the correct, updated version of the memory point that should have been produced — the one we focus on in this evaluation.
|
||||
{updated_memory}
|
||||
|
||||
3. **Original Memory Content:**
|
||||
This is the original version of the target memory before the update.
|
||||
{original_memory}
|
||||
|
||||
# Evaluation Criteria
|
||||
|
||||
Please make your judgment **strictly based on the content update of the “Target Memory for Update.”**
|
||||
Use the following categories:
|
||||
|
||||
### Correct Update
|
||||
|
||||
* **Generated Memories** **contains all information points** from the “Target Memory for Update,” accurately and completely reflecting the intended update.
|
||||
* **Key fields** (e.g., date, time, values, proper nouns, etc.) must match exactly.
|
||||
* The **original memory** is effectively replaced or marked as outdated.
|
||||
* Synonymous or slightly rephrased expressions are acceptable.
|
||||
|
||||
### Hallucinated Update
|
||||
|
||||
* **Factual error:** The **Generated Memories** includes a new memory related to the “Target Memory for Update,” but its content contains factual mistakes or contradictions compared to the correct update.
|
||||
|
||||
### Omitted Update
|
||||
|
||||
* **Completely omitted:** The **Generated Memories** contains no new memory related to the “Target Memory for Update.”
|
||||
* **Partially omitted:** A related new memory was generated in **Generated Memories**, but it **misses key information** that should have been included.
|
||||
|
||||
### Other
|
||||
|
||||
Used for update failures that do **not clearly fall** into the above categories of “Hallucination” or “Omission.”
|
||||
|
||||
# Output Requirements
|
||||
|
||||
Please return your evaluation strictly in the following JSON format and provide a concise explanation.
|
||||
|
||||
```json
|
||||
{{
|
||||
"reason": "Briefly explain your reasoning here and why it fits this category.",
|
||||
"evaluation_result": "Correct | Hallucination | Omission | Other"
|
||||
}}
|
||||
```
|
||||
|
||||
EVALUATION_PROMPT_FOR_QUESTION: |
|
||||
You are an **evaluation expert for AI memory system question answering**.
|
||||
Based **only** on the provided **“Question”**, **“Reference Answer”**, and **“Key Memory Points”** (the essential facts needed to derive the reference answer), strictly evaluate the **accuracy** of the **“Memory System Response.”** Classify it as one of **“Correct”**, **“Hallucination”**, or **“Omission.”** Do **not** use any external knowledge or subjective inference. Finally, output your judgment **strictly** in the specified JSON format.
|
||||
|
||||
# Evaluation Criteria
|
||||
|
||||
## Answer Type Classification
|
||||
|
||||
### 1. Correct
|
||||
|
||||
* The “Memory System Response” accurately answers the “Question,” and its content is **semantically equivalent** to the “Reference Answer.”
|
||||
* It contains **no contradictions** with the “Key Memory Points” or “Reference Answer.”
|
||||
* It introduces **no unsupported details** beyond the “Key Memory Points” that could alter the conclusion.
|
||||
* Synonyms, paraphrasing, and reasonable summarization are acceptable.
|
||||
|
||||
### 2. Hallucination
|
||||
|
||||
* The “Memory System Response” includes information or facts that **contradict or are inconsistent** with the “Reference Answer” or the “Key Memory Points.”
|
||||
* When the “Reference Answer” is labeled as *unknown/uncertain*, yet the response provides a specific verifiable fact or conclusion.
|
||||
* Extra irrelevant information that does **not change** the conclusion is **not** considered hallucination by itself; however, if it **changes or misleads** the conclusion, or **contradicts** the “Key Memory Points,” it should be judged as a **Hallucination**.
|
||||
|
||||
### 3. Omission
|
||||
|
||||
* The response is **incomplete** compared to the “Reference Answer.”
|
||||
* It explicitly states “don’t know,” “can’t remember,” or “no related memory,” even though relevant information exists in the “Key Memory Points.”
|
||||
* For multi-element questions, **all elements must be correct and present**; omission of **any** element is considered an **Omission**.
|
||||
|
||||
## Priority Rules (Conflict Handling)
|
||||
|
||||
* If the response contains **both missing necessary information** and **fabricated/contradictory information**, classify it as **Hallucination**.
|
||||
* If there is **no fabrication/contradiction** but some necessary information is missing, classify it as **Omission**.
|
||||
* Only when the meaning is **fully equivalent** to the reference answer should it be classified as **Correct**.
|
||||
|
||||
## Detailed Guidelines and Tolerance
|
||||
|
||||
* Equivalent expressions of numbers, times, and units are acceptable, but the **numerical values themselves must not differ**.
|
||||
* For multi-element questions, **all elements must be complete and accurate**; missing any element counts as **Omission**.
|
||||
* If the reference answer is *“unknown / cannot be determined”* and the system provides a definite fact, that is a **Hallucination**.
|
||||
If the system also answers *“unknown”* (without guessing), it may be **Correct**.
|
||||
* The evaluation must rely **only** on the *Reference Answer*, *Key Memory Points*, and *System Response* — no external context, world knowledge, or speculative reasoning is allowed.
|
||||
|
||||
# Information for Evaluation
|
||||
|
||||
* **Question:**
|
||||
{question}
|
||||
|
||||
* **Reference Answer:**
|
||||
{reference_answer}
|
||||
|
||||
* **Key Memory Points:**
|
||||
{key_memory_points}
|
||||
|
||||
* **Memory System Response:**
|
||||
{response}
|
||||
|
||||
# Output Requirements
|
||||
|
||||
Please provide your evaluation result **strictly** in the JSON format below.
|
||||
Do **not** add any extra explanation or comments outside the JSON block.
|
||||
|
||||
```json
|
||||
{{
|
||||
"reasoning": "Provide a concise and traceable evaluation rationale: first compare the system’s response with the Key Memory Points (which were correctly used, which were missing, and whether there was any fabrication/contradiction), then assess its consistency with the Reference Answer, and finally state the classification basis.",
|
||||
"evaluation_result": "Correct | Hallucination | Omission"
|
||||
}}
|
||||
```
|
||||
62
bench/halumem/llms.py
Normal file
62
bench/halumem/llms.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from tenacity import retry, stop_after_attempt, wait_random_exponential, before_sleep_log
|
||||
|
||||
from reme_ai.core.llm import OpenAILLM
|
||||
from reme_ai.core.schema import Message
|
||||
from reme_ai.core.utils import load_env
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
load_env()
|
||||
|
||||
WAIT_TIME_LOWER = 1
|
||||
WAIT_TIME_UPPER = 60
|
||||
RETRY_TIMES = 5
|
||||
|
||||
|
||||
@retry(
|
||||
wait=wait_random_exponential(min=WAIT_TIME_LOWER, max=WAIT_TIME_UPPER),
|
||||
stop=stop_after_attempt(3),
|
||||
reraise=True,
|
||||
before_sleep=before_sleep_log(logger, logging.WARNING),
|
||||
)
|
||||
async def llm_request(prompt, **kwargs) -> str:
|
||||
llm = OpenAILLM(model_name="qwen3-max")
|
||||
assistant_message = await llm.chat(
|
||||
messages=[
|
||||
Message(
|
||||
**{
|
||||
"role": "user",
|
||||
"content": prompt,
|
||||
},
|
||||
),
|
||||
],
|
||||
**kwargs,
|
||||
)
|
||||
return assistant_message.content
|
||||
|
||||
|
||||
@retry(
|
||||
wait=wait_random_exponential(min=WAIT_TIME_LOWER, max=WAIT_TIME_UPPER),
|
||||
stop=stop_after_attempt(RETRY_TIMES),
|
||||
reraise=True,
|
||||
before_sleep=before_sleep_log(logger, logging.WARNING),
|
||||
)
|
||||
async def llm_request_for_json(prompt, **kwargs):
|
||||
content = await llm_request(prompt, **kwargs)
|
||||
|
||||
match = re.search(r"```json\s*(\{.*?\})\s*```", content, re.DOTALL)
|
||||
if not match:
|
||||
raise ValueError(f"No JSON block found in model output: {content}")
|
||||
|
||||
json_str = match.group(1).strip()
|
||||
return json.loads(json_str)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
r = asyncio.run(llm_request_for_json('hello? answer in ```json\n{"answer": "..."}```'))
|
||||
print(r)
|
||||
|
|
@ -16,7 +16,7 @@ def init_logger(log_dir: str = "logs", level: str = "INFO") -> None:
|
|||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
# Generate filename based on the current timestamp
|
||||
current_ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
current_ts = datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
|
||||
log_filename = f"{current_ts}.log"
|
||||
log_filepath = os.path.join(log_dir, log_filename)
|
||||
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ class ChromaVectorStore(BaseVectorStore):
|
|||
self.client = chromadb.HttpClient(host=host, port=port)
|
||||
else:
|
||||
if path is None:
|
||||
path = "./chroma_db"
|
||||
path = "./chroma_vector_store"
|
||||
logger.info(f"Initializing local ChromaDB at {path}")
|
||||
self.client = chromadb.PersistentClient(
|
||||
path=path,
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ class BaseMemoryAgent(BaseOp, metaclass=ABCMeta):
|
|||
|
||||
self.messages: list[Message] = []
|
||||
self.success: bool = True
|
||||
|
||||
self.retrieved_nodes: list[MemoryNode] = []
|
||||
self.memory_nodes: list[MemoryNode | str] = []
|
||||
|
||||
def _build_tool_call(self) -> ToolCall:
|
||||
|
|
@ -130,7 +132,7 @@ class BaseMemoryAgent(BaseOp, metaclass=ABCMeta):
|
|||
tool_copy.tool_call.id = tool_call.id
|
||||
tool_list.append(tool_copy)
|
||||
kwargs.update(tool_call.argument_dict)
|
||||
self.submit_async_task(tool_copy.call, **kwargs)
|
||||
self.submit_async_task(tool_copy.call, retrieved_nodes=self.retrieved_nodes, **kwargs)
|
||||
if self.tool_call_interval > 0:
|
||||
await asyncio.sleep(self.tool_call_interval)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@ class ReMeRetriever(BaseMemoryAgent):
|
|||
"""Memory agent that retrieves and builds messages with meta memory context."""
|
||||
|
||||
def __init__(self, meta_memories: list[dict] | None = None, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
super().__init__(prompt_name="", **kwargs)
|
||||
# super().__init__(prompt_name="reme_retriever2", **kwargs)
|
||||
self.meta_memories: list[dict] = meta_memories or []
|
||||
|
||||
async def _read_meta_memories(self) -> str:
|
||||
|
|
@ -31,17 +32,35 @@ class ReMeRetriever(BaseMemoryAgent):
|
|||
async def build_messages(self) -> List[Message]:
|
||||
"""Build messages with system prompt and user message."""
|
||||
meta_memory_info = await self._read_meta_memories()
|
||||
if self.context.get("query"):
|
||||
context = self.context.query
|
||||
elif self.context.get("messages"):
|
||||
messages = [Message(**m) if isinstance(m, dict) else m for m in self.context.messages]
|
||||
context = self.description + format_messages(messages)
|
||||
else:
|
||||
raise ValueError("input must have either `query` or `messages`")
|
||||
system_prompt = self.prompt_format(
|
||||
prompt_name="system_prompt",
|
||||
now_time=get_now_time(),
|
||||
meta_memory_info=meta_memory_info,
|
||||
)
|
||||
|
||||
messages = [Message(role=Role.SYSTEM, content=system_prompt)]
|
||||
if self.context.get("query"):
|
||||
messages.append(Message(role=Role.USER, content=self.context.query))
|
||||
elif self.context.get("messages"):
|
||||
messages.extend([Message(**m) for m in self.context.messages])
|
||||
else:
|
||||
raise ValueError("input must have either `query` or `messages`")
|
||||
|
||||
return messages
|
||||
|
||||
async def build_messages2(self) -> List[Message]:
|
||||
"""Build messages with system prompt and user message."""
|
||||
if self.context.get("query"):
|
||||
context = self.context.query
|
||||
elif self.context.get("messages"):
|
||||
context = format_messages(self.context.messages)
|
||||
else:
|
||||
raise ValueError("input must have either `query` or `messages`")
|
||||
|
||||
system_prompt = self.prompt_format(
|
||||
prompt_name="system_prompt",
|
||||
now_time=get_now_time(),
|
||||
meta_memory_info=await self._read_meta_memories(),
|
||||
context=context,
|
||||
)
|
||||
|
||||
|
|
@ -49,4 +68,5 @@ class ReMeRetriever(BaseMemoryAgent):
|
|||
Message(role=Role.SYSTEM, content=system_prompt),
|
||||
Message(role=Role.USER, content=self.get_prompt("user_message")),
|
||||
]
|
||||
|
||||
return messages
|
||||
|
|
|
|||
|
|
@ -6,10 +6,9 @@ tool: |
|
|||
semantic searches across different memory types to find the most relevant memories.
|
||||
|
||||
system_prompt: |
|
||||
You are a memory agent. Please analyze the context, retrieve relevant memories when needed, and return a summary of the retrieved memories to assist in answering the user's question.
|
||||
You are a memory agent. Please analyze the context, retrieve relevant memories when needed, and directly answer the user's question based on the retrieved information.
|
||||
|
||||
## Context
|
||||
{context}
|
||||
**CRITICAL**: You must ONLY answer based on the retrieved memories. DO NOT fabricate, infer, or add any information that is not explicitly present in the retrieved memories. If the retrieved memories do not contain enough information to answer the question, you must acknowledge this limitation.
|
||||
|
||||
## Current Time
|
||||
{now_time}
|
||||
|
|
@ -34,6 +33,7 @@ system_prompt: |
|
|||
* Choose the optimal combination strategy based on the retrieval scenario.
|
||||
- **Important**: When retrieving tool-related memories (`memory_type` is "tool"), the query must use the tool’s exact name (not a description or paraphrase of the problem).
|
||||
- If retrieval results include a `ref_memory_id` and more details are needed—or if vector retrieval proves insufficient—use `read_history_memory` with the `ref_memory_id` as the `memory_id` parameter.
|
||||
- **Important**: When using `read_history_memory` with multiple `ref_memory_ids`, ensure all IDs are unique and do not provide duplicate IDs.
|
||||
|
||||
3. **Iterate if necessary**:
|
||||
- If the initial retrieval fails, try alternative phrasings or perspectives.
|
||||
|
|
@ -43,8 +43,5 @@ system_prompt: |
|
|||
|
||||
4. **Output** the result:
|
||||
- If no retrieval is needed, output `<NO_RETRIEVAL_NEEDED>`.
|
||||
- If relevant memories are found, clearly summarize the retrieved information.
|
||||
- If multiple attempts still yield no relevant memory, output `<NO_RELEVANT_MEMORY>`.
|
||||
|
||||
user_message: |
|
||||
Please analyze the context, retrieve relevant memories when needed, and return a summary of the retrieved memories to assist in answering the user's question.
|
||||
- If relevant memories are found and you can answer the user's question, provide a concise, direct answer **strictly based on the retrieved memories only**. DO NOT add any information, inference, or speculation beyond what is explicitly stated in the retrieved memories.
|
||||
- If after multiple retrieval attempts from various angles you still cannot find relevant information, output `<NO_RELEVANT_MEMORY>`.
|
||||
|
|
|
|||
49
reme_ai/mem_agent/retriever/reme_retriever2.yaml
Normal file
49
reme_ai/mem_agent/retriever/reme_retriever2.yaml
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
tool: |
|
||||
Retrieve relevant memories from the memory bank to assist in answering questions.
|
||||
Use this tool when you need to search for historical information, user preferences,
|
||||
procedural knowledge, or any other stored memories that may help answer the current query.
|
||||
The agent will analyze the context, determine what information is needed, and perform
|
||||
semantic searches across different memory types to find the most relevant memories.
|
||||
|
||||
system_prompt: |
|
||||
You are a memory retrieval agent. Please analyze the context, retrieve relevant information from the memory bank when needed, and return a summary of the retrieved memories to assist in answering the user's question.
|
||||
|
||||
## Context
|
||||
{context}
|
||||
|
||||
## Current Time
|
||||
{now_time}
|
||||
|
||||
## Available Meta-Memories
|
||||
Format: "- <memory_type>(<memory_target>): <description>"
|
||||
{meta_memory_info}
|
||||
|
||||
## Your Tasks
|
||||
1. **Analyze** the conversation context to determine whether retrieval is necessary:
|
||||
- If the question can be answered directly from the existing context, output `<NO_RETRIEVAL_NEEDED>` and stop.
|
||||
- If additional information is required, proceed with retrieval.
|
||||
- Consider which types of meta-memories from the "Available Meta-Memories" list are most relevant.
|
||||
|
||||
2. **Retrieve** relevant memories using `vector_retrieve_memory`:
|
||||
- Select `memory_type` and `memory_target` from the "Available Meta-Memories" list.
|
||||
- Clearly identify the needed information and construct appropriate queries.
|
||||
- Design queries flexibly based on actual needs:
|
||||
* Generate different queries for different `memory_type`/`memory_target` combinations.
|
||||
* For the same combination, generate multiple queries with varied phrasings or angles if needed.
|
||||
* Use the combination strategy that best fits the retrieval scenario.
|
||||
- **Important**: When retrieving tool memories (`memory_type` is "tool"), use the actual tool name as the query (not a description or question).
|
||||
- If retrieval results include a `ref_memory_id` and more detail is needed—or if vector retrieval proves insufficient—use `read_history_memory` with the `ref_memory_id` as the `memory_id` parameter.
|
||||
|
||||
3. **Iterate if necessary**:
|
||||
- If the initial retrieval yields no matches, try alternative phrasings or perspectives.
|
||||
- If multiple memory types exist, attempt retrieval across different types.
|
||||
- Before concluding that no relevant memory exists, perform at least 2–3 additional retrieval attempts using varied phrasings or angles.
|
||||
- If repeated vector retrievals still fail to provide adequate information, use `read_history_memory` to fetch the original message content.
|
||||
|
||||
4. **Output** the result:
|
||||
- If no retrieval is needed, output `<NO_RETRIEVAL_NEEDED>`.
|
||||
- If relevant memories are found, clearly summarize the retrieved information.
|
||||
- If multiple attempts still yield no relevant memories, output `<NO_RELEVANT_MEMORY>`.
|
||||
|
||||
user_message: |
|
||||
Please analyze the context, retrieve relevant information from the memory bank when needed, and return a summary of the retrieved memories to assist in answering the user's question.
|
||||
|
|
@ -11,6 +11,10 @@ from ...core.utils import get_now_time, format_messages
|
|||
class PersonalSummarizer(BaseMemoryAgent):
|
||||
"""Extracts and stores personal information about individuals from conversations."""
|
||||
|
||||
def __init__(self, recent_top_k: int = 20, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.recent_top_k: int = recent_top_k
|
||||
|
||||
memory_type: MemoryType = MemoryType.PERSONAL
|
||||
|
||||
def _build_tool_call(self) -> ToolCall:
|
||||
|
|
@ -43,11 +47,22 @@ class PersonalSummarizer(BaseMemoryAgent):
|
|||
},
|
||||
)
|
||||
|
||||
async def _retrieve_recent_memories(self) -> str:
|
||||
"""Retrieve recent memories sorted by time_modified."""
|
||||
from ...mem_tool import RetrieveRecentMemory
|
||||
|
||||
op = RetrieveRecentMemory(top_k=self.recent_top_k)
|
||||
await op.call(memory_type="personal", memory_target=self.memory_target, retrieved_nodes=self.retrieved_nodes)
|
||||
return op.output
|
||||
|
||||
async def build_messages(self) -> list[Message]:
|
||||
"""Construct messages with context, memory_target, and memory_type information."""
|
||||
await self._retrieve_recent_memories()
|
||||
|
||||
system_prompt = self.prompt_format(
|
||||
prompt_name="system_prompt",
|
||||
now_time=get_now_time(),
|
||||
recent_memories="\n".join([n.format_memory() for n in self.retrieved_nodes]),
|
||||
context=self.description + "\n" + format_messages(self.get_messages()),
|
||||
memory_type=self.memory_type.value,
|
||||
memory_target=self.memory_target,
|
||||
|
|
|
|||
|
|
@ -8,12 +8,17 @@ tool: |
|
|||
system_prompt: |
|
||||
You are a professional memory agent. Your task is to update the main agent's {memory_type} memory regarding {memory_target} based on the context.
|
||||
|
||||
**CRITICAL**: You must extract and store information STRICTLY based on what is explicitly stated in the context. DO NOT infer, assume, fabricate, or add any information that is not directly present in the dialogue. Only extract facts that are clearly and explicitly mentioned.
|
||||
|
||||
## Context:
|
||||
{context}
|
||||
|
||||
## Current Time:
|
||||
{now_time}
|
||||
|
||||
## Recent Memories:
|
||||
{recent_memories}
|
||||
|
||||
## Memory Objective:
|
||||
You are managing **{memory_type}** memories about **{memory_target}** for the main agent. Focus on extracting and storing information directly related to this person’s preferences, habits, personal background, and significant facts.
|
||||
|
||||
|
|
@ -22,7 +27,8 @@ system_prompt: |
|
|||
1. **Analyze and Extract** potential memories from the dialogue context:
|
||||
- Determine whether the conversation contains important, memorable information, including but not limited to: user preferences, habits, or personal details; key facts, decisions, or conclusions; relationships or contextual background related to people or topics.
|
||||
- If the dialogue is casual chatter or contains no valuable information, output `<NO_MEMORY_NEEDED>` and stop.
|
||||
- Extract key information using clear and concise phrasing.
|
||||
- Extract key information using clear and concise phrasing **strictly based on what is explicitly stated in the context**.
|
||||
- **Important**: DO NOT infer, assume, or add any information beyond what is directly mentioned in the conversation.
|
||||
- Each memory entry must be self-contained and understandable without additional context.
|
||||
- Avoid storing trivial or temporary information.
|
||||
- Before proceeding, list all extracted memories in your response.
|
||||
|
|
@ -32,12 +38,13 @@ system_prompt: |
|
|||
- Retrieve related memories for comparison to check for duplication or associations.
|
||||
|
||||
3. **Compare and Decide** on memory operations:
|
||||
- Compare the newly extracted memories with historical ones to ensure the final memory store contains no duplicates or contradictions.
|
||||
- Compare the newly extracted memories with **both Recent Memories and historical memories** retrieved in the previous step.
|
||||
- Check for duplication, redundancy, and contradictions across both recent and historical memory sources.
|
||||
- Choose the appropriate operation based on the situation:
|
||||
- If the information already exists and is consistent: skip—no action needed.
|
||||
- If existing memory needs supplementation or correction: use `update_memory` to update it.
|
||||
- If existing memory is outdated or incorrect: use `delete_memory` to remove it.
|
||||
- If the information is entirely new: use `add_memory` to add it to the memory store.
|
||||
- If the information already exists in Recent Memories or historical memories and is consistent: skip—no action needed.
|
||||
- If existing memory (recent or historical) needs supplementation or correction: use `update_memory` to update it.
|
||||
- If existing memory (recent or historical) is outdated or incorrect: use `delete_memory` to remove it.
|
||||
- If the information is entirely new and not present in either Recent Memories or historical memories: use `add_memory` to add it to the memory store.
|
||||
|
||||
4. **Output** the result:
|
||||
- If no memory operation is required, output `<NO_MEMORY_NEEDED>`.
|
||||
|
|
@ -46,9 +53,10 @@ system_prompt: |
|
|||
## Guidelines:
|
||||
- Be selective: store only truly important information.
|
||||
- Stay concise: each memory should be clear and atomic.
|
||||
- Be accurate: ensure extracted content faithfully reflects the original context.
|
||||
- **Be strictly accurate**: ensure extracted content faithfully reflects ONLY what is explicitly stated in the original context. DO NOT infer, extrapolate, or fabricate any details.
|
||||
- Avoid redundancy: always check for similar existing memories before adding new ones.
|
||||
- Include relevant metadata (e.g., timestamps) when appropriate.
|
||||
- **No assumptions**: Only store information that is directly and clearly stated in the conversation.
|
||||
|
||||
user_message: |
|
||||
Please analyze the context to determine whether important information should be extracted and stored as memory, and perform memory addition, deletion, or update operations when necessary.
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
"""Orchestrator for complete memory summarization workflow across all memory types."""
|
||||
|
||||
from typing import List
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from ..base_memory_agent import BaseMemoryAgent
|
||||
from ...core.context import C
|
||||
from ...core.enumeration import Role
|
||||
from ...core.enumeration import Role, MemoryType
|
||||
from ...core.schema import Message, MemoryNode, ToolCall
|
||||
from ...core.utils import get_now_time, format_messages
|
||||
|
||||
|
|
@ -51,22 +49,16 @@ class ReMeSummarizer(BaseMemoryAgent):
|
|||
},
|
||||
)
|
||||
|
||||
async def _add_history_memory(self) -> MemoryNode:
|
||||
"""Store conversation history and return the memory node."""
|
||||
from ...mem_tool import AddHistoryMemory
|
||||
|
||||
op = AddHistoryMemory()
|
||||
await op.call(messages=self.get_messages())
|
||||
return op.memory_nodes[0]
|
||||
|
||||
@staticmethod
|
||||
async def _read_identity_memory() -> str:
|
||||
async def _read_identity_memory(self) -> str:
|
||||
"""Retrieve agent's self-perception memory."""
|
||||
from ...mem_tool import ReadIdentityMemory
|
||||
if self.enable_identity_memory:
|
||||
from ...mem_tool import ReadIdentityMemory
|
||||
|
||||
op = ReadIdentityMemory()
|
||||
await op.call()
|
||||
return op.output
|
||||
op = ReadIdentityMemory()
|
||||
await op.call()
|
||||
return op.output
|
||||
else:
|
||||
return ""
|
||||
|
||||
async def _read_meta_memories(self) -> str:
|
||||
"""Fetch all meta-memory entries that define specialized memory agents."""
|
||||
|
|
@ -79,29 +71,26 @@ class ReMeSummarizer(BaseMemoryAgent):
|
|||
await op.call()
|
||||
return str(op.output)
|
||||
|
||||
async def build_messages(self) -> List[Message]:
|
||||
async def build_messages(self) -> list[Message]:
|
||||
"""Construct initial messages with context, identity, and meta-memory information."""
|
||||
memory_node: MemoryNode = await self._add_history_memory()
|
||||
self.context["ref_memory_id"] = memory_node.memory_id
|
||||
messages = [Message(**m) if isinstance(m, dict) else m for m in self.context.messages]
|
||||
self.context["messages_formated"] = self.description + "\n" + format_messages(messages)
|
||||
self.context["ref_memory_id"] = MemoryNode(
|
||||
memory_type=MemoryType.HISTORY,
|
||||
content=self.context["messages_formated"],
|
||||
).memory_id
|
||||
|
||||
now_time = get_now_time()
|
||||
identity_memory = await self._read_identity_memory()
|
||||
meta_memory_info = await self._read_meta_memories()
|
||||
context = self.description + "\n" + format_messages(self.get_messages())
|
||||
logger.info(
|
||||
f"now_time={now_time} "
|
||||
f"memory_node={memory_node.content[:100]}... "
|
||||
f"identity_memory={identity_memory} "
|
||||
f"meta_memory_info={meta_memory_info} "
|
||||
f"context={context[:100]}",
|
||||
)
|
||||
logger.info(f"now_time={now_time} identity_memory={identity_memory} meta_memory_info={meta_memory_info}")
|
||||
|
||||
system_prompt = self.prompt_format(
|
||||
prompt_name="system_prompt",
|
||||
now_time=now_time,
|
||||
identity_memory=identity_memory,
|
||||
meta_memory_info=meta_memory_info,
|
||||
context=context,
|
||||
context=self.context["messages_formated"],
|
||||
)
|
||||
|
||||
user_message = self.get_prompt("user_message")
|
||||
|
|
@ -118,16 +107,12 @@ class ReMeSummarizer(BaseMemoryAgent):
|
|||
|
||||
if system_messages:
|
||||
system_message = system_messages[0]
|
||||
now_time = get_now_time()
|
||||
identity_memory = await self._read_identity_memory()
|
||||
meta_memory_info = await self._read_meta_memories()
|
||||
context = self.description + "\n" + format_messages(self.get_messages())
|
||||
system_message.content = self.prompt_format(
|
||||
prompt_name="system_prompt",
|
||||
now_time=now_time,
|
||||
identity_memory=identity_memory,
|
||||
meta_memory_info=meta_memory_info,
|
||||
context=context,
|
||||
now_time=get_now_time(),
|
||||
identity_memory=await self._read_identity_memory(),
|
||||
meta_memory_info=await self._read_meta_memories(),
|
||||
context=self.context["messages_formated"],
|
||||
)
|
||||
|
||||
return await super()._reasoning_step(messages, step, **kwargs)
|
||||
|
|
@ -140,6 +125,7 @@ class ReMeSummarizer(BaseMemoryAgent):
|
|||
messages=self.context.get("messages", []),
|
||||
description=self.context.get("description"),
|
||||
ref_memory_id=self.context["ref_memory_id"],
|
||||
messages_formated=self.context["messages_formated"],
|
||||
author=self.author,
|
||||
**kwargs,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -78,6 +78,16 @@ class BaseMemoryTool(BaseOp, metaclass=ABCMeta):
|
|||
"""Get the reference memory ID from context."""
|
||||
return self.context.get("ref_memory_id", "")
|
||||
|
||||
@property
|
||||
def messages_formated(self) -> str:
|
||||
"""Get the formated messages from context."""
|
||||
return self.context.get("messages_formated", "")
|
||||
|
||||
@property
|
||||
def retrieved_nodes(self) -> list[MemoryNode]:
|
||||
"""Get the retrieved nodes from context."""
|
||||
return self.context.get("retrieved_nodes")
|
||||
|
||||
@property
|
||||
def author(self) -> str:
|
||||
"""Get the author from context."""
|
||||
|
|
|
|||
|
|
@ -43,7 +43,9 @@ class ReadHistoryMemory(BaseMemoryTool):
|
|||
ref_memory_id = self.context.get("ref_memory_id", "")
|
||||
ref_memory_ids: list[str] = [ref_memory_id] if ref_memory_id else []
|
||||
|
||||
# Remove empty IDs and duplicates
|
||||
ref_memory_ids = [mid for mid in ref_memory_ids if mid]
|
||||
ref_memory_ids = list(dict.fromkeys(ref_memory_ids)) # Remove duplicates while preserving order
|
||||
|
||||
if not ref_memory_ids:
|
||||
self.output = "No valid reference memory IDs provided for reading."
|
||||
|
|
|
|||
|
|
@ -8,4 +8,4 @@ ref_memory_id: |
|
|||
Reference memory ID to query the original history dialogue.
|
||||
|
||||
ref_memory_ids: |
|
||||
List of reference memory IDs to query the original history dialogues.
|
||||
List of reference memory IDs to query the original history dialogues. Please provide unique IDs without duplicates.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ tool: |
|
|||
- Meta information: "I am very happy"
|
||||
- Personal preferences: "John prefers dark mode", "Alice works in PST timezone"
|
||||
- Procedural knowledge: "To deploy, run build then push", "Always validate input before processing"
|
||||
- Tool usage tips: "search_tool works best with short queries", "Use cache tool for frequently accessed data"
|
||||
|
||||
**CRITICAL**: Only add memories based on explicitly stated facts. DO NOT store inferred, assumed, or fabricated information.
|
||||
|
||||
tool_multiple: |
|
||||
Add multiple memories to the vector store for future retrieval.
|
||||
|
|
@ -12,19 +13,17 @@ tool_multiple: |
|
|||
Each memory can include when_to_use conditions and metadata for better organization and retrieval.
|
||||
Examples: storing multiple user preferences, multiple procedural steps, or multiple tool usage tips.
|
||||
|
||||
**CRITICAL**: Only add memories based on explicitly stated facts. DO NOT store inferred, assumed, or fabricated information in any of the memory entries.
|
||||
|
||||
when_to_use: |
|
||||
Optional condition description for when to retrieve this memory.
|
||||
This field is used for vector embedding to improve retrieval accuracy by providing contextual information.
|
||||
Examples:
|
||||
- "when user asks about authentication"
|
||||
- "when deploying to production"
|
||||
- "when using search_tool"
|
||||
- "when handling error cases"
|
||||
|
||||
memory_content: |
|
||||
The content of the memory to store.
|
||||
Should be a clear, concise statement that captures the information to remember.
|
||||
Keep it focused on a single piece of information for better retrieval accuracy.
|
||||
**Must be strictly accurate and based only on explicitly stated facts - no inference or fabrication.**
|
||||
|
||||
memories: |
|
||||
A list of memory objects to store.
|
||||
|
|
|
|||
|
|
@ -75,11 +75,11 @@ class AddSummaryMemory(AddMemory):
|
|||
) -> MemoryNode:
|
||||
"""Build MemoryNode from content, when_to_use, and metadata."""
|
||||
node = MemoryNode(
|
||||
memory_type=MemoryType.SUMMARY,
|
||||
memory_type=MemoryType.HISTORY,
|
||||
memory_target="",
|
||||
when_to_use="",
|
||||
content=memory_content,
|
||||
ref_memory_id=self.ref_memory_id,
|
||||
when_to_use=memory_content,
|
||||
content=self.messages_formated,
|
||||
ref_memory_id="",
|
||||
author=self.author,
|
||||
metadata=metadata or {},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,17 +3,7 @@ tool: |
|
|||
Use this tool to store a summarized version of the provided context.
|
||||
The LLM should first summarize the context, then call this tool with the summarized content.
|
||||
|
||||
This tool is specifically designed for storing summaries of conversations, events, or information
|
||||
that has been condensed from a larger context. Examples:
|
||||
- Summarizing a long conversation: "User discussed project requirements for a web app with authentication"
|
||||
- Summarizing a decision: "Team decided to use PostgreSQL for the database after evaluating options"
|
||||
- Summarizing an event: "Successfully deployed version 2.0 to production with new features"
|
||||
|
||||
summary_memory: |
|
||||
The summarized content to store as memory.
|
||||
Should be a clear, concise summary that captures the key information from the context.
|
||||
Keep it focused and informative - aim for 1-3 sentences that convey the essential points.
|
||||
Examples:
|
||||
- "User prefers Python for backend development and has experience with FastAPI framework"
|
||||
- "Project deadline is January 15th, requires authentication, payment integration, and admin dashboard"
|
||||
- "Bug in user registration was caused by missing email validation, fixed by adding regex check"
|
||||
|
|
|
|||
|
|
@ -16,17 +16,14 @@ class RetrieveRecentMemory(BaseMemoryTool):
|
|||
Uses memory_type and memory_target from context (self.memory_type, self.memory_target).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
top_k: int = 20,
|
||||
**kwargs,
|
||||
):
|
||||
def __init__(self, top_k: int = 20, **kwargs):
|
||||
"""Initialize RetrieveRecentMemory.
|
||||
|
||||
Args:
|
||||
top_k: Max memories to retrieve.
|
||||
**kwargs: Additional args for BaseMemoryTool.
|
||||
"""
|
||||
kwargs["enable_multiple"] = False
|
||||
super().__init__(**kwargs)
|
||||
self.top_k: int = top_k
|
||||
|
||||
|
|
@ -60,19 +57,29 @@ class RetrieveRecentMemory(BaseMemoryTool):
|
|||
Outputs formatted results or error message.
|
||||
"""
|
||||
if not self.memory_type or not self.memory_target:
|
||||
self.output = "memory_type and memory_target are required for retrieval."
|
||||
return
|
||||
raise RuntimeError("memory_type and memory_target are required for retrieval.")
|
||||
|
||||
# Retrieve recent memories
|
||||
memory_nodes: list[MemoryNode] = await self._retrieve_recent()
|
||||
|
||||
# Deduplicate and format output
|
||||
memory_nodes = deduplicate_memories(memory_nodes)
|
||||
self.memory_nodes = memory_nodes
|
||||
|
||||
if not memory_nodes:
|
||||
self.output = "No memory_nodes found."
|
||||
# Build set of historical memory_ids for fast lookup
|
||||
retrieved_memory_ids = {node.memory_id for node in self.retrieved_nodes if node.memory_id}
|
||||
|
||||
# Filter out already retrieved memories by memory_id
|
||||
new_memory_nodes = [node for node in memory_nodes if node.memory_id not in retrieved_memory_ids]
|
||||
|
||||
# Update retrieved_nodes in context with new memories
|
||||
self.retrieved_nodes.extend(new_memory_nodes)
|
||||
|
||||
# Set output to new memories only (after deduplication)
|
||||
self.memory_nodes = new_memory_nodes
|
||||
|
||||
if not new_memory_nodes:
|
||||
self.output = "No new memory_nodes found (duplicates removed)."
|
||||
else:
|
||||
self.output = "\n".join([m.format_memory() for m in memory_nodes])
|
||||
self.output = "\n".join([m.format_memory() for m in new_memory_nodes])
|
||||
|
||||
logger.info(f"Retrieved {len(memory_nodes)} recent memory_nodes")
|
||||
logger.info(f"Retrieved {len(memory_nodes)} memory_nodes, {len(new_memory_nodes)} new after deduplication")
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ tool: |
|
|||
- You need to refine or improve the clarity of stored information
|
||||
Memory ID can be obtained from previous memory retrieval results.
|
||||
|
||||
**CRITICAL**: Only update memories with explicitly stated facts. DO NOT introduce inferred, assumed, or fabricated information in the updated content.
|
||||
|
||||
tool_multiple: |
|
||||
Update multiple memories in the vector store by replacing old memories with new content.
|
||||
Use this tool for batch updates when:
|
||||
|
|
@ -16,6 +18,8 @@ tool_multiple: |
|
|||
- You need to refine or improve multiple stored memories at once
|
||||
Memory IDs can be obtained from previous memory retrieval results.
|
||||
|
||||
**CRITICAL**: Only update memories with explicitly stated facts. DO NOT introduce inferred, assumed, or fabricated information in any of the updated contents.
|
||||
|
||||
memory_id: |
|
||||
The unique identifier (memory_id) of the old memory to be replaced.
|
||||
This ID is returned when memories are retrieved or added.
|
||||
|
|
@ -28,6 +32,7 @@ memory_content: |
|
|||
The new content of the memory to store.
|
||||
Should be a clear, concise statement that captures the updated information to remember.
|
||||
Keep it focused on a single piece of information for better retrieval accuracy.
|
||||
**Must be strictly accurate and based only on explicitly stated facts - no inference or fabrication.**
|
||||
|
||||
memories: |
|
||||
A list of memory update objects.
|
||||
|
|
|
|||
|
|
@ -237,11 +237,22 @@ class VectorRetrieveMemory(BaseMemoryTool):
|
|||
|
||||
# Deduplicate and format output
|
||||
memory_nodes = deduplicate_memories(memory_nodes)
|
||||
self.memory_nodes = memory_nodes
|
||||
|
||||
if not memory_nodes:
|
||||
self.output = "No memory_nodes found matching the query."
|
||||
# Build set of historical memory_ids for fast lookup
|
||||
retrieved_memory_ids = {node.memory_id for node in self.retrieved_nodes if node.memory_id}
|
||||
|
||||
# Filter out already retrieved memories by memory_id
|
||||
new_memory_nodes = [node for node in memory_nodes if node.memory_id not in retrieved_memory_ids]
|
||||
|
||||
# Update retrieved_nodes in context with new memories
|
||||
self.retrieved_nodes.extend(new_memory_nodes)
|
||||
|
||||
# Set output to new memories only (after deduplication)
|
||||
self.memory_nodes = new_memory_nodes
|
||||
|
||||
if not new_memory_nodes:
|
||||
self.output = "No new memory_nodes found matching the query (duplicates removed)."
|
||||
else:
|
||||
self.output = "\n".join([m.format_memory() for m in memory_nodes])
|
||||
self.output = "\n".join([m.format_memory() for m in new_memory_nodes])
|
||||
|
||||
logger.info(f"Retrieved {len(memory_nodes)} memory_nodes")
|
||||
logger.info(f"Retrieved {len(memory_nodes)} memory_nodes, {len(new_memory_nodes)} new after deduplication")
|
||||
|
|
|
|||
|
|
@ -3,25 +3,29 @@ tool: |
|
|||
Use this tool to find relevant memories based on semantic similarity to the query.
|
||||
The search returns the most relevant memories ranked by similarity score.
|
||||
|
||||
Note: Within the same session, this tool automatically deduplicates results across multiple calls.
|
||||
If you call this tool multiple times, only new memories (not previously retrieved) will be returned.
|
||||
This prevents redundant information in subsequent retrievals.
|
||||
|
||||
tool_multiple: |
|
||||
Retrieve memories from the memory store using multiple queries with vector similarity search.
|
||||
Use this tool to find relevant memories based on semantic similarity to multiple queries.
|
||||
This is useful when you need to search for different types of information in a single operation.
|
||||
The search returns the most relevant memories ranked by similarity score for each query.
|
||||
|
||||
Note: Within the same session, this tool automatically deduplicates results across multiple calls.
|
||||
If you call this tool multiple times, only new memories (not previously retrieved) will be returned.
|
||||
This prevents redundant information in subsequent retrievals.
|
||||
|
||||
memory_type: |
|
||||
The type of memory to search for. Must be one of:
|
||||
- "identity": Information about the AI agent's identity, role, or characteristics
|
||||
- "personal": Information about users, their preferences, or personal details
|
||||
- "procedural": Step-by-step instructions, workflows, or how-to knowledge
|
||||
- "tool": Tool usage tips, examples, and best practices
|
||||
|
||||
memory_target: |
|
||||
The target of the memory to search within.
|
||||
- For "personal" memory: the person's name or identifier (e.g., "john", "alice")
|
||||
- For "procedural" memory: the process or task name (e.g., "deployment", "authentication")
|
||||
- For "tool" memory: the tool name (e.g., "search_tool", "calculator")
|
||||
- For "identity" memory: typically "self" or the agent's identifier
|
||||
|
||||
query: |
|
||||
The query text for vector similarity search.
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ class ReMe(Application):
|
|||
"year": "The `year` information associated with the memory(Optional)",
|
||||
"month": "The `month` information associated with the memory(Optional)",
|
||||
"day": "The `day` information associated with the memory(Optional)",
|
||||
"hour": "The `hour` information associated with the memory(Optional)",
|
||||
# "hour": "The `hour` information associated with the memory(Optional)",
|
||||
# "year": "The year when the memory content occurred(Optional)",
|
||||
# "month": "The month when the memory content occurred(Optional)",
|
||||
# "day": "The day when the memory content occurred(Optional)",
|
||||
|
|
@ -110,7 +110,6 @@ class ReMe(Application):
|
|||
personal_summarizer = PersonalSummarizer(
|
||||
tools=[
|
||||
VectorRetrieveMemory(
|
||||
enable_summary_memory=False,
|
||||
add_memory_type_target=False,
|
||||
metadata_desc=None,
|
||||
top_k=15,
|
||||
|
|
@ -144,6 +143,7 @@ class ReMe(Application):
|
|||
description: str = "",
|
||||
user_id: str = "",
|
||||
assistant_id: str = "",
|
||||
top_k: int = 20,
|
||||
**kwargs,
|
||||
):
|
||||
"""Retrieves relevant memories based on the query and specified memory mode."""
|
||||
|
|
@ -155,7 +155,7 @@ class ReMe(Application):
|
|||
"year": "The year to filter memories(Optional)",
|
||||
"month": "The month to filter memories(Optional)",
|
||||
"day": "The day to filter memories(Optional)",
|
||||
"hour": "The hour to filter memories(Optional)",
|
||||
# "hour": "The hour to filter memories(Optional)",
|
||||
}
|
||||
meta_memories = [
|
||||
{
|
||||
|
|
@ -168,17 +168,15 @@ class ReMe(Application):
|
|||
meta_memories=meta_memories,
|
||||
tools=[
|
||||
VectorRetrieveMemory(
|
||||
enable_summary_memory=True,
|
||||
add_memory_type_target=True,
|
||||
metadata_desc=metadata_retrieve,
|
||||
top_k=20,
|
||||
top_k=top_k,
|
||||
),
|
||||
ReadHistoryMemory(),
|
||||
],
|
||||
)
|
||||
|
||||
await reme_retriever.call(query=query, messages=messages, description=description, **kwargs)
|
||||
|
||||
return reme_retriever.output
|
||||
|
||||
else:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue