mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-15 23:31:05 +00:00
refactor(benchmark): 重构长记忆评估中的模型配置
- 将原有的 eval_model_name 替换为专门的 retrieve_model_name 用于检索操作 - 添加对 qwen-max 模型配置的支持 - 更新参数解析器以支持新的检索模型参数 - 修改最大并发数默认值从 1 提升到 4 - 调整样本数量默认值从 4 减少到 1 - 统一模型参数命名规范，区分摘要、检索和评估模型 - 优化内存处理器初始化逻辑，支持独立的检索模型配置
This commit is contained in:
方应
parent
526d301051
commit
bb59a36f2c
2 changed files with 40 additions and 22 deletions
|
|
@ -191,7 +191,7 @@ async def answer_question_with_memories(
|
|||
question: str,
|
||||
memories: str,
|
||||
user_id: str = None,
|
||||
model_name: str = "qwen3-30b-a3b-instruct-2507",
|
||||
eval_model_name: str = "qwen3-30b-a3b-instruct-2507",
|
||||
):
|
||||
"""
|
||||
Answer a question using retrieved memories with PROMPT_MEMZERO_JSON template.
|
||||
|
|
@ -201,7 +201,7 @@ async def answer_question_with_memories(
|
|||
question: The question to answer
|
||||
memories: The retrieved memories (formatted as context)
|
||||
user_id: Optional user ID for context formatting
|
||||
model_name: Model name to use for LLM request
|
||||
eval_model_name: Model name to use for LLM request
|
||||
|
||||
Returns:
|
||||
dict with 'reasoning' and 'answer' fields
|
||||
|
|
@ -223,7 +223,7 @@ async def answer_question_with_memories(
|
|||
question=question,
|
||||
)
|
||||
|
||||
result = await reme.get_llm(model_name).simple_request_for_json(
|
||||
result = await reme.get_llm(eval_model_name).simple_request_for_json(
|
||||
prompt=prompt,
|
||||
model_name=None,
|
||||
)
|
||||
|
|
@ -236,6 +236,7 @@ async def evaluation_for_memory_accuracy(
|
|||
dialogue: str,
|
||||
golden_memories: list[dict],
|
||||
candidate_memory: dict,
|
||||
eval_model_name: str = "qwen-flash",
|
||||
):
|
||||
"""
|
||||
Memory Accuracy Evaluation - Check if an extracted memory is accurate.
|
||||
|
|
@ -245,7 +246,7 @@ async def evaluation_for_memory_accuracy(
|
|||
dialogue: The formatted dialogue string
|
||||
golden_memories: List of golden memory points from the session
|
||||
candidate_memory: The extracted memory to evaluate
|
||||
model_name: Model name to use for LLM request
|
||||
eval_model_name: Model name to use for LLM request
|
||||
|
||||
Returns:
|
||||
dict with 'accuracy_score' (0/1/2), 'is_included_in_golden_memories' (true/false), and 'reason'
|
||||
|
|
@ -265,7 +266,7 @@ async def evaluation_for_memory_accuracy(
|
|||
candidate_memory=candidate_content,
|
||||
)
|
||||
|
||||
result = await reme.get_llm("qwen-flash").simple_request_for_json(
|
||||
result = await reme.get_llm(eval_model_name).simple_request_for_json(
|
||||
prompt=prompt,
|
||||
model_name=None,
|
||||
)
|
||||
|
|
@ -454,7 +455,7 @@ class MemoryProcessor:
|
|||
question=query,
|
||||
memories=memories,
|
||||
user_id=user_id,
|
||||
model_name=self.eval_model_name,
|
||||
eval_model_name=self.eval_model_name,
|
||||
)
|
||||
|
||||
# Add original memories to the result
|
||||
|
|
|
|||
|
|
@ -41,8 +41,9 @@ class EvalConfig:
|
|||
max_concurrency: int = 1
|
||||
batch_size: int = 30
|
||||
output_dir: str = "cache/bench_results/longmemeval_reme"
|
||||
reme_model_name: str = "qwen-flash"
|
||||
eval_model_name: str = "qwen3-max"
|
||||
reme_model_name: str = "qwen-flash" # summary模型
|
||||
retrieve_model_name: str = "qwen-max" # retrieve模型
|
||||
eval_model_name: str = "qwen-max" # 评估/判断模型
|
||||
algo_version: str = "v1"
|
||||
samples_per_type: int = -1 # Number of samples per question type, -1 for all
|
||||
enable_thinking_params: bool = False
|
||||
|
|
@ -253,7 +254,7 @@ async def answer_question_with_memories(
|
|||
question: str,
|
||||
memories: str,
|
||||
user_id: str = None,
|
||||
model_name: str = "qwen3-max",
|
||||
model_name: str = "qwen-max",
|
||||
):
|
||||
"""
|
||||
Answer a question using retrieved memories with PROMPT_MEMZERO_JSON template.
|
||||
|
|
@ -285,7 +286,7 @@ async def answer_question_with_memories(
|
|||
question=question,
|
||||
)
|
||||
|
||||
result = await reme.get_llm("qwen-flash").simple_request_for_json(
|
||||
result = await reme.default_llm.simple_request_for_json(
|
||||
prompt=prompt,
|
||||
model_name=model_name,
|
||||
)
|
||||
|
|
@ -303,12 +304,14 @@ class MemoryProcessor:
|
|||
self,
|
||||
reme: ReMe,
|
||||
reme_model_name: str = "qwen-flash",
|
||||
eval_model_name: str = "qwen3-max",
|
||||
retrieve_model_name: str = "qwen-max",
|
||||
eval_model_name: str = "qwen-max",
|
||||
algo_version: str = "v1",
|
||||
enable_thinking_params: bool = False,
|
||||
):
|
||||
self.reme = reme
|
||||
self.reme_model_name = reme_model_name
|
||||
self.retrieve_model_name = retrieve_model_name
|
||||
self.eval_model_name = eval_model_name
|
||||
self.algo_version = algo_version
|
||||
self.enable_thinking_params = enable_thinking_params
|
||||
|
|
@ -368,7 +371,7 @@ class MemoryProcessor:
|
|||
|
||||
# Retrieve memories from ReMe using new API
|
||||
result = await self.reme.retrieve_memory(
|
||||
llm_config_name="qwen-max-t",
|
||||
llm_config_name=self.retrieve_model_name,
|
||||
query=query,
|
||||
retrieve_top_k=top_k,
|
||||
user_name=user_id,
|
||||
|
|
@ -571,6 +574,10 @@ class LongMemEvalEvaluator:
|
|||
"backend": "openai",
|
||||
"model_name": "qwen-flash",
|
||||
},
|
||||
"qwen-max": {
|
||||
"backend": "openai",
|
||||
"model_name": "qwen3-max",
|
||||
},
|
||||
}
|
||||
|
||||
# Load evaluation prompts path
|
||||
|
|
@ -651,6 +658,7 @@ class LongMemEvalEvaluator:
|
|||
memory_processor = MemoryProcessor(
|
||||
reme,
|
||||
self.config.reme_model_name,
|
||||
self.config.retrieve_model_name,
|
||||
self.config.eval_model_name,
|
||||
self.config.algo_version,
|
||||
self.config.enable_thinking_params,
|
||||
|
|
@ -777,7 +785,7 @@ class LongMemEvalEvaluator:
|
|||
print(f"Samples per type: {self.config.samples_per_type} (-1 = all)")
|
||||
print(f"Questions to process: {total_questions} | Top-K: {self.config.top_k}")
|
||||
print(f"Max Concurrency: {self.config.max_concurrency}")
|
||||
print(f"ReMe Model: {self.config.reme_model_name} | Eval Model: {self.config.eval_model_name}")
|
||||
print(f"Summary Model: {self.config.reme_model_name} | Retrieve Model: {self.config.retrieve_model_name} | Eval Model: {self.config.eval_model_name}")
|
||||
print(f"Algo Version: {self.config.algo_version}")
|
||||
print("=" * 80 + "\n")
|
||||
|
||||
|
|
@ -908,7 +916,8 @@ async def main_async(
|
|||
batch_size: int = 30,
|
||||
output_dir: str = "bench_results/longmemeval_reme",
|
||||
reme_model_name: str = "qwen-flash",
|
||||
eval_model_name: str = "qwen3-max",
|
||||
retrieve_model_name: str = "qwen-max",
|
||||
eval_model_name: str = "qwen-max",
|
||||
algo_version: str = "v1",
|
||||
samples_per_type: int = -1,
|
||||
enable_thinking_params: bool = False,
|
||||
|
|
@ -923,6 +932,7 @@ async def main_async(
|
|||
batch_size=batch_size,
|
||||
output_dir=output_dir,
|
||||
reme_model_name=reme_model_name,
|
||||
retrieve_model_name=retrieve_model_name,
|
||||
eval_model_name=eval_model_name,
|
||||
algo_version=algo_version,
|
||||
samples_per_type=samples_per_type,
|
||||
|
|
@ -943,7 +953,8 @@ def main(
|
|||
batch_size: int = 30,
|
||||
output_dir: str = "bench_results/longmemeval_reme",
|
||||
reme_model_name: str = "qwen-flash",
|
||||
eval_model_name: str = "qwen3-max",
|
||||
retrieve_model_name: str = "qwen-max",
|
||||
eval_model_name: str = "qwen-max",
|
||||
algo_version: str = "v1",
|
||||
samples_per_type: int = -1,
|
||||
enable_thinking_params: bool = False,
|
||||
|
|
@ -959,6 +970,7 @@ def main(
|
|||
batch_size=batch_size,
|
||||
output_dir=output_dir,
|
||||
reme_model_name=reme_model_name,
|
||||
retrieve_model_name=retrieve_model_name,
|
||||
eval_model_name=eval_model_name,
|
||||
algo_version=algo_version,
|
||||
samples_per_type=samples_per_type,
|
||||
|
|
@ -1001,7 +1013,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument(
|
||||
"--max_concurrency",
|
||||
type=int,
|
||||
default=1,
|
||||
default=4,
|
||||
help="Maximum concurrent question processing (default: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
|
|
@ -1019,16 +1031,20 @@ if __name__ == "__main__":
|
|||
parser.add_argument(
|
||||
"--reme_model_name",
|
||||
type=str,
|
||||
# default="gpt-4o-mini-2024-07-18",
|
||||
default="qwen-flash",
|
||||
help="Model name for ReMe operations (default: qwen-flash)",
|
||||
help="Model name for ReMe summary operations (default: qwen-flash)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--retrieve_model_name",
|
||||
type=str,
|
||||
default="qwen-max",
|
||||
help="Model name for memory retrieval (default: qwen-max)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--eval_model_name",
|
||||
type=str,
|
||||
# default="gpt-4o-mini-2024-07-18",
|
||||
default="qwen-max",
|
||||
help="Model name for evaluation/judgment (default: qwen3-max)",
|
||||
default="qwen-flash",
|
||||
help="Model name for evaluation/judgment (default: qwen-max)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--algo_version",
|
||||
|
|
@ -1039,7 +1055,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument(
|
||||
"--samples_per_type",
|
||||
type=int,
|
||||
default=4,
|
||||
default=1,
|
||||
help="Number of samples per question type, -1 for all (default: -1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
|
|
@ -1061,6 +1077,7 @@ if __name__ == "__main__":
|
|||
batch_size=args.batch_size,
|
||||
output_dir=args.output_dir,
|
||||
reme_model_name=args.reme_model_name,
|
||||
retrieve_model_name=args.retrieve_model_name,
|
||||
eval_model_name=args.eval_model_name,
|
||||
algo_version=args.algo_version,
|
||||
samples_per_type=args.samples_per_type,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue