diff --git a/benchmark/halumem/eval_reme.py b/benchmark/halumem/eval_reme.py index 14c0cb64..8cc27e53 100644 --- a/benchmark/halumem/eval_reme.py +++ b/benchmark/halumem/eval_reme.py @@ -577,6 +577,8 @@ class MemoryIntegrityEvaluator: ) # Build result record + if eval_result is None: + eval_result = {} integrity_result = { **memory_point, "integrity_score": eval_result.get("score"), @@ -619,9 +621,12 @@ class MemoryAccuracyEvaluator: dialogue=formatted_dialogue, golden_memories=memory_points, candidate_memory=memory, + eval_model_name=self.eval_model_name, ) # Build result record + if eval_result is None: + eval_result = {} accuracy_result = { "memory_content": memory.get("content", memory.get("memory_content", str(memory))), "memory_id": memory.get("memory_id", ""), diff --git a/benchmark/longmemeval/eval_longmemeval_reme.py b/benchmark/longmemeval/eval_longmemeval_reme.py index ada3372b..c529b434 100644 --- a/benchmark/longmemeval/eval_longmemeval_reme.py +++ b/benchmark/longmemeval/eval_longmemeval_reme.py @@ -41,8 +41,9 @@ class EvalConfig: max_concurrency: int = 1 batch_size: int = 30 output_dir: str = "cache/bench_results/longmemeval_reme" - reme_model_name: str = "qwen-flash" - eval_model_name: str = "qwen3-max" + reme_model_name: str = "qwen-flash" # summary模型 + retrieve_model_name: str = "qwen-max" # retrieve模型 + eval_model_name: str = "qwen-max" # 评估/判断模型 algo_version: str = "v1" samples_per_type: int = -1 # Number of samples per question type, -1 for all enable_thinking_params: bool = False @@ -253,7 +254,7 @@ async def answer_question_with_memories( question: str, memories: str, user_id: str = None, - model_name: str = "qwen3-max", + model_name: str = "qwen-max", ): """ Answer a question using retrieved memories with PROMPT_MEMZERO_JSON template. @@ -285,7 +286,7 @@ async def answer_question_with_memories( question=question, ) - result = await reme.get_llm("qwen-flash").simple_request_for_json( + result = await reme.default_llm.simple_request_for_json( prompt=prompt, model_name=model_name, ) @@ -303,12 +304,14 @@ class MemoryProcessor: self, reme: ReMe, reme_model_name: str = "qwen-flash", - eval_model_name: str = "qwen3-max", + retrieve_model_name: str = "qwen-max", + eval_model_name: str = "qwen-max", algo_version: str = "v1", enable_thinking_params: bool = False, ): self.reme = reme self.reme_model_name = reme_model_name + self.retrieve_model_name = retrieve_model_name self.eval_model_name = eval_model_name self.algo_version = algo_version self.enable_thinking_params = enable_thinking_params @@ -368,7 +371,7 @@ class MemoryProcessor: # Retrieve memories from ReMe using new API result = await self.reme.retrieve_memory( - llm_config_name="qwen-max-t", + llm_config_name=self.retrieve_model_name, query=query, retrieve_top_k=top_k, user_name=user_id, @@ -571,6 +574,10 @@ class LongMemEvalEvaluator: "backend": "openai", "model_name": "qwen-flash", }, + "qwen-max": { + "backend": "openai", + "model_name": "qwen3-max", + }, } # Load evaluation prompts path @@ -651,6 +658,7 @@ class LongMemEvalEvaluator: memory_processor = MemoryProcessor( reme, self.config.reme_model_name, + self.config.retrieve_model_name, self.config.eval_model_name, self.config.algo_version, self.config.enable_thinking_params, @@ -777,7 +785,10 @@ class LongMemEvalEvaluator: print(f"Samples per type: {self.config.samples_per_type} (-1 = all)") print(f"Questions to process: {total_questions} | Top-K: {self.config.top_k}") print(f"Max Concurrency: {self.config.max_concurrency}") - print(f"ReMe Model: {self.config.reme_model_name} | Eval Model: {self.config.eval_model_name}") + print( + f"Summary Model: {self.config.reme_model_name} | Retrieve Model: {self.config.retrieve_model_name} " + f"| Eval Model: {self.config.eval_model_name}", + ) print(f"Algo Version: {self.config.algo_version}") print("=" * 80 + "\n") @@ -908,7 +919,8 @@ async def main_async( batch_size: int = 30, output_dir: str = "bench_results/longmemeval_reme", reme_model_name: str = "qwen-flash", - eval_model_name: str = "qwen3-max", + retrieve_model_name: str = "qwen-max", + eval_model_name: str = "qwen-max", algo_version: str = "v1", samples_per_type: int = -1, enable_thinking_params: bool = False, @@ -923,6 +935,7 @@ async def main_async( batch_size=batch_size, output_dir=output_dir, reme_model_name=reme_model_name, + retrieve_model_name=retrieve_model_name, eval_model_name=eval_model_name, algo_version=algo_version, samples_per_type=samples_per_type, @@ -943,7 +956,8 @@ def main( batch_size: int = 30, output_dir: str = "bench_results/longmemeval_reme", reme_model_name: str = "qwen-flash", - eval_model_name: str = "qwen3-max", + retrieve_model_name: str = "qwen-max", + eval_model_name: str = "qwen-max", algo_version: str = "v1", samples_per_type: int = -1, enable_thinking_params: bool = False, @@ -959,6 +973,7 @@ def main( batch_size=batch_size, output_dir=output_dir, reme_model_name=reme_model_name, + retrieve_model_name=retrieve_model_name, eval_model_name=eval_model_name, algo_version=algo_version, samples_per_type=samples_per_type, @@ -976,8 +991,7 @@ if __name__ == "__main__": parser.add_argument( "--data_path", type=str, - # default="/Users/zhouwk/PycharmProjects/MemAgent/dataset/longmemeval/longmemeval_s_cleaned.json", - default="/Users/zhouwk/PycharmProjects/MemAgent/dataset/longmemeval/longmemeval_oracle.json", + required=True, help="Path to LongMemEval JSON file", ) parser.add_argument( @@ -1001,7 +1015,7 @@ if __name__ == "__main__": parser.add_argument( "--max_concurrency", type=int, - default=1, + default=4, help="Maximum concurrent question processing (default: 1)", ) parser.add_argument( @@ -1019,16 +1033,20 @@ if __name__ == "__main__": parser.add_argument( "--reme_model_name", type=str, - # default="gpt-4o-mini-2024-07-18", default="qwen-flash", - help="Model name for ReMe operations (default: qwen-flash)", + help="Model name for ReMe summary operations (default: qwen-flash)", + ) + parser.add_argument( + "--retrieve_model_name", + type=str, + default="qwen-max", + help="Model name for memory retrieval (default: qwen-max)", ) parser.add_argument( "--eval_model_name", type=str, - # default="gpt-4o-mini-2024-07-18", - default="qwen-max", - help="Model name for evaluation/judgment (default: qwen3-max)", + default="qwen-flash", + help="Model name for evaluation/judgment (default: qwen-max)", ) parser.add_argument( "--algo_version", @@ -1039,7 +1057,7 @@ if __name__ == "__main__": parser.add_argument( "--samples_per_type", type=int, - default=4, + default=1, help="Number of samples per question type, -1 for all (default: -1)", ) parser.add_argument( @@ -1061,8 +1079,9 @@ if __name__ == "__main__": batch_size=args.batch_size, output_dir=args.output_dir, reme_model_name=args.reme_model_name, + retrieve_model_name=args.retrieve_model_name, eval_model_name=args.eval_model_name, algo_version=args.algo_version, samples_per_type=args.samples_per_type, enable_thinking_params=args.enable_thinking_params, - ) \ No newline at end of file + ) diff --git a/reme/reme.py b/reme/reme.py index faec7ed5..6c5525ee 100644 --- a/reme/reme.py +++ b/reme/reme.py @@ -647,4 +647,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main()