From 526d301051af4164f118261a8b7508a2ecd31587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=96=B9=E5=BA=94?= Date: Tue, 3 Mar 2026 10:55:44 +0800 Subject: [PATCH 1/5] reformat 2 files --- benchmark/longmemeval/eval_longmemeval_reme.py | 2 +- reme/reme.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/longmemeval/eval_longmemeval_reme.py b/benchmark/longmemeval/eval_longmemeval_reme.py index ada3372b..98635850 100644 --- a/benchmark/longmemeval/eval_longmemeval_reme.py +++ b/benchmark/longmemeval/eval_longmemeval_reme.py @@ -1065,4 +1065,4 @@ if __name__ == "__main__": algo_version=args.algo_version, samples_per_type=args.samples_per_type, enable_thinking_params=args.enable_thinking_params, - ) \ No newline at end of file + ) diff --git a/reme/reme.py b/reme/reme.py index faec7ed5..6c5525ee 100644 --- a/reme/reme.py +++ b/reme/reme.py @@ -647,4 +647,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() From bb59a36f2c3e36646944ce59df6afe378bfb2fd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=96=B9=E5=BA=94?= Date: Tue, 3 Mar 2026 11:26:45 +0800 Subject: [PATCH 2/5] =?UTF-8?q?refactor(benchmark):=20=E9=87=8D=E6=9E=84?= =?UTF-8?q?=E9=95=BF=E8=AE=B0=E5=BF=86=E8=AF=84=E4=BC=B0=E4=B8=AD=E7=9A=84?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 将原有的 eval_model_name 替换为专门的 retrieve_model_name 用于检索操作 - 添加对 qwen-max 模型配置的支持 - 更新参数解析器以支持新的检索模型参数 - 修改最大并发数默认值从 1 提升到 4 - 调整样本数量默认值从 4 减少到 1 - 统一模型参数命名规范，区分摘要、检索和评估模型 - 优化内存处理器初始化逻辑，支持独立的检索模型配置 --- benchmark/halumem/eval_reme.py | 13 ++--- .../longmemeval/eval_longmemeval_reme.py | 49 +++++++++++++------ 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/benchmark/halumem/eval_reme.py b/benchmark/halumem/eval_reme.py index 685f8249..5c4cf40e 100644 --- a/benchmark/halumem/eval_reme.py +++ b/benchmark/halumem/eval_reme.py @@ -191,7 +191,7 @@ async def answer_question_with_memories( question: str, memories: str, user_id: str = None, - model_name: str = "qwen3-30b-a3b-instruct-2507", + eval_model_name: str = "qwen3-30b-a3b-instruct-2507", ): """ Answer a question using retrieved memories with PROMPT_MEMZERO_JSON template. @@ -201,7 +201,7 @@ async def answer_question_with_memories( question: The question to answer memories: The retrieved memories (formatted as context) user_id: Optional user ID for context formatting - model_name: Model name to use for LLM request + eval_model_name: Model name to use for LLM request Returns: dict with 'reasoning' and 'answer' fields @@ -223,7 +223,7 @@ async def answer_question_with_memories( question=question, ) - result = await reme.get_llm(model_name).simple_request_for_json( + result = await reme.get_llm(eval_model_name).simple_request_for_json( prompt=prompt, model_name=None, ) @@ -236,6 +236,7 @@ async def evaluation_for_memory_accuracy( dialogue: str, golden_memories: list[dict], candidate_memory: dict, + eval_model_name: str = "qwen-flash", ): """ Memory Accuracy Evaluation - Check if an extracted memory is accurate. @@ -245,7 +246,7 @@ async def evaluation_for_memory_accuracy( dialogue: The formatted dialogue string golden_memories: List of golden memory points from the session candidate_memory: The extracted memory to evaluate - model_name: Model name to use for LLM request + eval_model_name: Model name to use for LLM request Returns: dict with 'accuracy_score' (0/1/2), 'is_included_in_golden_memories' (true/false), and 'reason' @@ -265,7 +266,7 @@ async def evaluation_for_memory_accuracy( candidate_memory=candidate_content, ) - result = await reme.get_llm("qwen-flash").simple_request_for_json( + result = await reme.get_llm(eval_model_name).simple_request_for_json( prompt=prompt, model_name=None, ) @@ -454,7 +455,7 @@ class MemoryProcessor: question=query, memories=memories, user_id=user_id, - model_name=self.eval_model_name, + eval_model_name=self.eval_model_name, ) # Add original memories to the result diff --git a/benchmark/longmemeval/eval_longmemeval_reme.py b/benchmark/longmemeval/eval_longmemeval_reme.py index 98635850..b8a57c65 100644 --- a/benchmark/longmemeval/eval_longmemeval_reme.py +++ b/benchmark/longmemeval/eval_longmemeval_reme.py @@ -41,8 +41,9 @@ class EvalConfig: max_concurrency: int = 1 batch_size: int = 30 output_dir: str = "cache/bench_results/longmemeval_reme" - reme_model_name: str = "qwen-flash" - eval_model_name: str = "qwen3-max" + reme_model_name: str = "qwen-flash" # summary模型 + retrieve_model_name: str = "qwen-max" # retrieve模型 + eval_model_name: str = "qwen-max" # 评估/判断模型 algo_version: str = "v1" samples_per_type: int = -1 # Number of samples per question type, -1 for all enable_thinking_params: bool = False @@ -253,7 +254,7 @@ async def answer_question_with_memories( question: str, memories: str, user_id: str = None, - model_name: str = "qwen3-max", + model_name: str = "qwen-max", ): """ Answer a question using retrieved memories with PROMPT_MEMZERO_JSON template. @@ -285,7 +286,7 @@ async def answer_question_with_memories( question=question, ) - result = await reme.get_llm("qwen-flash").simple_request_for_json( + result = await reme.default_llm.simple_request_for_json( prompt=prompt, model_name=model_name, ) @@ -303,12 +304,14 @@ class MemoryProcessor: self, reme: ReMe, reme_model_name: str = "qwen-flash", - eval_model_name: str = "qwen3-max", + retrieve_model_name: str = "qwen-max", + eval_model_name: str = "qwen-max", algo_version: str = "v1", enable_thinking_params: bool = False, ): self.reme = reme self.reme_model_name = reme_model_name + self.retrieve_model_name = retrieve_model_name self.eval_model_name = eval_model_name self.algo_version = algo_version self.enable_thinking_params = enable_thinking_params @@ -368,7 +371,7 @@ class MemoryProcessor: # Retrieve memories from ReMe using new API result = await self.reme.retrieve_memory( - llm_config_name="qwen-max-t", + llm_config_name=self.retrieve_model_name, query=query, retrieve_top_k=top_k, user_name=user_id, @@ -571,6 +574,10 @@ class LongMemEvalEvaluator: "backend": "openai", "model_name": "qwen-flash", }, + "qwen-max": { + "backend": "openai", + "model_name": "qwen3-max", + }, } # Load evaluation prompts path @@ -651,6 +658,7 @@ class LongMemEvalEvaluator: memory_processor = MemoryProcessor( reme, self.config.reme_model_name, + self.config.retrieve_model_name, self.config.eval_model_name, self.config.algo_version, self.config.enable_thinking_params, @@ -777,7 +785,7 @@ class LongMemEvalEvaluator: print(f"Samples per type: {self.config.samples_per_type} (-1 = all)") print(f"Questions to process: {total_questions} | Top-K: {self.config.top_k}") print(f"Max Concurrency: {self.config.max_concurrency}") - print(f"ReMe Model: {self.config.reme_model_name} | Eval Model: {self.config.eval_model_name}") + print(f"Summary Model: {self.config.reme_model_name} | Retrieve Model: {self.config.retrieve_model_name} | Eval Model: {self.config.eval_model_name}") print(f"Algo Version: {self.config.algo_version}") print("=" * 80 + "\n") @@ -908,7 +916,8 @@ async def main_async( batch_size: int = 30, output_dir: str = "bench_results/longmemeval_reme", reme_model_name: str = "qwen-flash", - eval_model_name: str = "qwen3-max", + retrieve_model_name: str = "qwen-max", + eval_model_name: str = "qwen-max", algo_version: str = "v1", samples_per_type: int = -1, enable_thinking_params: bool = False, @@ -923,6 +932,7 @@ async def main_async( batch_size=batch_size, output_dir=output_dir, reme_model_name=reme_model_name, + retrieve_model_name=retrieve_model_name, eval_model_name=eval_model_name, algo_version=algo_version, samples_per_type=samples_per_type, @@ -943,7 +953,8 @@ def main( batch_size: int = 30, output_dir: str = "bench_results/longmemeval_reme", reme_model_name: str = "qwen-flash", - eval_model_name: str = "qwen3-max", + retrieve_model_name: str = "qwen-max", + eval_model_name: str = "qwen-max", algo_version: str = "v1", samples_per_type: int = -1, enable_thinking_params: bool = False, @@ -959,6 +970,7 @@ def main( batch_size=batch_size, output_dir=output_dir, reme_model_name=reme_model_name, + retrieve_model_name=retrieve_model_name, eval_model_name=eval_model_name, algo_version=algo_version, samples_per_type=samples_per_type, @@ -1001,7 +1013,7 @@ if __name__ == "__main__": parser.add_argument( "--max_concurrency", type=int, - default=1, + default=4, help="Maximum concurrent question processing (default: 1)", ) parser.add_argument( @@ -1019,16 +1031,20 @@ if __name__ == "__main__": parser.add_argument( "--reme_model_name", type=str, - # default="gpt-4o-mini-2024-07-18", default="qwen-flash", - help="Model name for ReMe operations (default: qwen-flash)", + help="Model name for ReMe summary operations (default: qwen-flash)", + ) + parser.add_argument( + "--retrieve_model_name", + type=str, + default="qwen-max", + help="Model name for memory retrieval (default: qwen-max)", ) parser.add_argument( "--eval_model_name", type=str, - # default="gpt-4o-mini-2024-07-18", - default="qwen-max", - help="Model name for evaluation/judgment (default: qwen3-max)", + default="qwen-flash", + help="Model name for evaluation/judgment (default: qwen-max)", ) parser.add_argument( "--algo_version", @@ -1039,7 +1055,7 @@ if __name__ == "__main__": parser.add_argument( "--samples_per_type", type=int, - default=4, + default=1, help="Number of samples per question type, -1 for all (default: -1)", ) parser.add_argument( @@ -1061,6 +1077,7 @@ if __name__ == "__main__": batch_size=args.batch_size, output_dir=args.output_dir, reme_model_name=args.reme_model_name, + retrieve_model_name=args.retrieve_model_name, eval_model_name=args.eval_model_name, algo_version=args.algo_version, samples_per_type=args.samples_per_type, From 7b66423d1933a705cb9f091282ab28de2785996e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=96=B9=E5=BA=94?= Date: Tue, 3 Mar 2026 11:27:17 +0800 Subject: [PATCH 3/5] =?UTF-8?q?fix(benchmark):=20=E7=A7=BB=E9=99=A4?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=B7=AF=E5=BE=84=E9=BB=98=E8=AE=A4=E5=80=BC?= =?UTF-8?q?=E5=B9=B6=E8=AE=BE=E4=B8=BA=E5=BF=85=E5=A1=AB=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 将LongMemEval评估脚本中的data_path参数改为必需参数 - 将HaluMem评估脚本中的data_path参数改为必需参数 - 删除了硬编码的默认文件路径配置 - 强制用户显式指定数据集文件路径以避免路径错误 --- benchmark/halumem/eval_reme.py | 3 +-- benchmark/longmemeval/eval_longmemeval_reme.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/benchmark/halumem/eval_reme.py b/benchmark/halumem/eval_reme.py index 5c4cf40e..14c0cb64 100644 --- a/benchmark/halumem/eval_reme.py +++ b/benchmark/halumem/eval_reme.py @@ -1421,8 +1421,7 @@ if __name__ == "__main__": parser.add_argument( "--data_path", type=str, - # required=True, - default="/Users/zhouwk/PycharmProjects/MemAgent/dataset/halumem/HaluMem-Medium.jsonl", + required=True, help="Path to HaluMem JSONL file", ) parser.add_argument( diff --git a/benchmark/longmemeval/eval_longmemeval_reme.py b/benchmark/longmemeval/eval_longmemeval_reme.py index b8a57c65..2edc204c 100644 --- a/benchmark/longmemeval/eval_longmemeval_reme.py +++ b/benchmark/longmemeval/eval_longmemeval_reme.py @@ -988,8 +988,7 @@ if __name__ == "__main__": parser.add_argument( "--data_path", type=str, - # default="/Users/zhouwk/PycharmProjects/MemAgent/dataset/longmemeval/longmemeval_s_cleaned.json", - default="/Users/zhouwk/PycharmProjects/MemAgent/dataset/longmemeval/longmemeval_oracle.json", + required=True, help="Path to LongMemEval JSON file", ) parser.add_argument( From 45f268432b079a6af95fc4c952688264ff58a681 Mon Sep 17 00:00:00 2001 From: Zhouwk <57825291+nitwtog@users.noreply.github.com> Date: Tue, 3 Mar 2026 12:20:43 +0800 Subject: [PATCH 4/5] Update __init__.py --- reme/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reme/__init__.py b/reme/__init__.py index 2621a539..d5304a7f 100644 --- a/reme/__init__.py +++ b/reme/__init__.py @@ -8,7 +8,7 @@ from .reme import ReMe from .reme_cli import ReMeCli from .reme_fb import ReMeFb -__version__ = "0.3.0.1" +__version__ = "0.3.0.2" __all__ = [ "config", From b9b54c5fcd8f2d2541da53d7fbaa4a89e59b7daa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=96=B9=E5=BA=94?= Date: Tue, 3 Mar 2026 12:28:33 +0800 Subject: [PATCH 5/5] =?UTF-8?q?fix(benchmark):=20=E4=BF=AE=E5=A4=8DReMe?= =?UTF-8?q?=E8=AF=84=E4=BC=B0=E4=B8=AD=E7=9A=84=E6=A8=A1=E5=9E=8B=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E5=92=8C=E7=A9=BA=E5=80=BC=E5=A4=84=E7=90=86=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 移除了retrieve_memory调用中不需要的llm_config_name参数 - 修复了长字符串打印的换行格式问题 - 添加了eval_result为空时的初始化处理 - 在accuracy评估中加入了eval_model_name参数传递 --- benchmark/halumem/eval_reme.py | 5 +++++ benchmark/longmemeval/eval_longmemeval_reme.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmark/halumem/eval_reme.py b/benchmark/halumem/eval_reme.py index 14c0cb64..8cc27e53 100644 --- a/benchmark/halumem/eval_reme.py +++ b/benchmark/halumem/eval_reme.py @@ -577,6 +577,8 @@ class MemoryIntegrityEvaluator: ) # Build result record + if eval_result is None: + eval_result = {} integrity_result = { **memory_point, "integrity_score": eval_result.get("score"), @@ -619,9 +621,12 @@ class MemoryAccuracyEvaluator: dialogue=formatted_dialogue, golden_memories=memory_points, candidate_memory=memory, + eval_model_name=self.eval_model_name, ) # Build result record + if eval_result is None: + eval_result = {} accuracy_result = { "memory_content": memory.get("content", memory.get("memory_content", str(memory))), "memory_id": memory.get("memory_id", ""), diff --git a/benchmark/longmemeval/eval_longmemeval_reme.py b/benchmark/longmemeval/eval_longmemeval_reme.py index 2edc204c..c529b434 100644 --- a/benchmark/longmemeval/eval_longmemeval_reme.py +++ b/benchmark/longmemeval/eval_longmemeval_reme.py @@ -785,7 +785,10 @@ class LongMemEvalEvaluator: print(f"Samples per type: {self.config.samples_per_type} (-1 = all)") print(f"Questions to process: {total_questions} | Top-K: {self.config.top_k}") print(f"Max Concurrency: {self.config.max_concurrency}") - print(f"Summary Model: {self.config.reme_model_name} | Retrieve Model: {self.config.retrieve_model_name} | Eval Model: {self.config.eval_model_name}") + print( + f"Summary Model: {self.config.reme_model_name} | Retrieve Model: {self.config.retrieve_model_name} " + f"| Eval Model: {self.config.eval_model_name}", + ) print(f"Algo Version: {self.config.algo_version}") print("=" * 80 + "\n")