feat(benchmark): 添加ReMe模型名称配置选项

- 在EvalConfig中新增reme_model_name参数,默认值为qwen-flash
- 修改ReMe实例化方式,传入配置的模型名称参数
- 更新函数签名以支持reme_model_name参数传递
- 调整命令行参数解析,区分reme_model_name和eval_model_name
- 将默认reme模型从硬编码改为可配置参数
This commit is contained in:
方应 2026-01-30 17:17:06 +08:00
parent 07905a0358
commit 61071ab654

View file

@ -41,6 +41,7 @@ class EvalConfig:
max_concurrency: int = 2
batch_size: int = 20
output_dir: str = "bench_results/reme"
reme_model_name: str = "qwen-flash"
eval_model_name: str = "qwen3-max"
algo_version: str = "v1"
@ -540,7 +541,7 @@ class HaluMemEvaluator:
def __init__(self, config: EvalConfig):
self.config = config
self.reme = ReMe()
self.reme = ReMe(llm={"model_name": self.config.reme_model_name})
# Load evaluation prompts into ReMe's prompt handler
prompts_yaml_path = Path(__file__).parent / "eval_reme.yaml"
@ -872,6 +873,7 @@ async def main_async(
top_k: int,
user_num: int,
max_concurrency: int,
reme_model_name: str= "qwen-flash",
eval_model_name: str = "qwen3-max",
algo_version: str = "v1"
):
@ -881,6 +883,7 @@ async def main_async(
top_k=top_k,
user_num=user_num,
max_concurrency=max_concurrency,
reme_model_name=reme_model_name,
eval_model_name=eval_model_name,
algo_version=algo_version
)
@ -895,6 +898,7 @@ def main(
top_k: int,
user_num: int,
max_concurrency: int,
reme_model_name: str= "qwen-flash",
eval_model_name: str = "qwen3-max",
algo_version: str = "v1"
):
@ -904,6 +908,7 @@ def main(
top_k=top_k,
user_num=user_num,
max_concurrency=max_concurrency,
reme_model_name=reme_model_name,
eval_model_name=eval_model_name,
algo_version=algo_version
))
@ -941,10 +946,15 @@ if __name__ == "__main__":
help="Maximum concurrent user processing (default: 100)"
)
parser.add_argument(
"--eval_model_name",
"--reme_model_name",
type=str,
default="qwen-flash",
# default="qwen3-235b-a22b-instruct-2507",
help="Model name for ReMe (default: qwen-flash)"
)
parser.add_argument(
"--eval_model_name",
type=str,
default="qwen3-max",
help="Model name for evaluation (default: qwen3-max)"
)
parser.add_argument(
@ -961,6 +971,7 @@ if __name__ == "__main__":
top_k=args.top_k,
user_num=args.user_num,
max_concurrency=args.max_concurrency,
reme_model_name=args.reme_model_name,
eval_model_name=args.eval_model_name,
algo_version=args.algo_version
)