From d9017b54078f1df3fb0830e57bb4d0bcf1ba64bf Mon Sep 17 00:00:00 2001 From: sa-buc Date: Thu, 30 Jul 2026 19:44:27 +0800 Subject: [PATCH] feat: support injection_config to disable runtime state injection in benchmarks - Add InjectionConfig passthrough in AsAgentWrapper.reply() - Disable inject_runtime_state in BaseAgenticAnswerStep to avoid wall-clock time conflicting with benchmark query_time anchors - Disable inject_runtime_state in beam/lme llm_judge calls --- reme/components/agent_wrapper/as_agent_wrapper.py | 3 ++- reme/steps/benchmark/base/agentic_answer.py | 3 +++ reme/steps/benchmark/beam/llm_judge.py | 8 ++++++-- reme/steps/benchmark/lme/llm_judge.py | 1 + 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/reme/components/agent_wrapper/as_agent_wrapper.py b/reme/components/agent_wrapper/as_agent_wrapper.py index 41886a05..369e8438 100644 --- a/reme/components/agent_wrapper/as_agent_wrapper.py +++ b/reme/components/agent_wrapper/as_agent_wrapper.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, TYPE_CHECKING from uuid import uuid4 -from agentscope.agent import Agent, ContextConfig, ReActConfig +from agentscope.agent import Agent, ContextConfig, InjectionConfig, ReActConfig from agentscope.agent._config import ModelConfig from agentscope.event import ( DataBlockDeltaEvent, @@ -337,6 +337,7 @@ class AsAgentWrapper(BaseAgentWrapper): model_config=ModelConfig(**(kwargs.get("model_config") or {})), context_config=ContextConfig(**(kwargs.get("context_config") or {})), react_config=ReActConfig(**(kwargs.get("react_config") or {})), + injection_config=InjectionConfig(**(kwargs.get("injection_config") or {})), ) if isinstance(inputs, str): diff --git a/reme/steps/benchmark/base/agentic_answer.py b/reme/steps/benchmark/base/agentic_answer.py index cf69bbc5..66da9bc5 100644 --- a/reme/steps/benchmark/base/agentic_answer.py +++ b/reme/steps/benchmark/base/agentic_answer.py @@ -54,6 +54,9 @@ class BaseAgenticAnswerStep(BaseStep): "system_prompt": sys_prompt, "job_tools": ["search", "add_draft", "read_all_draft"], "react_config": {"max_iters": self.MAX_ITERATION}, + # Benchmarks anchor "now" via query_time in the system prompt; the + # real wall-clock time injected by AgentScope would conflict with it. + "injection_config": {"inject_runtime_state": False}, "tool_context_id": tool_context_id, } diff --git a/reme/steps/benchmark/beam/llm_judge.py b/reme/steps/benchmark/beam/llm_judge.py index bf51f345..52a88095 100644 --- a/reme/steps/benchmark/beam/llm_judge.py +++ b/reme/steps/benchmark/beam/llm_judge.py @@ -77,7 +77,11 @@ async def _llm_equivalence(agent_wrapper, reference: str, system: str) -> bool: ) user_prompt = f"First snippet: {reference}\n\nSecond snippet: {system}" - result = await agent_wrapper.reply(user_prompt, system_prompt=system_prompt) + result = await agent_wrapper.reply( + user_prompt, + system_prompt=system_prompt, + injection_config={"inject_runtime_state": False}, + ) raw = (result.get("result") or "").strip().lower() return "yes" in raw @@ -277,7 +281,7 @@ class BeamRubricJudgeStep(BaseStep): for item in rubric: prompt = judge_template.replace("", item).replace("", llm_response) - result = await self.agent_wrapper.reply(prompt) + result = await self.agent_wrapper.reply(prompt, injection_config={"inject_runtime_state": False}) raw = (result.get("result") or "").strip() try: diff --git a/reme/steps/benchmark/lme/llm_judge.py b/reme/steps/benchmark/lme/llm_judge.py index 9c13c5a5..0aa43ca0 100644 --- a/reme/steps/benchmark/lme/llm_judge.py +++ b/reme/steps/benchmark/lme/llm_judge.py @@ -59,6 +59,7 @@ class LmeAnswerJudgeStep(BaseStep): result = await self.agent_wrapper.reply( user_prompt, system_prompt=self.prompt_format(judge_prompt_key), + injection_config={"inject_runtime_state": False}, ) raw_answer = (result.get("result") or "").strip()