feat: agentscope dual-version compat & benchmark improvements

- Add version_tuple utility for semantic version comparison
- AsAgentWrapper: version-aware InjectionConfig, max_iters doubling,
  and token usage collection (reply vs reply_stream) for AS>=2.0.5/<2.0.5
- Default inject_runtime_state=False in wrapper to avoid benchmark
  time-anchor conflicts; remove per-callsite injection_config overrides
- longmemeval run.py: support question_ids filter in dataset config
- Fix unused import in test_evaluation_interface; format fixes

benchmark/longmemeval/config_flip_test.yaml

@@ -0,0 +1,74 @@
+dataset:
+  path: benchmark/datasets/longmemeval/longmemeval_s_reme_cleaned.json
+  start_index: 0
+  num_items: 500
+  max_sessions: 0
+  question_types: []
+  workspace_root: benchmark/memory_workspaces/longmemeval-s
+  question_ids:
+  - b86304ba
+  - 0862e8bf_abs
+  - 19b5f2b3_abs
+  - 29f2956b_abs
+  - b5ef892d
+  - aae3761f
+  - c4a1ceb8
+  - gpt4_2ba83207
+  - gpt4_194be4b3
+  - 5a7937c8
+  - 10d9b85a
+  - 60bf93ed_abs
+  - edced276_abs
+  - afdc33df
+  - a89d7624
+  - 1c0ddc50
+  - 0a34ad58
+  - 9aaed6a3
+  - 7405e8b1
+  - 5025383b
+  - a1cc6108
+  - b46e15ed
+  - 9a707b81
+  - gpt4_7f6b06db
+  - gpt4_fa19884d
+  - 0bc8ad93
+  - gpt4_8279ba03
+  - gpt4_2487a7cb
+  - gpt4_76048e76
+  - 0bb5a684
+  - 2a1811e2
+  - gpt4_5dcc0aab
+  - gpt4_6ed717ea
+  - gpt4_70e84552
+  - 982b5123
+  - gpt4_4edbafa2
+  - gpt4_483dd43c
+  - gpt4_b4a80587
+  - gpt4_9a159967
+  - 993da5e2
+  - a3045048
+  - gpt4_cd90e484
+  - gpt4_1a1dc16d
+  - gpt4_2f584639
+  - 8c18457d
+  - 982b5123_abs
+  - 830ce83f
+  - 184da446
+  - 6a27ffc2
+  - a2f3aa27
+  - 0ddfec37_abs
+  - f685340e_abs
+evaluation:
+  num_workers: 32
+  filter_future_sessions: true
+reme:
+  config: lme.yaml
+  dream_trigger_hour: 23
+  dream_scan_days: 2
+  dream_max_units: 5
+output:
+  dir: benchmark/results/longmemeval
+  log_dir: logs
+  log_prefix: longmemeval
+  log_to_console: true
+  log_to_file: true

benchmark/longmemeval/run.py

@@ -571,6 +571,16 @@ def main(
             f"Filtered by question_types={question_types}: {before_filter} -> {len(items_with_idx)} items",
         )
 
+    # Filter by question_id if specified
+    question_ids = dataset_cfg.get("question_ids") or []
+    if question_ids:
+        qid_set = set(question_ids)
+        before_filter = len(items_with_idx)
+        items_with_idx = [(idx, item) for idx, item in items_with_idx if item.get("question_id") in qid_set]
+        logger.info(
+            f"Filtered by question_ids ({len(qid_set)} ids): {before_filter} -> {len(items_with_idx)} items",
+        )
+
     logger.info(
         "Evaluating %d item(s) starting from index %d%s",
         len(items_with_idx),

reme/components/agent_wrapper/as_agent_wrapper.py

@@ -11,7 +11,8 @@ from pathlib import Path
 from typing import Any, TYPE_CHECKING
 from uuid import uuid4
 
-from agentscope.agent import Agent, ContextConfig, InjectionConfig, ReActConfig
+import agentscope
+from agentscope.agent import Agent, ContextConfig, ReActConfig
 from agentscope.agent._config import ModelConfig
 from agentscope.event import (
     DataBlockDeltaEvent,
@@ -60,11 +61,20 @@ from ..as_llm import BaseAsLLM
 from ..component_registry import R
 from ...enumeration import ChunkEnum
 from ...schema import StreamChunk, TokenUsage
-from ...utils import AsStateHandler
+from ...utils import AsStateHandler, version_tuple
 
 if TYPE_CHECKING:
     from ..job.base_job import BaseJob
 
+# agentscope>=2.0.5 reworked the agent loop: InjectionConfig (runtime-state
+# injection) was introduced, and token usage moved from per-call
+# ModelCallEndEvent fields onto the aggregated ``Msg.usage`` of the reply.
+_AS_GE_2_0_5 = version_tuple(agentscope.__version__) >= (2, 0, 5)
+if _AS_GE_2_0_5:
+    from agentscope.agent import InjectionConfig
+else:
+    InjectionConfig = None
+
 _UUID_RE = re.compile(
     r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$",
     re.IGNORECASE,
@@ -328,6 +338,28 @@ class AsAgentWrapper(BaseAgentWrapper):
         perm_mode = PermissionMode(kwargs.get("permission_mode", "bypass"))
         state = await self._load_state(kwargs, perm_mode)
 
+        version_kwargs: dict[str, Any] = {}
+        react_config = dict(kwargs.get("react_config") or {})
+        if _AS_GE_2_0_5:
+            # Default to no runtime-state injection: the injected wall-clock
+            # "ground truth" time conflicts with callers that anchor time
+            # explicitly (benchmarks, historical ingestion). Callers can still
+            # opt back in by passing injection_config explicitly.
+            injection = {"inject_runtime_state": False, **(kwargs.get("injection_config") or {})}
+            version_kwargs["injection_config"] = InjectionConfig(**injection)
+            # agentscope>=2.0.5 counts Reasoning and Acting passes separately,
+            # so one full ReAct round costs two iterations. Callers express
+            # max_iters in reasoning rounds; double it here so the effective
+            # budget matches agentscope<2.0.5 semantics.
+            react_config["max_iters"] = 2 * int(react_config.get("max_iters", ReActConfig().max_iters))
+        elif kwargs.get("injection_config"):
+            # agentscope<2.0.5 has no runtime-state injection at all, so a
+            # request to disable it is already satisfied; other fields cannot
+            # be honored and are dropped.
+            self.logger.debug(
+                f"injection_config requires agentscope>=2.0.5; ignored on {agentscope.__version__}",
+            )
+
         agent = Agent(
             name=self.name,
             system_prompt=system_prompt,
@@ -336,8 +368,8 @@ class AsAgentWrapper(BaseAgentWrapper):
             state=state,
             model_config=ModelConfig(**(kwargs.get("model_config") or {})),
             context_config=ContextConfig(**(kwargs.get("context_config") or {})),
-            react_config=ReActConfig(**(kwargs.get("react_config") or {})),
-            injection_config=InjectionConfig(**(kwargs.get("injection_config") or {})),
+            react_config=ReActConfig(**react_config),
+            **version_kwargs,
         )
 
         if isinstance(inputs, str):
@@ -350,7 +382,23 @@ class AsAgentWrapper(BaseAgentWrapper):
         agent, inputs = await self._build_agent(inputs, **kwargs)
 
         await agent.observe(inputs)
-        last_msg = await agent.reply()
+        if _AS_GE_2_0_5:
+            last_msg = await agent.reply()
+            usage = self._agentscope_usage(last_msg.usage) if getattr(last_msg, "usage", None) else None
+        else:
+            # agentscope<2.0.5 reports usage per model call instead of on the
+            # reply message, so accumulate ModelCallEndEvent fields here.
+            usages: list[TokenUsage] = []
+            async for event in agent.reply_stream():
+                if isinstance(event, ModelCallEndEvent):
+                    usages.append(
+                        TokenUsage(
+                            input_tokens=event.input_tokens,
+                            output_tokens=event.output_tokens,
+                        ),
+                    )
+            last_msg = agent.state.context[-1]
+            usage = TokenUsage.combine(usages) if usages else None
         await self._dump_state(agent.state)
 
         result = {
@@ -359,7 +407,6 @@ class AsAgentWrapper(BaseAgentWrapper):
             "result": last_msg.get_text_content(),
         }
 
-        usage = self._agentscope_usage(last_msg.usage) if hasattr(last_msg, "usage") and last_msg.usage else None
         if usage is None:
             self.logger.error("AgentScope did not return token usage; token accounting is unavailable for this reply.")
 
@@ -376,7 +423,9 @@ class AsAgentWrapper(BaseAgentWrapper):
             result["structured_output"] = res.content
             if res.usage is None:
                 usage = None
-                self.logger.error("AgentScope did not return structured-output token usage; token accounting is unavailable.")
+                self.logger.error(
+                    "AgentScope did not return structured-output token usage; token accounting is unavailable.",
+                )
             elif usage is not None:
                 usage = TokenUsage.combine([usage, self._agentscope_usage(res.usage)])
 

reme/steps/benchmark/base/agentic_answer.py

@@ -25,6 +25,8 @@ class BaseAgenticAnswerStep(BaseStep):
         The agent's final answer text.
     """
 
+    # Reasoning-round budget. The AgentScope wrapper converts it to the
+    # backend's iteration-counting semantics; other backends ignore it.
     MAX_ITERATION = 10
     TOOL_CONTEXT_PREFIX: str = "content_agentic_answer"
 
@@ -54,9 +56,6 @@ class BaseAgenticAnswerStep(BaseStep):
             "system_prompt": sys_prompt,
             "job_tools": ["search", "add_draft", "read_all_draft"],
             "react_config": {"max_iters": self.MAX_ITERATION},
-            # Benchmarks anchor "now" via query_time in the system prompt; the
-            # real wall-clock time injected by AgentScope would conflict with it.
-            "injection_config": {"inject_runtime_state": False},
             "tool_context_id": tool_context_id,
         }
 

reme/steps/benchmark/beam/llm_judge.py

@@ -77,11 +77,7 @@ async def _llm_equivalence(agent_wrapper, reference: str, system: str) -> bool:
     )
     user_prompt = f"First snippet: {reference}\n\nSecond snippet: {system}"
 
-    result = await agent_wrapper.reply(
-        user_prompt,
-        system_prompt=system_prompt,
-        injection_config={"inject_runtime_state": False},
-    )
+    result = await agent_wrapper.reply(user_prompt, system_prompt=system_prompt)
     raw = (result.get("result") or "").strip().lower()
     return "yes" in raw
 
@@ -281,7 +277,7 @@ class BeamRubricJudgeStep(BaseStep):
         for item in rubric:
             prompt = judge_template.replace("<rubric_item>", item).replace("<llm_response>", llm_response)
 
-            result = await self.agent_wrapper.reply(prompt, injection_config={"inject_runtime_state": False})
+            result = await self.agent_wrapper.reply(prompt)
             raw = (result.get("result") or "").strip()
 
             try:

reme/steps/benchmark/lme/llm_judge.py

@@ -59,7 +59,6 @@ class LmeAnswerJudgeStep(BaseStep):
         result = await self.agent_wrapper.reply(
             user_prompt,
             system_prompt=self.prompt_format(judge_prompt_key),
-            injection_config={"inject_runtime_state": False},
         )
 
         raw_answer = (result.get("result") or "").strip()

reme/utils/__init__.py

@@ -2,6 +2,7 @@
 
 from .common_utils import (
     hash_text,
+    version_tuple,
     execute_stream_task,
     mock_reme_server,
     call_action,
@@ -19,6 +20,7 @@ from .counter import global_counter_add, global_counter_get, global_counter_get_
 
 __all__ = [
     "hash_text",
+    "version_tuple",
     "execute_stream_task",
     "mock_reme_server",
     "call_action",

reme/utils/common_utils.py

@@ -22,6 +22,20 @@ def hash_text(text: str, encoding: str = "utf-8") -> str:
     return hashlib.sha256(text.encode(encoding)).hexdigest()
 
 
+def version_tuple(version: str) -> tuple[int, ...]:
+    """Parse the leading numeric parts of a version string into a tuple.
+
+    Non-numeric suffixes are ignored, e.g. "2.0.4.post1" -> (2, 0, 4),
+    so tuples compare correctly across pre/post releases.
+    """
+    parts: list[int] = []
+    for part in version.split("."):
+        if not part.isdigit():
+            break
+        parts.append(int(part))
+    return tuple(parts)
+
+
 def _format_chunk(
     chunk: StreamChunk,
     output_format: Literal["str", "bytes", "chunk"],

tests/unit/test_evaluation_interface.py

@@ -9,7 +9,6 @@ from reme.components.job import BaseJob, StreamJob
 from reme.utils import global_counter_add
 from reme.utils.evaluation_interface import (
     check_agent_token_count,
-    check_agent_token_usage,
     check_job_count,
     track_agent_token_usage,
     track_agent_token_counts,

tests/unit/test_token_usage.py

@@ -2,6 +2,7 @@
 
 from types import SimpleNamespace
 
+from agentscope.event import ModelCallEndEvent
 from agentscope.model._model_usage import ChatUsage
 import pytest
 
@@ -25,7 +26,7 @@ def test_provider_usage_keeps_only_input_and_output_tokens():
             "cache_read_input_tokens": 20,
             "cache_creation_input_tokens": 30,
             "reasoning_output_tokens": 2,
-        }
+        },
     )
 
     assert usage.model_dump() == {
@@ -43,7 +44,7 @@ def test_provider_usage_uses_reported_input_without_cache_adjustment():
             "output_tokens": 4,
             "cached_input_tokens": 20,
             "reasoning_output_tokens": 2,
-        }
+        },
     )
 
     assert usage.input_tokens == 60
@@ -106,7 +107,7 @@ def test_token_counter_is_a_per_agent_metric_tree(tmp_path):
 
 @pytest.mark.asyncio
 async def test_agentscope_reply_records_final_message_usage(tmp_path, monkeypatch):
-    """AgentScope 2.0.5 exposes full-reply usage on the final message."""
+    """Reply usage is recorded: from the final message (>=2.0.5) or model-call events (<2.0.5)."""
     context = ApplicationContext(workspace_dir=str(tmp_path))
     wrapper = AsAgentWrapper(name="research", as_llm="", app_context=context)
     message = SimpleNamespace(
@@ -118,6 +119,9 @@ async def test_agentscope_reply_records_final_message_usage(tmp_path, monkeypatc
         state=SimpleNamespace(session_id="session-1", context=[message]),
         observe=lambda _inputs: _async_none(),
         reply=lambda: _async_value(message),
+        reply_stream=lambda: _async_events(
+            [ModelCallEndEvent(reply_id="r1", input_tokens=10, output_tokens=4)],
+        ),
     )
 
     async def build_agent(inputs, **_kwargs):
@@ -129,12 +133,15 @@ async def test_agentscope_reply_records_final_message_usage(tmp_path, monkeypatc
     result = await wrapper.reply("hello")
 
     assert result["usage"] == {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14}
-    assert global_counter_get_all(context.metadata, ["__token_counter", "research"])["children"]["total_tokens"]["value"] == 14
+    assert (
+        global_counter_get_all(context.metadata, ["__token_counter", "research"])["children"]["total_tokens"]["value"]
+        == 14
+    )
 
 
 @pytest.mark.asyncio
 async def test_agentscope_reply_without_usage_leaves_token_counters_unset(tmp_path, monkeypatch):
-    """AgentScope 2.0.4 messages without usage remain visibly unavailable."""
+    """Replies without any usage information remain visibly unavailable."""
     context = ApplicationContext(workspace_dir=str(tmp_path))
     wrapper = AsAgentWrapper(name="research", as_llm="", app_context=context)
     message = SimpleNamespace(model_dump=lambda: {"text": "answer"}, get_text_content=lambda: "answer")
@@ -142,6 +149,7 @@ async def test_agentscope_reply_without_usage_leaves_token_counters_unset(tmp_pa
         state=SimpleNamespace(session_id="session-1", context=[message]),
         observe=lambda _inputs: _async_none(),
         reply=lambda: _async_value(message),
+        reply_stream=lambda: _async_events([]),
     )
 
     async def build_agent(inputs, **_kwargs):
@@ -164,6 +172,11 @@ async def _async_value(value):
     return value
 
 
+async def _async_events(events):
+    for event in events:
+        yield event
+
+
 @pytest.mark.asyncio
 async def test_agentscope_stream_reply_does_not_record_token_usage(tmp_path, monkeypatch):
     """Only non-streaming AgentScope replies contribute to token accounting."""