diff --git a/benchmark/longmemeval/config_flip_test.yaml b/benchmark/longmemeval/config_flip_test.yaml new file mode 100644 index 00000000..ef62eed1 --- /dev/null +++ b/benchmark/longmemeval/config_flip_test.yaml @@ -0,0 +1,74 @@ +dataset: + path: benchmark/datasets/longmemeval/longmemeval_s_reme_cleaned.json + start_index: 0 + num_items: 500 + max_sessions: 0 + question_types: [] + workspace_root: benchmark/memory_workspaces/longmemeval-s + question_ids: + - b86304ba + - 0862e8bf_abs + - 19b5f2b3_abs + - 29f2956b_abs + - b5ef892d + - aae3761f + - c4a1ceb8 + - gpt4_2ba83207 + - gpt4_194be4b3 + - 5a7937c8 + - 10d9b85a + - 60bf93ed_abs + - edced276_abs + - afdc33df + - a89d7624 + - 1c0ddc50 + - 0a34ad58 + - 9aaed6a3 + - 7405e8b1 + - 5025383b + - a1cc6108 + - b46e15ed + - 9a707b81 + - gpt4_7f6b06db + - gpt4_fa19884d + - 0bc8ad93 + - gpt4_8279ba03 + - gpt4_2487a7cb + - gpt4_76048e76 + - 0bb5a684 + - 2a1811e2 + - gpt4_5dcc0aab + - gpt4_6ed717ea + - gpt4_70e84552 + - 982b5123 + - gpt4_4edbafa2 + - gpt4_483dd43c + - gpt4_b4a80587 + - gpt4_9a159967 + - 993da5e2 + - a3045048 + - gpt4_cd90e484 + - gpt4_1a1dc16d + - gpt4_2f584639 + - 8c18457d + - 982b5123_abs + - 830ce83f + - 184da446 + - 6a27ffc2 + - a2f3aa27 + - 0ddfec37_abs + - f685340e_abs +evaluation: + num_workers: 32 + filter_future_sessions: true +reme: + config: lme.yaml + dream_trigger_hour: 23 + dream_scan_days: 2 + dream_max_units: 5 +output: + dir: benchmark/results/longmemeval + log_dir: logs + log_prefix: longmemeval + log_to_console: true + log_to_file: true diff --git a/benchmark/longmemeval/run.py b/benchmark/longmemeval/run.py index 08ad8cde..0e682bf1 100644 --- a/benchmark/longmemeval/run.py +++ b/benchmark/longmemeval/run.py @@ -571,6 +571,16 @@ def main( f"Filtered by question_types={question_types}: {before_filter} -> {len(items_with_idx)} items", ) + # Filter by question_id if specified + question_ids = dataset_cfg.get("question_ids") or [] + if question_ids: + qid_set = set(question_ids) + before_filter = len(items_with_idx) + items_with_idx = [(idx, item) for idx, item in items_with_idx if item.get("question_id") in qid_set] + logger.info( + f"Filtered by question_ids ({len(qid_set)} ids): {before_filter} -> {len(items_with_idx)} items", + ) + logger.info( "Evaluating %d item(s) starting from index %d%s", len(items_with_idx), diff --git a/reme/components/agent_wrapper/as_agent_wrapper.py b/reme/components/agent_wrapper/as_agent_wrapper.py index 369e8438..3e1cf1bc 100644 --- a/reme/components/agent_wrapper/as_agent_wrapper.py +++ b/reme/components/agent_wrapper/as_agent_wrapper.py @@ -11,7 +11,8 @@ from pathlib import Path from typing import Any, TYPE_CHECKING from uuid import uuid4 -from agentscope.agent import Agent, ContextConfig, InjectionConfig, ReActConfig +import agentscope +from agentscope.agent import Agent, ContextConfig, ReActConfig from agentscope.agent._config import ModelConfig from agentscope.event import ( DataBlockDeltaEvent, @@ -60,11 +61,20 @@ from ..as_llm import BaseAsLLM from ..component_registry import R from ...enumeration import ChunkEnum from ...schema import StreamChunk, TokenUsage -from ...utils import AsStateHandler +from ...utils import AsStateHandler, version_tuple if TYPE_CHECKING: from ..job.base_job import BaseJob +# agentscope>=2.0.5 reworked the agent loop: InjectionConfig (runtime-state +# injection) was introduced, and token usage moved from per-call +# ModelCallEndEvent fields onto the aggregated ``Msg.usage`` of the reply. +_AS_GE_2_0_5 = version_tuple(agentscope.__version__) >= (2, 0, 5) +if _AS_GE_2_0_5: + from agentscope.agent import InjectionConfig +else: + InjectionConfig = None + _UUID_RE = re.compile( r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE, @@ -328,6 +338,28 @@ class AsAgentWrapper(BaseAgentWrapper): perm_mode = PermissionMode(kwargs.get("permission_mode", "bypass")) state = await self._load_state(kwargs, perm_mode) + version_kwargs: dict[str, Any] = {} + react_config = dict(kwargs.get("react_config") or {}) + if _AS_GE_2_0_5: + # Default to no runtime-state injection: the injected wall-clock + # "ground truth" time conflicts with callers that anchor time + # explicitly (benchmarks, historical ingestion). Callers can still + # opt back in by passing injection_config explicitly. + injection = {"inject_runtime_state": False, **(kwargs.get("injection_config") or {})} + version_kwargs["injection_config"] = InjectionConfig(**injection) + # agentscope>=2.0.5 counts Reasoning and Acting passes separately, + # so one full ReAct round costs two iterations. Callers express + # max_iters in reasoning rounds; double it here so the effective + # budget matches agentscope<2.0.5 semantics. + react_config["max_iters"] = 2 * int(react_config.get("max_iters", ReActConfig().max_iters)) + elif kwargs.get("injection_config"): + # agentscope<2.0.5 has no runtime-state injection at all, so a + # request to disable it is already satisfied; other fields cannot + # be honored and are dropped. + self.logger.debug( + f"injection_config requires agentscope>=2.0.5; ignored on {agentscope.__version__}", + ) + agent = Agent( name=self.name, system_prompt=system_prompt, @@ -336,8 +368,8 @@ class AsAgentWrapper(BaseAgentWrapper): state=state, model_config=ModelConfig(**(kwargs.get("model_config") or {})), context_config=ContextConfig(**(kwargs.get("context_config") or {})), - react_config=ReActConfig(**(kwargs.get("react_config") or {})), - injection_config=InjectionConfig(**(kwargs.get("injection_config") or {})), + react_config=ReActConfig(**react_config), + **version_kwargs, ) if isinstance(inputs, str): @@ -350,7 +382,23 @@ class AsAgentWrapper(BaseAgentWrapper): agent, inputs = await self._build_agent(inputs, **kwargs) await agent.observe(inputs) - last_msg = await agent.reply() + if _AS_GE_2_0_5: + last_msg = await agent.reply() + usage = self._agentscope_usage(last_msg.usage) if getattr(last_msg, "usage", None) else None + else: + # agentscope<2.0.5 reports usage per model call instead of on the + # reply message, so accumulate ModelCallEndEvent fields here. + usages: list[TokenUsage] = [] + async for event in agent.reply_stream(): + if isinstance(event, ModelCallEndEvent): + usages.append( + TokenUsage( + input_tokens=event.input_tokens, + output_tokens=event.output_tokens, + ), + ) + last_msg = agent.state.context[-1] + usage = TokenUsage.combine(usages) if usages else None await self._dump_state(agent.state) result = { @@ -359,7 +407,6 @@ class AsAgentWrapper(BaseAgentWrapper): "result": last_msg.get_text_content(), } - usage = self._agentscope_usage(last_msg.usage) if hasattr(last_msg, "usage") and last_msg.usage else None if usage is None: self.logger.error("AgentScope did not return token usage; token accounting is unavailable for this reply.") @@ -376,7 +423,9 @@ class AsAgentWrapper(BaseAgentWrapper): result["structured_output"] = res.content if res.usage is None: usage = None - self.logger.error("AgentScope did not return structured-output token usage; token accounting is unavailable.") + self.logger.error( + "AgentScope did not return structured-output token usage; token accounting is unavailable.", + ) elif usage is not None: usage = TokenUsage.combine([usage, self._agentscope_usage(res.usage)]) diff --git a/reme/steps/benchmark/base/agentic_answer.py b/reme/steps/benchmark/base/agentic_answer.py index 66da9bc5..27b9f26c 100644 --- a/reme/steps/benchmark/base/agentic_answer.py +++ b/reme/steps/benchmark/base/agentic_answer.py @@ -25,6 +25,8 @@ class BaseAgenticAnswerStep(BaseStep): The agent's final answer text. """ + # Reasoning-round budget. The AgentScope wrapper converts it to the + # backend's iteration-counting semantics; other backends ignore it. MAX_ITERATION = 10 TOOL_CONTEXT_PREFIX: str = "content_agentic_answer" @@ -54,9 +56,6 @@ class BaseAgenticAnswerStep(BaseStep): "system_prompt": sys_prompt, "job_tools": ["search", "add_draft", "read_all_draft"], "react_config": {"max_iters": self.MAX_ITERATION}, - # Benchmarks anchor "now" via query_time in the system prompt; the - # real wall-clock time injected by AgentScope would conflict with it. - "injection_config": {"inject_runtime_state": False}, "tool_context_id": tool_context_id, } diff --git a/reme/steps/benchmark/beam/llm_judge.py b/reme/steps/benchmark/beam/llm_judge.py index 52a88095..bf51f345 100644 --- a/reme/steps/benchmark/beam/llm_judge.py +++ b/reme/steps/benchmark/beam/llm_judge.py @@ -77,11 +77,7 @@ async def _llm_equivalence(agent_wrapper, reference: str, system: str) -> bool: ) user_prompt = f"First snippet: {reference}\n\nSecond snippet: {system}" - result = await agent_wrapper.reply( - user_prompt, - system_prompt=system_prompt, - injection_config={"inject_runtime_state": False}, - ) + result = await agent_wrapper.reply(user_prompt, system_prompt=system_prompt) raw = (result.get("result") or "").strip().lower() return "yes" in raw @@ -281,7 +277,7 @@ class BeamRubricJudgeStep(BaseStep): for item in rubric: prompt = judge_template.replace("", item).replace("", llm_response) - result = await self.agent_wrapper.reply(prompt, injection_config={"inject_runtime_state": False}) + result = await self.agent_wrapper.reply(prompt) raw = (result.get("result") or "").strip() try: diff --git a/reme/steps/benchmark/lme/llm_judge.py b/reme/steps/benchmark/lme/llm_judge.py index 0aa43ca0..9c13c5a5 100644 --- a/reme/steps/benchmark/lme/llm_judge.py +++ b/reme/steps/benchmark/lme/llm_judge.py @@ -59,7 +59,6 @@ class LmeAnswerJudgeStep(BaseStep): result = await self.agent_wrapper.reply( user_prompt, system_prompt=self.prompt_format(judge_prompt_key), - injection_config={"inject_runtime_state": False}, ) raw_answer = (result.get("result") or "").strip() diff --git a/reme/utils/__init__.py b/reme/utils/__init__.py index 6fbbe767..ecbf3ab8 100644 --- a/reme/utils/__init__.py +++ b/reme/utils/__init__.py @@ -2,6 +2,7 @@ from .common_utils import ( hash_text, + version_tuple, execute_stream_task, mock_reme_server, call_action, @@ -19,6 +20,7 @@ from .counter import global_counter_add, global_counter_get, global_counter_get_ __all__ = [ "hash_text", + "version_tuple", "execute_stream_task", "mock_reme_server", "call_action", diff --git a/reme/utils/common_utils.py b/reme/utils/common_utils.py index fd03eb65..6c6bf549 100644 --- a/reme/utils/common_utils.py +++ b/reme/utils/common_utils.py @@ -22,6 +22,20 @@ def hash_text(text: str, encoding: str = "utf-8") -> str: return hashlib.sha256(text.encode(encoding)).hexdigest() +def version_tuple(version: str) -> tuple[int, ...]: + """Parse the leading numeric parts of a version string into a tuple. + + Non-numeric suffixes are ignored, e.g. "2.0.4.post1" -> (2, 0, 4), + so tuples compare correctly across pre/post releases. + """ + parts: list[int] = [] + for part in version.split("."): + if not part.isdigit(): + break + parts.append(int(part)) + return tuple(parts) + + def _format_chunk( chunk: StreamChunk, output_format: Literal["str", "bytes", "chunk"], diff --git a/tests/unit/test_evaluation_interface.py b/tests/unit/test_evaluation_interface.py index a5904866..086528a7 100644 --- a/tests/unit/test_evaluation_interface.py +++ b/tests/unit/test_evaluation_interface.py @@ -9,7 +9,6 @@ from reme.components.job import BaseJob, StreamJob from reme.utils import global_counter_add from reme.utils.evaluation_interface import ( check_agent_token_count, - check_agent_token_usage, check_job_count, track_agent_token_usage, track_agent_token_counts, diff --git a/tests/unit/test_token_usage.py b/tests/unit/test_token_usage.py index 81b595b1..3ba5c311 100644 --- a/tests/unit/test_token_usage.py +++ b/tests/unit/test_token_usage.py @@ -2,6 +2,7 @@ from types import SimpleNamespace +from agentscope.event import ModelCallEndEvent from agentscope.model._model_usage import ChatUsage import pytest @@ -25,7 +26,7 @@ def test_provider_usage_keeps_only_input_and_output_tokens(): "cache_read_input_tokens": 20, "cache_creation_input_tokens": 30, "reasoning_output_tokens": 2, - } + }, ) assert usage.model_dump() == { @@ -43,7 +44,7 @@ def test_provider_usage_uses_reported_input_without_cache_adjustment(): "output_tokens": 4, "cached_input_tokens": 20, "reasoning_output_tokens": 2, - } + }, ) assert usage.input_tokens == 60 @@ -106,7 +107,7 @@ def test_token_counter_is_a_per_agent_metric_tree(tmp_path): @pytest.mark.asyncio async def test_agentscope_reply_records_final_message_usage(tmp_path, monkeypatch): - """AgentScope 2.0.5 exposes full-reply usage on the final message.""" + """Reply usage is recorded: from the final message (>=2.0.5) or model-call events (<2.0.5).""" context = ApplicationContext(workspace_dir=str(tmp_path)) wrapper = AsAgentWrapper(name="research", as_llm="", app_context=context) message = SimpleNamespace( @@ -118,6 +119,9 @@ async def test_agentscope_reply_records_final_message_usage(tmp_path, monkeypatc state=SimpleNamespace(session_id="session-1", context=[message]), observe=lambda _inputs: _async_none(), reply=lambda: _async_value(message), + reply_stream=lambda: _async_events( + [ModelCallEndEvent(reply_id="r1", input_tokens=10, output_tokens=4)], + ), ) async def build_agent(inputs, **_kwargs): @@ -129,12 +133,15 @@ async def test_agentscope_reply_records_final_message_usage(tmp_path, monkeypatc result = await wrapper.reply("hello") assert result["usage"] == {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14} - assert global_counter_get_all(context.metadata, ["__token_counter", "research"])["children"]["total_tokens"]["value"] == 14 + assert ( + global_counter_get_all(context.metadata, ["__token_counter", "research"])["children"]["total_tokens"]["value"] + == 14 + ) @pytest.mark.asyncio async def test_agentscope_reply_without_usage_leaves_token_counters_unset(tmp_path, monkeypatch): - """AgentScope 2.0.4 messages without usage remain visibly unavailable.""" + """Replies without any usage information remain visibly unavailable.""" context = ApplicationContext(workspace_dir=str(tmp_path)) wrapper = AsAgentWrapper(name="research", as_llm="", app_context=context) message = SimpleNamespace(model_dump=lambda: {"text": "answer"}, get_text_content=lambda: "answer") @@ -142,6 +149,7 @@ async def test_agentscope_reply_without_usage_leaves_token_counters_unset(tmp_pa state=SimpleNamespace(session_id="session-1", context=[message]), observe=lambda _inputs: _async_none(), reply=lambda: _async_value(message), + reply_stream=lambda: _async_events([]), ) async def build_agent(inputs, **_kwargs): @@ -164,6 +172,11 @@ async def _async_value(value): return value +async def _async_events(events): + for event in events: + yield event + + @pytest.mark.asyncio async def test_agentscope_stream_reply_does_not_record_token_usage(tmp_path, monkeypatch): """Only non-streaming AgentScope replies contribute to token accounting."""