diff --git a/benchmark/beam/config.yaml b/benchmark/beam/config.yaml index cad4b6fd..b54f1f85 100644 --- a/benchmark/beam/config.yaml +++ b/benchmark/beam/config.yaml @@ -3,14 +3,14 @@ dataset: beam_root: "benchmark/beam/dataset/BEAM" # BEAM dataset root - chat_size: "1M" # 100K | 500K | 1M | 10M (dataset variant) + chat_size: "100K" # 100K | 500K | 1M | 10M (dataset variant) case_ids: [] # empty = all cases; or ["1", "2", "3"] start_index: 0 # first case index (for pagination) num_items: 0 # 0 = all cases; >0 = limit workspace_root: "benchmark/beam/workspaces/beam" # workspace root for case workspaces evaluation: - num_workers: 18 # 0 = auto; 1 = sequential; >1 = parallel (per-case) + num_workers: 20 # 0 = auto; 1 = sequential; >1 = parallel (per-case) compress_session: false # true = compress session chunks in search_v2 (query-aware); false = no compression reme: diff --git a/benchmark/beam/run.py b/benchmark/beam/run.py index a75f2ddf..4a4488a1 100644 --- a/benchmark/beam/run.py +++ b/benchmark/beam/run.py @@ -386,13 +386,22 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False app = Application(**cfg) await app.start() + from reme.utils.evaluation_interface import check_agent_token_usage # noqa: E402 + + _MEM_AGENT_NAMES = ("default", "bench") sessions_ingested = 0 + memory_token_usage: dict[str, dict[str, int | None]] = {} try: if not eval_only: - # ── Phase 1: Ingest sessions ────────────────────────────── + # ── Phase 1: Ingest sessions (with token tracking) ───────── sessions = load_beam_chat(chat_path, chat_size, case_id) logger.info(f"[Case {case_id}] Loaded {len(sessions)} sessions from chat.json") + # Snapshot token counters before memory construction + mem_token_start = { + name: check_agent_token_usage(name, app.context) for name in _MEM_AGENT_NAMES + } + for i, session in enumerate(sessions): logger.info( f"[Case {case_id}] Ingesting session {i+1}/{len(sessions)}: " @@ -419,6 +428,17 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False await app.run_job("digest_update") logger.info(f"[Case {case_id}] Ingestion complete.") + # Compute memory construction token deltas + for name in _MEM_AGENT_NAMES: + end_usage = check_agent_token_usage(name, app.context) + delta: dict[str, int | None] = {} + for metric in _TOKEN_USAGE_METRICS: + current = end_usage[metric] + start = mem_token_start[name][metric] + delta[metric] = None if current is None else current - (start or 0) + memory_token_usage[name] = delta + logger.info(f"[Case {case_id}] Memory construction token usage: {memory_token_usage}") + # ── Phase 2: Answer + Judge probing questions ─────────────── with open(probing_questions_path, encoding="utf-8") as f: probing_questions = json.load(f) @@ -497,6 +517,7 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False "sessions_ingested": sessions_ingested, "total_questions": len(all_question_results), "questions": all_question_results, + "memory_token_usage": memory_token_usage, } @@ -700,10 +721,14 @@ def main( # pylint: disable=too-many-statements all_binary_scores: list[float] = [] all_tool_call_totals: list[int] = [] all_token_usages: list[dict[str, int | None]] = [] + all_memory_token_usages: list[dict[str, dict[str, int | None]]] = [] for case_result in results: if "error" in case_result: continue + mem_usage = case_result.get("memory_token_usage", {}) + if mem_usage: + all_memory_token_usages.append(mem_usage) for q in case_result.get("questions", []): judgment = q.get("agentic_judgment", {}) score = judgment.get("llm_judge_score", 0.0) @@ -726,6 +751,24 @@ def main( # pylint: disable=too-many-statements all_tool_call_totals.append(sum(metadata.get("tool_counts", {}).values())) all_token_usages.append(metadata.get("token_usage", {})) + # Memory construction token usage summary + if all_memory_token_usages: + print("\n ── Memory Construction Token Usage ──") + for agent_name in ("default", "bench"): + for metric in _TOKEN_USAGE_METRICS: + values = [ + usage[agent_name][metric] + for usage in all_memory_token_usages + if usage.get(agent_name, {}).get(metric) is not None + ] + if values: + total = sum(values) + mean, std = _mean_and_std(values) + print(f" {agent_name}/{metric}: total={total} mean={mean:.2f} std={std:.2f} ({len(values)} cases)") + else: + print(f" {agent_name}/{metric}: unavailable") + print() + print("\n ── AGENTIC ──") if all_scores: for qtype in sorted(type_scores.keys()): @@ -760,7 +803,14 @@ def main( # pylint: disable=too-many-statements continue n_qs = case_result.get("total_questions", 0) n_sessions = case_result.get("sessions_ingested", 0) + mem_usage = case_result.get("memory_token_usage", {}) parts = [f"Case {case_id}: {n_sessions} sessions, {n_qs} questions"] + # Append memory construction total tokens if available + for agent_name in ("default", "bench"): + agent_usage = mem_usage.get(agent_name, {}) + total = agent_usage.get("total_tokens") + if total is not None: + parts.append(f"mem_{agent_name}_tokens={total}") questions = case_result.get("questions", []) scores = [q.get("agentic_judgment", {}).get("llm_judge_score", 0.0) for q in questions] if scores: diff --git a/benchmark/results_md/beam.md b/benchmark/results_md/beam.md index c66dd8cd..b78aff0a 100644 --- a/benchmark/results_md/beam.md +++ b/benchmark/results_md/beam.md @@ -5,19 +5,30 @@ ### 100K -| 题型 | Prompted(limit=15) | Agentic | -|---|---|---| -| abstention | 0.525 | 0.575 | -| contradiction_resolution | 0.100 | 0.384 | -| event_ordering | 0.403 | 0.465 | -| information_extraction | 0.618 | 0.884 | -| instruction_following | 0.481 | 0.719 | -| knowledge_update | 0.637 | 0.650 | -| multi_session_reasoning | 0.444 | 0.633 | -| preference_following | 0.706 | 0.829 | -| summarization | 0.423 | 0.617 | -| temporal_reasoning | 0.344 | 0.550 | -| **OVERALL** | **0.468** | **0.631** | +agentscope==2.0.4.post1, conda reme 环境, 20 并发, 从头构建 memory +(2026-08-04, 20 cases / 400 Qs, 总耗时 61.8 min) + +| 题型 | Agentic | Binary | input tok/q | output tok/q | total tok/q | tool calls/q | +|---|---|---|---|---|---|---| +| abstention | 0.500 | 0.500 | 90,460 | 1,121 | 91,581 | 4.15 | +| contradiction_resolution | 0.338 | 0.300 | 27,951 | 751 | 28,703 | 2.48 | +| event_ordering | 0.533 | 0.472 | 143,223 | 5,658 | 148,880 | 4.55 | +| information_extraction | 0.833 | 0.796 | 38,650 | 794 | 39,444 | 2.62 | +| instruction_following | 0.794 | 0.762 | 32,991 | 782 | 33,773 | 2.35 | +| knowledge_update | 0.688 | 0.675 | 26,218 | 716 | 26,934 | 2.08 | +| multi_session_reasoning | 0.644 | 0.584 | 76,239 | 3,229 | 79,469 | 4.00 | +| preference_following | 0.858 | 0.842 | 28,486 | 865 | 29,350 | 2.17 | +| summarization | 0.581 | 0.412 | 83,303 | 1,866 | 85,168 | 3.88 | +| temporal_reasoning | 0.581 | 0.550 | 35,142 | 1,213 | 36,354 | 2.42 | +| **OVERALL** | **0.635** | **0.589** | **58,266** | **1,699** | **59,966** | **3.07** | + +**Memory Construction Token 消耗（default agent / qwen3.6-flash, per case）** + +| 指标 | 均值 ± 标准差 | +|---|---| +| input_tokens | 2,172,316 ± 975,656 | +| output_tokens | 136,697 ± 37,484 | +| total_tokens | 2,309,013 ± 1,009,612 | ### 1M diff --git a/reme/steps/benchmark/lme/agentic_answer.yaml b/reme/steps/benchmark/lme/agentic_answer.yaml index 0be584c6..9d125086 100644 --- a/reme/steps/benchmark/lme/agentic_answer.yaml +++ b/reme/steps/benchmark/lme/agentic_answer.yaml @@ -13,15 +13,16 @@ system_prompt: | 1. Use `search` to locate possibly relevant content. It is a hybrid retrieval tool combining BM25 keyword matching and vector semantic search, ensuring diverse recall. Call it multiple times with different query phrasings to gather comprehensive information; do not stop after the first relevant result unless you are confident the information is sufficient. 2. When a retrieved chunk is strongly related to the query but ambiguous because of compression, call `read` on its path to recover the full original session, and ground your answer on that. 3. Use `add_draft` to save key findings during the search, and `read_all_draft` to review all saved notes before answering. - 4. Stop searching and generate the final answer once you are confident that further searches will not yield additional useful information. + 4. After finding a preliminary answer, continue searching from multiple angles to check for potentially conflicting, supplementary, or updated information. The goal is to improve the accuracy and completeness of the final answer. + 5. Stop searching and generate the final answer once you are confident that further searches will not yield additional useful information. - Your total tool calls should be at most 9 times. ## Answer Rules - Answer based ONLY on retrieved or read context. - If memories contain contradictory information, weigh the context, query, and most importantly the timestamp of each memory to determine the final answer. - Output ONLY the direct factual answer — no reasoning, no search process, no elaboration. - - Do NOT invent or assume information that isn't in the memories. - - If information is not found or not sufficient after multiple searches, reply: 'Information not found.' + - Do NOT invent or assume information that isn't in the memories. Do NOT extrapolate or fabricate connections from related but non-direct evidence. + - If information is not found or not sufficient after multiple searches, or if the maximum iteration count is reached without finding answer, reply: 'Information not found.' temporal_hint: | Current time context: {query_time}