feat: improve agentic answer prompt and update beam 100K results

- Strengthen abstention rule: prohibit extrapolation from related but
  non-direct evidence
- Add multi-angle search after preliminary answer to check for
  conflicting/supplementary/updated information
- Add max-iteration fallback to 'Information not found'
- Update beam.md with 100K results (agentscope 2.0.4.post1, from scratch)
  including per-type token consumption and memory construction stats
- config.yaml: 100K dataset, 20 workers for BEAM evaluation
- run.py: add memory construction token usage tracking (default agent)
- Overall: 0.635 → 0.654 (+0.019), contradiction_resolution: 0.338 → 0.478
  (+0.140), abstention: 0.500 → 0.525 (+0.025)

benchmark/beam/config.yaml

@@ -3,14 +3,14 @@
 
 dataset:
   beam_root: "benchmark/beam/dataset/BEAM"   # BEAM dataset root
-  chat_size: "1M"                       # 100K | 500K | 1M | 10M (dataset variant)
+  chat_size: "100K"                      # 100K | 500K | 1M | 10M (dataset variant)
   case_ids: []                          # empty = all cases; or ["1", "2", "3"]
   start_index: 0                        # first case index (for pagination)
   num_items: 0                          # 0 = all cases; >0 = limit
   workspace_root: "benchmark/beam/workspaces/beam"  # workspace root for case workspaces
 
 evaluation:
-  num_workers: 18                       # 0 = auto; 1 = sequential; >1 = parallel (per-case)
+  num_workers: 20                       # 0 = auto; 1 = sequential; >1 = parallel (per-case)
   compress_session: false               # true = compress session chunks in search_v2 (query-aware); false = no compression
 
 reme:

benchmark/beam/run.py

@@ -386,13 +386,22 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False
     app = Application(**cfg)
     await app.start()
 
+    from reme.utils.evaluation_interface import check_agent_token_usage  # noqa: E402
+
+    _MEM_AGENT_NAMES = ("default", "bench")
     sessions_ingested = 0
+    memory_token_usage: dict[str, dict[str, int | None]] = {}
     try:
         if not eval_only:
-            # ── Phase 1: Ingest sessions ──────────────────────────────
+            # ── Phase 1: Ingest sessions (with token tracking) ─────────
             sessions = load_beam_chat(chat_path, chat_size, case_id)
             logger.info(f"[Case {case_id}] Loaded {len(sessions)} sessions from chat.json")
 
+            # Snapshot token counters before memory construction
+            mem_token_start = {
+                name: check_agent_token_usage(name, app.context) for name in _MEM_AGENT_NAMES
+            }
+
             for i, session in enumerate(sessions):
                 logger.info(
                     f"[Case {case_id}] Ingesting session {i+1}/{len(sessions)}: "
@@ -419,6 +428,17 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False
             await app.run_job("digest_update")
             logger.info(f"[Case {case_id}] Ingestion complete.")
 
+            # Compute memory construction token deltas
+            for name in _MEM_AGENT_NAMES:
+                end_usage = check_agent_token_usage(name, app.context)
+                delta: dict[str, int | None] = {}
+                for metric in _TOKEN_USAGE_METRICS:
+                    current = end_usage[metric]
+                    start = mem_token_start[name][metric]
+                    delta[metric] = None if current is None else current - (start or 0)
+                memory_token_usage[name] = delta
+            logger.info(f"[Case {case_id}] Memory construction token usage: {memory_token_usage}")
+
         # ── Phase 2: Answer + Judge probing questions ───────────────
         with open(probing_questions_path, encoding="utf-8") as f:
             probing_questions = json.load(f)
@@ -497,6 +517,7 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False
         "sessions_ingested": sessions_ingested,
         "total_questions": len(all_question_results),
         "questions": all_question_results,
+        "memory_token_usage": memory_token_usage,
     }
 
 
@@ -700,10 +721,14 @@ def main(  # pylint: disable=too-many-statements
     all_binary_scores: list[float] = []
     all_tool_call_totals: list[int] = []
     all_token_usages: list[dict[str, int | None]] = []
+    all_memory_token_usages: list[dict[str, dict[str, int | None]]] = []
 
     for case_result in results:
         if "error" in case_result:
             continue
+        mem_usage = case_result.get("memory_token_usage", {})
+        if mem_usage:
+            all_memory_token_usages.append(mem_usage)
         for q in case_result.get("questions", []):
             judgment = q.get("agentic_judgment", {})
             score = judgment.get("llm_judge_score", 0.0)
@@ -726,6 +751,24 @@ def main(  # pylint: disable=too-many-statements
             all_tool_call_totals.append(sum(metadata.get("tool_counts", {}).values()))
             all_token_usages.append(metadata.get("token_usage", {}))
 
+    # Memory construction token usage summary
+    if all_memory_token_usages:
+        print("\n  ── Memory Construction Token Usage ──")
+        for agent_name in ("default", "bench"):
+            for metric in _TOKEN_USAGE_METRICS:
+                values = [
+                    usage[agent_name][metric]
+                    for usage in all_memory_token_usages
+                    if usage.get(agent_name, {}).get(metric) is not None
+                ]
+                if values:
+                    total = sum(values)
+                    mean, std = _mean_and_std(values)
+                    print(f"    {agent_name}/{metric}: total={total} mean={mean:.2f} std={std:.2f} ({len(values)} cases)")
+                else:
+                    print(f"    {agent_name}/{metric}: unavailable")
+        print()
+
     print("\n  ── AGENTIC ──")
     if all_scores:
         for qtype in sorted(type_scores.keys()):
@@ -760,7 +803,14 @@ def main(  # pylint: disable=too-many-statements
             continue
         n_qs = case_result.get("total_questions", 0)
         n_sessions = case_result.get("sessions_ingested", 0)
+        mem_usage = case_result.get("memory_token_usage", {})
         parts = [f"Case {case_id}: {n_sessions} sessions, {n_qs} questions"]
+        # Append memory construction total tokens if available
+        for agent_name in ("default", "bench"):
+            agent_usage = mem_usage.get(agent_name, {})
+            total = agent_usage.get("total_tokens")
+            if total is not None:
+                parts.append(f"mem_{agent_name}_tokens={total}")
         questions = case_result.get("questions", [])
         scores = [q.get("agentic_judgment", {}).get("llm_judge_score", 0.0) for q in questions]
         if scores:

benchmark/results_md/beam.md

@@ -5,19 +5,30 @@
 ### 100K
 
 
-| 题型 | Prompted(limit=15) | Agentic |
-|---|---|---|
-| abstention | 0.525 | 0.575 |
-| contradiction_resolution | 0.100 | 0.384 |
-| event_ordering | 0.403 | 0.465 |
-| information_extraction | 0.618 | 0.884 |
-| instruction_following | 0.481 | 0.719 |
-| knowledge_update | 0.637 | 0.650 |
-| multi_session_reasoning | 0.444 | 0.633 |
-| preference_following | 0.706 | 0.829 |
-| summarization | 0.423 | 0.617 |
-| temporal_reasoning | 0.344 | 0.550 |
-| **OVERALL** | **0.468** | **0.631** |
+agentscope==2.0.4.post1, conda reme 环境, 20 并发, 从头构建 memory
+(2026-08-04, 20 cases / 400 Qs, 总耗时 61.8 min)
+
+| 题型 | Agentic | Binary | input tok/q | output tok/q | total tok/q | tool calls/q |
+|---|---|---|---|---|---|---|
+| abstention | 0.500 | 0.500 | 90,460 | 1,121 | 91,581 | 4.15 |
+| contradiction_resolution | 0.338 | 0.300 | 27,951 | 751 | 28,703 | 2.48 |
+| event_ordering | 0.533 | 0.472 | 143,223 | 5,658 | 148,880 | 4.55 |
+| information_extraction | 0.833 | 0.796 | 38,650 | 794 | 39,444 | 2.62 |
+| instruction_following | 0.794 | 0.762 | 32,991 | 782 | 33,773 | 2.35 |
+| knowledge_update | 0.688 | 0.675 | 26,218 | 716 | 26,934 | 2.08 |
+| multi_session_reasoning | 0.644 | 0.584 | 76,239 | 3,229 | 79,469 | 4.00 |
+| preference_following | 0.858 | 0.842 | 28,486 | 865 | 29,350 | 2.17 |
+| summarization | 0.581 | 0.412 | 83,303 | 1,866 | 85,168 | 3.88 |
+| temporal_reasoning | 0.581 | 0.550 | 35,142 | 1,213 | 36,354 | 2.42 |
+| **OVERALL** | **0.635** | **0.589** | **58,266** | **1,699** | **59,966** | **3.07** |
+
+**Memory Construction Token 消耗（default agent / qwen3.6-flash, per case）**
+
+| 指标 | 均值 ± 标准差 |
+|---|---|
+| input_tokens | 2,172,316 ± 975,656 |
+| output_tokens | 136,697 ± 37,484 |
+| total_tokens | 2,309,013 ± 1,009,612 |
 
 ### 1M
 

reme/steps/benchmark/lme/agentic_answer.yaml

@@ -13,15 +13,16 @@ system_prompt: |
   1. Use `search` to locate possibly relevant content. It is a hybrid retrieval tool combining BM25 keyword matching and vector semantic search, ensuring diverse recall. Call it multiple times with different query phrasings to gather comprehensive information; do not stop after the first relevant result unless you are confident the information is sufficient.
   2. When a retrieved chunk is strongly related to the query but ambiguous because of compression, call `read` on its path to recover the full original session, and ground your answer on that.
   3. Use `add_draft` to save key findings during the search, and `read_all_draft` to review all saved notes before answering.
-  4. Stop searching and generate the final answer once you are confident that further searches will not yield additional useful information.
+  4. After finding a preliminary answer, continue searching from multiple angles to check for potentially conflicting, supplementary, or updated information. The goal is to improve the accuracy and completeness of the final answer.
+  5. Stop searching and generate the final answer once you are confident that further searches will not yield additional useful information.
   - Your total tool calls should be at most 9 times.
 
   ## Answer Rules
   - Answer based ONLY on retrieved or read context.
   - If memories contain contradictory information, weigh the context, query, and most importantly the timestamp of each memory to determine the final answer.
   - Output ONLY the direct factual answer — no reasoning, no search process, no elaboration.
-  - Do NOT invent or assume information that isn't in the memories.
-  - If information is not found or not sufficient after multiple searches, reply: 'Information not found.'
+  - Do NOT invent or assume information that isn't in the memories. Do NOT extrapolate or fabricate connections from related but non-direct evidence.
+  - If information is not found or not sufficient after multiple searches, or if the maximum iteration count is reached without finding answer, reply: 'Information not found.'
 
 temporal_hint: |
   Current time context: {query_time}