From ef6b0d74e177b366a8a7dac008d1da84d224fb2e Mon Sep 17 00:00:00 2001 From: sa-buc Date: Wed, 5 Aug 2026 13:24:22 +0800 Subject: [PATCH] feat(read): add session-aware formatting for read tool and update BEAM eval - Add truncate_session_output in _file_io.py to render jsonl session lines as [speaker @ time] content before byte-budget truncation - Add read_step_format_session flag to ReadStep, honoring injected job kwargs (precedence) and YAML fallback - Inject read_step_format_session=True into BaseAgenticAnswerStep so agentic answer reads render session transcripts human-readably - Refine BEAM agentic_answer prompt: continue multi-angle search after preliminary answer, forbid fabrication/extrapolation - Update BEAM config to 1M variant and add sequential 100K-eval / 1M-build shell script - Refresh benchmark/results_md/beam.md with latest results --- benchmark/beam/config.yaml | 2 +- benchmark/beam/run_beam_100k_eval_1m_build.sh | 110 ++++++++++++++++++ benchmark/results_md/beam.md | 83 ++++++------- reme/steps/benchmark/base/agentic_answer.py | 2 +- reme/steps/benchmark/beam/agentic_answer.yaml | 4 +- reme/steps/file_io/_file_io.py | 42 +++++++ reme/steps/file_io/read.py | 43 ++++++- 7 files changed, 234 insertions(+), 52 deletions(-) create mode 100644 benchmark/beam/run_beam_100k_eval_1m_build.sh diff --git a/benchmark/beam/config.yaml b/benchmark/beam/config.yaml index b54f1f85..423697a5 100644 --- a/benchmark/beam/config.yaml +++ b/benchmark/beam/config.yaml @@ -3,7 +3,7 @@ dataset: beam_root: "benchmark/beam/dataset/BEAM" # BEAM dataset root - chat_size: "100K" # 100K | 500K | 1M | 10M (dataset variant) + chat_size: "1M" # 100K | 500K | 1M | 10M (dataset variant) case_ids: [] # empty = all cases; or ["1", "2", "3"] start_index: 0 # first case index (for pagination) num_items: 0 # 0 = all cases; >0 = limit diff --git a/benchmark/beam/run_beam_100k_eval_1m_build.sh b/benchmark/beam/run_beam_100k_eval_1m_build.sh new file mode 100644 index 00000000..6fe6b80e --- /dev/null +++ b/benchmark/beam/run_beam_100k_eval_1m_build.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Sequential BEAM evaluation: 100K eval-only → 1M full build +# Usage: bash benchmark/beam/run_beam_100k_eval_1m_build.sh +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +CONFIG="$PROJECT_ROOT/benchmark/beam/config.yaml" +LOG_DIR="$PROJECT_ROOT/logs" +TS=$(date +%Y%m%d_%H%M%S) +LOG_FILE="$LOG_DIR/beam_sequential_${TS}.log" +mkdir -p "$LOG_DIR" + +echo "=== BEAM Sequential Evaluation ===" | tee "$LOG_FILE" +echo "Started at: $(date)" | tee -a "$LOG_FILE" +echo "Log file: $LOG_FILE" | tee -a "$LOG_FILE" + +# ── Phase 1: BEAM 100K eval-only (20 workers) ── +echo "" | tee -a "$LOG_FILE" +echo "==========================================" | tee -a "$LOG_FILE" +echo " Phase 1: BEAM 100K eval-only" | tee -a "$LOG_FILE" +echo "==========================================" | tee -a "$LOG_FILE" + +# Write 100K eval-only config +cat > "$CONFIG" <<'EOF' +# BEAM evaluation configuration +# This file controls what/how to evaluate. + +dataset: + beam_root: "benchmark/beam/dataset/BEAM" # BEAM dataset root + chat_size: "100K" # 100K | 500K | 1M | 10M (dataset variant) + case_ids: [] # empty = all cases; or ["1", "2", "3"] + start_index: 0 # first case index (for pagination) + num_items: 0 # 0 = all cases; >0 = limit + workspace_root: "benchmark/beam/workspaces/beam" # workspace root for case workspaces + +evaluation: + num_workers: 20 # 0 = auto; 1 = sequential; >1 = parallel (per-case) + compress_session: false # true = compress session chunks in search_v2 (query-aware); false = no compression + +reme: + config: "beam.yaml" # reme config (in reme/config/) + +output: + dir: "benchmark/beam/results" + log_dir: "logs" # log directory (relative to project root) + log_prefix: "beam" # benchmark name used in log filenames + log_to_console: true + log_to_file: true +EOF + +echo "Config written for 100K eval-only" | tee -a "$LOG_FILE" +cd "$PROJECT_ROOT" +conda run -n reme --no-capture-output python benchmark/beam/run.py --eval_only 2>&1 | tee -a "$LOG_FILE" +PHASE1_EXIT=${PIPESTATUS[0]} + +if [ $PHASE1_EXIT -ne 0 ]; then + echo "Phase 1 (100K eval-only) FAILED with exit code $PHASE1_EXIT" | tee -a "$LOG_FILE" + exit $PHASE1_EXIT +fi + +echo "" | tee -a "$LOG_FILE" +echo "Phase 1 (100K eval-only) completed at $(date)" | tee -a "$LOG_FILE" + +# ── Phase 2: BEAM 1M full build (20 workers) ── +echo "" | tee -a "$LOG_FILE" +echo "==========================================" | tee -a "$LOG_FILE" +echo " Phase 2: BEAM 1M full build" | tee -a "$LOG_FILE" +echo "==========================================" | tee -a "$LOG_FILE" + +# Write 1M full build config +cat > "$CONFIG" <<'EOF' +# BEAM evaluation configuration +# This file controls what/how to evaluate. + +dataset: + beam_root: "benchmark/beam/dataset/BEAM" # BEAM dataset root + chat_size: "1M" # 100K | 500K | 1M | 10M (dataset variant) + case_ids: [] # empty = all cases; or ["1", "2", "3"] + start_index: 0 # first case index (for pagination) + num_items: 0 # 0 = all cases; >0 = limit + workspace_root: "benchmark/beam/workspaces/beam" # workspace root for case workspaces + +evaluation: + num_workers: 20 # 0 = auto; 1 = sequential; >1 = parallel (per-case) + compress_session: false # true = compress session chunks in search_v2 (query-aware); false = no compression + +reme: + config: "beam.yaml" # reme config (in reme/config/) + +output: + dir: "benchmark/beam/results" + log_dir: "logs" # log directory (relative to project root) + log_prefix: "beam" # benchmark name used in log filenames + log_to_console: true + log_to_file: true +EOF + +echo "Config written for 1M full build" | tee -a "$LOG_FILE" +conda run -n reme --no-capture-output python benchmark/beam/run.py 2>&1 | tee -a "$LOG_FILE" +PHASE2_EXIT=${PIPESTATUS[0]} + +if [ $PHASE2_EXIT -ne 0 ]; then + echo "Phase 2 (1M full build) FAILED with exit code $PHASE2_EXIT" | tee -a "$LOG_FILE" + exit $PHASE2_EXIT +fi + +echo "" | tee -a "$LOG_FILE" +echo "Phase 2 (1M full build) completed at $(date)" | tee -a "$LOG_FILE" +echo "" | tee -a "$LOG_FILE" +echo "=== ALL DONE at $(date) ===" | tee -a "$LOG_FILE" diff --git a/benchmark/results_md/beam.md b/benchmark/results_md/beam.md index b78aff0a..651690d7 100644 --- a/benchmark/results_md/beam.md +++ b/benchmark/results_md/beam.md @@ -5,61 +5,50 @@ ### 100K -agentscope==2.0.4.post1, conda reme 环境, 20 并发, 从头构建 memory -(2026-08-04, 20 cases / 400 Qs, 总耗时 61.8 min) +agentscope==2.0.4.post1, conda reme 环境, 20 并发, eval-only（复用已构建 memory） +(2026-08-05, 20 cases / 400 Qs, 总耗时 46.0 min) | 题型 | Agentic | Binary | input tok/q | output tok/q | total tok/q | tool calls/q | |---|---|---|---|---|---|---| -| abstention | 0.500 | 0.500 | 90,460 | 1,121 | 91,581 | 4.15 | -| contradiction_resolution | 0.338 | 0.300 | 27,951 | 751 | 28,703 | 2.48 | -| event_ordering | 0.533 | 0.472 | 143,223 | 5,658 | 148,880 | 4.55 | -| information_extraction | 0.833 | 0.796 | 38,650 | 794 | 39,444 | 2.62 | -| instruction_following | 0.794 | 0.762 | 32,991 | 782 | 33,773 | 2.35 | -| knowledge_update | 0.688 | 0.675 | 26,218 | 716 | 26,934 | 2.08 | -| multi_session_reasoning | 0.644 | 0.584 | 76,239 | 3,229 | 79,469 | 4.00 | -| preference_following | 0.858 | 0.842 | 28,486 | 865 | 29,350 | 2.17 | -| summarization | 0.581 | 0.412 | 83,303 | 1,866 | 85,168 | 3.88 | -| temporal_reasoning | 0.581 | 0.550 | 35,142 | 1,213 | 36,354 | 2.42 | -| **OVERALL** | **0.635** | **0.589** | **58,266** | **1,699** | **59,966** | **3.07** | +| abstention | 0.550 | 0.550 | 96,031 | 1,070 | 97,101 | 4.58 | +| contradiction_resolution | 0.438 | 0.412 | 32,263 | 872 | 33,135 | 2.48 | +| event_ordering | 0.501 | 0.423 | 140,195 | 5,163 | 145,358 | 4.70 | +| information_extraction | 0.873 | 0.832 | 50,245 | 883 | 51,128 | 3.15 | +| instruction_following | 0.750 | 0.725 | 37,986 | 848 | 38,834 | 2.67 | +| knowledge_update | 0.688 | 0.675 | 31,198 | 651 | 31,849 | 2.27 | +| multi_session_reasoning | 0.626 | 0.584 | 85,038 | 4,563 | 89,601 | 4.28 | +| preference_following | 0.925 | 0.912 | 34,281 | 989 | 35,270 | 2.50 | +| summarization | 0.623 | 0.461 | 89,657 | 2,056 | 91,713 | 4.12 | +| temporal_reasoning | 0.637 | 0.625 | 34,563 | 1,049 | 35,612 | 2.52 | +| **OVERALL** | **0.661** | **0.620** | **63,146** | **1,814** | **64,960** | **3.33** | -**Memory Construction Token 消耗（default agent / qwen3.6-flash, per case）** +Memory Construction 平均 token 消耗（default agent, 20 cases 全量构建）: -| 指标 | 均值 ± 标准差 | -|---|---| -| input_tokens | 2,172,316 ± 975,656 | -| output_tokens | 136,697 ± 37,484 | -| total_tokens | 2,309,013 ± 1,009,612 | +| Agent | input tok/case | output tok/case | total tok/case | +|---|---|---|---| +| default | 2,172,316 | 136,697 | 2,309,013 | ### 1M -| 题型 | Prompted(limit=15) | Prompted Binary | Agentic | Agentic Binary | -|---|---|---|---|---| -| abstention | 0.464 | 0.464 | 0.464 | 0.464 | -| contradiction_resolution | 0.079 | 0.068 | 0.379 | 0.346 | -| event_ordering | 0.455 | 0.334 | 0.535 | 0.442 | -| information_extraction | 0.653 | 0.589 | 0.795 | 0.758 | -| instruction_following | 0.541 | 0.524 | 0.774 | 0.758 | -| knowledge_update | 0.571 | 0.507 | 0.693 | 0.679 | -| multi_session_reasoning | 0.426 | 0.324 | 0.656 | 0.602 | -| preference_following | 0.718 | 0.676 | 0.805 | 0.796 | -| summarization | 0.516 | 0.303 | 0.672 | 0.504 | -| temporal_reasoning | 0.198 | 0.169 | 0.462 | 0.448 | -| **OVERALL** | **0.462** | **0.396** | **0.623** | **0.580** | +agentscope==2.0.4.post1, conda reme 环境, 20 并发, 全量构建 memory +(2026-08-05, 35 cases / 700 Qs, 总耗时 459.2 min) -在auto-memory中加入Source Index机制 +| 题型 | Agentic | Binary | input tok/q | output tok/q | total tok/q | tool calls/q | +|---|---|---|---|---|---|---| +| abstention | 0.429 | 0.429 | 118,707 | 1,178 | 119,886 | 4.20 | +| contradiction_resolution | 0.391 | 0.364 | 49,787 | 810 | 50,597 | 2.50 | +| event_ordering | 0.558 | 0.456 | 201,514 | 3,889 | 205,403 | 4.79 | +| information_extraction | 0.809 | 0.772 | 78,950 | 894 | 79,844 | 3.00 | +| instruction_following | 0.852 | 0.832 | 55,757 | 924 | 56,681 | 2.81 | +| knowledge_update | 0.779 | 0.771 | 45,981 | 665 | 46,646 | 2.37 | +| multi_session_reasoning | 0.658 | 0.612 | 138,133 | 2,873 | 141,006 | 4.40 | +| preference_following | 0.798 | 0.777 | 51,796 | 920 | 52,716 | 2.53 | +| summarization | 0.693 | 0.537 | 158,794 | 2,905 | 161,700 | 4.44 | +| temporal_reasoning | 0.536 | 0.536 | 100,176 | 3,148 | 103,324 | 3.90 | +| **OVERALL** | **0.650** | **0.609** | **99,959** | **1,821** | **101,780** | **3.49** | -### 1M (2026-07-28, 35 cases / 700 Qs) +Memory Construction 平均 token 消耗（default agent, 35 cases 全量构建）: -| 题型 | Agentic | Agentic Binary | -|---|---|---| -| abstention | 0.457 | 0.457 | -| contradiction_resolution | 0.405 | 0.371 | -| event_ordering | 0.566 | 0.478 | -| information_extraction | 0.782 | 0.742 | -| instruction_following | 0.860 | 0.843 | -| knowledge_update | 0.704 | 0.679 | -| multi_session_reasoning | 0.663 | 0.603 | -| preference_following | 0.827 | 0.807 | -| summarization | 0.686 | 0.532 | -| temporal_reasoning | 0.511 | 0.500 | -| **OVERALL** | **0.646** | **0.601** | \ No newline at end of file +| Agent | input tok/case | output tok/case | total tok/case | +|---|---|---|---| +| default | 31,943,817 | 1,417,061 | 33,360,878 | \ No newline at end of file diff --git a/reme/steps/benchmark/base/agentic_answer.py b/reme/steps/benchmark/base/agentic_answer.py index eb06a55c..53c854f4 100644 --- a/reme/steps/benchmark/base/agentic_answer.py +++ b/reme/steps/benchmark/base/agentic_answer.py @@ -34,7 +34,7 @@ class BaseAgenticAnswerStep(BaseStep): MAX_ITERATION = 10 TOOL_CONTEXT_PREFIX: str = "content_agentic_answer" JOB_TOOLS: list[str] = ["search", "add_draft", "read_all_draft", "read"] - INJECTED_JOB_KWARGS: dict = {} + INJECTED_JOB_KWARGS: dict = {"read_step_format_session": True} def _injected_job_kwargs(self, query: str) -> dict: # pylint: disable=unused-argument """Server-owned kwargs injected into every job tool call. diff --git a/reme/steps/benchmark/beam/agentic_answer.yaml b/reme/steps/benchmark/beam/agentic_answer.yaml index 15e4fb06..153d211b 100644 --- a/reme/steps/benchmark/beam/agentic_answer.yaml +++ b/reme/steps/benchmark/beam/agentic_answer.yaml @@ -6,12 +6,14 @@ system_prompt: | - You can call 'search' tool to search multiple times (at least once) with different queries to gather comprehensive information. - Do not stop after the first relevant one unless you are confident that the information is sufficient. - You can use 'read' tool to read specific parts of a specific file. + - After finding a preliminary answer, continue searching from multiple angles to check for potentially conflicting, supplementary, or updated information. ## Draft Tool - Use 'add_draft' to save key findings during search, and 'read_all_draft' to review all saved notes before answering. ## Answer Rules - Answer based ONLY on retrieved context. - Output ONLY the direct factual answer — no reasoning, no search process, no elaboration. - - If information is not founded or not sufficient after multiple searches, reply: 'Information not found.' + - Do NOT invent or assume information that isn't in the memories. Do NOT extrapolate or fabricate connections from related but non-direct evidence. + - If information is not found or not sufficient after multiple searches, or if the maximum iteration count is reached without finding answer, reply: 'Information not found.' temporal_hint: | Current time context: {query_time} diff --git a/reme/steps/file_io/_file_io.py b/reme/steps/file_io/_file_io.py index 1a979449..82c5264d 100644 --- a/reme/steps/file_io/_file_io.py +++ b/reme/steps/file_io/_file_io.py @@ -8,6 +8,7 @@ import aiofiles import aiofiles.os from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES, TRUNCATION_NOTICE_MARKER +from ...schema import FileChunk from ...utils import get_logger logger = get_logger(log_to_file=False) @@ -214,3 +215,44 @@ def truncate_text_output( except Exception: logger.warning("truncate_text_output failed, returning original text", exc_info=True) return text + + +def truncate_session_output( + text: str, + *, + start_line: int = 1, + total_lines: int = 0, + max_bytes: int = DEFAULT_MAX_BYTES, + file_path: str | None = None, + encoding: str = "utf-8", +) -> str: + """Render session jsonl lines then truncate by bytes preserving line integrity. + + Each line of *text* is expected to be a serialized ``Msg`` JSON from a raw + session transcript (``*.jsonl``). Lines are first rendered via + :func:`~reme.steps.index._source_format.render_session_chunk_lines` — one + message per line as ``[speaker @ created_at] content`` with internal + newlines flattened — and then truncated with the same byte-budget logic + as :func:`truncate_text_output`. + """ + if not text or max_bytes <= 0: + return text + + # Render each jsonl line into a human-readable [speaker @ time] content line. + # Import locally to avoid a hard dependency from file_io on the index package + # at module load time; the import is safe (no circular dependency). + from ..index._source_format import render_session_chunk_lines + + chunk = FileChunk(text=text) + rendered_lines = render_session_chunk_lines(chunk) + rendered_text = "\n".join(rendered_lines) + + # Reuse the same byte-budget truncation as truncate_text_output. + return truncate_text_output( + rendered_text, + start_line=start_line, + total_lines=total_lines, + max_bytes=max_bytes, + file_path=file_path, + encoding=encoding, + ) diff --git a/reme/steps/file_io/read.py b/reme/steps/file_io/read.py index 589a56ac..71bdac43 100644 --- a/reme/steps/file_io/read.py +++ b/reme/steps/file_io/read.py @@ -2,9 +2,10 @@ from pathlib import Path -from ._file_io import read_file_lines_safe, read_file_safe, truncate_text_output +from ._file_io import read_file_lines_safe, read_file_safe, truncate_session_output, truncate_text_output from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path from ..base_step import BaseStep +from ..index._source_format import is_session_path from ...components import R from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES from ...utils import expand_links, render_expansion_lines @@ -21,6 +22,14 @@ class ReadStep(BaseStep): neighbors (out/in link targets) with name/description meta, fetched via the file_store. Same rendering as SearchStep. max_neighbors_per_direction (int, default 10): cap per direction. + read_step_format_session (bool, default False): when true and the + resolved path is a raw session transcript (``*.jsonl`` under the + dialog dir), render each jsonl line as + ``[speaker @ created_at] content`` before truncation via + :func:`truncate_session_output`. Non-session files fall back to + the standard :func:`truncate_text_output` path. Injected at + runtime via ``injected_job_kwargs`` (takes precedence) or set in + the YAML ``steps`` config as a fallback. Permission: honors the request-scoped ``_allowed_paths`` constraint injected by the server into the RuntimeContext. @@ -93,6 +102,29 @@ class ReadStep(BaseStep): self._fail(f"read failed: {e}", path=str(target)) return None + def _truncate_with_session_format(self, excerpt: str, target: Path, start_line: int, total: int) -> str: + """Truncate *excerpt* using session-aware rendering when *target* is a session file. + + Falls back to :func:`truncate_text_output` when the path is not under + the workspace or is not a raw session transcript. + """ + try: + rel_path = str(target.relative_to(self.workspace_path)) + except ValueError: + self.logger.info(f"[{self.name}] skip session format: path outside workspace_path path={target}") + return truncate_text_output(excerpt, start_line=start_line, total_lines=total, file_path=str(target)) + + dialog_dir = self.config_value("dialog_dir") + if not is_session_path(rel_path, dialog_dir): + return truncate_text_output(excerpt, start_line=start_line, total_lines=total, file_path=str(target)) + + return truncate_session_output( + excerpt, + start_line=start_line, + total_lines=total, + file_path=str(target), + ) + # pylint: disable=too-many-return-statements async def execute(self): assert self.context is not None @@ -100,6 +132,10 @@ class ReadStep(BaseStep): start_line, end_line = self.context.get("start_line"), self.context.get("end_line") with_neighbors: bool = bool(self.kwargs.get("with_neighbors", False)) max_neighbors_per_direction: int = int(self.kwargs.get("max_neighbors_per_direction", 10)) + format_session: bool = bool( + self.context.get("read_step_format_session") + or self.kwargs.get("read_step_format_session", False), + ) # Validate inputs and target before touching the filesystem twice. target = self._resolve_target(raw) @@ -152,7 +188,10 @@ class ReadStep(BaseStep): return None e = min(total, requested_end if requested_end is not None else total) - text = truncate_text_output(excerpt, start_line=s, total_lines=total, file_path=str(target)) + if format_session: + text = self._truncate_with_session_format(excerpt, target, s, total) + else: + text = truncate_text_output(excerpt, start_line=s, total_lines=total, file_path=str(target)) self.context.response.success = True self.context.response.answer = text