mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-02 07:13:50 +00:00
feat(read): add session-aware formatting for read tool and update BEAM eval
- Add truncate_session_output in _file_io.py to render jsonl session lines as [speaker @ time] content before byte-budget truncation - Add read_step_format_session flag to ReadStep, honoring injected job kwargs (precedence) and YAML fallback - Inject read_step_format_session=True into BaseAgenticAnswerStep so agentic answer reads render session transcripts human-readably - Refine BEAM agentic_answer prompt: continue multi-angle search after preliminary answer, forbid fabrication/extrapolation - Update BEAM config to 1M variant and add sequential 100K-eval / 1M-build shell script - Refresh benchmark/results_md/beam.md with latest results
This commit is contained in:
sa-buc
parent
bb7a436d51
commit
ef6b0d74e1
7 changed files with 234 additions and 52 deletions
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
dataset:
|
||||
beam_root: "benchmark/beam/dataset/BEAM" # BEAM dataset root
|
||||
chat_size: "100K" # 100K | 500K | 1M | 10M (dataset variant)
|
||||
chat_size: "1M" # 100K | 500K | 1M | 10M (dataset variant)
|
||||
case_ids: [] # empty = all cases; or ["1", "2", "3"]
|
||||
start_index: 0 # first case index (for pagination)
|
||||
num_items: 0 # 0 = all cases; >0 = limit
|
||||
|
|
|
|||
110
benchmark/beam/run_beam_100k_eval_1m_build.sh
Normal file
110
benchmark/beam/run_beam_100k_eval_1m_build.sh
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
#!/usr/bin/env bash
|
||||
# Sequential BEAM evaluation: 100K eval-only → 1M full build
|
||||
# Usage: bash benchmark/beam/run_beam_100k_eval_1m_build.sh
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
CONFIG="$PROJECT_ROOT/benchmark/beam/config.yaml"
|
||||
LOG_DIR="$PROJECT_ROOT/logs"
|
||||
TS=$(date +%Y%m%d_%H%M%S)
|
||||
LOG_FILE="$LOG_DIR/beam_sequential_${TS}.log"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
echo "=== BEAM Sequential Evaluation ===" | tee "$LOG_FILE"
|
||||
echo "Started at: $(date)" | tee -a "$LOG_FILE"
|
||||
echo "Log file: $LOG_FILE" | tee -a "$LOG_FILE"
|
||||
|
||||
# ── Phase 1: BEAM 100K eval-only (20 workers) ──
|
||||
echo "" | tee -a "$LOG_FILE"
|
||||
echo "==========================================" | tee -a "$LOG_FILE"
|
||||
echo " Phase 1: BEAM 100K eval-only" | tee -a "$LOG_FILE"
|
||||
echo "==========================================" | tee -a "$LOG_FILE"
|
||||
|
||||
# Write 100K eval-only config
|
||||
cat > "$CONFIG" <<'EOF'
|
||||
# BEAM evaluation configuration
|
||||
# This file controls what/how to evaluate.
|
||||
|
||||
dataset:
|
||||
beam_root: "benchmark/beam/dataset/BEAM" # BEAM dataset root
|
||||
chat_size: "100K" # 100K | 500K | 1M | 10M (dataset variant)
|
||||
case_ids: [] # empty = all cases; or ["1", "2", "3"]
|
||||
start_index: 0 # first case index (for pagination)
|
||||
num_items: 0 # 0 = all cases; >0 = limit
|
||||
workspace_root: "benchmark/beam/workspaces/beam" # workspace root for case workspaces
|
||||
|
||||
evaluation:
|
||||
num_workers: 20 # 0 = auto; 1 = sequential; >1 = parallel (per-case)
|
||||
compress_session: false # true = compress session chunks in search_v2 (query-aware); false = no compression
|
||||
|
||||
reme:
|
||||
config: "beam.yaml" # reme config (in reme/config/)
|
||||
|
||||
output:
|
||||
dir: "benchmark/beam/results"
|
||||
log_dir: "logs" # log directory (relative to project root)
|
||||
log_prefix: "beam" # benchmark name used in log filenames
|
||||
log_to_console: true
|
||||
log_to_file: true
|
||||
EOF
|
||||
|
||||
echo "Config written for 100K eval-only" | tee -a "$LOG_FILE"
|
||||
cd "$PROJECT_ROOT"
|
||||
conda run -n reme --no-capture-output python benchmark/beam/run.py --eval_only 2>&1 | tee -a "$LOG_FILE"
|
||||
PHASE1_EXIT=${PIPESTATUS[0]}
|
||||
|
||||
if [ $PHASE1_EXIT -ne 0 ]; then
|
||||
echo "Phase 1 (100K eval-only) FAILED with exit code $PHASE1_EXIT" | tee -a "$LOG_FILE"
|
||||
exit $PHASE1_EXIT
|
||||
fi
|
||||
|
||||
echo "" | tee -a "$LOG_FILE"
|
||||
echo "Phase 1 (100K eval-only) completed at $(date)" | tee -a "$LOG_FILE"
|
||||
|
||||
# ── Phase 2: BEAM 1M full build (20 workers) ──
|
||||
echo "" | tee -a "$LOG_FILE"
|
||||
echo "==========================================" | tee -a "$LOG_FILE"
|
||||
echo " Phase 2: BEAM 1M full build" | tee -a "$LOG_FILE"
|
||||
echo "==========================================" | tee -a "$LOG_FILE"
|
||||
|
||||
# Write 1M full build config
|
||||
cat > "$CONFIG" <<'EOF'
|
||||
# BEAM evaluation configuration
|
||||
# This file controls what/how to evaluate.
|
||||
|
||||
dataset:
|
||||
beam_root: "benchmark/beam/dataset/BEAM" # BEAM dataset root
|
||||
chat_size: "1M" # 100K | 500K | 1M | 10M (dataset variant)
|
||||
case_ids: [] # empty = all cases; or ["1", "2", "3"]
|
||||
start_index: 0 # first case index (for pagination)
|
||||
num_items: 0 # 0 = all cases; >0 = limit
|
||||
workspace_root: "benchmark/beam/workspaces/beam" # workspace root for case workspaces
|
||||
|
||||
evaluation:
|
||||
num_workers: 20 # 0 = auto; 1 = sequential; >1 = parallel (per-case)
|
||||
compress_session: false # true = compress session chunks in search_v2 (query-aware); false = no compression
|
||||
|
||||
reme:
|
||||
config: "beam.yaml" # reme config (in reme/config/)
|
||||
|
||||
output:
|
||||
dir: "benchmark/beam/results"
|
||||
log_dir: "logs" # log directory (relative to project root)
|
||||
log_prefix: "beam" # benchmark name used in log filenames
|
||||
log_to_console: true
|
||||
log_to_file: true
|
||||
EOF
|
||||
|
||||
echo "Config written for 1M full build" | tee -a "$LOG_FILE"
|
||||
conda run -n reme --no-capture-output python benchmark/beam/run.py 2>&1 | tee -a "$LOG_FILE"
|
||||
PHASE2_EXIT=${PIPESTATUS[0]}
|
||||
|
||||
if [ $PHASE2_EXIT -ne 0 ]; then
|
||||
echo "Phase 2 (1M full build) FAILED with exit code $PHASE2_EXIT" | tee -a "$LOG_FILE"
|
||||
exit $PHASE2_EXIT
|
||||
fi
|
||||
|
||||
echo "" | tee -a "$LOG_FILE"
|
||||
echo "Phase 2 (1M full build) completed at $(date)" | tee -a "$LOG_FILE"
|
||||
echo "" | tee -a "$LOG_FILE"
|
||||
echo "=== ALL DONE at $(date) ===" | tee -a "$LOG_FILE"
|
||||
|
|
@ -5,61 +5,50 @@
|
|||
### 100K
|
||||
|
||||
|
||||
agentscope==2.0.4.post1, conda reme 环境, 20 并发, 从头构建 memory
|
||||
(2026-08-04, 20 cases / 400 Qs, 总耗时 61.8 min)
|
||||
agentscope==2.0.4.post1, conda reme 环境, 20 并发, eval-only（复用已构建 memory）
|
||||
(2026-08-05, 20 cases / 400 Qs, 总耗时 46.0 min)
|
||||
|
||||
| 题型 | Agentic | Binary | input tok/q | output tok/q | total tok/q | tool calls/q |
|
||||
|---|---|---|---|---|---|---|
|
||||
| abstention | 0.500 | 0.500 | 90,460 | 1,121 | 91,581 | 4.15 |
|
||||
| contradiction_resolution | 0.338 | 0.300 | 27,951 | 751 | 28,703 | 2.48 |
|
||||
| event_ordering | 0.533 | 0.472 | 143,223 | 5,658 | 148,880 | 4.55 |
|
||||
| information_extraction | 0.833 | 0.796 | 38,650 | 794 | 39,444 | 2.62 |
|
||||
| instruction_following | 0.794 | 0.762 | 32,991 | 782 | 33,773 | 2.35 |
|
||||
| knowledge_update | 0.688 | 0.675 | 26,218 | 716 | 26,934 | 2.08 |
|
||||
| multi_session_reasoning | 0.644 | 0.584 | 76,239 | 3,229 | 79,469 | 4.00 |
|
||||
| preference_following | 0.858 | 0.842 | 28,486 | 865 | 29,350 | 2.17 |
|
||||
| summarization | 0.581 | 0.412 | 83,303 | 1,866 | 85,168 | 3.88 |
|
||||
| temporal_reasoning | 0.581 | 0.550 | 35,142 | 1,213 | 36,354 | 2.42 |
|
||||
| **OVERALL** | **0.635** | **0.589** | **58,266** | **1,699** | **59,966** | **3.07** |
|
||||
| abstention | 0.550 | 0.550 | 96,031 | 1,070 | 97,101 | 4.58 |
|
||||
| contradiction_resolution | 0.438 | 0.412 | 32,263 | 872 | 33,135 | 2.48 |
|
||||
| event_ordering | 0.501 | 0.423 | 140,195 | 5,163 | 145,358 | 4.70 |
|
||||
| information_extraction | 0.873 | 0.832 | 50,245 | 883 | 51,128 | 3.15 |
|
||||
| instruction_following | 0.750 | 0.725 | 37,986 | 848 | 38,834 | 2.67 |
|
||||
| knowledge_update | 0.688 | 0.675 | 31,198 | 651 | 31,849 | 2.27 |
|
||||
| multi_session_reasoning | 0.626 | 0.584 | 85,038 | 4,563 | 89,601 | 4.28 |
|
||||
| preference_following | 0.925 | 0.912 | 34,281 | 989 | 35,270 | 2.50 |
|
||||
| summarization | 0.623 | 0.461 | 89,657 | 2,056 | 91,713 | 4.12 |
|
||||
| temporal_reasoning | 0.637 | 0.625 | 34,563 | 1,049 | 35,612 | 2.52 |
|
||||
| **OVERALL** | **0.661** | **0.620** | **63,146** | **1,814** | **64,960** | **3.33** |
|
||||
|
||||
**Memory Construction Token 消耗（default agent / qwen3.6-flash, per case）**
|
||||
Memory Construction 平均 token 消耗（default agent, 20 cases 全量构建）:
|
||||
|
||||
| 指标 | 均值 ± 标准差 |
|
||||
|---|---|
|
||||
| input_tokens | 2,172,316 ± 975,656 |
|
||||
| output_tokens | 136,697 ± 37,484 |
|
||||
| total_tokens | 2,309,013 ± 1,009,612 |
|
||||
| Agent | input tok/case | output tok/case | total tok/case |
|
||||
|---|---|---|---|
|
||||
| default | 2,172,316 | 136,697 | 2,309,013 |
|
||||
|
||||
### 1M
|
||||
|
||||
| 题型 | Prompted(limit=15) | Prompted Binary | Agentic | Agentic Binary |
|
||||
|---|---|---|---|---|
|
||||
| abstention | 0.464 | 0.464 | 0.464 | 0.464 |
|
||||
| contradiction_resolution | 0.079 | 0.068 | 0.379 | 0.346 |
|
||||
| event_ordering | 0.455 | 0.334 | 0.535 | 0.442 |
|
||||
| information_extraction | 0.653 | 0.589 | 0.795 | 0.758 |
|
||||
| instruction_following | 0.541 | 0.524 | 0.774 | 0.758 |
|
||||
| knowledge_update | 0.571 | 0.507 | 0.693 | 0.679 |
|
||||
| multi_session_reasoning | 0.426 | 0.324 | 0.656 | 0.602 |
|
||||
| preference_following | 0.718 | 0.676 | 0.805 | 0.796 |
|
||||
| summarization | 0.516 | 0.303 | 0.672 | 0.504 |
|
||||
| temporal_reasoning | 0.198 | 0.169 | 0.462 | 0.448 |
|
||||
| **OVERALL** | **0.462** | **0.396** | **0.623** | **0.580** |
|
||||
agentscope==2.0.4.post1, conda reme 环境, 20 并发, 全量构建 memory
|
||||
(2026-08-05, 35 cases / 700 Qs, 总耗时 459.2 min)
|
||||
|
||||
在auto-memory中加入Source Index机制
|
||||
| 题型 | Agentic | Binary | input tok/q | output tok/q | total tok/q | tool calls/q |
|
||||
|---|---|---|---|---|---|---|
|
||||
| abstention | 0.429 | 0.429 | 118,707 | 1,178 | 119,886 | 4.20 |
|
||||
| contradiction_resolution | 0.391 | 0.364 | 49,787 | 810 | 50,597 | 2.50 |
|
||||
| event_ordering | 0.558 | 0.456 | 201,514 | 3,889 | 205,403 | 4.79 |
|
||||
| information_extraction | 0.809 | 0.772 | 78,950 | 894 | 79,844 | 3.00 |
|
||||
| instruction_following | 0.852 | 0.832 | 55,757 | 924 | 56,681 | 2.81 |
|
||||
| knowledge_update | 0.779 | 0.771 | 45,981 | 665 | 46,646 | 2.37 |
|
||||
| multi_session_reasoning | 0.658 | 0.612 | 138,133 | 2,873 | 141,006 | 4.40 |
|
||||
| preference_following | 0.798 | 0.777 | 51,796 | 920 | 52,716 | 2.53 |
|
||||
| summarization | 0.693 | 0.537 | 158,794 | 2,905 | 161,700 | 4.44 |
|
||||
| temporal_reasoning | 0.536 | 0.536 | 100,176 | 3,148 | 103,324 | 3.90 |
|
||||
| **OVERALL** | **0.650** | **0.609** | **99,959** | **1,821** | **101,780** | **3.49** |
|
||||
|
||||
### 1M (2026-07-28, 35 cases / 700 Qs)
|
||||
Memory Construction 平均 token 消耗（default agent, 35 cases 全量构建）:
|
||||
|
||||
| 题型 | Agentic | Agentic Binary |
|
||||
|---|---|---|
|
||||
| abstention | 0.457 | 0.457 |
|
||||
| contradiction_resolution | 0.405 | 0.371 |
|
||||
| event_ordering | 0.566 | 0.478 |
|
||||
| information_extraction | 0.782 | 0.742 |
|
||||
| instruction_following | 0.860 | 0.843 |
|
||||
| knowledge_update | 0.704 | 0.679 |
|
||||
| multi_session_reasoning | 0.663 | 0.603 |
|
||||
| preference_following | 0.827 | 0.807 |
|
||||
| summarization | 0.686 | 0.532 |
|
||||
| temporal_reasoning | 0.511 | 0.500 |
|
||||
| **OVERALL** | **0.646** | **0.601** |
|
||||
| Agent | input tok/case | output tok/case | total tok/case |
|
||||
|---|---|---|---|
|
||||
| default | 31,943,817 | 1,417,061 | 33,360,878 |
|
||||
|
|
@ -34,7 +34,7 @@ class BaseAgenticAnswerStep(BaseStep):
|
|||
MAX_ITERATION = 10
|
||||
TOOL_CONTEXT_PREFIX: str = "content_agentic_answer"
|
||||
JOB_TOOLS: list[str] = ["search", "add_draft", "read_all_draft", "read"]
|
||||
INJECTED_JOB_KWARGS: dict = {}
|
||||
INJECTED_JOB_KWARGS: dict = {"read_step_format_session": True}
|
||||
|
||||
def _injected_job_kwargs(self, query: str) -> dict: # pylint: disable=unused-argument
|
||||
"""Server-owned kwargs injected into every job tool call.
|
||||
|
|
|
|||
|
|
@ -6,12 +6,14 @@ system_prompt: |
|
|||
- You can call 'search' tool to search multiple times (at least once) with different queries to gather comprehensive information.
|
||||
- Do not stop after the first relevant one unless you are confident that the information is sufficient.
|
||||
- You can use 'read' tool to read specific parts of a specific file.
|
||||
- After finding a preliminary answer, continue searching from multiple angles to check for potentially conflicting, supplementary, or updated information.
|
||||
## Draft Tool
|
||||
- Use 'add_draft' to save key findings during search, and 'read_all_draft' to review all saved notes before answering.
|
||||
## Answer Rules
|
||||
- Answer based ONLY on retrieved context.
|
||||
- Output ONLY the direct factual answer — no reasoning, no search process, no elaboration.
|
||||
- If information is not founded or not sufficient after multiple searches, reply: 'Information not found.'
|
||||
- Do NOT invent or assume information that isn't in the memories. Do NOT extrapolate or fabricate connections from related but non-direct evidence.
|
||||
- If information is not found or not sufficient after multiple searches, or if the maximum iteration count is reached without finding answer, reply: 'Information not found.'
|
||||
|
||||
temporal_hint: |
|
||||
Current time context: {query_time}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import aiofiles
|
|||
import aiofiles.os
|
||||
|
||||
from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES, TRUNCATION_NOTICE_MARKER
|
||||
from ...schema import FileChunk
|
||||
from ...utils import get_logger
|
||||
|
||||
logger = get_logger(log_to_file=False)
|
||||
|
|
@ -214,3 +215,44 @@ def truncate_text_output(
|
|||
except Exception:
|
||||
logger.warning("truncate_text_output failed, returning original text", exc_info=True)
|
||||
return text
|
||||
|
||||
|
||||
def truncate_session_output(
|
||||
text: str,
|
||||
*,
|
||||
start_line: int = 1,
|
||||
total_lines: int = 0,
|
||||
max_bytes: int = DEFAULT_MAX_BYTES,
|
||||
file_path: str | None = None,
|
||||
encoding: str = "utf-8",
|
||||
) -> str:
|
||||
"""Render session jsonl lines then truncate by bytes preserving line integrity.
|
||||
|
||||
Each line of *text* is expected to be a serialized ``Msg`` JSON from a raw
|
||||
session transcript (``*.jsonl``). Lines are first rendered via
|
||||
:func:`~reme.steps.index._source_format.render_session_chunk_lines` — one
|
||||
message per line as ``[speaker @ created_at] content`` with internal
|
||||
newlines flattened — and then truncated with the same byte-budget logic
|
||||
as :func:`truncate_text_output`.
|
||||
"""
|
||||
if not text or max_bytes <= 0:
|
||||
return text
|
||||
|
||||
# Render each jsonl line into a human-readable [speaker @ time] content line.
|
||||
# Import locally to avoid a hard dependency from file_io on the index package
|
||||
# at module load time; the import is safe (no circular dependency).
|
||||
from ..index._source_format import render_session_chunk_lines
|
||||
|
||||
chunk = FileChunk(text=text)
|
||||
rendered_lines = render_session_chunk_lines(chunk)
|
||||
rendered_text = "\n".join(rendered_lines)
|
||||
|
||||
# Reuse the same byte-budget truncation as truncate_text_output.
|
||||
return truncate_text_output(
|
||||
rendered_text,
|
||||
start_line=start_line,
|
||||
total_lines=total_lines,
|
||||
max_bytes=max_bytes,
|
||||
file_path=file_path,
|
||||
encoding=encoding,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@
|
|||
|
||||
from pathlib import Path
|
||||
|
||||
from ._file_io import read_file_lines_safe, read_file_safe, truncate_text_output
|
||||
from ._file_io import read_file_lines_safe, read_file_safe, truncate_session_output, truncate_text_output
|
||||
from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path
|
||||
from ..base_step import BaseStep
|
||||
from ..index._source_format import is_session_path
|
||||
from ...components import R
|
||||
from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES
|
||||
from ...utils import expand_links, render_expansion_lines
|
||||
|
|
@ -21,6 +22,14 @@ class ReadStep(BaseStep):
|
|||
neighbors (out/in link targets) with name/description meta,
|
||||
fetched via the file_store. Same rendering as SearchStep.
|
||||
max_neighbors_per_direction (int, default 10): cap per direction.
|
||||
read_step_format_session (bool, default False): when true and the
|
||||
resolved path is a raw session transcript (``*.jsonl`` under the
|
||||
dialog dir), render each jsonl line as
|
||||
``[speaker @ created_at] content`` before truncation via
|
||||
:func:`truncate_session_output`. Non-session files fall back to
|
||||
the standard :func:`truncate_text_output` path. Injected at
|
||||
runtime via ``injected_job_kwargs`` (takes precedence) or set in
|
||||
the YAML ``steps`` config as a fallback.
|
||||
|
||||
Permission: honors the request-scoped ``_allowed_paths`` constraint
|
||||
injected by the server into the RuntimeContext.
|
||||
|
|
@ -93,6 +102,29 @@ class ReadStep(BaseStep):
|
|||
self._fail(f"read failed: {e}", path=str(target))
|
||||
return None
|
||||
|
||||
def _truncate_with_session_format(self, excerpt: str, target: Path, start_line: int, total: int) -> str:
|
||||
"""Truncate *excerpt* using session-aware rendering when *target* is a session file.
|
||||
|
||||
Falls back to :func:`truncate_text_output` when the path is not under
|
||||
the workspace or is not a raw session transcript.
|
||||
"""
|
||||
try:
|
||||
rel_path = str(target.relative_to(self.workspace_path))
|
||||
except ValueError:
|
||||
self.logger.info(f"[{self.name}] skip session format: path outside workspace_path path={target}")
|
||||
return truncate_text_output(excerpt, start_line=start_line, total_lines=total, file_path=str(target))
|
||||
|
||||
dialog_dir = self.config_value("dialog_dir")
|
||||
if not is_session_path(rel_path, dialog_dir):
|
||||
return truncate_text_output(excerpt, start_line=start_line, total_lines=total, file_path=str(target))
|
||||
|
||||
return truncate_session_output(
|
||||
excerpt,
|
||||
start_line=start_line,
|
||||
total_lines=total,
|
||||
file_path=str(target),
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-return-statements
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
|
|
@ -100,6 +132,10 @@ class ReadStep(BaseStep):
|
|||
start_line, end_line = self.context.get("start_line"), self.context.get("end_line")
|
||||
with_neighbors: bool = bool(self.kwargs.get("with_neighbors", False))
|
||||
max_neighbors_per_direction: int = int(self.kwargs.get("max_neighbors_per_direction", 10))
|
||||
format_session: bool = bool(
|
||||
self.context.get("read_step_format_session")
|
||||
or self.kwargs.get("read_step_format_session", False),
|
||||
)
|
||||
|
||||
# Validate inputs and target before touching the filesystem twice.
|
||||
target = self._resolve_target(raw)
|
||||
|
|
@ -152,7 +188,10 @@ class ReadStep(BaseStep):
|
|||
return None
|
||||
e = min(total, requested_end if requested_end is not None else total)
|
||||
|
||||
text = truncate_text_output(excerpt, start_line=s, total_lines=total, file_path=str(target))
|
||||
if format_session:
|
||||
text = self._truncate_with_session_format(excerpt, target, s, total)
|
||||
else:
|
||||
text = truncate_text_output(excerpt, start_line=s, total_lines=total, file_path=str(target))
|
||||
|
||||
self.context.response.success = True
|
||||
self.context.response.answer = text
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue