From db7f718236a2b966816d0356bb915492323b2ab2 Mon Sep 17 00:00:00 2001 From: sa-buc Date: Tue, 28 Jul 2026 11:40:20 +0800 Subject: [PATCH] feat(beam): enhance auto_memory with source markers and pilot ingest tooling --- benchmark/beam/_pilot_ingest.py | 65 ++++++++++ benchmark/beam/check_source_markers.py | 128 +++++++++++++++++++ benchmark/results_md/beam.md | 20 ++- reme/config/beam.yaml | 4 + reme/steps/benchmark/beam/auto_memory.py | 135 ++++++++++++++++++++- reme/steps/benchmark/beam/auto_memory.yaml | 48 ++++---- tests/unit/test_beam_auto_memory.py | 128 +++++++++++++++++++ 7 files changed, 501 insertions(+), 27 deletions(-) create mode 100644 benchmark/beam/_pilot_ingest.py create mode 100644 benchmark/beam/check_source_markers.py create mode 100644 tests/unit/test_beam_auto_memory.py diff --git a/benchmark/beam/_pilot_ingest.py b/benchmark/beam/_pilot_ingest.py new file mode 100644 index 00000000..6387654d --- /dev/null +++ b/benchmark/beam/_pilot_ingest.py @@ -0,0 +1,65 @@ +"""Pilot: ingest a few BEAM sessions and inspect source-marker quality. + +Temporary helper for prompt tuning — ingests the first N sessions of one +case into an isolated pilot workspace (never the real benchmark one). + +Usage: + python benchmark/beam/_pilot_ingest.py [case_id] [n_sessions] +""" + +import asyncio +import shutil +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from run import _PROJECT_ROOT, load_beam_chat, load_eval_config # noqa: E402 + +PILOT_ROOT = _PROJECT_ROOT / "benchmark/memory_workspaces/beam_pilot" + + +async def main(case_id: str, n_sessions: int): + from reme import Application + from reme.config import resolve_app_config + + eval_config = load_eval_config() + chat_size = eval_config["dataset"]["chat_size"] + beam_root = _PROJECT_ROOT / eval_config["dataset"].get("beam_root", "benchmark/datasets/BEAM") + chat_path = beam_root / "chats" / chat_size / case_id / "chat.json" + + case_dir = PILOT_ROOT / f"{chat_size}_{case_id}" + if case_dir.exists(): + shutil.rmtree(case_dir) + case_dir.mkdir(parents=True, exist_ok=True) + + cfg = resolve_app_config( + config=eval_config["reme"]["config"], + workspace_dir=str(case_dir / ".reme"), + log_to_console=False, + log_to_file=False, + enable_logo=False, + ) + app = Application(**cfg) + await app.start() + try: + sessions = load_beam_chat(chat_path, chat_size, case_id)[:n_sessions] + for i, session in enumerate(sessions): + print(f"[pilot] ingesting session {i + 1}/{len(sessions)}: " + f"{session['session_id']} msgs={len(session['messages'])}", flush=True) + resp = await app.run_job( + "auto_memory", + messages=session["messages"], + session_id=session["session_id"], + date=session["date"], + ) + print(f"[pilot] success={resp.success} chunks={resp.metadata.get('chunks')} " + f"answer={str(resp.answer)[:300]}", flush=True) + await app.run_job("index_update") + finally: + await app.close() + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1] if len(sys.argv) > 1 else "1", + int(sys.argv[2]) if len(sys.argv) > 2 else 2)) diff --git a/benchmark/beam/check_source_markers.py b/benchmark/beam/check_source_markers.py new file mode 100644 index 00000000..c0087f64 --- /dev/null +++ b/benchmark/beam/check_source_markers.py @@ -0,0 +1,128 @@ +"""Check source markers in BEAM auto-memory notes. + +Scans daily notes under one or more case workspaces and validates every +source marker against the required format:: + + [[#L-L(,L-L)*]] + +Reported problems: + +- ``legacy_format``: old single-bracket ``[path:1-2]`` markers. +- ``bad_anchor``: ``[[...jsonl#...]]`` whose anchor is not L-prefixed + hyphen ranges (colon separators, missing ``L``, spaces, bare line...). +- ``labelled``: marker prefixed with ``Source``/``来源`` labels. +- ``missing_file``: linked session file does not exist in the workspace. +- ``out_of_range``: cited line numbers exceed the session file length, + or start > end. + +Usage: + python benchmark/beam/check_source_markers.py [workspace_root ...] + (default: benchmark/memory_workspaces/beam) +""" + +import re +import sys +from pathlib import Path + +_PROJECT_ROOT = Path(__file__).parent.parent.parent + +WIKILINK_RE = re.compile(r"\[\[([^\[\]\|\n]+?)\]\]") +VALID_ANCHOR_RE = re.compile(r"^L(\d+)-L(\d+)(,L(\d+)-L(\d+))*$") +RANGE_RE = re.compile(r"L(\d+)-L(\d+)") +# Old format: [session/dialog/x.jsonl:1-2,5-6] (single brackets, colon) +LEGACY_RE = re.compile(r"(? str: + """Return the note body, skipping the leading YAML frontmatter block.""" + if text.startswith("---\n"): + end = text.find("\n---", 4) + if end != -1: + return text[end + 4 :] + return text + + +def _session_line_count(workspace: Path, rel_path: str) -> int | None: + f = workspace / rel_path + if not f.is_file(): + return None + with open(f, encoding="utf-8") as fh: + return sum(1 for _ in fh) + + +def check_note(note: Path, workspace: Path) -> tuple[list[dict], int]: + """Return (problems, valid_marker_count) for one note file.""" + text = _split_frontmatter(note.read_text(encoding="utf-8")) + rel_note = str(note.relative_to(workspace)) + problems: list[dict] = [] + valid = 0 + + def _add(kind: str, marker: str): + problems.append({"note": rel_note, "kind": kind, "marker": marker.strip()[:160]}) + + for m in LEGACY_RE.finditer(text): + _add("legacy_format", m.group(0)) + + for m in LABEL_RE.finditer(text): + _add("labelled", m.group(0)) + + line_counts: dict[str, int | None] = {} + for m in WIKILINK_RE.finditer(text): + inner = m.group(1) + if "#" not in inner: + continue # plain file link (e.g. day-index), not a line marker + target, anchor = inner.split("#", 1) + if not target.endswith(".jsonl"): + continue + if not VALID_ANCHOR_RE.match(anchor): + _add("bad_anchor", m.group(0)) + continue + if target not in line_counts: + line_counts[target] = _session_line_count(workspace, target) + n_lines = line_counts[target] + if n_lines is None: + _add("missing_file", m.group(0)) + continue + ok = True + for r in RANGE_RE.finditer(anchor): + start, end = int(r.group(1)), int(r.group(2)) + if start < 1 or end > n_lines or start > end: + _add("out_of_range", f"{m.group(0)} (file has {n_lines} lines)") + ok = False + break + if ok: + valid += 1 + return problems, valid + + +def main(roots: list[str]) -> int: + """Scan every case workspace under *roots* and print a problem report.""" + total_valid = 0 + all_problems: list[dict] = [] + n_notes = 0 + for root in roots: + root_path = Path(root) + if not root_path.is_absolute(): + root_path = _PROJECT_ROOT / root + for workspace in sorted(root_path.glob("*/.reme")): + for note in sorted(workspace.rglob("daily/*/*.md")): + n_notes += 1 + problems, valid = check_note(note, workspace) + total_valid += valid + all_problems.extend({**p, "workspace": workspace.parent.name} for p in problems) + + print(f"Scanned {n_notes} notes: {total_valid} valid markers, {len(all_problems)} problems") + by_kind: dict[str, int] = {} + for p in all_problems: + by_kind[p["kind"]] = by_kind.get(p["kind"], 0) + 1 + for kind, count in sorted(by_kind.items(), key=lambda kv: -kv[1]): + print(f" {kind}: {count}") + print() + for p in all_problems: + print(f"[{p['kind']}] {p['workspace']}/{p['note']}\n {p['marker']}") + return 1 if all_problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:] or ["benchmark/memory_workspaces/beam"])) diff --git a/benchmark/results_md/beam.md b/benchmark/results_md/beam.md index 97adff3c..c66dd8cd 100644 --- a/benchmark/results_md/beam.md +++ b/benchmark/results_md/beam.md @@ -33,4 +33,22 @@ | preference_following | 0.718 | 0.676 | 0.805 | 0.796 | | summarization | 0.516 | 0.303 | 0.672 | 0.504 | | temporal_reasoning | 0.198 | 0.169 | 0.462 | 0.448 | -| **OVERALL** | **0.462** | **0.396** | **0.623** | **0.580** | \ No newline at end of file +| **OVERALL** | **0.462** | **0.396** | **0.623** | **0.580** | + +在auto-memory中加入Source Index机制 + +### 1M (2026-07-28, 35 cases / 700 Qs) + +| 题型 | Agentic | Agentic Binary | +|---|---|---| +| abstention | 0.457 | 0.457 | +| contradiction_resolution | 0.405 | 0.371 | +| event_ordering | 0.566 | 0.478 | +| information_extraction | 0.782 | 0.742 | +| instruction_following | 0.860 | 0.843 | +| knowledge_update | 0.704 | 0.679 | +| multi_session_reasoning | 0.663 | 0.603 | +| preference_following | 0.827 | 0.807 | +| summarization | 0.686 | 0.532 | +| temporal_reasoning | 0.511 | 0.500 | +| **OVERALL** | **0.646** | **0.601** | \ No newline at end of file diff --git a/reme/config/beam.yaml b/reme/config/beam.yaml index 805f8769..1aae28ed 100644 --- a/reme/config/beam.yaml +++ b/reme/config/beam.yaml @@ -164,6 +164,10 @@ jobs: - messages steps: - backend: beam_auto_memory_step + # Long sessions are split into turn-aligned chunks and fed to the + # agent incrementally; each chunk holds at most this many words. + # <= 0 disables splitting. + max_chunk_words: 10000 # ── Reindex (full rebuild) ── reindex: diff --git a/reme/steps/benchmark/beam/auto_memory.py b/reme/steps/benchmark/beam/auto_memory.py index 1a934224..dd57ab12 100644 --- a/reme/steps/benchmark/beam/auto_memory.py +++ b/reme/steps/benchmark/beam/auto_memory.py @@ -5,8 +5,61 @@ from datetime import datetime, timedelta from agentscope.message import Msg from ...evolve.auto_memory import AutoMemoryStep, _normalize_msg_timestamp +from ...file_io import validate_session_id from ....components import R +# Runtime-context key carrying the 0-based line offset of the current chunk +# inside the full session file (chunked ingestion of long sessions). +_LINE_OFFSET_KEY = "beam_line_offset" + + +def _msg_word_count(msg: Msg) -> int: + """Whitespace-separated word count of a message's text content.""" + return len((msg.get_text_content() or "").split()) + + +def split_turn_chunks(messages: list[Msg], max_words: int) -> list[tuple[int, list[Msg]]]: + """Split *messages* into chunks of complete turns bounded by *max_words*. + + A turn starts at every ``user`` message and spans all following non-user + messages (assistant replies). Chunks only break at turn boundaries, so a + ``user + assistant`` exchange is never split across two chunks. A single + turn larger than *max_words* still becomes its own (oversized) chunk. + + Returns ``[(offset, msgs), ...]`` where ``offset`` is the 0-based index of + the chunk's first message in *messages* — i.e. its line number in the + session file minus 1. ``max_words <= 0`` disables splitting. + """ + if not messages: + return [] + if max_words <= 0: + return [(0, list(messages))] + + # Group into turns: each user message opens a new turn. + turns: list[list[Msg]] = [] + for msg in messages: + if msg.role == "user" or not turns: + turns.append([msg]) + else: + turns[-1].append(msg) + + chunks: list[tuple[int, list[Msg]]] = [] + current: list[Msg] = [] + current_words = 0 + offset = 0 + for turn in turns: + turn_words = sum(_msg_word_count(m) for m in turn) + if current and current_words + turn_words > max_words: + chunks.append((offset, current)) + offset += len(current) + current = [] + current_words = 0 + current.extend(turn) + current_words += turn_words + if current: + chunks.append((offset, current)) + return chunks + def _parse_iso_seconds(value: str) -> datetime | None: """Parse an ISO-8601 timestamp that is precise to at least seconds. @@ -121,8 +174,75 @@ class BeamAutoMemoryStep(AutoMemoryStep): Date pinning for ``daily_write`` is handled by the base class through the agent wrapper's server-owned ``injected_job_kwargs``. + + Long sessions are ingested incrementally: the message list is split into + chunks of complete turns (``max_chunk_words`` step config, default 10000 + words per chunk) and each chunk runs one agent pass — the first pass + creates the daily note, later passes merge into it. Line numbers shown to + the agent always refer to the ORIGINAL session file, not the chunk. """ + async def execute(self): + assert self.context is not None + raw_messages = self.context.get("messages") or [] + session_id: str = self.context.get("session_id", "") + max_words = int(self.kwargs.get("max_chunk_words", 10000) or 0) + + messages = self._build_messages(raw_messages) + chunks = split_turn_chunks(messages, max_words) + + if len(chunks) <= 1: + self.context[_LINE_OFFSET_KEY] = 0 + await super().execute() + return + + # Persist the FULL session file up-front so every chunk's [Ln] labels + # (and the note's source markers) match the final on-disk layout. The + # per-chunk saves inside the base execute become no-op appends. + if session_id and validate_session_id(session_id) is None: + await self._save_session_messages(session_id, messages) + + base_hint = self.context.get("memory_hint", "") or "" + answers: list[str] = [] + failed_chunks = 0 + self.logger.info( + f"[{self.name}] chunked ingestion session_id={session_id!r} " + f"messages={len(messages)} chunks={len(chunks)} max_chunk_words={max_words}", + ) + for part, (offset, chunk) in enumerate(chunks, start=1): + part_hint = ( + f"This is part {part}/{len(chunks)} of one long session " + f"(lines {offset + 1}-{offset + len(chunk)} of the session file). " + "Earlier parts were already recorded; extract ONLY from the turns shown below." + ) + self.context["messages"] = chunk + self.context["memory_hint"] = f"{base_hint}\n{part_hint}".strip() + self.context[_LINE_OFFSET_KEY] = offset + # One bad chunk (e.g. transiently corrupted note frontmatter) must + # not lose the remaining chunks of the session. + try: + await super().execute() + ok = bool(self.context.response.success) + summary = self.context.response.answer or "" + except Exception as exc: # pylint: disable=broad-exception-caught + ok = False + summary = f"Error: {exc}" + self.logger.warning( + f"[{self.name}] chunk {part}/{len(chunks)} failed session_id={session_id!r}: {exc}", + ) + if not ok: + failed_chunks += 1 + answers.append(f"[part {part}/{len(chunks)}] {summary}") + + # Restore caller-visible context and aggregate the response. + self.context["messages"] = raw_messages + self.context["memory_hint"] = base_hint + self.context.response.success = failed_chunks < len(chunks) + self.context.response.answer = "\n".join(answers) + self.context.response.metadata.update( + {"n_messages": len(messages), "chunks": len(chunks), "failed_chunks": failed_chunks}, + ) + def _build_messages(self, raw_messages: list) -> list[Msg]: # Interpolate timestamps: if any message carries created_at, fill in # the rest via linear interpolation so the whole session has coherent @@ -135,15 +255,18 @@ class BeamAutoMemoryStep(AutoMemoryStep): def _format_history(self, messages: list[Msg]) -> str: # Annotate every turn with its physical line number in the session # file so the agent can cite information sources as - # [:-]. A BEAM batch is already + # [[#L-L]]. A BEAM batch is already # in chronological order, matching the one-message-per-line layout # written by AutoMemoryStep._save_session_messages, so number each turn - # by its position (no re-sorting). + # by its position plus the chunk's offset into the full session file + # (no re-sorting). When the session is ingested in chunks, the offset + # keeps [Ln] labels aligned with the ORIGINAL file, not the chunk. session_id = self.context.get("session_id", "") if self.context is not None else "" + line_offset = int(self.context.get(_LINE_OFFSET_KEY, 0) or 0) if self.context is not None else 0 source_path = self._session_source_path(session_id) turns: list[str] = [] - for line, msg in enumerate(messages, start=1): + for line, msg in enumerate(messages, start=line_offset + 1): text = (msg.get_text_content() or "").strip() if not text: continue @@ -153,9 +276,13 @@ class BeamAutoMemoryStep(AutoMemoryStep): if not turns: return "(empty)" + first_line = line_offset + 1 + last_line = line_offset + len(messages) preamble = ( f"Source file: {source_path}\n" - "(Each turn below is prefixed with [Ln] = its line number in the source file.)" + f"(Excerpt covering lines {first_line}-{last_line} of the source file. Each turn below is " + "prefixed with [Ln] = its line number in the source file; copy these numbers verbatim " + "into source markers.)" ) return preamble + "\n\n" + "\n\n".join(turns) diff --git a/reme/steps/benchmark/beam/auto_memory.yaml b/reme/steps/benchmark/beam/auto_memory.yaml index 24639494..7b8421ac 100644 --- a/reme/steps/benchmark/beam/auto_memory.yaml +++ b/reme/steps/benchmark/beam/auto_memory.yaml @@ -7,12 +7,13 @@ system_prompt: | 2. **Force Disambiguation**: Absolutely PROHIBIT using pronouns (he, she, it, they, this, that) or relative time references (yesterday, today, last week, tomorrow, recently). Always use full names, explicit dates, and specific references. 3. **Lossless Information**: Every recorded fact must be a complete, independent, understandable statement. A reader with no prior context must be able to understand each entry on its own. 4. **Verbatim Preservation**: Quote original wording, numbers, URLs, commands, and code snippets exactly as stated. - 5. **Source Attribution**: Mark the source of factual content using the `Source file` path (copied verbatim) and the `[Ln]` line numbers of the turns the fact was derived from, joining the start and end line with a hyphen `-` (NOT a colon), e.g. `[session/dialog/beam_1M_1_batch1.jsonl:1-2]`. - - **No `Source` label**: write the bracket marker DIRECTLY at the end of the entry (or heading). Do NOT prefix it with `Source`, `Source:`, `source`, `来源`, or any other word — output only the bare `[...]` marker. - - **Full path, NO `L` prefix**: EVERY marker MUST begin with the full session file path, then a colon, then BARE line numbers — e.g. `[session/dialog/beam_1M_2_batch1.jsonl:29-34,53-58]`. The `[Ln]` labels prefixing each input turn (e.g. `[L29 | user @ ...]`) are INPUT annotations only; NEVER copy that `L` into your marker and NEVER drop the file path. A marker that omits the `.jsonl:` path, or writes numbers with an `L` prefix, or uses a colon instead of a hyphen inside a range, is INVALID. ✗ WRONG: `[L29-34, L53-58]`, `[29-34]`, `[L1:L2]`, `Source: [L29-34]`. ✓ RIGHT: `[session/dialog/beam_1M_2_batch1.jsonl:29-34,53-58]`. Each line number equals the message index in the conversation (one message = one line in the source file); newlines inside a message's own content do NOT affect its line number. - - **Merge same-file ranges into ONE marker**: instead of back-to-back markers like `[...jsonl:1-2][...jsonl:5-6]`, write a single marker with comma-separated ranges `[...jsonl:1-2,5-6]`. + 5. **Source Attribution**: Mark the source of factual content with a wikilink-style marker built from the `Source file` path (copied verbatim) and the `[Ln]` line numbers of the turns the fact was derived from. The EXACT format is `[[#L-L(,L-L)*]]`, e.g. `[[session/dialog/beam_1M_1_batch1.jsonl#L1-L2]]`. + - **No `Source` label**: write the marker DIRECTLY at the end of the entry (or heading). Do NOT prefix it with `Source`, `Source:`, `source`, `来源`, or any other word — output only the bare `[[...]]` marker. + - **Double brackets + `#` + `L`-prefixed hyphen ranges**: EVERY marker MUST be wrapped in DOUBLE square brackets `[[...]]`, begin with the full session file path, then a hash `#`, then one or more line ranges. Each range is `L-L` — BOTH endpoints carry the `L` prefix and are joined by a hyphen `-` (NEVER a colon). A single line is written as a one-line range, e.g. `L41-L41`. This applies to EVERY comma-separated range too: `L69-L72,L77` is INVALID — write `L69-L72,L77-L77`. Multiple ranges of the same file go into ONE marker, separated by commas with NO spaces. ✗ WRONG: `[session/dialog/....jsonl:29-34]` (old single-bracket colon format), `[[...jsonl#29-34]]` (numbers missing `L`), `[[...jsonl#L29-34]]` (end missing `L`), `[[...jsonl#L29:L34]]` (colon instead of hyphen), `[[L29-L34]]` (path dropped), `[[...jsonl#L41]]` (bare single line — write `L41-L41`), `[[...jsonl#L69-L72,L77]]` (second range is a bare line — write `L77-L77`), `Source: [[...jsonl#L29-L34]]` (label added). ✓ RIGHT: `[[session/dialog/beam_1M_2_batch1.jsonl#L29-L34,L53-L58]]`. + - **Line numbers come from the `[Ln]` labels**: the `[Ln]` label prefixing each input turn (e.g. `[L29 | user @ ...]`) is that message's line number in the ORIGINAL source file — copy those numbers verbatim. The excerpt you see may START at a line far greater than 1; NEVER renumber from 1 and NEVER cite line numbers outside the excerpt's stated range. Newlines inside a message's own content do NOT affect its line number. + - **Merge same-file ranges into ONE marker**: instead of back-to-back markers like `[[...jsonl#L1-L2]][[...jsonl#L5-L6]]`, write a single marker with comma-separated ranges `[[...jsonl#L1-L2,L5-L6]]`. - **Hoist shared markers, never repeat**: when several consecutive bullets share the SAME marker, do NOT append it to every bullet. Attach ONE marker to the group instead — either append it to the section heading (where it then covers every bullet beneath that heading until a different marker appears), or merge those bullets into a single entry carrying one trailing marker. Repeating an identical marker across sibling bullets is PROHIBITED — it wastes tokens. - - **Cite enough self-contained context**: the cited `[Ln]` range must be self-contained. Do NOT cite only the single line that states the key detail; widen the range to include the surrounding turns (e.g. the question or setup turn that gives the fact its meaning) so that a reader who opens only that line range understands the fact correctly and is not misled by missing context. + - **Cite enough self-contained context**: the cited line range must be self-contained. Do NOT cite only the single line that states the key detail; widen the range to include the surrounding turns (e.g. the question or setup turn that gives the fact its meaning) so that a reader who opens only that line range understands the fact correctly and is not misled by missing context. ## What to Extract @@ -28,7 +29,7 @@ system_prompt: | ## Body Format - Use structured sections with headings. Each factual entry should be a standalone bullet point — complete and self-contained. Group entries by topic. Attach source markers coarsely (see the Source Attribution principle): prefer ONE marker per section heading or per grouped entry, write only the bare `[...]` marker with no `Source` label, and never repeat an identical marker across consecutive bullets. Do NOT add a `## Metadata` section. + Use structured sections with headings. Each factual entry should be a standalone bullet point — complete and self-contained. Group entries by topic. Attach source markers coarsely (see the Source Attribution principle): prefer ONE marker per section heading or per grouped entry, write only the bare `[[...]]` marker with no `Source` label, and never repeat an identical marker across consecutive bullets. Do NOT add a `## Metadata` section. ## Frontmatter Rules @@ -53,12 +54,13 @@ system_prompt_zh: | 2. **强制消歧**：绝对禁止使用代词（他、她、它、他们、这个、那个）或相对时间（昨天、今天、上周、明天、最近）。必须使用全名、明确日期和具体引用。 3. **无损信息**：每条记录的事实必须是完整的、独立的、可理解的陈述。没有上下文的读者必须能单独理解每条记录。 4. **原文保留**：逐字引用原始措辞、数字、URL、命令和代码片段。 - 5. **来源标注**：为事实内容标注来源，使用对话中显示的 `Source file` 路径（逐字复制）与该事实所依据对话轮次的 `[Ln]` 行号，起始行与结束行之间用连字符 `-` 连接（不是冒号 `:`），例如 `[session/dialog/beam_1M_1_batch1.jsonl:1-2]`。 - - **不要 `Source` 标签**：把方括号标记**直接**写在条目（或标题）末尾。不要在前面加 `Source`、`Source:`、`source`、`来源` 或任何其他词——只输出裸的 `[...]` 标记。 - - **完整路径、禁止 `L` 前缀**：每个标记都**必须**以完整的 session 文件路径开头，然后接冒号，再接**纯数字**行号——例如 `[session/dialog/beam_1M_2_batch1.jsonl:29-34,53-58]`。输入中每条对话轮次前面的 `[Ln]` 标签（如 `[L29 | user @ ...]`）**仅是输入标注**；绝不能把那个 `L` 抄进你的标记，也绝不能丢掉文件路径。凡是省略 `.jsonl:` 路径、或数字带 `L` 前缀、或在区间内用冒号而非连字符的标记，都是**无效的**。✗ 错误：`[L29-34, L53-58]`、`[29-34]`、`[L1:L2]`、`Source: [L29-34]`。✓ 正确：`[session/dialog/beam_1M_2_batch1.jsonl:29-34,53-58]`。每个行号就是该消息在对话中的序号（一条消息对应源文件中的一行）；消息正文内部的换行不影响其行号。 - - **同一文件多区间合并进一个标记**：不要写成相邻的多个标记 `[...jsonl:1-2][...jsonl:5-6]`，而要写成用逗号分隔区间的单个标记 `[...jsonl:1-2,5-6]`。 + 5. **来源标注**：为事实内容标注来源，使用 wikilink 风格的标记，由对话中显示的 `Source file` 路径（逐字复制）与该事实所依据对话轮次的 `[Ln]` 行号拼成。格式**必须**为 `[[#L-L(,L-L)*]]`，例如 `[[session/dialog/beam_1M_1_batch1.jsonl#L1-L2]]`。 + - **不要 `Source` 标签**：把标记**直接**写在条目（或标题）末尾。不要在前面加 `Source`、`Source:`、`source`、`来源` 或任何其他词——只输出裸的 `[[...]]` 标记。 + - **双层方括号 + `#` + 带 `L` 前缀的连字符区间**：每个标记都**必须**用双层方括号 `[[...]]` 包裹，以完整的 session 文件路径开头，然后接 `#`，再接一个或多个行号区间。每个区间写作 `L-L`——两端都带 `L` 前缀，用连字符 `-` 连接（**绝不**用冒号）。单行引用写成单行区间，如 `L41-L41`；逗号分隔的每一个区间也同样适用：`L69-L72,L77` 是**无效的**，应写成 `L69-L72,L77-L77`。同一文件的多个区间放入**同一个**标记，用逗号分隔且不加空格。✗ 错误：`[session/dialog/....jsonl:29-34]`（旧的单括号冒号格式）、`[[...jsonl#29-34]]`（数字缺 `L`）、`[[...jsonl#L29-34]]`（末端缺 `L`）、`[[...jsonl#L29:L34]]`（用了冒号）、`[[L29-L34]]`（丢了路径）、`[[...jsonl#L41]]`（裸单行，应写 `L41-L41`）、`[[...jsonl#L69-L72,L77]]`（第二个区间是裸单行，应写 `L77-L77`）、`Source: [[...jsonl#L29-L34]]`（加了标签）。✓ 正确：`[[session/dialog/beam_1M_2_batch1.jsonl#L29-L34,L53-L58]]`。 + - **行号来自 `[Ln]` 标签**：输入中每条对话轮次前的 `[Ln]` 标签（如 `[L29 | user @ ...]`）就是该消息在**原始**源文件中的行号——请逐字复制这些数字。你看到的片段可能从远大于 1 的行号开始；**绝不**要从 1 重新编号，也**绝不**要引用片段声明范围之外的行号。消息正文内部的换行不影响其行号。 + - **同一文件多区间合并进一个标记**：不要写成相邻的多个标记 `[[...jsonl#L1-L2]][[...jsonl#L5-L6]]`，而要写成用逗号分隔区间的单个标记 `[[...jsonl#L1-L2,L5-L6]]`。 - **上提共享标记，绝不重复**：当连续多个条目共享**相同**标记时，不要给每个条目都追加。而是给整组只标注一个标记——要么把标记追加到分节标题上（此时它覆盖该标题下、直到出现不同标记之前的所有条目），要么把这些条目合并为一条并只在末尾带一个标记。在同级条目上重复相同标记是**被禁止的**——会浪费 token。 - - **引用充分且自洽的上下文**：所标注的 `[Ln]` 区间必须自洽完整。不要只标注陈述关键信息的那一句，要把区间扩大到包含周边轮次（例如赋予该事实意义的提问或铺垫轮次），使得仅打开该行号区间的读者也能正确理解这条事实，不会因缺失上下文而断章取义。 + - **引用充分且自洽的上下文**：所标注的行号区间必须自洽完整。不要只标注陈述关键信息的那一句，要把区间扩大到包含周边轮次（例如赋予该事实意义的提问或铺垫轮次），使得仅打开该行号区间的读者也能正确理解这条事实，不会因缺失上下文而断章取义。 ## 提取什么 @@ -74,7 +76,7 @@ system_prompt_zh: | ## 正文格式 - 使用带标题的结构化分节。每条事实记录应是独立的列表项——完整且自包含。按主题分组。粗粒度地标注来源（见来源标注原则）：优先每个分节标题或每组条目只用一个标记，只写裸的 `[...]` 标记且不带 `Source` 标签，并且绝不在连续条目上重复相同标记。不要添加 `## Metadata` 部分。 + 使用带标题的结构化分节。每条事实记录应是独立的列表项——完整且自包含。按主题分组。粗粒度地标注来源（见来源标注原则）：优先每个分节标题或每组条目只用一个标记，只写裸的 `[[...]]` 标记且不带 `Source` 标签，并且绝不在连续条目上重复相同标记。不要添加 `## Metadata` 部分。 ## Frontmatter 规则 @@ -94,7 +96,7 @@ user_message_create: | Today: {today} Extra hint: {note} Session ID: {session_id} - Session file: {session_file} ← copy THIS exact path into every source marker (e.g. `[{session_file}:1-2,5-6]`) + Session file: {session_file} ← copy THIS exact path into every source marker (e.g. `[[{session_file}#L1-L2,L5-L6]]`) # Recent Conversation @@ -130,7 +132,7 @@ user_message_create: | ⚠️ CRITICAL: You MUST invoke `daily_write` as a **tool call** (function call). Do NOT write `daily_write name=...` as plain text — that will NOT execute the tool and NO memory will be saved. Body requirements — follow the system prompt's Extraction Principles, Body Format, and Frontmatter Rules (lossless entries; no pronouns; absolute dates; grouped by topic; no `## Metadata`). Two rules that MUST NOT be violated: - - Source markers: put a bare `[session/dialog/....jsonl:1-2,5-6]` at the END of an entry or heading — full path, no `L` prefix, no `Source` label, coarse (hoist a shared marker, never repeat it across bullets), and cite a self-contained line range. + - Source markers: put a bare `[[session/dialog/....jsonl#L1-L2,L5-L6]]` at the END of an entry or heading — double brackets, full path + `#`, `L`-prefixed hyphen ranges, no `Source` label, coarse (hoist a shared marker, never repeat it across bullets), and cite a self-contained line range copied verbatim from the `[Ln]` labels. - `description`: a thorough, lossless summary — specific enough that the description alone conveys all key information. ## Step 3 — Summary @@ -144,7 +146,7 @@ user_message_create_zh: | 今天：{today} 额外提示：{note} Session ID：{session_id} - Session 文件：{session_file} ← 请把这个完整路径原样复制进每个来源标记（例如 `[{session_file}:1-2,5-6]`） + Session 文件：{session_file} ← 请把这个完整路径原样复制进每个来源标记（例如 `[[{session_file}#L1-L2,L5-L6]]`） # 最近的对话 @@ -180,7 +182,7 @@ user_message_create_zh: | ⚠️ 关键：你**必须**以**工具调用**（函数调用）方式调用 `daily_write`。不要将 `daily_write name=...` 作为纯文本输出——那样**不会**执行工具，也**不会**保存任何记忆。 正文要求——遵循 system prompt 的提取原则、正文格式与 Frontmatter 规则（无损；不用代词；绝对日期；按主题分组；无 `## Metadata`）。以下两条**绝不能违反**： - - 来源标记：在条目或标题**末尾**放一个裸的 `[session/dialog/....jsonl:1-2,5-6]`——完整路径、无 `L` 前缀、无 `Source` 标签、粒度粗（上提共享标记、绝不在连续条目上重复），且引用自洽完整的行号区间。 + - 来源标记：在条目或标题**末尾**放一个裸的 `[[session/dialog/....jsonl#L1-L2,L5-L6]]`——双层方括号、完整路径 + `#`、带 `L` 前缀的连字符区间、无 `Source` 标签、粒度粗（上提共享标记、绝不在连续条目上重复），且行号逐字复制自 `[Ln]` 标签、引用自洽完整的行号区间。 - `description`：详尽的无损总结——具体到仅凭 description 就能传达全部核心信息。 ## 步骤 3 — 总结 @@ -195,7 +197,7 @@ user_message_update: | Today: {today} Extra hint: {note} Target path: {note_path} - Session file: {session_file} ← copy THIS exact path into every source marker (e.g. `[{session_file}:1-2,5-6]`) + Session file: {session_file} ← copy THIS exact path into every source marker (e.g. `[[{session_file}#L1-L2,L5-L6]]`) # Recent Conversation @@ -221,7 +223,7 @@ user_message_update: | The note already has content. Extract new information from the conversation and merge into it. - Extraction rules (apply to ALL new entries) — follow the system prompt's Extraction Principles and Body Format (lossless; no pronouns; absolute dates; grouped by topic). Source markers MUST be a bare `[session/dialog/....jsonl:1-2,5-6]` — full path, no `L` prefix, no `Source` label, coarse (never repeated across bullets), self-contained range. + Extraction rules (apply to ALL new entries) — follow the system prompt's Extraction Principles and Body Format (lossless; no pronouns; absolute dates; grouped by topic). Source markers MUST be a bare `[[session/dialog/....jsonl#L1-L2,L5-L6]]` — double brackets, full path + `#`, `L`-prefixed hyphen ranges copied verbatim from the `[Ln]` labels, no `Source` label, coarse (never repeated across bullets), self-contained range. Merge rules: - **Timeline / history entries**: append only, never delete existing entries. @@ -230,6 +232,7 @@ user_message_update: | Execution: 1. **Call the `edit` tool** for each section that needs updating, with parameters: `path`="{note_path}", `old`=, `new`=. You may call `edit` multiple times. Each call MUST be a tool call, not text. + - ⚠️ `edit` is for the note BODY only. NEVER touch the frontmatter block (the `---`-delimited YAML at the top of the file) with `edit` — a malformed YAML edit corrupts the whole note. All frontmatter changes MUST go through the `frontmatter_update` tool. 2. After body changes, **call the `frontmatter_update` tool** to refresh frontmatter, with parameters: `path`="{note_path}", `metadata`={{"name": "", "description": ""}}. - Keep the existing `name` only when it is already the best concise topic/event filename stem. The system will rename the file after your final response. - Do not add today's date or the daily directory date to `name`. @@ -239,7 +242,7 @@ user_message_update: | The file exists but its body is empty. Extract and write the full content by **calling the `write` tool** with parameters: `path`="{note_path}", `name`=, `description`=, `content`=. - Body requirements — follow the system prompt's Extraction Principles, Body Format, and Frontmatter Rules (lossless; no pronouns; absolute dates; grouped by topic; no `## Metadata`). Source markers MUST be a bare `[session/dialog/....jsonl:1-2,5-6]` — full path, no `L` prefix, no `Source` label, coarse (never repeated across bullets), self-contained range. `description` must be a thorough, lossless summary that alone conveys all key information. + Body requirements — follow the system prompt's Extraction Principles, Body Format, and Frontmatter Rules (lossless; no pronouns; absolute dates; grouped by topic; no `## Metadata`). Source markers MUST be a bare `[[session/dialog/....jsonl#L1-L2,L5-L6]]` — double brackets, full path + `#`, `L`-prefixed hyphen ranges copied verbatim from the `[Ln]` labels, no `Source` label, coarse (never repeated across bullets), self-contained range. `description` must be a thorough, lossless summary that alone conveys all key information. ## Step 4 — Summary @@ -254,7 +257,7 @@ user_message_update_zh: | 今天：{today} 额外提示：{note} 目标路径：{note_path} - Session 文件：{session_file} ← 请把这个完整路径原样复制进每个来源标记（例如 `[{session_file}:1-2,5-6]`） + Session 文件：{session_file} ← 请把这个完整路径原样复制进每个来源标记（例如 `[[{session_file}#L1-L2,L5-L6]]`） # 最近的对话 @@ -280,7 +283,7 @@ user_message_update_zh: | 笔记已有内容。从对话中提取新信息并合并进去。 - 提取规则（适用于所有新条目）——遵循 system prompt 的提取原则与正文格式（无损；不用代词；绝对日期；按主题分组）。来源标记必须是裸的 `[session/dialog/....jsonl:1-2,5-6]`——完整路径、无 `L` 前缀、无 `Source` 标签、粒度粗（不在连续条目上重复）、自洽完整的行号区间。 + 提取规则（适用于所有新条目）——遵循 system prompt 的提取原则与正文格式（无损；不用代词；绝对日期；按主题分组）。来源标记必须是裸的 `[[session/dialog/....jsonl#L1-L2,L5-L6]]`——双层方括号、完整路径 + `#`、带 `L` 前缀的连字符区间（行号逐字复制自 `[Ln]` 标签）、无 `Source` 标签、粒度粗（不在连续条目上重复）、自洽完整的行号区间。 合并规则： - **时间线 / 历史条目**：仅追加，永远不删除已有条目。 @@ -289,6 +292,7 @@ user_message_update_zh: | 执行： 1. **调用 `edit` 工具**更新需要修改的部分，参数为：`path`="{note_path}"、`old`=<原文片段>、`new`=<替换片段>。可以多次调用 `edit`。每次调用必须是工具调用，不是文本。 + - ⚠️ `edit` 只能用于笔记**正文**。**绝不**要用 `edit` 碰 frontmatter 块（文件顶部由 `---` 包围的 YAML）——格式错误的 YAML 会损坏整个笔记。所有 frontmatter 修改都必须通过 `frontmatter_update` 工具。 2. 正文变更后，**调用 `frontmatter_update` 工具**刷新 frontmatter，参数为：`path`="{note_path}"、`metadata`={{"name": "<更新后的文件名 stem>", "description": "<更新后的总结>"}}。 - 只有当前 `name` 已经是最合适的简洁主题/事件文件名 stem 时才保留。系统会在你最终回复后负责重命名文件。 - 不要在 `name` 中加入今天日期或日记目录日期。 @@ -298,7 +302,7 @@ user_message_update_zh: | 文件存在但正文为空。提取并**调用 `write` 工具**一次性写入完整内容，参数为：`path`="{note_path}"、`name`=、`description`=、`content`=<正文>。 - 正文要求——遵循 system prompt 的提取原则、正文格式与 Frontmatter 规则（无损；不用代词；绝对日期；按主题分组；无 `## Metadata`）。来源标记必须是裸的 `[session/dialog/....jsonl:1-2,5-6]`——完整路径、无 `L` 前缀、无 `Source` 标签、粒度粗（不在连续条目上重复）、自洽完整的行号区间。`description` 必须是仅凭自身即可传达全部核心信息的详尽无损总结。 + 正文要求——遵循 system prompt 的提取原则、正文格式与 Frontmatter 规则（无损；不用代词；绝对日期；按主题分组；无 `## Metadata`）。来源标记必须是裸的 `[[session/dialog/....jsonl#L1-L2,L5-L6]]`——双层方括号、完整路径 + `#`、带 `L` 前缀的连字符区间（行号逐字复制自 `[Ln]` 标签）、无 `Source` 标签、粒度粗（不在连续条目上重复）、自洽完整的行号区间。`description` 必须是仅凭自身即可传达全部核心信息的详尽无损总结。 ## 步骤 4 — 总结 diff --git a/tests/unit/test_beam_auto_memory.py b/tests/unit/test_beam_auto_memory.py new file mode 100644 index 00000000..193a04b5 --- /dev/null +++ b/tests/unit/test_beam_auto_memory.py @@ -0,0 +1,128 @@ +"""Unit tests for BEAM auto-memory chunked ingestion and line numbering.""" + +# pylint: disable=missing-class-docstring,missing-function-docstring,protected-access + +from agentscope.message import Msg + +from reme.components.runtime_context import RuntimeContext +from reme.steps.benchmark.beam.auto_memory import ( + BeamAutoMemoryStep, + split_turn_chunks, +) + + +def _msg(role: str, words: int, created_at: str = "2024-03-01T09:00:00") -> Msg: + return Msg( + name=role, + role=role, + content=[{"type": "text", "text": " ".join(["w"] * words)}], + created_at=created_at, + ) + + +def _dialog(n_turns: int, words_per_msg: int) -> list[Msg]: + messages: list[Msg] = [] + for _ in range(n_turns): + messages.append(_msg("user", words_per_msg)) + messages.append(_msg("assistant", words_per_msg)) + return messages + + +class TestSplitTurnChunks: + def test_empty(self): + assert not split_turn_chunks([], 100) + + def test_disabled_returns_single_chunk(self): + messages = _dialog(3, 10) + chunks = split_turn_chunks(messages, 0) + assert len(chunks) == 1 + assert chunks[0] == (0, messages) + + def test_under_limit_single_chunk(self): + messages = _dialog(3, 10) # 60 words total + chunks = split_turn_chunks(messages, 100) + assert len(chunks) == 1 + assert chunks[0][0] == 0 + assert chunks[0][1] == messages + + def test_splits_at_turn_boundaries(self): + # 4 turns x 20 words each; limit 40 -> 2 turns per chunk + messages = _dialog(4, 10) + chunks = split_turn_chunks(messages, 40) + assert len(chunks) == 2 + offsets = [offset for offset, _ in chunks] + assert offsets == [0, 4] + # Every chunk starts with a user message and ends with an assistant + for _, chunk in chunks: + assert chunk[0].role == "user" + assert chunk[-1].role == "assistant" + # No message lost or duplicated, order preserved + flattened = [m for _, chunk in chunks for m in chunk] + assert flattened == messages + + def test_never_splits_inside_a_turn(self): + # One turn alone exceeds the limit -> becomes its own oversized chunk + messages = [ + _msg("user", 5), + _msg("assistant", 5), + _msg("user", 50), + _msg("assistant", 50), # 100-word turn > limit 60 + _msg("user", 5), + _msg("assistant", 5), + ] + chunks = split_turn_chunks(messages, 60) + assert [offset for offset, _ in chunks] == [0, 2, 4] + assert [len(chunk) for _, chunk in chunks] == [2, 2, 2] + + def test_offsets_are_original_indices(self): + messages = _dialog(5, 30) # 60 words per turn + chunks = split_turn_chunks(messages, 120) + # 2 turns per chunk -> offsets 0, 4, 8 + assert [offset for offset, _ in chunks] == [0, 4, 8] + for offset, chunk in chunks: + for i, msg in enumerate(chunk): + assert msg is messages[offset + i] + + def test_multi_assistant_turn_stays_together(self): + messages = [ + _msg("user", 10), + _msg("assistant", 10), + _msg("assistant", 10), + _msg("user", 10), + _msg("assistant", 10), + ] + chunks = split_turn_chunks(messages, 30) + assert [offset for offset, _ in chunks] == [0, 3] + assert len(chunks[0][1]) == 3 + + +class TestFormatHistoryLineNumbers: + def _step(self, offset: int) -> BeamAutoMemoryStep: + step = BeamAutoMemoryStep(name="beam_auto_memory_step", backend="beam_auto_memory_step") + step.context = RuntimeContext( + session_id="beam_1M_1_batch1", + beam_line_offset=offset, + ) + return step + + def test_numbers_start_at_one_without_offset(self): + step = self._step(0) + history = step._format_history(_dialog(2, 3)) + assert "[L1 | user @" in history + assert "[L4 | assistant @" in history + assert "lines 1-4" in history + assert "session/dialog/beam_1M_1_batch1.jsonl" in history + + def test_numbers_use_original_file_offset(self): + step = self._step(40) + history = step._format_history(_dialog(2, 3)) + assert "[L41 | user @" in history + assert "[L44 | assistant @" in history + assert "lines 41-44" in history + assert "[L1 " not in history + + def test_build_messages_passes_msg_objects_through(self): + step = self._step(0) + messages = _dialog(2, 3) + rebuilt = step._build_messages(messages) + assert [m.id for m in rebuilt] == [m.id for m in messages]