From ed89e59ead1e19728060d6bbe002bcd9d64a1941 Mon Sep 17 00:00:00 2001 From: sa-buc Date: Thu, 23 Jul 2026 15:38:18 +0800 Subject: [PATCH] feat(auto_memory): add source line-number markers for note traceability - Add _format_history hook in AutoMemoryStep with line-number annotation - Override in BeamAutoMemoryStep to prefix each turn with [Ln] for citation - Add session_file variable to prompt templates for source marker paths - Simplify repeated extraction rules by referencing system prompt - Enhance agentic_answer search strategy (multi-search, read tool hint) - Add warning log on ReadStep failure --- reme/steps/benchmark/beam/agentic_answer.yaml | 2 + reme/steps/benchmark/beam/auto_memory.py | 30 +++++++ reme/steps/benchmark/beam/auto_memory.yaml | 81 +++++++------------ reme/steps/evolve/auto_memory.py | 16 +++- reme/steps/file_io/read.py | 1 + 5 files changed, 76 insertions(+), 54 deletions(-) diff --git a/reme/steps/benchmark/beam/agentic_answer.yaml b/reme/steps/benchmark/beam/agentic_answer.yaml index ab4f00aa..15e4fb06 100644 --- a/reme/steps/benchmark/beam/agentic_answer.yaml +++ b/reme/steps/benchmark/beam/agentic_answer.yaml @@ -4,6 +4,8 @@ system_prompt: | ## Search Strategy - You can call 'search' tool to search multiple times (at least once) with different queries to gather comprehensive information. + - Do not stop after the first relevant one unless you are confident that the information is sufficient. + - You can use 'read' tool to read specific parts of a specific file. ## Draft Tool - Use 'add_draft' to save key findings during search, and 'read_all_draft' to review all saved notes before answering. ## Answer Rules diff --git a/reme/steps/benchmark/beam/auto_memory.py b/reme/steps/benchmark/beam/auto_memory.py index b0013dc5..1a934224 100644 --- a/reme/steps/benchmark/beam/auto_memory.py +++ b/reme/steps/benchmark/beam/auto_memory.py @@ -131,3 +131,33 @@ class BeamAutoMemoryStep(AutoMemoryStep): [item if not isinstance(item, dict) else dict(item) for item in raw_messages], ) return [self._to_msg(item) for item in interpolated] + + def _format_history(self, messages: list[Msg]) -> str: + # Annotate every turn with its physical line number in the session + # file so the agent can cite information sources as + # [:-]. A BEAM batch is already + # in chronological order, matching the one-message-per-line layout + # written by AutoMemoryStep._save_session_messages, so number each turn + # by its position (no re-sorting). + session_id = self.context.get("session_id", "") if self.context is not None else "" + source_path = self._session_source_path(session_id) + + turns: list[str] = [] + for line, msg in enumerate(messages, start=1): + text = (msg.get_text_content() or "").strip() + if not text: + continue + speaker = msg.name or msg.role or "?" + turns.append(f"[L{line} | {speaker} @ {msg.created_at}]\n{text}") + + if not turns: + return "(empty)" + + preamble = ( + f"Source file: {source_path}\n" + "(Each turn below is prefixed with [Ln] = its line number in the source file.)" + ) + return preamble + "\n\n" + "\n\n".join(turns) + + def _reply_extra_kwargs(self, day: str) -> dict: + return {"tool_defaults": {"daily_write": {"date": day}}} diff --git a/reme/steps/benchmark/beam/auto_memory.yaml b/reme/steps/benchmark/beam/auto_memory.yaml index a9c59c27..24639494 100644 --- a/reme/steps/benchmark/beam/auto_memory.yaml +++ b/reme/steps/benchmark/beam/auto_memory.yaml @@ -7,6 +7,12 @@ system_prompt: | 2. **Force Disambiguation**: Absolutely PROHIBIT using pronouns (he, she, it, they, this, that) or relative time references (yesterday, today, last week, tomorrow, recently). Always use full names, explicit dates, and specific references. 3. **Lossless Information**: Every recorded fact must be a complete, independent, understandable statement. A reader with no prior context must be able to understand each entry on its own. 4. **Verbatim Preservation**: Quote original wording, numbers, URLs, commands, and code snippets exactly as stated. + 5. **Source Attribution**: Mark the source of factual content using the `Source file` path (copied verbatim) and the `[Ln]` line numbers of the turns the fact was derived from, joining the start and end line with a hyphen `-` (NOT a colon), e.g. `[session/dialog/beam_1M_1_batch1.jsonl:1-2]`. + - **No `Source` label**: write the bracket marker DIRECTLY at the end of the entry (or heading). Do NOT prefix it with `Source`, `Source:`, `source`, `来源`, or any other word — output only the bare `[...]` marker. + - **Full path, NO `L` prefix**: EVERY marker MUST begin with the full session file path, then a colon, then BARE line numbers — e.g. `[session/dialog/beam_1M_2_batch1.jsonl:29-34,53-58]`. The `[Ln]` labels prefixing each input turn (e.g. `[L29 | user @ ...]`) are INPUT annotations only; NEVER copy that `L` into your marker and NEVER drop the file path. A marker that omits the `.jsonl:` path, or writes numbers with an `L` prefix, or uses a colon instead of a hyphen inside a range, is INVALID. ✗ WRONG: `[L29-34, L53-58]`, `[29-34]`, `[L1:L2]`, `Source: [L29-34]`. ✓ RIGHT: `[session/dialog/beam_1M_2_batch1.jsonl:29-34,53-58]`. Each line number equals the message index in the conversation (one message = one line in the source file); newlines inside a message's own content do NOT affect its line number. + - **Merge same-file ranges into ONE marker**: instead of back-to-back markers like `[...jsonl:1-2][...jsonl:5-6]`, write a single marker with comma-separated ranges `[...jsonl:1-2,5-6]`. + - **Hoist shared markers, never repeat**: when several consecutive bullets share the SAME marker, do NOT append it to every bullet. Attach ONE marker to the group instead — either append it to the section heading (where it then covers every bullet beneath that heading until a different marker appears), or merge those bullets into a single entry carrying one trailing marker. Repeating an identical marker across sibling bullets is PROHIBITED — it wastes tokens. + - **Cite enough self-contained context**: the cited `[Ln]` range must be self-contained. Do NOT cite only the single line that states the key detail; widen the range to include the surrounding turns (e.g. the question or setup turn that gives the fact its meaning) so that a reader who opens only that line range understands the fact correctly and is not misled by missing context. ## What to Extract @@ -22,11 +28,11 @@ system_prompt: | ## Body Format - Use structured sections with headings. Each factual entry should be a standalone bullet point — complete and self-contained. Group entries by topic. Include a `## Metadata` section at the end with extracted keywords, persons, entities, and topics. + Use structured sections with headings. Each factual entry should be a standalone bullet point — complete and self-contained. Group entries by topic. Attach source markers coarsely (see the Source Attribution principle): prefer ONE marker per section heading or per grouped entry, write only the bare `[...]` marker with no `Source` label, and never repeat an identical marker across consecutive bullets. Do NOT add a `## Metadata` section. ## Frontmatter Rules - - `name` = a concise, stable topic/event filename stem, such as `cold-remedies` or `project-kickoff-decision`. Do not include today's date or the daily directory date; the outer daily path already records the date. For existing notes, update it when a better filename is clearly warranted. + - `name` = a concise, stable topic/event filename stem in kebab-case, such as `cold-remedies` or `project-kickoff-decision`. No date, no slash/backslash, none of `< > : " | ? *`. Do not include today's date or the daily directory date; the outer daily path already records the date. For existing notes, update it when a better filename is clearly warranted. - `description` = a thorough, lossless summary; vague descriptions like "notes" / "misc" are unacceptable. The description alone should convey all key information. - **Never set `status`** — it is a field reserved for downstream processing. @@ -37,6 +43,7 @@ system_prompt: | - **NEVER** write tool invocations as text strings such as `daily_write name=...`. Text output does NOT execute tools — no memory will be saved. - **ALWAYS** use the tool call mechanism provided by the runtime. The system handles parameter passing automatically. - If you write a tool name and its parameters as text, NO tool will be executed and NO memory will be saved. + system_prompt_zh: | 你是自动记忆提取系统。你的任务是从对话中提取所有有价值的信息，并记录到结构化的日记记忆中。确保**完整覆盖**——每一条值得保留的信息都必须被捕获。 @@ -46,6 +53,12 @@ system_prompt_zh: | 2. **强制消歧**：绝对禁止使用代词（他、她、它、他们、这个、那个）或相对时间（昨天、今天、上周、明天、最近）。必须使用全名、明确日期和具体引用。 3. **无损信息**：每条记录的事实必须是完整的、独立的、可理解的陈述。没有上下文的读者必须能单独理解每条记录。 4. **原文保留**：逐字引用原始措辞、数字、URL、命令和代码片段。 + 5. **来源标注**：为事实内容标注来源，使用对话中显示的 `Source file` 路径（逐字复制）与该事实所依据对话轮次的 `[Ln]` 行号，起始行与结束行之间用连字符 `-` 连接（不是冒号 `:`），例如 `[session/dialog/beam_1M_1_batch1.jsonl:1-2]`。 + - **不要 `Source` 标签**：把方括号标记**直接**写在条目（或标题）末尾。不要在前面加 `Source`、`Source:`、`source`、`来源` 或任何其他词——只输出裸的 `[...]` 标记。 + - **完整路径、禁止 `L` 前缀**：每个标记都**必须**以完整的 session 文件路径开头，然后接冒号，再接**纯数字**行号——例如 `[session/dialog/beam_1M_2_batch1.jsonl:29-34,53-58]`。输入中每条对话轮次前面的 `[Ln]` 标签（如 `[L29 | user @ ...]`）**仅是输入标注**；绝不能把那个 `L` 抄进你的标记，也绝不能丢掉文件路径。凡是省略 `.jsonl:` 路径、或数字带 `L` 前缀、或在区间内用冒号而非连字符的标记，都是**无效的**。✗ 错误：`[L29-34, L53-58]`、`[29-34]`、`[L1:L2]`、`Source: [L29-34]`。✓ 正确：`[session/dialog/beam_1M_2_batch1.jsonl:29-34,53-58]`。每个行号就是该消息在对话中的序号（一条消息对应源文件中的一行）；消息正文内部的换行不影响其行号。 + - **同一文件多区间合并进一个标记**：不要写成相邻的多个标记 `[...jsonl:1-2][...jsonl:5-6]`，而要写成用逗号分隔区间的单个标记 `[...jsonl:1-2,5-6]`。 + - **上提共享标记，绝不重复**：当连续多个条目共享**相同**标记时，不要给每个条目都追加。而是给整组只标注一个标记——要么把标记追加到分节标题上（此时它覆盖该标题下、直到出现不同标记之前的所有条目），要么把这些条目合并为一条并只在末尾带一个标记。在同级条目上重复相同标记是**被禁止的**——会浪费 token。 + - **引用充分且自洽的上下文**：所标注的 `[Ln]` 区间必须自洽完整。不要只标注陈述关键信息的那一句，要把区间扩大到包含周边轮次（例如赋予该事实意义的提问或铺垫轮次），使得仅打开该行号区间的读者也能正确理解这条事实，不会因缺失上下文而断章取义。 ## 提取什么 @@ -61,11 +74,11 @@ system_prompt_zh: | ## 正文格式 - 使用带标题的结构化分节。每条事实记录应是独立的列表项——完整且自包含。按主题分组。在末尾添加 `## Metadata` 部分，列出提取的关键词、人物、实体和主题。 + 使用带标题的结构化分节。每条事实记录应是独立的列表项——完整且自包含。按主题分组。粗粒度地标注来源（见来源标注原则）：优先每个分节标题或每组条目只用一个标记，只写裸的 `[...]` 标记且不带 `Source` 标签，并且绝不在连续条目上重复相同标记。不要添加 `## Metadata` 部分。 ## Frontmatter 规则 - - `name` = 简洁、稳定的主题/事件文件名 stem，例如 `cold-remedies` 或 `project-kickoff-decision`。不要包含今天日期或日记目录日期；外层日记路径已经记录日期。对已有笔记，如果明显有更好的文件名，就更新它。 + - `name` = 简洁、稳定的主题/事件文件名 stem，采用 kebab-case，例如 `cold-remedies` 或 `project-kickoff-decision`。不含日期、不含 slash/反斜杠、不含 `< > : " | ? *`。不要包含今天日期或日记目录日期；外层日记路径已经记录日期。对已有笔记，如果明显有更好的文件名，就更新它。 - `description` = 详尽的无损总结；模糊的描述如 "notes" / "misc" 不可接受。仅凭 description 就应能传达全部核心信息。 - **永远不要设置 `status`**——它是下游处理保留的字段。 @@ -81,6 +94,7 @@ user_message_create: | Today: {today} Extra hint: {note} Session ID: {session_id} + Session file: {session_file} ← copy THIS exact path into every source marker (e.g. `[{session_file}:1-2,5-6]`) # Recent Conversation @@ -115,15 +129,8 @@ user_message_create: | ⚠️ CRITICAL: You MUST invoke `daily_write` as a **tool call** (function call). Do NOT write `daily_write name=...` as plain text — that will NOT execute the tool and NO memory will be saved. - Body requirements: - - Each factual entry must be a complete, independent, self-contained statement (lossless restatement). - - PROHIBIT pronouns (he/she/it/they/this/that) — use full names and explicit references. - - PROHIBIT relative time (yesterday/today/last week) — use absolute dates (YYYY-MM-DD). - - Group entries by topic with headings. - - End with a `## Metadata` section containing: keywords, persons, entities, topic. - - Filename and description rules: - - `name`: concise, stable topic/event filename stem in kebab-case. No date, no slash/backslash, no `< > : " | ? *`. + Body requirements — follow the system prompt's Extraction Principles, Body Format, and Frontmatter Rules (lossless entries; no pronouns; absolute dates; grouped by topic; no `## Metadata`). Two rules that MUST NOT be violated: + - Source markers: put a bare `[session/dialog/....jsonl:1-2,5-6]` at the END of an entry or heading — full path, no `L` prefix, no `Source` label, coarse (hoist a shared marker, never repeat it across bullets), and cite a self-contained line range. - `description`: a thorough, lossless summary — specific enough that the description alone conveys all key information. ## Step 3 — Summary @@ -137,6 +144,7 @@ user_message_create_zh: | 今天：{today} 额外提示：{note} Session ID：{session_id} + Session 文件：{session_file} ← 请把这个完整路径原样复制进每个来源标记（例如 `[{session_file}:1-2,5-6]`） # 最近的对话 @@ -171,15 +179,8 @@ user_message_create_zh: | ⚠️ 关键：你**必须**以**工具调用**（函数调用）方式调用 `daily_write`。不要将 `daily_write name=...` 作为纯文本输出——那样**不会**执行工具，也**不会**保存任何记忆。 - 正文要求： - - 每条事实记录必须是完整的、独立的、自包含的陈述（无损复述）。 - - 禁止使用代词（他/她/它/他们/这个/那个）——必须使用全名和明确引用。 - - 禁止使用相对时间（昨天/今天/上周）——必须使用绝对日期（YYYY-MM-DD）。 - - 按主题分组并使用标题。 - - 末尾添加 `## Metadata` 部分，包含：keywords、persons、entities、topic。 - - 文件名和描述规则： - - `name`：简洁、稳定的主题/事件文件名 stem，采用 kebab-case。不含日期、不含 slash/反斜杠、不含 `< > : " | ? *`。 + 正文要求——遵循 system prompt 的提取原则、正文格式与 Frontmatter 规则（无损；不用代词；绝对日期；按主题分组；无 `## Metadata`）。以下两条**绝不能违反**： + - 来源标记：在条目或标题**末尾**放一个裸的 `[session/dialog/....jsonl:1-2,5-6]`——完整路径、无 `L` 前缀、无 `Source` 标签、粒度粗（上提共享标记、绝不在连续条目上重复），且引用自洽完整的行号区间。 - `description`：详尽的无损总结——具体到仅凭 description 就能传达全部核心信息。 ## 步骤 3 — 总结 @@ -194,6 +195,7 @@ user_message_update: | Today: {today} Extra hint: {note} Target path: {note_path} + Session file: {session_file} ← copy THIS exact path into every source marker (e.g. `[{session_file}:1-2,5-6]`) # Recent Conversation @@ -219,16 +221,12 @@ user_message_update: | The note already has content. Extract new information from the conversation and merge into it. - Extraction rules (apply to ALL new entries): - - Each new fact must be a complete, independent, self-contained statement (lossless restatement). - - PROHIBIT pronouns (he/she/it/they/this/that) — use full names and explicit references. - - PROHIBIT relative time (yesterday/today/last week) — use absolute dates (YYYY-MM-DD). + Extraction rules (apply to ALL new entries) — follow the system prompt's Extraction Principles and Body Format (lossless; no pronouns; absolute dates; grouped by topic). Source markers MUST be a bare `[session/dialog/....jsonl:1-2,5-6]` — full path, no `L` prefix, no `Source` label, coarse (never repeated across bullets), self-contained range. Merge rules: - **Timeline / history entries**: append only, never delete existing entries. - **Current-state entries** (progress, blockers, next steps, open questions): rewrite the entire section to reflect the latest snapshot. - **Everything else**: merge and deduplicate — keep all old facts, add new facts, remove exact duplicates. - - **Metadata section**: merge keywords, persons, entities, topics — union of old and new values. Execution: 1. **Call the `edit` tool** for each section that needs updating, with parameters: `path`="{note_path}", `old`=, `new`=. You may call `edit` multiple times. Each call MUST be a tool call, not text. @@ -241,16 +239,7 @@ user_message_update: | The file exists but its body is empty. Extract and write the full content by **calling the `write` tool** with parameters: `path`="{note_path}", `name`=, `description`=, `content`=. - Body requirements: - - Each factual entry must be a complete, independent, self-contained statement. - - PROHIBIT pronouns — use full names and explicit references. - - PROHIBIT relative time — use absolute dates (YYYY-MM-DD). - - Group entries by topic with headings. - - End with a `## Metadata` section containing: keywords, persons, entities, topic. - - Filename and description rules: - - `name`: concise, stable topic/event filename stem in kebab-case. No date. - - `description`: a thorough, lossless summary — the description alone should convey all key information. + Body requirements — follow the system prompt's Extraction Principles, Body Format, and Frontmatter Rules (lossless; no pronouns; absolute dates; grouped by topic; no `## Metadata`). Source markers MUST be a bare `[session/dialog/....jsonl:1-2,5-6]` — full path, no `L` prefix, no `Source` label, coarse (never repeated across bullets), self-contained range. `description` must be a thorough, lossless summary that alone conveys all key information. ## Step 4 — Summary @@ -265,6 +254,7 @@ user_message_update_zh: | 今天：{today} 额外提示：{note} 目标路径：{note_path} + Session 文件：{session_file} ← 请把这个完整路径原样复制进每个来源标记（例如 `[{session_file}:1-2,5-6]`） # 最近的对话 @@ -290,16 +280,12 @@ user_message_update_zh: | 笔记已有内容。从对话中提取新信息并合并进去。 - 提取规则（适用于所有新条目）： - - 每条新事实必须是完整的、独立的、自包含的陈述（无损复述）。 - - 禁止使用代词（他/她/它/他们/这个/那个）——必须使用全名和明确引用。 - - 禁止使用相对时间（昨天/今天/上周）——必须使用绝对日期（YYYY-MM-DD）。 + 提取规则（适用于所有新条目）——遵循 system prompt 的提取原则与正文格式（无损；不用代词；绝对日期；按主题分组）。来源标记必须是裸的 `[session/dialog/....jsonl:1-2,5-6]`——完整路径、无 `L` 前缀、无 `Source` 标签、粒度粗（不在连续条目上重复）、自洽完整的行号区间。 合并规则： - **时间线 / 历史条目**：仅追加，永远不删除已有条目。 - **当下状态类条目**（进度、卡点、下一步、未决问题）：整段重写，反映最新快照。 - **其余内容**：合并去重——保留全部旧事实，添加新事实，去除完全重复项。 - - **Metadata 部分**：合并 keywords、persons、entities、topics——取旧值和新值的并集。 执行： 1. **调用 `edit` 工具**更新需要修改的部分，参数为：`path`="{note_path}"、`old`=<原文片段>、`new`=<替换片段>。可以多次调用 `edit`。每次调用必须是工具调用，不是文本。 @@ -312,16 +298,7 @@ user_message_update_zh: | 文件存在但正文为空。提取并**调用 `write` 工具**一次性写入完整内容，参数为：`path`="{note_path}"、`name`=、`description`=、`content`=<正文>。 - 正文要求： - - 每条事实记录必须是完整的、独立的、自包含的陈述。 - - 禁止使用代词——必须使用全名和明确引用。 - - 禁止使用相对时间——必须使用绝对日期（YYYY-MM-DD）。 - - 按主题分组并使用标题。 - - 末尾添加 `## Metadata` 部分，包含：keywords、persons、entities、topic。 - - 文件名和描述规则： - - `name`：简洁、稳定的主题/事件文件名 stem，采用 kebab-case。不含日期。 - - `description`：详尽的无损总结——仅凭 description 就应能传达全部核心信息。 + 正文要求——遵循 system prompt 的提取原则、正文格式与 Frontmatter 规则（无损；不用代词；绝对日期；按主题分组；无 `## Metadata`）。来源标记必须是裸的 `[session/dialog/....jsonl:1-2,5-6]`——完整路径、无 `L` 前缀、无 `Source` 标签、粒度粗（不在连续条目上重复）、自洽完整的行号区间。`description` 必须是仅凭自身即可传达全部核心信息的详尽无损总结。 ## 步骤 4 — 总结 diff --git a/reme/steps/evolve/auto_memory.py b/reme/steps/evolve/auto_memory.py index b0ceef88..0277fd5f 100644 --- a/reme/steps/evolve/auto_memory.py +++ b/reme/steps/evolve/auto_memory.py @@ -71,8 +71,11 @@ class AutoMemoryStep(BaseStep): def _session_path(self, session_id: str) -> Path: return self.file_store.workspace_path / self._session_dir() / "dialog" / f"{session_id}.jsonl" + def _session_source_path(self, session_id: str) -> str: + return f"{self._session_dir()}/dialog/{session_id}.jsonl" + def _session_link(self, session_id: str) -> str: - return f"[[{self._session_dir()}/dialog/{session_id}.jsonl]]" + return f"[[{self._session_source_path(session_id)}]]" def _daily_note_path(self, day: str, name: str) -> str: return f"{self.config_value('daily_dir')}/{day}/{name}.md" @@ -240,6 +243,14 @@ class AutoMemoryStep(BaseStep): """ return {} + def _format_history(self, messages: list[Msg]) -> str: + """Render the conversation history injected into the prompt. + + Overridable hook: subclasses can annotate messages (e.g. with their + source line numbers in the session file) before rendering. + """ + return format_history(messages) + # pylint: disable=too-many-return-statements async def execute(self): assert self.context is not None @@ -308,7 +319,8 @@ class AutoMemoryStep(BaseStep): note=memory_hint or "(none)", note_path=note_path, session_id=session_id, - history=format_history(messages), + session_file=self._session_source_path(session_id), + history=self._format_history(messages), ) self.logger.info(f"[{self.name}] agent start path={note_path} template={template_key}") diff --git a/reme/steps/file_io/read.py b/reme/steps/file_io/read.py index 56fcecbe..589a56ac 100644 --- a/reme/steps/file_io/read.py +++ b/reme/steps/file_io/read.py @@ -31,6 +31,7 @@ class ReadStep(BaseStep): assert self.context is not None self.context.response.success = False self.context.response.answer = f"Error: {message}" + self.logger.warning(f"[{self.name}] FAILED: {message} | meta={meta}") if meta: self.context.response.metadata.update(meta)