refactor(memory): move ingestor from memory to steps.jobs module
Some checks failed
Pre-commit / run (ubuntu-latest) (push) Has been cancelled

- Update import path in memory/__init__.py to reference ingestor
  from ..steps.jobs instead of local memory module
- Remove ingestor.py file from memory module as it's now located
  in steps.jobs
- Remove ingestor.yaml configuration file from memory module
- Update reme_backup.py to import Summarizer from .steps.jobs.summarizer
  instead of .memory.summarizer
- Remove summarizer.py and summarizer.yaml files that were also
  part of the memory module cleanup
This commit is contained in:
huangsen 2026-05-17 17:37:10 +08:00
parent c9345455a9
commit 744ba5edc3
13 changed files with 892 additions and 395 deletions

2
.gitignore vendored
View file

@ -44,3 +44,5 @@ meta_memory/*
memories/*
.reme/*
/vault
reme2.md
structure.md

View file

@ -60,7 +60,7 @@ name these tools resolve at boot.
"""
from . import retriever # noqa: F401 -- @R.register("hybrid")
from . import ingestor # noqa: F401 -- @R.register("ingestor")
from ..steps.jobs import ingestor # noqa: F401 -- @R.register("ingestor")
from . import maintainer # noqa: F401 -- @R.register("maintainer")
# Tool surfaces — each module's @R.register decorators fire on import.

View file

@ -1,324 +0,0 @@
"""Summarizer module for memory summarization operations."""
import datetime
import json
import zoneinfo
from agentscope.agent import ReActAgent
from agentscope.message import Msg
from agentscope.token import HuggingFaceTokenCounter
from agentscope.tool import Toolkit
from loguru import logger
from ..component import BaseStep
from ..schema import AsMsgStat, AsBlockStat
class Summarizer(BaseStep):
"""Summarizer step for summarizing memory messages."""
def __init__(
self,
working_dir: str,
memory_dir: str,
memory_compact_threshold: int,
toolkit: Toolkit | None = None,
console_enabled: bool = False,
timezone: str | None = None,
add_thinking_block: bool = True,
as_token_counter: HuggingFaceTokenCounter | None = None,
**kwargs,
):
"""Initialize the summarizer step.
Args:
working_dir: Working directory path.
memory_dir: Memory directory path for storing summaries.
memory_compact_threshold: Token threshold for memory compaction.
toolkit: Optional toolkit for the agent.
console_enabled: Whether to enable console output.
timezone: Optional timezone string for date formatting.
add_thinking_block: Whether to include thinking blocks in output.
as_token_counter: Optional token counter instance.
**kwargs: Additional keyword arguments passed to BaseStep.
"""
super().__init__(**kwargs)
self.working_dir: str = working_dir
self.memory_dir: str = memory_dir
self.memory_compact_threshold: int = memory_compact_threshold
self.toolkit: Toolkit | None = toolkit
self.console_enabled: bool = console_enabled
self.timezone: str | None = timezone
self.add_thinking_block: bool = add_thinking_block
self._as_token_counter: HuggingFaceTokenCounter | None = as_token_counter
def _get_current_datetime(self) -> datetime.datetime:
"""Get current datetime with timezone, fallback to local time if timezone is invalid."""
if self.timezone:
try:
return datetime.datetime.now(zoneinfo.ZoneInfo(self.timezone))
except Exception as e:
self.logger.error(f"Invalid timezone: {self.timezone}, falling back to local time error={e}")
return datetime.datetime.now()
async def _count_str_token(self, text: str) -> int:
"""Count tokens in a string."""
return await self.as_token_counter.count(messages=[], text=text)
async def _format_tool_result_output(self, output: str | list[dict]) -> tuple[str, int]:
"""Convert tool result output to string."""
if isinstance(output, str):
return output, await self._count_str_token(output)
textual_parts = []
total_token_count = 0
for block in output:
try:
if not isinstance(block, dict) or "type" not in block:
logger.warning(
f"Invalid block: {block}, expected a dict with 'type' key, skipped.",
)
continue
block_type = block["type"]
if block_type == "text":
textual_parts.append(block.get("text", ""))
total_token_count += await self._count_str_token(textual_parts[-1])
elif block_type in ["image", "audio", "video"]:
source = block.get("source", {})
if source.get("type") == "base64":
data = source.get("data", "")
total_token_count += len(data) // 4 if data else 10
else:
url = source.get("url", "")
total_token_count += await self._count_str_token(url) if url else 10
textual_parts.append(f"[{block_type}] {url}")
elif block_type == "file":
file_path = block.get("path", "") or block.get("url", "")
file_name = block.get("name", file_path)
textual_parts.append(f"[file] {file_name}: {file_path}")
total_token_count += await self._count_str_token(file_path)
else:
logger.warning(
f"Unsupported block type '{block_type}' in tool result, skipped.",
)
except Exception as e:
logger.warning(
f"Failed to process block {block}: {e}, skipped.",
)
return "\n".join(textual_parts), total_token_count
async def _stat_message(self, message: Msg) -> AsMsgStat:
"""Analyze a message and generate block statistics."""
blocks = []
if isinstance(message.content, str):
blocks.append(
AsBlockStat(
block_type="text",
text=message.content,
token_count=await self._count_str_token(message.content),
),
)
return AsMsgStat(
name=message.name or message.role,
role=message.role,
content=blocks,
timestamp=message.timestamp or "",
metadata=message.metadata or {},
)
for block in message.content:
block_type = block.get("type", "unknown")
if block_type == "text":
text = block.get("text", "")
token_count = await self._count_str_token(text)
blocks.append(
AsBlockStat(
block_type=block_type,
text=text,
token_count=token_count,
),
)
elif block_type == "thinking":
thinking = block.get("thinking", "")
token_count = await self._count_str_token(thinking)
blocks.append(
AsBlockStat(
block_type=block_type,
text=thinking,
token_count=token_count,
),
)
elif block_type in ("image", "audio", "video"):
source = block.get("source", {})
url = source.get("url", "")
if source.get("type") == "base64":
data = source.get("data", "")
token_count = len(data) // 4 if data else 10
else:
token_count = await self._count_str_token(url) if url else 10
blocks.append(
AsBlockStat(
block_type=block_type,
text="",
token_count=token_count,
media_url=url,
),
)
elif block_type == "tool_use":
tool_name = block.get("name", "")
tool_input = block.get("input", "")
try:
input_str = json.dumps(tool_input, ensure_ascii=False)
except (TypeError, ValueError):
input_str = str(tool_input)
token_count = await self._count_str_token(tool_name + input_str)
blocks.append(
AsBlockStat(
block_type=block_type,
text="",
token_count=token_count,
tool_name=tool_name,
tool_input=input_str,
),
)
elif block_type == "tool_result":
tool_name = block.get("name", "")
output = block.get("output", "")
formatted_output, token_count = await self._format_tool_result_output(output)
blocks.append(
AsBlockStat(
block_type=block_type,
text="",
token_count=token_count,
tool_name=tool_name,
tool_output=formatted_output,
),
)
else:
logger.warning(f"Unsupported block type {block_type}, skipped.")
return AsMsgStat(
name=message.name or message.role,
role=message.role,
content=blocks,
timestamp=message.timestamp or "",
metadata=message.metadata or {},
)
async def _count_msgs_token(self, messages: list[Msg]) -> int:
"""Count total token count of a list of messages."""
total = 0
for msg in messages:
stat = await self._stat_message(msg)
total += stat.total_tokens
return total
async def _format_msgs_to_str(
self,
messages: list[Msg],
memory_compact_threshold: int,
include_thinking: bool = True,
) -> str:
"""Format list of messages to a single formatted string.
Messages are processed in reverse order (newest first) and older
messages are skipped when token count exceeds memory_compact_threshold.
Args:
messages: List of Msg objects to format.
memory_compact_threshold: Maximum token count before skipping older messages.
include_thinking: Whether to include thinking blocks in output.
"""
if not messages:
return ""
formatted_parts: list[str] = []
total_token_count = 0
for i in range(len(messages) - 1, -1, -1):
stat = await self._stat_message(messages[i])
formatted_content = stat.format(include_thinking=include_thinking)
content_token_count = await self._count_str_token(formatted_content)
is_latest = i == len(messages) - 1
if not is_latest and total_token_count + content_token_count > memory_compact_threshold:
logger.info(
f"Skipping older messages: adding {content_token_count} tokens would exceed threshold "
f"{memory_compact_threshold} (current: {total_token_count})",
)
break
if is_latest and content_token_count > memory_compact_threshold:
logger.warning(
f"Latest message alone ({content_token_count} tokens) exceeds threshold "
f"{memory_compact_threshold}, including it anyway.",
)
formatted_parts.append(formatted_content)
total_token_count += content_token_count
formatted_parts.reverse()
return "\n\n".join(formatted_parts)
async def execute(self):
"""Execute the summarization step."""
messages: list[Msg] = self.context.data.get("messages", [])
if not messages:
return ""
before_token_count = await self._count_msgs_token(messages)
history_formatted_str: str = await self._format_msgs_to_str(
messages=messages,
memory_compact_threshold=self.memory_compact_threshold,
include_thinking=self.add_thinking_block,
)
after_token_count = await self._count_str_token(history_formatted_str)
logger.info(f"Summarizer before_token_count={before_token_count} after_token_count={after_token_count}")
if not history_formatted_str:
logger.warning(f"No history to summarize. messages={messages}")
return ""
agent = ReActAgent(
name="reme_summarizer",
model=self.as_llm.model,
sys_prompt="You are a helpful assistant.",
formatter=self.as_llm_formatter.formatter,
toolkit=self.toolkit,
)
agent.set_console_output_enabled(self.console_enabled)
user_message: str = f"# conversation\n{history_formatted_str}\n\n" + self.prompt_format(
"user_message",
date=self._get_current_datetime().strftime("%Y-%m-%d"),
working_dir=self.working_dir,
memory_dir=self.memory_dir,
)
summary_msg: Msg = await agent.reply(
Msg(
name="reme",
role="user",
content=user_message,
),
)
for i, (msg, _) in enumerate(agent.memory.content):
logger.info(f"Summarizer memory[{i}]: {msg.content}")
history_summary: str = summary_msg.get_text_content()
logger.info(f"Summarizer Result:\n{history_summary}")
return history_summary

View file

@ -1,62 +0,0 @@
user_message: |
Memory Pre-compression Flush Cycle.
The current session is about to enter the automatic compression phase. Please capture persistent memory AND session reflections, then write them to disk.
Current date: {date}
Working directory: {working_dir}
# Task
Immediately store persistent memory and reflections to: {memory_dir}/YYYY-MM-DD.md
# Workflow
1. Extract and synthesize content from the current session:
- Persistent Memory: Facts, user profile updates, project states, and important events.
- Experience Reflection: Reusable thinking logic derived from user feedback, successful problem-solving strategies, mistakes made/pitfalls to avoid, and actionable insights for future interactions.
2. `read` {memory_dir}/YYYY-MM-DD.md (if the file doesnt exist, an error message will be returned)
- If the file doesnt exist, use `write` tool directly.
- If the file exists, intelligently merge new information with existing content, prefer using `edit` to update specific sections.
- Use `write` to overwrite the entire file only if substantial restructuring is required.
# Principles
- Intelligently merge new information with existing content:
- Categorize clearly (e.g., separate "Factual Memory" from "Reflections & Logic").
- Avoid duplicating already recorded information.
- Enrich existing entries with new details where relevant.
- Maintain chronological order wherever applicable.
- Always preserve timestamps and any date/time-related context.
- Add only genuinely new or meaningfully enriching information.
- Reflections MUST focus on forming reusable cognitive frameworks based on user feedback, aiming to improve future task execution.
- Keep entries concise yet complete.
- If theres nothing to store or reflect on, respond with [SILENT].
user_message_zh: |
预压缩内存刷新轮次。
当前会话即将进入自动压缩阶段;请将持久化记忆与经验反思捕获并写入磁盘。
当前日期:{date}
工作目录:{working_dir}
# 任务
立即存储持久化记忆与反思(使用路径 {memory_dir}/YYYY-MM-DD.md
# 工作流程
1. 从当前会话中提取并综合两类内容:
- 持久化记忆:客观事实、用户信息更新、项目状态及重要事件。
- 经验反思:基于用户反馈形成的可复用思考逻辑、成功的问题解决策略、犯下的错误/应避免的陷阱,以及对未来交互有帮助的行动指南。
2. `read` {memory_dir}/YYYY-MM-DD.md如文件不存在会返回错误提示
- 若文件不存在,直接使用 `write` 工具写入。
- 若文件已存在,智能合并新信息与现有内容,尽可能使用 `edit` 更新特定部分,仅在需要大幅重构时使用 `write` 覆盖整个文件。
# 原则
- 智能合并新信息与现有内容:
- 将内容进行清晰的分类(例如明确区分“事实记忆”与“反思与逻辑”)。
- 避免重复已记录的信息。
- 在相关时丰富现有条目的新细节。
- 在适用时保持时间顺序。
- 始终保留时间戳、日期和时间相关上下文。
- 仅添加真正新的或有丰富价值的信息。
- 反思内容必须侧重于根据用户反馈构建可复用的思维逻辑,以改善未来的任务执行。
- 保持条目简洁但完整。
- 若无任何新内容可存储或反思,请回复 [SILENT]。

View file

@ -13,7 +13,7 @@ from .application import Application
from .component import R, RuntimeContext
from .config import parse_args
from .enumeration import ComponentEnum
from .memory.summarizer import Summarizer
from .steps.jobs.summarizer import Summarizer
from .utils import run_coro_safely

64
reme2/steps/crud/read.py Normal file
View file

@ -0,0 +1,64 @@
"""``read`` — return the text content of a vault file.
Counterpart to ``write``. Suitable for any operation that needs the
full file body (markdown or otherwise) wikilink edits, workspace
read-modify-write loops, etc.
``path`` accepts a short link or vault-relative path; resolution
goes through ``path_resolver.resolve_to_absolute``. Short-link
ambiguity is surfaced as ``error="ambiguous"`` with the candidate
list the read is **not** executed in that case.
Returns ``{exists, content}`` (``content`` omitted when ``exists=False``).
"""
from __future__ import annotations
from agentscope.tool import ToolResponse
from ..base_step import BaseStep
from ..runtime_response import _set_answer, _tool_response
from ...component import R
from ...enumeration import ComponentEnum
from ...utils import path_resolver
async def _read(file_store, path: str, encoding: str) -> dict:
try:
target = await path_resolver.resolve_to_absolute(file_store, path)
except path_resolver.PathAmbiguous as e:
return {"path": path, "error": "ambiguous", "candidates": e.candidates}
except path_resolver.PathNotFound:
return {"path": path, "exists": False}
if not target.is_file():
return {"path": path, "exists": False}
return {
"path": path,
"exists": True,
"content": target.read_text(encoding=encoding),
}
@R.register("read")
class FileRead(BaseStep):
"""Return the text content of a vault file."""
component_type = ComponentEnum.STEP
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
encoding: str = self.context.get("encoding") or "utf-8"
assert path, "path is required"
payload = await _read(self.file_store, path, encoding)
self.context.response.success = payload.get("exists", False) and "error" not in payload
_set_answer(self.context, payload)
async def file_read(self, path: str, encoding: str = "utf-8") -> ToolResponse:
"""Return the text content of the vault file at ``path``."""
payload = await _read(self.file_store, path, encoding)
ok = payload.get("exists", False) and "error" not in payload
return _tool_response("read", ok, payload, audit=self.audit)

72
reme2/steps/crud/write.py Normal file
View file

@ -0,0 +1,72 @@
"""``write`` — write text content to a vault file.
Counterpart to ``read``. Whole-file replace (or create); use this for
any operation that needs to put a string of bytes at a known path.
The watcher / parser pick up the change asynchronously.
``path`` **must be a vault-relative path** with a directory component
short links and absolute paths are rejected (same rule as
``upload``). File creation needs an unambiguous primary key, and a
short link can't promise that without a graph entry.
``overwrite=True`` is the default since most write flows are
intentional replacements.
"""
from __future__ import annotations
from pathlib import Path
from agentscope.tool import ToolResponse
from ..base_step import BaseStep
from ..runtime_response import _set_answer, _tool_response
from ...component import R
from ...enumeration import ComponentEnum
from ...utils import path_resolver
def _write(file_store, path: str, content: str, overwrite: bool, encoding: str) -> dict:
if Path(path).is_absolute():
return {"path": path, "error": "path must be vault-relative"}
if path_resolver.is_short_path(path):
return {"path": path, "error": "path must include a directory component"}
target = path_resolver.to_absolute(file_store, path)
if target.exists() and not overwrite:
return {"path": path, "error": "destination exists; pass overwrite=True"}
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(content, encoding=encoding)
return {"path": path, "size": target.stat().st_size}
@R.register("write")
class FileWrite(BaseStep):
"""Write text content to a vault file."""
component_type = ComponentEnum.STEP
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
content: str = self.context.get("content", "") or ""
overwrite: bool = bool(self.context.get("overwrite", True))
encoding: str = self.context.get("encoding") or "utf-8"
assert path, "path is required"
payload = _write(self.file_store, path, content, overwrite, encoding)
self.context.response.success = "error" not in payload
_set_answer(self.context, payload)
async def file_write(
self,
path: str,
content: str,
overwrite: bool = True,
encoding: str = "utf-8",
) -> ToolResponse:
"""Write ``content`` to the vault file at ``path``."""
payload = _write(self.file_store, path, content, overwrite, encoding)
ok = "error" not in payload
return _tool_response("write", ok, payload, audit=self.audit)

View file

View file

@ -28,14 +28,14 @@ from agentscope.message import Msg
from agentscope.tool import Toolkit
from pydantic import BaseModel, Field
from ..steps.runtime_response import _set_answer, _to_jsonable
from . import memory_io
from .memory_io import create_file
from ..runtime_response import _set_answer, _to_jsonable
from ...memory import memory_io
from ...memory.memory_io import create_file
from .agent_toolkit import build_agent_toolkit
from ..component import R
from ..component.base_step import BaseStep
from ..enumeration import ComponentEnum
from ..utils.wikilink_resolver import extract_wikilinks
from ...component import R
from ...component import BaseStep
from ...enumeration import ComponentEnum
from ...utils import extract_wikilinks
class IngestResult(BaseModel):

View file

@ -0,0 +1,283 @@
# Summarizer 设计方案 (v4)
## 1. 定位
**Summarizer = Agent 的"任务工作区"周期同步器**
- 触发方:宿主 agent;触发时机:上下文/对话轮次达阈值(agent 自决)
- 双重产出:
1. **持久化**:把 in-progress 任务沉到 vault 的 workspace folder(folder + summary note + 任意类型 materials)
2. **回灌 context**:把完整的 summary note 内容随结果一起返回,agent 可在 compact / new-session 时直接注入新 context
- 不做:不压缩 agent context、不 distill、不动 events/topics、不感知阈值
对照 `structure.md`:summarizer 写的是 **Warm Summary**,且**主动把 Warm Summary 反哺 Hot Context**。
---
## 2. 工作区物理形态
```
<vault>/daily/<YYYY-MM-DD>/<slug>/
├── <slug>.md # workspace summary note (即 folder note)
├── <material1>.md # 文本材料
├── <material2>.pdf # 任意类型 (pdf / doc / png / json / csv ...)
└── ...
```
- `<slug>` = task title 的 kebab-case ≤ 60 字符;同一任务在同一天 slug 必须稳定
- `<slug>.md` 与父目录同名 — 即是 **summary note**,也是 reme L1 视角的 **folder note**(同一份文件,两个名字)
- Materials 不限文件类型
> **术语统一**:本方案中 "summary note" 与 "folder note" 指同一份 `<slug>.md` 文件。前者是 summarizer 视角的语义名(内容是 workspace summary),后者是 reme L1 视角的结构名(同名约定下的目录索引)。
---
## 3. Summary note 结构
```markdown
---
title: <一行任务标题>
description: <2-3 句:任务是什么为什么>
status: active | completed
created: <YYYY-MM-DD>
updated: <YYYY-MM-DD>
inherits: [[daily/<earlier-date>/<earlier-slug>/<earlier-slug>]] # 可选,仅 INHERIT 时
---
## Objective
<长期稳定目标 创建时定一次,scope 变化才改>
## Plan
<整体计划/方法/分阶段路线 滚动维护,wholesale 重写>
## Progress
- <YYYY-MM-DD HH:MM> <一条进度>
<append-only>
## Findings
- <关键发现/事实/数据点 next-session 必须知道的"已确认结论">
<append-only>
## Decisions
- <关键决策与原因>
<append-only>
## Next
- [ ] <todo>
- [x] <done todo>
<wholesale 替换>
## Materials
- [[<material1>]] — <一句话说明> # md 用 wikilink (folder note 短链)
- [<material2.pdf>](material2.pdf) — <说明> # 非 md 用 markdown 链接 + 显式扩展名
<union 去重>
```
字段哲学:
- **Append-only**:Progress / Findings / Decisions
- **Wholesale-replace**:Plan / Next
- **Set-once**:Objective(scope shift 才改)
- **Union**:Materials
---
## 4. Materials 来源(双轨)
| 来源 | 谁写 | 落盘方式 | 进 Materials list |
|---|---|---|---|
| **Agent 主动落盘** | Agent 在工作中通过 `write` / `upload` 把片段、文档、产物存到 `daily/<date>/<slug>/` | 任意工具 | summarizer 触发时 `list daily/<date>/<slug>/`,把所有非 index 文件加进 Materials |
| **Summarizer 自动抽** | Summarizer 从对话里抽关键文本片段(报错堆栈、用户原话、产出代码块、调研结论)| `write``daily/<date>/<slug>/<auto-name>.md` | 写完 append 到 Materials |
**UPDATE 路径必须先 `list`** workspace folder,把 agent 已放进去的新文件补进 Materials,避免丢失。
非 md 材料只能由 agent 通过 `upload` 放,summarizer 不生成非 md 文件。
---
## 5. 决策树
```
[1] messages 非空? ─否─▶ SKIP, used_llm=false, return
│是
[2] LLM 读 messages: 是不是 in-progress 多步任务?
否 ─▶ "[SKIP]", no tool calls
│是
[3] LLM 推断稳定 task title → slug
[4] list daily/<today>/
┌── 今天已有 <slug>/ ─▶ UPDATE
│ (a) read daily/<today>/<slug>/<slug>.md
│ (b) list daily/<today>/<slug>/ (含非 md)
│ (c) merge 各 section:
│ Objective: 保留
│ Plan: wholesale 重写
│ Progress: append (旧 + 新)
│ Findings: append
│ Decisions: append
│ Next: wholesale 替换
│ Materials: union(旧 list, 新文件 list, 自动抽片段)
│ (d) [可选] 写关键片段成新 md material (防冲突)
│ (e) write 新 index (overwrite=True 覆盖旧 index 是预期)
├── 今天没有, 扫近 N (默认 7) 天 daily/ 找 status=active 同 title ─▶ INHERIT
│ (a) property:read 候选 index 看 title/status
│ (b) read predecessor index → 拷 Objective + Plan
│ (c) [可选] 写关键片段 (防冲突)
│ (d) write 新 index:
│ Inherits: [[daily/<earlier-date>/<earlier-slug>/<earlier-slug>]]
│ Objective: 拷
│ Plan: 拷
│ 其他 sections: 全新
│ Materials: 仅自己生成 (旧 materials 通过 Inherits 链回去)
│ (e) predecessor 的 status 不动
└── 没找到 ─▶ CREATE
(a) [可选] 写关键片段 (防冲突)
(b) write 新 index (各 section 全新)
[5] 完成态判定:
- 默认 status=active
- 仅当对话有非常明确的"任务结束/已交付"信号 → completed
- 翻转能力保留, prompt 写死保守判定
[6] LLM 输出最终回复 (一行 summary):
"<created|inherited|updated> daily/<date>/<slug>/<slug>.md (+M materials)"
或 "[SKIP]"
[7] Step 在 LLM 完成后, 用 file_store 直接 read 出
刚写的 daily/<date>/<slug>/<slug>.md 完整内容,
填进 SummarizerResult.summary 字段
```
---
## 6. Toolkit (LLM 可用)
复用现有 6 个 crud,无需新增:
| 工具 | 用途 |
|---|---|
| `list(path, recursive)` | 扫今天目录、扫 workspace folder(含非 md)、扫 inherit 候选 |
| `stat(path)` | **写前防冲突检查** |
| `read(path)` | UPDATE 读旧 index;INHERIT 读 predecessor |
| `write(path, content, overwrite)` | 写 index;写自动抽 material(默认 `overwrite=False`)|
| `property:read(path)` | 扫 inherit 候选只看 frontmatter |
| `property:update(path, **fields)` | 翻 status / 改 updated 不重写正文 |
`upload` 不在 toolkit 里 — summarizer 不拷非 md 材料。
---
## 7. Step 参数 & 输出契约
### 7.1 `__init__` 参数
```python
class Summarizer(BaseStep):
def __init__(
self,
toolkit: Toolkit | None = None,
console_enabled: bool = False,
timezone: str | None = None,
inherit_window_days: int = 7, # INHERIT 扫描窗口
**kwargs,
):
...
```
### 7.2 输出契约(扩展)
```python
class SummarizerResult(BaseModel):
used_llm: bool
applied: list[dict] # 成功的 tool calls
failed: list[dict] # 失败的 tool calls
skipped: bool # True = LLM 判定非任务
actions: str # LLM 一行 action 陈述
# —— 服务 agent 上下文管理的核心字段 ——
workspace: str | None = None # daily/<date>/<slug>/ (folder 相对路径,带尾 /)
summary: str | None = None # summary note **完整 markdown 内容**
# (含 frontmatter + 全部 sections)
# SKIP 或 失败时为 None
success = len(failed) == 0
```
**`summary` 字段是关键**:agent 在 compact context / 新 session 启动时,可直接把 `result.summary` 作为 system context 注入,不必再发 read 请求。
填充时机:LLM 完成所有 tool call 后,Step 在 `execute()` 末尾用 `file_store` **直接** read 出 note 文件(从 `actions` 行正则提取 path),确保拿到的是 agent 刚刚写完的最新版本。
> **Input/Output 字段对齐**:`workspace` 一词在两边语义统一 — 输入时是 caller 给的 hint(任务标题或 `daily/.../` 路径),输出时是确定的 folder 相对路径。
---
## 8. 冲突管理(LLM 自管)
`write` 默认 `overwrite=False`(已支持,见 `reme2/steps/crud/write.py:55`)。LLM 协议:
| 场景 | LLM 应做 |
|---|---|
| **写 index `<slug>.md`** | UPDATE 路径:已 `read` 确认是同一任务 → 显式传 `overwrite=True`<br>CREATE / INHERIT:`stat` 不存在再写;意外存在 → 加后缀消歧 |
| **写自动抽的 material** | 名字 LLM 自取(描述性);`stat` 检查;若存在 → 改名(如 `auth-error-2.md`)再写;不允许 silent overwrite |
| **agent 已写过同名** | 同上 — `stat` 拦下,LLM 改名 |
---
## 9. 关键设计决策汇总
| # | 决策 | 选定 |
|---|---|---|
| D1 | workspace 形态 | folder + summary note(= folder note) + 同目录 materials |
| D2 | 术语 | "summary note" / "folder note" = 同一份 `<slug>.md`,本方案统一 |
| D3 | summary note 字段 | Objective / Plan / Progress / Findings / Decisions / Next + Materials + (Inherits) |
| D4 | append-only | Progress / Findings / Decisions |
| D5 | wholesale-replace | Plan / Next |
| D6 | Materials 来源 | 双轨 — agent 主动 + summarizer 自动抽(仅 md) |
| D7 | Materials 文件类型 | 任意 — md 用 wikilink,非 md 用 markdown 链接 + 扩展名 |
| D8 | INHERIT 扫描窗口 | 参数化 `inherit_window_days`,默认 7 |
| D9 | completed 翻转 | 保留能力,prompt 写死保守判定 |
| D10 | Predecessor status | INHERIT 时不动 |
| D11 | Material 自动文件名 | LLM 自取(描述性) |
| D12 | 冲突管理 | `write` 默认 `overwrite=False`,LLM 自己 `stat` 检查并改名 |
| D13 | 触发器 | agent 自决,summarizer 不感知阈值 |
| D14 | 与 ingestor 关系 | summarizer 写 daily/ workspace(warm);ingestor 写 events/topics(cold)— 解耦 |
| D15 | 输出含完整 note | `SummarizerResult.summary` 携带刚写完的 summary note 全文,服务 agent 上下文管理 |
| D16 | summary 字段填充 | Step 在 LLM 完成后用 `file_store` 直读,不依赖 LLM 自己回传 |
| D17 | input/output 字段对齐 | `workspace` 一词在 input(hint) 与 output(folder path) 共用,`actions` 字段承载 LLM 一行陈述 |
---
## 10. 改动清单
### `reme2/steps/jobs/summarizer.py`
1. `__init__``inherit_window_days: int = 7`
2. `execute()`:
- 输入字段:`task_hint``workspace`(语义统一)
- `prompt_format("user_message", ...)` 多传 `inherit_window_days` + `workspace`
- LLM 完成后,从 `actions`(原 `agent_summary`)解析出 note path(正则匹配)
- 用 `path_resolver.to_absolute(file_store, note_path).read_text()` 拿完整内容
- 填进 `SummarizerResult.summary` / `workspace`
- SKIP / 失败 → 两个字段 None
3. `SummarizerResult` 字段重命名:`agent_summary``actions`,`workspace_path``workspace`,`note``summary`,删除 `note_path`
### `reme2/steps/jobs/summarizer.yaml`
完全重写 `system_prompt` + `user_message`:
- 路径模板改 folder 形态 `daily/<date>/<slug>/<slug>.md`
- 字段扩展 Objective / Plan / Progress / Findings / Decisions / Next + Materials
- Materials 引用规则:md wikilink,非 md markdown link + 扩展名
- 决策树明确 UPDATE / INHERIT / CREATE
- INHERIT 窗口用 `{inherit_window_days}` 占位
- Workspace hint 用 `{workspace}` 占位 (替代旧的 `{task_hint}`)
- 冲突管理协议写明 `stat` 先行 + 改名
- 完成态保守判定
- 输出格式 `<action> daily/<date>/<slug>/<slug>.md (+M materials)``[SKIP]`
- 强调:LLM 输出的 path **必须用相对 vault 的形式**,Step 后续会据此 read 完整 note
### note path 提取协议(实施细节)
让 LLM 在 actions 行严格输出 `<action> <relative-path> (+M materials)`,Step 用正则 `r"daily/\d{4}-\d{2}-\d{2}/[^/\s]+/[^/\s]+\.md"` 提取 path。SKIP 行不提取。提取失败 → `summary=None` 但不算 step 失败(只是回灌 context 失败,持久化已完成)。

View file

@ -0,0 +1,242 @@
"""Summarizer — workspace-sync ReAct agent.
Watches the agent's recent conversation and persists in-progress
tasks as a workspace **folder** inside reme's vault, so future
agent invocations can pick the work back up. Pure sync mechanism
does **not** compress the agent's context (compression is the
agent's own concern).
Workspace layout: ``daily/<YYYY-MM-DD>/<slug>/<slug>.md`` plus
sibling materials of any file type (md / pdf / doc / csv / ...).
The note filename equals its parent folder name, which makes it a
folder note in reme L1 short wikilinks like ``[[<slug>]]`` from
sibling files resolve back to it. Frontmatter carries
``title`` / ``description`` / ``status`` / ``created`` / ``updated``
plus an optional ``inherits`` wikilink for cross-day continuation;
body splits into ``Objective`` / ``Plan`` / ``Progress`` /
``Findings`` / ``Decisions`` / ``Next`` / ``Materials`` sections.
Inputs (from RuntimeContext):
messages (list[Msg], required): conversation slice to inspect.
workspace (str, optional): caller-supplied workspace hint
(task title or vault-relative ``daily/.../`` path) to bias
slug selection and disambiguate same-day tasks.
Output (written to context.response.answer):
{
"skipped": True if the agent reported [SKIP],
"actions": one-line action statement from the agent,
"workspace": vault-relative folder path, or None,
"summary": full markdown content of the summary note,
for the calling agent to reload into a
compacted context. None when SKIP / failed.
}
The toolkit bound for the agent is built in-place from registered
crud / property steps (``list``, ``read``, ``write``, ``stat``,
``property:read``, ``property:update``); each call accumulates into
the audit list returned in ``applied`` / ``failed``.
"""
from __future__ import annotations
import datetime
import re
import zoneinfo
from pathlib import Path
from agentscope.agent import ReActAgent
from agentscope.message import Msg
from agentscope.tool import Toolkit
from pydantic import BaseModel, Field
from ..base_step import BaseStep
from ..runtime_response import _set_answer
from ...component import R
from ...enumeration import ComponentEnum
from ...utils import path_resolver
_NOTE_PATH_RE = re.compile(r"daily/\d{4}-\d{2}-\d{2}/[^/\s]+/[^/\s]+\.md")
_WORKSPACE_TOOLS: tuple[tuple[str, str], ...] = (
("list", "file_list"),
("read", "file_read"),
("write", "file_write"),
("stat", "file_stat"),
("property:read", "property_read"),
("property:update", "property_update"),
)
def _build_toolkit(
app_context,
audit: list[dict],
toolkit: Toolkit | None = None,
) -> Toolkit:
"""Bind workspace-relevant step tool methods into a single Toolkit.
Each bound instance shares the same ``audit`` list so every tool
call's outcome lands in one trail.
"""
toolkit = toolkit or Toolkit()
for step_name, method_name in _WORKSPACE_TOOLS:
cls = R.get(ComponentEnum.STEP, step_name)
if cls is None:
continue
instance = cls(app_context=app_context)
instance.audit = audit # type: ignore[attr-defined]
toolkit.register_tool_function(
getattr(instance, method_name),
namesake_strategy="override",
)
return toolkit
def _format_history(messages: list[Msg]) -> str:
"""Render the conversation as a speaker-tagged transcript.
Skips messages whose text content is empty (tool-only frames
don't help the LLM judge task state).
"""
if not messages:
return "(empty)"
lines: list[str] = []
for msg in messages:
speaker = msg.name or msg.role or "?"
text = (msg.get_text_content() or "").strip()
if not text:
continue
lines.append(f"[{speaker}]\n{text}")
return "\n\n".join(lines) or "(no text)"
class SummarizerResult(BaseModel):
"""Audit trail + context-management payload for a single workspace-sync call."""
used_llm: bool = Field(default=False)
applied: list[dict] = Field(default_factory=list)
failed: list[dict] = Field(default_factory=list)
skipped: bool = Field(default=False)
actions: str = Field(
default="",
description="One-line action statement from the agent (e.g. "
"'updated daily/2026-05-15/auth-refactor/auth-refactor.md (+2 materials)' or '[SKIP]').",
)
workspace: str | None = Field(
default=None,
description="Vault-relative folder path of the synced workspace, "
"e.g. 'daily/2026-05-15/auth-refactor/'. None when SKIP / failed.",
)
summary: str | None = Field(
default=None,
description="Full markdown content of the workspace summary note. Lets the calling "
"agent reload the warm summary into a freshly compacted context without an extra read.",
)
@R.register("summarizer")
class Summarizer(BaseStep):
"""Drive workspace sync via a ReAct agent."""
component_type = ComponentEnum.STEP
def __init__(
self,
toolkit: Toolkit | None = None,
console_enabled: bool = False,
timezone: str | None = None,
inherit_window_days: int = 7,
**kwargs,
):
super().__init__(**kwargs)
self.toolkit = toolkit
self.console_enabled = console_enabled
self.timezone = timezone
self.inherit_window_days = inherit_window_days
def _now(self) -> datetime.datetime:
if self.timezone:
try:
return datetime.datetime.now(zoneinfo.ZoneInfo(self.timezone))
except Exception as e:
self.logger.error(
f"Invalid timezone {self.timezone!r}, falling back to local time: {e}",
)
return datetime.datetime.now()
def _working_dir(self) -> Path:
wd = getattr(self.file_store, "working_dir", None)
return Path(wd).resolve() if wd else Path.cwd().resolve()
async def execute(self):
assert self.context is not None
messages: list[Msg] = self.context.get("messages") or []
workspace_hint: str = self.context.get("workspace", "") or ""
if not messages:
result = SummarizerResult(used_llm=False, skipped=True)
self.context.response.success = True
_set_answer(self.context, result.model_dump())
return
audit: list[dict] = []
toolkit = _build_toolkit(self.app_context, audit=audit, toolkit=self.toolkit)
agent = ReActAgent(
name="reme_summarizer",
model=self.as_llm,
sys_prompt=self.prompt_format("system_prompt"),
formatter=self.as_llm_formatter,
toolkit=toolkit,
)
agent.set_console_output_enabled(self.console_enabled)
user_message: str = self.prompt_format(
"user_message",
today=self._now().strftime("%Y-%m-%d"),
working_dir=str(self._working_dir()),
inherit_window_days=self.inherit_window_days,
workspace=workspace_hint or "(none)",
history=_format_history(messages),
)
final_msg: Msg = await agent.reply(
Msg(name="reme", role="user", content=user_message),
)
actions = (final_msg.get_text_content() or "").strip()
result = SummarizerResult(used_llm=True, actions=actions)
for entry in audit:
(result.applied if entry.get("ok") else result.failed).append(entry)
if not audit and "[SKIP]" in actions.upper():
result.skipped = True
# Reload the freshly written note so the calling agent can drop
# it back into a compacted context without an extra read trip.
if not result.skipped and not result.failed:
self._reload_note(result, actions)
self.context.response.success = len(result.failed) == 0
_set_answer(self.context, result.model_dump())
def _reload_note(self, result: SummarizerResult, actions: str) -> None:
"""Parse the agent's action line for the note path and read the
full file back into ``result.summary``. Best-effort: a parse miss
leaves the context-management fields as None but does not fail
the step (persistence already succeeded)."""
match = _NOTE_PATH_RE.search(actions)
if not match:
return
note_path = match.group(0)
try:
absolute = path_resolver.to_absolute(self.file_store, note_path)
text = absolute.read_text(encoding="utf-8")
except Exception as e:
self.logger.warning(f"summarizer: could not reload note {note_path!r}: {e}")
return
result.workspace = str(Path(note_path).parent) + "/"
result.summary = text

View file

@ -0,0 +1,220 @@
system_prompt: |
You are reme's workspace synchronizer. You watch the agent's recent
conversation and persist any **task in progress** as a workspace
folder inside reme's vault, so future agent invocations can pick the
work back up — and so the calling agent can immediately reload the
resulting summary into its own context.
## Workspace location
<vault>/daily/<YYYY-MM-DD>/<slug>/
├── <slug>.md # workspace summary note (this is what you write)
├── <material1>.md # markdown materials
├── <material2>.pdf # arbitrary files (pdf / doc / png / json / csv ...)
└── ...
- ``<slug>`` = kebab-case of the task title, max 60 chars.
The note filename **must equal the parent folder name** — this is
the folder-note convention reme L1 relies on for short-link
resolution.
- One workspace folder per task per day. Same task across days
inherits via the ``inherits:`` frontmatter wikilink.
- Materials accept any file type, not just markdown.
## Summary note shape
```markdown
---
title: <one-line task title>
description: <2-3 sentences: what and why>
status: active | completed
created: <YYYY-MM-DD>
updated: <YYYY-MM-DD>
inherits: [[daily/<earlier-date>/<earlier-slug>/<earlier-slug>]] # only when INHERIT
---
## Objective
<stable long-term goal — set once at creation, only edit if scope shifts>
## Plan
<overall approach / phased roadmap — rolling, wholesale-rewritten each update>
## Progress
- <YYYY-MM-DD HH:MM> <one progress entry>
<append-only — never delete prior entries>
## Findings
- <key fact / data point / confirmed conclusion the next session must know>
<append-only>
## Decisions
- <one key choice and the why>
<append-only>
## Next
- [ ] <todo>
- [x] <done todo>
<wholesale-replaced each update — reflects the current todo list>
## Materials
- [[<name>]] — <one-line note> # markdown: wikilink, folder-note short link works
- [<name.pdf>](name.pdf) — <one-line> # non-markdown: standard markdown link + explicit ext
<union with prior list, deduped>
```
### Section discipline
- **Append-only** (never delete history): Progress / Findings / Decisions
- **Wholesale-replace** (current state only): Plan / Next
- **Set-once** (only re-write on scope shift): Objective
- **Union** (merge with what's already there): Materials
### Materials reference rules
- Markdown files (``.md``) → wikilink ``[[name]]``. Because the note is
a folder note of its parent directory, the short link resolves to a
sibling file in the same workspace folder.
- Non-markdown files (``.pdf``, ``.doc``, ``.png``, ``.csv``, ``.json``, ...)
→ standard markdown link with **explicit extension**:
``[name.pdf](name.pdf)``. Path is relative to the note's folder.
- Each entry gets a one-line description so future readers know what
the material is without opening it.
## Materials sources (dual-track)
Materials in the workspace folder come from two places — your job
treats them uniformly:
1. **Agent-pushed**: the calling agent may have already saved
snippets / docs / outputs into ``daily/<today>/<slug>/`` during
its work (via ``write`` or ``upload``). When updating an existing
workspace, you **must** ``list`` that folder first and fold every
non-index file into the Materials section.
2. **You-extracted**: when the conversation contains a key snippet
worth preserving (error trace, user's exact ask, generated code,
research conclusion), you may ``write`` it as a new ``.md`` file
in the workspace folder. Pick a descriptive filename
(e.g. ``auth-error-trace.md``, ``user-spec.md``) — never generic
names like ``notes.md``.
You only generate **markdown** materials. Non-markdown files (pdf etc.)
are exclusively agent-pushed.
## Decision tree per call
1. Read the conversation. Is the agent in the middle of a non-trivial,
multi-step task? (Casual Q&A, single-shot answers, idle chat → no.)
If **not** a task, respond with the literal text ``[SKIP]`` and stop.
2. If yes, pick a stable task title (consistent across calls — the
slug derives from it).
3. Branch by checking today's folder via ``list path=daily/<today>``:
**a) UPDATE** — today's ``daily/<today>/<slug>/`` already exists:
- ``read daily/<today>/<slug>/<slug>.md`` to load the prior note
- ``list daily/<today>/<slug>/`` to discover new materials the
agent pushed (any extension)
- merge per section discipline above; union the Materials list
- optionally ``write`` new extracted snippets as fresh ``.md``
materials (see conflict protocol below)
- ``write`` the new index with ``overwrite=True`` (this is the
one case where overwriting is expected)
**b) INHERIT** — today has no ``<slug>/`` folder, but scanning the
last {inherit_window_days} days of ``daily/<date>/`` finds an
``active`` workspace whose title matches this task:
- use ``property:read`` on candidate index files to cheaply
check ``title`` / ``status`` without reading the full body
- ``read`` the predecessor's index; copy ``Objective`` and
``Plan`` into the new note
- optionally ``write`` extracted snippets as fresh materials
- ``write`` the new index with the ``inherits:`` frontmatter
pointing at the predecessor; Progress / Findings / Decisions
/ Next start fresh; Materials list contains only what you
generate today (the predecessor's materials are reachable via
the inherits link)
- **do not** modify the predecessor's status — leave it active
**c) CREATE** — neither today's folder nor an inheritable
predecessor exists:
- optionally ``write`` extracted snippets as fresh materials
- ``write`` the new index from scratch with all sections filled
4. Set ``status: completed`` **only** when the conversation makes it
extremely explicit that the task is done (e.g. user says "task
complete", "we're done with this", "shipped"). When in doubt,
leave ``active``. False positives here are worse than false
negatives — a task wrongly marked completed will not be picked up
next session.
## Conflict protocol (write defaults to overwrite=False)
- **Index file** (``<slug>.md``):
- UPDATE path: you've already ``read`` it and confirmed identity,
so call ``write`` with ``overwrite=True`` explicitly.
- CREATE / INHERIT path: ``stat`` first; only write if it
doesn't exist. If it unexpectedly exists, append a disambiguating
suffix to the slug (e.g. ``<slug>-2``).
- **Material file**: pick a descriptive name; ``stat`` first; if it
exists, rename (``<name>-2.md``, ``<name>-3.md``, ...) and write.
Never silently overwrite a material — the agent or a prior call
may have put it there with content you'd lose.
## Tools
- ``list(path, recursive)`` — list files under a directory.
- ``read(path)`` → ``{exists, content}`` — full file text.
- ``write(path, content, overwrite=False)`` — create or replace.
Default is ``overwrite=False`` — pass ``overwrite=True`` only when
you intentionally replace an existing file you've already read.
- ``property:read(path)`` → ``{frontmatter}`` — cheap frontmatter peek
(use this when scanning inherit candidates).
- ``property:update(path, **fields)`` — set frontmatter fields without
rewriting the body (use for status/title flips).
- ``stat(path)`` — existence + mtime check (use before any write).
## Output
After completing your tool calls, respond with **exactly one line**
in the form:
<action> daily/<date>/<slug>/<slug>.md (+M materials)
where ``<action>`` is ``created`` / ``inherited`` / ``updated`` and
``M`` is the count of files in the Materials list of the index you
just wrote (including agent-pushed and you-extracted).
Or ``[SKIP]`` (literal, alone on the line) if the conversation is
not an in-progress task.
The path you emit will be parsed by reme to read the note back and
return its full content to the calling agent — so the path **must
exactly match** the index file you wrote, in vault-relative form.
user_message: |
Today: {today}
Vault working dir: {working_dir}
Daily root: daily/{today}/
Inherit scan window: last {inherit_window_days} days
Workspace hint (optional task title or daily/.../ path, may be empty): {workspace}
# Recent conversation
{history}
# Your job
Follow the decision tree from the system prompt. Either persist the
in-progress task as a workspace folder under daily/{today}/ (via
CREATE, INHERIT, or UPDATE), or respond with `[SKIP]` if no task is
in progress.
If a Workspace hint is provided, treat it as a strong suggestion for
the slug / target folder — but still verify the existing-folder
branch logic before deciding CREATE vs INHERIT vs UPDATE.
Remember: write defaults to overwrite=False. Always stat before
creating; only pass overwrite=True when intentionally replacing an
index you've already read.
Final answer must be a single line:
`<action> daily/{today}/<slug>/<slug>.md (+M materials)` or `[SKIP]`.