diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 263bd23a..02f8acdf 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -36,7 +36,7 @@ jobs: - name: Run tests4 unit tests run: | - pytest tests4/unittest \ + pytest tests4/unit \ -v \ --tb=long \ -s \ diff --git a/docs4/reme_design.md b/docs4/reme_design.md index 56d30120..d2e69904 100644 --- a/docs4/reme_design.md +++ b/docs4/reme_design.md @@ -88,11 +88,11 @@ reme4 search query="..." backend=mcp | crud | append | path="My Note" content="New line" | | crud | delete | path="My Note -| daily:crud | daily:xxx | 与 crud 参数保持一致 | +| daily_crud | daily_xxx | 与 crud 参数保持一致 | -- daily:resolve name=xxxx (符合一定规范 win下要求) -- daily:list date=xxxx 返回path -- daily:index +- daily_resolve name=xxxx (符合一定规范 win下要求) +- daily_list date=xxxx 返回path +- daily_index frontmatter read path frontmatter update path metadata={} diff --git a/reme4/components/file_parser/linked_file_parser.py b/reme4/components/file_parser/linked_file_parser.py index 229b9d14..bfd7522a 100644 --- a/reme4/components/file_parser/linked_file_parser.py +++ b/reme4/components/file_parser/linked_file_parser.py @@ -113,7 +113,7 @@ class LinkedFileParser(BaseFileParser): def __init__( self, encoding: str = "utf-8", - chunk_chars: int = 2000, + chunk_chars: int = 10000, embed_toc: bool = True, **kwargs, ): diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml index 42ac251e..09ce7a7f 100644 --- a/reme4/config/default.yaml +++ b/reme4/config/default.yaml @@ -107,7 +107,7 @@ jobs: expand_links: true max_links_per_direction: 10 - daily:create: + daily_create: backend: base description: "Provision a note slug under a daily folder: daily//.md" parameters: @@ -115,7 +115,7 @@ jobs: properties: slug: type: string - description: "the file stem" + description: "the file stem: event name or topic name" date: type: string description: "YYYY-MM-DD; empty = today" @@ -125,7 +125,7 @@ jobs: steps: - backend: daily_create_step - daily:list: + daily_list: backend: base description: "List notes under a single day." parameters: @@ -138,7 +138,7 @@ jobs: steps: - backend: daily_list_step - daily:reindex: + daily_reindex: backend: base description: "Rebuild the day-index page daily/.md." parameters: @@ -151,7 +151,7 @@ jobs: steps: - backend: daily_reindex_step - frontmatter:delete: + frontmatter_delete: backend: base description: "Drop keys from a file's frontmatter." parameters: @@ -171,7 +171,7 @@ jobs: steps: - backend: frontmatter_delete_step - frontmatter:read: + frontmatter_read: backend: base description: "Read a file's frontmatter as a dict." parameters: @@ -185,7 +185,7 @@ jobs: steps: - backend: frontmatter_read_step - frontmatter:update: + frontmatter_update: backend: base description: "Merge key-values into a file's frontmatter." parameters: @@ -369,6 +369,30 @@ jobs: steps: - backend: edit_step + auto_memory: + backend: base + description: "Auto-memory orchestrator" + parameters: + type: object + properties: + messages: + type: array + description: "messages" + items: + type: object + memory_hint: + type: string + description: "optional hint" + timezone: + type: string + description: "IANA timezone, e.g. Asia/Shanghai" + default: "Asia/Shanghai" + required: + - messages + steps: + - backend: auto_memory_planner_step + - backend: auto_memory_writer_step + components: tokenizer: default: @@ -382,6 +406,18 @@ components: model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-v4} dimensions: 1024 + as_llm: + default: + backend: ${LLM_BACKEND:-anthropic} + model_name: ${LLM_MODEL_NAME:-glm-5} + api_key: ${LLM_API_KEY:-} + client_kwargs: + base_url: ${LLM_BASE_URL:-https://dashscope.aliyuncs.com/apps/anthropic} + + as_llm_formatter: + default: + backend: ${LLM_FORMATTER_BACKEND:-anthropic} + file_graph: default: backend: local diff --git a/reme4/config/qwenpaw.yaml b/reme4/config/qwenpaw.yaml index 96d30ad9..5c61b8ba 100644 --- a/reme4/config/qwenpaw.yaml +++ b/reme4/config/qwenpaw.yaml @@ -185,7 +185,7 @@ jobs: steps: - backend: stat_step - frontmatter:read: + frontmatter_read: backend: base description: "Read a file's YAML frontmatter as a dict." parameters: @@ -197,7 +197,7 @@ jobs: required: - path steps: - - backend: frontmatter:read_step + - backend: frontmatter_read_step # ── Write Operations────────────────────────────────────────────────────────── write: @@ -267,7 +267,7 @@ jobs: steps: - backend: append_step - frontmatter:update: + frontmatter_update: backend: base description: "Merge keys into a file's YAML frontmatter." parameters: @@ -286,7 +286,7 @@ jobs: steps: - backend: frontmatter_update_step - frontmatter:delete: + frontmatter_delete: backend: base description: "Drop keys from a file's YAML frontmatter." parameters: @@ -369,7 +369,7 @@ jobs: - backend: download_step # ── Daily Operations (slug provisioning + day-index rollup) ────────── - daily:create: + daily_create: backend: base description: "Provision daily//.md (empty body, frontmatter {name: slug}); idempotent; refreshes the day index." parameters: @@ -387,7 +387,7 @@ jobs: steps: - backend: daily_create_step - daily:list: + daily_list: backend: base description: "List notes under a single day." parameters: @@ -400,7 +400,7 @@ jobs: steps: - backend: daily_list_step - daily:reindex: + daily_reindex: backend: base description: "Rebuild the day-index page daily/.md." parameters: diff --git a/reme4/steps/__init__.py b/reme4/steps/__init__.py index daa59838..afd7bbb0 100644 --- a/reme4/steps/__init__.py +++ b/reme4/steps/__init__.py @@ -4,8 +4,11 @@ from .base_step import BaseStep from .common.demo import DemoEchoStep1, DemoEchoStep2 from .common.health_check import HealthCheckStep from .common.help import HelpStep +from .common.llm_demo import LLMDemoStep from .common.stream_demo import StreamDemoStep1, StreamDemoStep2 from .common.version import VersionStep +from .evolve.auto_memory_planner import AutoMemoryPlannerStep +from .evolve.auto_memory_writer import AutoMemoryWriterStep from .file_io.daily_create import DailyCreateStep from .file_io.daily_list import DailyListStep from .file_io.daily_reindex import DailyReindexStep @@ -38,9 +41,13 @@ __all__ = [ "DemoEchoStep2", "HealthCheckStep", "HelpStep", + "LLMDemoStep", "StreamDemoStep1", "StreamDemoStep2", "VersionStep", + # evolve + "AutoMemoryPlannerStep", + "AutoMemoryWriterStep", # file_io "DeleteStep", "EditStep", diff --git a/reme4/steps/base_step.py b/reme4/steps/base_step.py index d60564e1..0b74620e 100644 --- a/reme4/steps/base_step.py +++ b/reme4/steps/base_step.py @@ -202,5 +202,12 @@ class BaseStep(ABC): tool_func=run_job, func_name=job_name, func_description=job.description, - json_schema=job.parameters, + json_schema={ + "type": "function", + "function": { + "name": job_name, + "description": job.description, + "parameters": job.parameters, + }, + }, ) diff --git a/reme4/steps/common/llm_demo.py b/reme4/steps/common/llm_demo.py new file mode 100644 index 00000000..fc6dda06 --- /dev/null +++ b/reme4/steps/common/llm_demo.py @@ -0,0 +1,78 @@ +"""Demo step that drives a ReActAgent via BaseStep.as_llm/as_llm_formatter.""" + +from agentscope.agent import ReActAgent +from agentscope.message import Msg, TextBlock +from agentscope.tool import Toolkit, ToolResponse + +from ..base_step import BaseStep +from ...components import R + + +def _add(a: float, b: float) -> ToolResponse: + """Add two numbers and return the sum. + + Args: + a: first addend + b: second addend + """ + return ToolResponse(content=[TextBlock(type="text", text=str(a + b))]) + + +@R.register("llm_demo_step") +class LLMDemoStep(BaseStep): + """Drive a ReActAgent powered by ``self.as_llm`` / ``self.as_llm_formatter``. + + Inputs (from RuntimeContext): + query (str, required): user message content. + sys_prompt (str, optional): system prompt for the agent. + use_add_tool (bool, optional): register the ``add`` tool when True. + console_enabled (bool, optional): mirror agent output to stdout. + + Output (written to context.response.answer):fa + The agent's final reply text. + """ + + DEFAULT_SYS_PROMPT = "You are a concise assistant. Reply in one short sentence." + + async def execute(self): + assert self.context is not None + query: str = self.context.get("query", "") + sys_prompt: str = self.context.get("sys_prompt") or self.DEFAULT_SYS_PROMPT + use_add_tool: bool = bool(self.context.get("use_add_tool", False)) + console_enabled: bool = bool(self.context.get("console_enabled", False)) + + if not query: + self.context.response.success = False + self.context.response.answer = "Skipped: empty query" + return self.context.response + + toolkit = Toolkit() + if use_add_tool: + toolkit.register_tool_function(_add) + + agent = ReActAgent( + name=self.name, + sys_prompt=sys_prompt, + model=self.as_llm, + formatter=self.as_llm_formatter, + toolkit=toolkit, + ) + agent.set_console_output_enabled(console_enabled) + + response: Msg = await agent.reply( + Msg(name="user", role="user", content=query), + ) + text = (response.get_text_content() or "").strip() + self.logger.info(f"[{self.name}] response: {text!r}") + + self.context.response.success = True + self.context.response.answer = text + self.context.response.metadata.update( + { + "query": query, + "sys_prompt": sys_prompt, + "use_add_tool": use_add_tool, + "response": text, + }, + ) + return self.context.response diff --git a/reme4/steps/evolve/__init__.py b/reme4/steps/evolve/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reme4/steps/evolve/_evolve.py b/reme4/steps/evolve/_evolve.py new file mode 100644 index 00000000..bac0a0d4 --- /dev/null +++ b/reme4/steps/evolve/_evolve.py @@ -0,0 +1,29 @@ +"""Shared helpers for evolve steps.""" + +import datetime +import zoneinfo + +from agentscope.message import Msg + + +def now(timezone: str | None = None) -> datetime.datetime: + """Return current datetime in the given IANA timezone, falling back to local.""" + if not timezone: + return datetime.datetime.now() + try: + return datetime.datetime.now(zoneinfo.ZoneInfo(timezone)) + except Exception: + return datetime.datetime.now() + + +def format_history(messages: list[Msg], include_timestamp: bool = True) -> str: + """Render a conversation slice as a human-readable transcript.""" + lines: list[str] = [] + for msg in messages: + text = (msg.get_text_content() or "").strip() + if not text: + continue + speaker = msg.name or msg.role or "?" + header = f"[{speaker} @ {msg.timestamp}]" if include_timestamp else f"[{speaker}]" + lines.append(f"{header}\n{text}") + return "\n\n".join(lines) or "(empty)" diff --git a/reme4/steps/evolve/auto_memory_planner.py b/reme4/steps/evolve/auto_memory_planner.py new file mode 100644 index 00000000..d39ff0d7 --- /dev/null +++ b/reme4/steps/evolve/auto_memory_planner.py @@ -0,0 +1,120 @@ +"""``auto_memory_planner`` — planner for the auto-memory system. + +Inspects the recent conversation, surveys today's existing daily notes +via ``daily_list``, reads any candidate that already covers the topic +via ``read``, and emits a list of daily-note upsert tasks +``(path, description)`` through structured output. The result is +exposed under ``response.metadata['memory_updates']`` for the +orchestrator (``auto_memory``) to feed into ``auto_memory_writer`` +one task at a time. + +The planner never writes notes itself — planning only. + +Inputs (from RuntimeContext): + messages (list[Msg], required): conversation slice to inspect. + memory_hint (str, optional): caller-supplied hint to bias filename + stem selection or disambiguate same-day tasks. + +Output (written to context.response): + answer: one-line human summary of what was planned. + metadata['memory_updates']: list of ``{path, description}`` dicts. +""" + +from agentscope.agent import ReActAgent +from agentscope.message import Msg +from agentscope.tool import Toolkit +from pydantic import BaseModel, Field + +from ._evolve import format_history, now +from ..base_step import BaseStep +from ...components import R + + +class MemoryUpdateTask(BaseModel): + """One daily-note upsert task emitted by the planner.""" + + path: str = Field( + description="Vault-relative note path, form `daily//.md`. " + "Reuse an existing path to upsert.", + ) + description: str = Field(description="Flat fact checklist — what to preserve, not how to categorize or format.") + + +class MemoryUpdatesPlan(BaseModel): + """Structured output emitted by the planner's finish-tool.""" + + memory_updates: list[MemoryUpdateTask] = Field( + default_factory=list, + description="List of daily-note upsert tasks; empty means nothing worth persisting.", + ) + + +@R.register("auto_memory_planner_step") +class AutoMemoryPlannerStep(BaseStep): + """Plan daily-note upsert tasks via a ReAct agent with structured output.""" + + def __init__(self, console_enabled: bool = False, **kwargs): + super().__init__(**kwargs) + self.console_enabled = console_enabled + self.planner_tools: list[str] = ["daily_list", "read"] + + async def execute(self): + assert self.context is not None + messages: list[Msg] = [ + item if isinstance(item, Msg) else Msg.from_dict(item) for item in self.context.get("messages", []) + ] + memory_hint: str = self.context.get("memory_hint", "") + current = now(self.context.get("timezone")) + + if not messages: + self.context.response.success = True + self.context.response.answer = "Skipped: no messages supplied" + self.context.response.metadata.update({"memory_updates": []}) + return + + toolkit = Toolkit() + for job_name in self.planner_tools: + self.add_as_tool(toolkit, job_name) + + agent = ReActAgent( + name="auto_memory_planner", + model=self.as_llm, + sys_prompt=self.prompt_format("system_prompt"), + formatter=self.as_llm_formatter, + toolkit=toolkit, + ) + agent.set_console_output_enabled(self.console_enabled) + + user_message: str = self.prompt_format( + "user_message", + today=current.strftime("%Y-%m-%d"), + vault_dir=str(self.file_store.vault_path), + note=memory_hint or "(none)", + history=format_history(messages), + ) + + final_msg: Msg = await agent.reply( + Msg(name="reme", role="user", content=user_message), + structured_model=MemoryUpdatesPlan, + ) + + meta: dict = final_msg.metadata if isinstance(final_msg.metadata, dict) else {} + raw_tasks = meta.get("memory_updates") or [] + cleaned: list[dict] = [] + for item in raw_tasks: + if not isinstance(item, dict): + continue + path = str(item.get("path") or "").strip() + description = str(item.get("description") or "").strip() + if path and description and path.endswith(".md") and ".." not in path.split("/"): + cleaned.append({"path": path, "description": description}) + + self.context.response.success = True + self.context.response.metadata.update({"memory_updates": cleaned, "count": len(cleaned)}) + if not cleaned: + self.context.response.answer = "[SKIP] No memory updates planned" + return + + lines = [f"Planned {len(cleaned)} memory update(s):"] + lines += [f"- {t['path']}: {t['description']}" for t in cleaned] + self.context.response.answer = "\n".join(lines) diff --git a/reme4/steps/evolve/auto_memory_planner.yaml b/reme4/steps/evolve/auto_memory_planner.yaml new file mode 100644 index 00000000..3e6b7284 --- /dev/null +++ b/reme4/steps/evolve/auto_memory_planner.yaml @@ -0,0 +1,142 @@ +system_prompt: | + You are the planner of an auto-memory system. Each invocation's task: inspect the recent conversation, then emit a list of daily-note upsert tasks via the `generate_response` finish tool. You never write notes yourself — a separate writer agent handles that. + +user_message: | + Today: {today} + Vault dir: {vault_dir} + Extra hint: {note} + + # Recent conversation + + {history} + + # Your task + + Inspect the conversation above and emit a list of daily-note upsert tasks. Follow the five-step workflow below, then submit your planned `memory_updates` by calling `generate_response`. + + ## Five steps per invocation + + ### Step 1 — Skip check + Is the conversation a substantive exchange that produced information worth long-term memory — such as user preferences, project decisions, technical facts, workflow knowledge, or status updates? Pure greetings or small talk with no such information → emit an empty `memory_updates` list (the orchestrator treats this as a skip). + When truly ambiguous, default to writing — losing memory is worse than an extra note. + + ### Step 2 — Survey today + Call `daily_list` to see all existing `daily//.md` files along with their `name`, `description`, and other metadata. + + ### Step 3 — Read candidates + For any existing note whose `name` / `description` looks relevant to this conversation, call `read path=daily//.md` to see its body. Skip clearly unrelated notes. + + ### Step 4 — Plan paths + + Decide one or more `(path, description)` pairs. Each `path` is the vault-relative note path of form `daily//.md`, where `` is the filename without the `.md` suffix. + + #### 4a. Filename stem naming conventions + + The stem is the note's filename without `.md` — its job is to **uniquely identify** this event/topic, not to cram in all context. Constraints: + + - **Format**: English kebab-case, composed of nouns / noun phrases, stable as a filename. + - **Length**: Usually 2-5 words, roughly 15-50 characters. **Too short and uninformative** (e.g. `bug`, `chat`, `misc`, `notes-1`) → unusable; **too long with progress/blockers/dates packed in** → unusable, those belong in the body and frontmatter. + - **Information density**: Reading the stem alone should identify which event/topic it refers to; progress, status, and context do not go into the stem. + - **Event type** (specific event / task / outage / debug / release): the stem identifies "which event". E.g.: `auth-middleware-rewrite`, `ingest-pipeline-oom`. + - **Topic type** (ongoing concept / knowledge domain / preference / tool recipe): the stem identifies "which topic". E.g.: `pytorch-distributed-training`, `pr-summary-style`. + + #### 4b. Reuse vs. create + + - When the conversation continues a thread you found in Step 3 (same logical event, same topic), **reuse the same path** (same stem under today's folder). Fragmenting one thread across multiple paths is the worst failure mode. + - When no existing note covers the topic, **create a new path** of form `daily//.md` with the stem following 4a conventions. + - A single conversation may span multiple unrelated topics — emit a separate task for each distinct event/topic rather than merging into one giant task. + + #### 4c. What goes in the description + + The description only answers "what facts to preserve" — how to format the body, structure sections, or merge updates is the writer's responsibility, not yours. + + The description is a flat fact checklist listing everything from the conversation worth preserving. Quote key original wording or numbers verbatim. Do not categorize the facts — categorization into note sections is the writer's job. + + Coverage should be comprehensive, including but not limited to: + - Durable facts about the user (role, project, responsibilities, tools, preferences, constraints, goals) + - Domain knowledge (concepts, decision conclusions and rationale, dependency versions, design constraints, system topology) + - Replayable operations (command sequences, script recipes, workflows, debugging steps) + - Current status (progress, blockers, next steps, open questions) + - Timeline events (events that occurred, decision moments) + + ### Step 5 — Submit + Call `generate_response` with `memory_updates=[{{path, description}}, ...]`. If Step 1 determined a skip, emit `[]`. This is your only way to finish — do not produce free text after Step 4; the structured payload from the finish tool is your entire deliverable. + + ## Boundaries + + - You never write notes — do not call `write`, `edit`, or `frontmatter_update`. Those tools belong to the writer. + - When surveying, stay in today's `daily/` folder — do not traverse the entire vault. + - Every `path` you emit must be of form `daily//.md` — never write outside today's daily folder. + - Emit exactly one `memory_updates` list per invocation. Do not call `generate_response` more than once. + + +system_prompt_zh: | + 你是自动记忆系统的规划者。每次调用的任务：研究最近的对话，然后通过 `generate_response` 完成工具发出一份日记 upsert 任务列表。你自己不写笔记——由独立的写入代理负责。 + +user_message_zh: | + 今天：{today} + Vault 目录：{vault_dir} + 额外提示：{note} + + # 最近的对话 + + {history} + + # 你的任务 + + 研究上面的对话，发出一份日记 upsert 任务列表。按下面的五步流程执行，最后通过调用 `generate_response` 提交计划好的 `memory_updates`。 + + ## 每次调用的五个步骤 + + ### 步骤 1 — 跳过检查 + 对话是否产生了值得长期记忆的信息——如用户偏好、项目决策、技术事实、工作流知识或状态更新？纯粹的寒暄或闲聊，且未透露任何此类信息 → 发出空的 `memory_updates` 列表（编排器会视为跳过）。 + 当真正模棱两可时，默认写入——丢失记忆比多写一条笔记更糟。 + + ### 步骤 2 — 概览今天 + 调用 `daily_list` 查看所有现存的 `daily//.md` 及其 `name`、`description` 和其他 metadata 信息。 + + ### 步骤 3 — 阅读候选 + 对任何 `name` / `description` 看起来与本次对话相关的现有笔记，调用 `read path=daily//.md` 直接阅读正文。明显无关的笔记跳过。 + + ### 步骤 4 — 规划 path + + 决定一个或多个 `(path, description)` 对。每个 `path` 是 vault 相对路径，形如 `daily//.md`；其中 `` 段就是笔记文件名去掉 `.md` 后缀的部分。 + + #### 4a. 文件名 stem 命名规范 + + stem 就是笔记文件名去掉 `.md` 的部分——它的职责是**唯一标识**这条 event/topic，不是塞进所有上下文。约束： + + - **格式**：英文 kebab-case，由名词 / 名词短语组成，稳定可作文件名。 + - **长度**：通常 2-5 个词，约 15-50 字符。**太短无信息**（如 `bug`、`chat`、`misc`、`notes-1`）→ 不可用；**太长把进度/卡点/日期都塞进去** → 不可用，那些属于正文与 frontmatter。 + - **信息含量**：读 stem 即可识别该 event/topic 是什么；进度、状态、上下文不进 stem。 + - **event 类型**（具体事件 / 任务 / 故障 / 调试 / 发布）：stem 标识"是哪件事"。例：`auth-middleware-rewrite`、`ingest-pipeline-oom`。 + - **topic 类型**（持续性的概念 / 知识领域 / 偏好 / 工具配方）：stem 标识"是哪类主题"。例：`pytorch-distributed-training`、`pr-summary-style`。 + + #### 4b. 复用 vs 新建 + + - 当对话延续一条你在步骤 3 中发现的现有线索时（同一逻辑事件、同一主题），**复用相同的 path**（即今天目录下相同的 stem）。把一条线索碎片化到多个 path 是最严重的失败模式。 + - 当没有现存笔记覆盖该主题时，**新建一个形如 `daily//.md` 的 path**，stem 段遵循 4a 规范。 + - 一次对话可能跨越多个无关主题——为每个不同的 event/topic 各发一个任务，而不是合成一个巨型任务。 + + #### 4c. description 的内容 + + description 只负责回答"要保留什么事实"——正文如何分 section、frontmatter 怎么写、UPDATE 如何合并都由写入者处理，不在你的职责内。 + + description 是一份扁平的事实清单，列出对话中全部值得保留的内容——关键处逐字引用原始措辞或数字。不要对事实做分类——分到笔记 section 是写入者的工作。 + + 覆盖范围要全面，包括但不限于： + - 用户身份相关的持久事实（角色、项目、职责、工具、偏好、约束、目标） + - 领域知识（概念、决策结论与理由、依赖版本、设计约束、系统拓扑） + - 可重放的操作（命令序列、脚本配方、工作流、调试步骤） + - 当下状态（进度、卡点、下一步、未决问题） + - 时间线事件（发生的事件、决策时刻） + + ### 步骤 5 — 提交 + 调用 `generate_response`，传入 `memory_updates=[{{path, description}}, ...]`。如果步骤 1 判定跳过，就发出 `[]`。这是你唯一的收尾方式——步骤 4 之后不要再产出自由文本；完成工具的结构化负载就是全部交付物。 + + ## 边界 + + - 你从不写笔记——不调用 `write`、`edit`、`frontmatter_update`。那些工具属于写入者。 + - 概览时只待在今天的 `daily/` 文件夹——不要走遍整个 vault。 + - 发出的每个 `path` 必须形如 `daily//.md`——绝不写到今天 daily 目录之外。 + - 每次调用只产出一份 `memory_updates` 列表。不要多次调用 `generate_response`。 diff --git a/reme4/steps/evolve/auto_memory_writer.py b/reme4/steps/evolve/auto_memory_writer.py new file mode 100644 index 00000000..6c0fbf4b --- /dev/null +++ b/reme4/steps/evolve/auto_memory_writer.py @@ -0,0 +1,89 @@ +"""``auto_memory_writer`` — execute daily-note upserts. + +Reads the ``memory_updates`` list produced by ``auto_memory_planner`` +from ``context.response.metadata['memory_updates']``, then iterates +over each ``{path, description}`` task: decides UPDATE vs CREATE by +probing the vault, and writes the note via ``frontmatter_read`` / +``frontmatter_update`` / ``read`` / ``edit`` / ``write``. + +A fresh ReAct agent is created per task to keep conversations isolated. + +Inputs (from RuntimeContext): + messages (list[Msg], required): conversation slice (context). + memory_hint (str, optional): caller-supplied note hint. + response.metadata['memory_updates'] (list[dict]): planner output. + +Output (written to context.response): + answer: one line per task — `` ``. + metadata['written_count']: number of tasks executed. +""" + +from agentscope.agent import ReActAgent +from agentscope.message import Msg +from agentscope.tool import Toolkit + +from ._evolve import format_history, now +from ..base_step import BaseStep +from ...components import R + + +@R.register("auto_memory_writer_step") +class AutoMemoryWriterStep(BaseStep): + """Execute note upserts from the planner's task list.""" + + def __init__(self, console_enabled: bool = False, **kwargs): + super().__init__(**kwargs) + self.console_enabled = console_enabled + self.writer_tools: list[str] = ["frontmatter_read", "frontmatter_update", "read", "edit", "write"] + + async def execute(self): + assert self.context is not None + memory_updates: list[dict] = self.context.response.metadata.get("memory_updates") or [] + if not memory_updates: + self.context.response.success = True + self.context.response.answer = "[SKIP] No memory updates to write" + return + + current = now(self.context.get("timezone")) + messages: list[Msg] = [ + item if isinstance(item, Msg) else Msg.from_dict(item) for item in self.context.get("messages", []) + ] + memory_hint: str = self.context.get("memory_hint", "") + + toolkit = Toolkit() + for job_name in self.writer_tools: + self.add_as_tool(toolkit, job_name) + + results: list[str] = [] + for task in memory_updates: + note_path = task.get("path", "") + description = task.get("description", "") + if not note_path or not description: + continue + + agent = ReActAgent( + name="auto_memory_writer", + model=self.as_llm, + sys_prompt=self.prompt_format("system_prompt"), + formatter=self.as_llm_formatter, + toolkit=toolkit, + ) + agent.set_console_output_enabled(self.console_enabled) + + user_message: str = self.prompt_format( + "user_message", + today=current.strftime("%Y-%m-%d"), + vault_dir=str(self.file_store.vault_path), + note=memory_hint or "(none)", + note_path=note_path, + writing_hint=description, + history=format_history(messages), + ) + + final_msg: Msg = await agent.reply(Msg(name="reme", role="user", content=user_message)) + result_line = (final_msg.get_text_content() or "").strip() + results.append(result_line) + + self.context.response.success = True + self.context.response.answer = "\n".join(results) if results else "[SKIP] No valid tasks" + self.context.response.metadata.update({"written_count": len(results)}) diff --git a/reme4/steps/evolve/auto_memory_writer.yaml b/reme4/steps/evolve/auto_memory_writer.yaml new file mode 100644 index 00000000..fc0f3912 --- /dev/null +++ b/reme4/steps/evolve/auto_memory_writer.yaml @@ -0,0 +1,167 @@ +system_prompt: | + You are the writer of the auto-memory system. The planner hands you exactly one target path + description per call; you create or update the corresponding daily note at that path. You never plan new paths or filenames yourself. + +user_message: | + Today: {today} + Vault directory: {vault_dir} + Extra hint: {note} + Target path: {note_path} + + # Writing hint to reference + + {writing_hint} + + # Recent conversation + + {history} + + # Your task + + Referring to the hint above, create or update the note at the target path. Follow the four-step process below. + + ## Output format: body + + Your job is to capture the facts from the recent conversation into the body — not a single one may be dropped, nor watered down by paraphrase. + + Coverage must be comprehensive, including but not limited to: + - Persistent facts about the user's identity (role, project, responsibilities, tools, preferences, constraints, goals) + - Domain knowledge (concepts, decisions and their rationale, dependency versions, design constraints, system topology) + - Replayable operations (command sequences, script recipes, workflows, debugging steps) + - Current state (progress, blockers, next steps, open questions) + - Timeline events (things that happened, decision moments) + + The body format is free-form — use whatever structure best fits the content. The only hard rule is **completeness**: every fact in writing_hint must appear in the body. Quote the original wording or numbers verbatim at critical points. + + Merge rules for UPDATE: + - Timeline / history entries: append only — never delete existing entries. + - Current-state entries (progress, blockers, next steps): rewrite the whole section to reflect the latest snapshot. + - Everything else: merge and dedupe (keep all old facts, add new facts, drop exact duplicates). + + ## Output format: frontmatter + + Only two required fields: + + ```yaml + --- + name: + description: + --- + ``` + + Rules: + - `name` must be the filename stem of the target path (the part between the last `/` and `.md`), copied verbatim. Do not Title-Case it; do not rewrite it. + - `description` must be an exhaustive summary: mention every key fact, decision, and state point in the body, so the description itself works as a reliable index entry. Vague descriptions like "notes" / "misc" / "various topics" are unacceptable. + - **Never set `status`** — it is a field reserved for the downstream distill stage; touching it will cause the note to be skipped on the next distill run. + - On UPDATE, refresh `description` to reflect the updated body. + + ## The four steps for each call + + ### Step 1 — Probe + Call `frontmatter_read path=`: + - Returns frontmatter dict → **UPDATE branch** (note already exists) + - Returns error / not-found → **CREATE branch** (does not yet exist) + + ### Step 2 — UPDATE branch + 1. `read path=` to view the current body. + 2. Plan the merge per the rules above (timeline appends, facts merge-and-dedupe, current-state rewritten in full). + 3. Execute the write — **prefer `edit` over `write`**; only fall back to `write` when the change is genuinely too sweeping for `edit` to express cleanly: + - **Default: `edit`** — `edit path= old= new=` for each changed section (frontmatter is unaffected). Use multiple `edit` calls if needed to cover several sections. + - After any body change, you must also refresh frontmatter → `frontmatter_update path= metadata={{"description": ""}}`. + - **Fallback: `write`** — only when the body has changed so extensively that multiple edits would be harder to get right than a full rewrite → `write path= name= description= content=`, resetting body and frontmatter in one shot. + + ### Step 2' — CREATE branch + 1. Write the full body, capturing every fact from writing_hint. + 2. Write the frontmatter with `name` and `description`. + 3. `write path= name= description= content=` — done in one call. + + ### Step 3 — Summarize what changed + State in one sentence what you did (which file you created / what you updated). This sentence is your final text output and must be strictly a single line. + + ## Boundaries + + - Each call targets exactly one path: the one assigned. Even if the conversation mentions other notes, do not touch them. + - `write` unconditionally overwrites both body and frontmatter — use it carefully. + +system_prompt_zh: | + 你是自动记忆系统的写入者。规划者每次调用会交给你恰好一个目标路径 + description；你在该路径创建或更新对应的日记。你从不规划新的路径或文件名。 + +user_message_zh: | + 今天：{today} + Vault 目录：{vault_dir} + 额外提示：{note} + 目标路径：{note_path} + + # 可以参考的提示 + + {writing_hint} + + # 最近的对话 + + {history} + + # 你的任务 + + 参考上面的提示，在目标路径创建或更新日记。按下面的四步流程执行。 + + ## 输出格式要求：正文 + + 你的职责是捕捉最近的对话中的事实写入正文——一条都不能漏，也不能被改写淡化。 + + 覆盖范围要全面，包括但不限于： + - 用户身份相关的持久事实（角色、项目、职责、工具、偏好、约束、目标） + - 领域知识（概念、决策结论与理由、依赖版本、设计约束、系统拓扑） + - 可重放的操作（命令序列、脚本配方、工作流、调试步骤） + - 当下状态（进度、卡点、下一步、未决问题） + - 时间线事件（发生的事件、决策时刻） + + 正文格式自由——用最适合内容的结构。唯一的硬性规则是**完整性**：writing_hint 中的每一条事实都必须出现在正文中。关键处逐字引用原始措辞或数字。 + + UPDATE 时的合并规则： + - 时间线 / 历史条目：仅追加，永远不删除已有条目。 + - 当下状态类条目（进度、卡点、下一步）：整段重写，反映最新快照。 + - 其余内容：合并去重（保留全部旧事实，添加新事实，去除完全重复项）。 + + ## 输出格式要求：frontmatter + + 只有两个必填字段： + + ```yaml + --- + name: <必须等于目标路径的文件名 stem，逐字照抄> + description: <正文的详细总结——具体到仅凭 description 就能传达笔记中的全部核心信息> + --- + ``` + + 规则： + - `name` 必须是目标路径的文件名 stem（最后一个 `/` 与 `.md` 之间的部分），逐字照抄。不要 Title-Case 化，不要改写。 + - `description` 必须是详尽的总结：提及正文中的每一个关键事实、决策和状态要点，使 description 本身就能作为可靠的索引条目。模糊的描述如 "notes" / "misc" / "各种主题" 不可接受。 + - **永远不要设置 `status`**——它是下游蒸馏阶段保留的字段；动了它会让笔记在下一次 distill 运行中被忽略。 + - UPDATE 时，刷新 `description` 以反映更新后的正文内容。 + + ## 每次调用的四个步骤 + + ### 步骤 1 — 探测 + 调用 `frontmatter_read path=<目标路径>`： + - 返回 frontmatter 字典 → **UPDATE 分支**（笔记已存在） + - 返回错误 / not-found → **CREATE 分支**（尚不存在） + + ### 步骤 2 — UPDATE 分支 + 1. `read path=<目标路径>` 查看当前正文。 + 2. 按上述规则规划合并（时间线追加，事实合并去重，当下状态整段重写）。 + 3. 执行写入——**优先使用 `edit`，而非 `write`**；仅当变更范围确实过大、edit 难以清晰表达时才退回到 `write`： + - **默认：`edit`** — `edit path=<目标路径> old=<原文片段> new=<替换片段>`，对每个变更区域分别调用（frontmatter 不受影响）。需要时可多次调用 `edit` 覆盖多个区域。 + - 正文变更后，必须同时刷新 frontmatter → `frontmatter_update path=<目标路径> metadata={{"description": "<更新后的总结>"}}`。 + - **退路：`write`** — 仅当正文改动极大、多次 edit 反而更难准确操作时 → `write path=<目标路径> name= description= content=<完整正文>`，一次性重置正文和 frontmatter。 + + ### 步骤 2' — CREATE 分支 + 1. 编写完整正文，捕捉 writing_hint 中的全部事实。 + 2. 编写 frontmatter，包含 `name` 和 `description`。 + 3. `write path=<目标路径> name= description= content=<正文内容>`——一次性完成。 + + ### 步骤 3 — 总结改动内容 + 用一句话说明你做了什么（创建了哪个文件 / 更新了哪些内容）。这句话是你最后一次文本输出，必须严格只有一行。 + + ## 边界 + + - 每次调用只针对一个目标路径：被分配的那个。即使对话里提到其他笔记，也不要碰。 + - `write` 会无条件覆盖正文和frontmatter，请谨慎使用。 diff --git a/reme4/steps/file_io/daily_list.py b/reme4/steps/file_io/daily_list.py index 0552e975..05d9bedc 100644 --- a/reme4/steps/file_io/daily_list.py +++ b/reme4/steps/file_io/daily_list.py @@ -37,11 +37,27 @@ class DailyListStep(BaseStep): """Keep only the user-facing keys (drop internal scan_notes fields, if any).""" return {"path": note["path"], "slug": note["slug"], "metadata": note["metadata"]} + @staticmethod + def _format_note_line(note: dict) -> str: + """Format a single note as ``- path: ... name: ... description: ... ``.""" + meta: dict = note.get("metadata", {}) + ordered_keys = [k for k in ("name", "description") if k in meta] + ordered_keys += [k for k in meta if k not in ("name", "description")] + parts = [f"- path: {note['path']}"] + for key in ordered_keys: + value = meta[key] + if value is None or value == "": + continue + value_str = str(value).replace("\r\n", " ").replace("\r", " ").replace("\n", " ") + parts.append(f"{key}: {value_str}") + return " ".join(parts) + async def execute(self): """Scan ``//`` and emit one projected record per note.""" assert self.context is not None day, daily_dir, vault_dir = self._collect_params() notes = [self._project(n) for n in scan_notes(vault_dir, day, daily_dir)] self.context.response.success = True - self.context.response.answer = f"Listed {len(notes)} note(s) for {day}" - self.context.response.metadata.update({"date": day, "notes": notes}) + lines = [self._format_note_line(n) for n in notes] + self.context.response.answer = "\n".join(lines) if lines else f"No notes found for {day}" + self.context.response.metadata.update({"date": day, "count": len(notes)}) diff --git a/reme4/steps/file_io/edit.py b/reme4/steps/file_io/edit.py index 4e6809aa..34160e7d 100644 --- a/reme4/steps/file_io/edit.py +++ b/reme4/steps/file_io/edit.py @@ -1,6 +1,7 @@ """Find-and-replace text in a markdown file body (front matter is preserved).""" import frontmatter +import yaml from ._file_io import ( NON_MD_WARNING, @@ -73,7 +74,11 @@ class EditStep(BaseStep): # Markdown: parse frontmatter and operate on body only. Non-markdown: # there's no frontmatter convention, so operate on the full text. if is_md: - post = frontmatter.loads(raw_text) + try: + post = frontmatter.loads(raw_text) + except yaml.YAMLError as exc: + self._fail(f"failed to parse frontmatter in {target}: {exc}", path=str(target)) + return None body = post.content not_found_msg = ( f"text to replace was not found in the body of {target} (front matter is excluded from edit)" diff --git a/reme4/steps/file_io/frontmatter_read.py b/reme4/steps/file_io/frontmatter_read.py index c01bf6f5..c9c28e82 100644 --- a/reme4/steps/file_io/frontmatter_read.py +++ b/reme4/steps/file_io/frontmatter_read.py @@ -11,6 +11,7 @@ doesn't exist; otherwise ``{exists: true, frontmatter: {...}}``. from pathlib import Path import frontmatter +import yaml from ..base_step import BaseStep from ...components import R @@ -37,7 +38,13 @@ class FrontmatterReadStep(BaseStep): self.context.response.metadata.update({"path": path, "error": "not markdown"}) return - meta = dict(frontmatter.loads(target.read_text(encoding="utf-8")).metadata) + try: + meta = dict(frontmatter.loads(target.read_text(encoding="utf-8")).metadata) + except yaml.YAMLError as exc: + self.context.response.success = False + self.context.response.answer = f"Error: failed to parse frontmatter in {path}: {exc}" + self.context.response.metadata.update({"path": path, "exists": True, "error": str(exc)}) + return self.context.response.success = True self.context.response.answer = f"Read frontmatter from {path} ({len(meta)} key(s))" self.context.response.metadata.update({"path": path, "exists": True, "frontmatter": meta}) diff --git a/reme4/steps/jobs/__init__.py b/reme4/steps/jobs/__init__.py index 952c3cd4..ff1927ec 100644 --- a/reme4/steps/jobs/__init__.py +++ b/reme4/steps/jobs/__init__.py @@ -1,10 +1,3 @@ -"""Jobs steps — composite ReAct-agent-driven workflows. - -Two steps: - - digester — cold-write: distill daily notes into digest/ (R-M-W via a ReAct agent). - synchronizer — hot-write: persist in-progress task as a daily note. -""" +"""Jobs steps — composite ReAct-agent-driven workflows.""" from . import digester # noqa: F401 -- @R.register("digester") -from . import synchronizer # noqa: F401 -- @R.register("synchronizer") diff --git a/reme4/steps/jobs/digester.py b/reme4/steps/jobs/digester.py index 641ae39e..f40b7f74 100644 --- a/reme4/steps/jobs/digester.py +++ b/reme4/steps/jobs/digester.py @@ -1,6 +1,6 @@ """Smart Digester — knowledge distillation from daily notes to digest/. -The Digester is the **cold-write** counterpart to Synchronizer (hot-write). +The Digester is the **cold-write** counterpart to AutoMemory (hot-write). It reads completed work in ``daily//.md`` note files, identifies entities / concepts / claims / methods worth preserving long-term, and sinks them into ``digest/`` as canonical-entry nodes so @@ -26,7 +26,7 @@ extra). After processing each daily, the agent must call (or ``metadata={"status": "skipped"}`` when intentionally bypassed). Convention: absent ≡ ``pending``, so the next pass finds residual work via ``file_list path=daily recursive=true`` + per-item ``frontmatter_read`` to filter for absent ``status``. -Only the digester writes ``status``; Synchronizer / hand-edits must +Only the digester writes ``status``; AutoMemory / hand-edits must leave it alone. No degraded path — distillation strictly requires an LLM. When ``as_llm`` diff --git a/reme4/steps/jobs/synchronizer.py b/reme4/steps/jobs/synchronizer.py deleted file mode 100644 index 0eabaede..00000000 --- a/reme4/steps/jobs/synchronizer.py +++ /dev/null @@ -1,276 +0,0 @@ -"""Synchronizer — daily-note sync ReAct agent. - -Watches the agent's recent conversation and persists in-progress -tasks as a daily note inside reme's vault_dir, so future agent -invocations can pick the work back up. Pure sync mechanism — does -**not** compress the agent's context (compression is the agent's -own concern). - -Note layout: a single markdown file ``daily//.md``. -Everything worth preserving (verbatim user prompt, key tool output, -intermediate data) goes inline inside this file — there are no -sibling materials. References to it use the full path relative to -the vault (``[[daily//.md]]``); short or no-extension -forms do not resolve. Frontmatter carries ``name`` / ``description`` -plus an optional ``inherits`` wikilink for cross-day continuation. -Body splits into ``Objective`` / ``Plan`` / ``Progress`` / -``Findings`` / ``Decisions`` / ``Next`` / ``References`` sections -(the last is a list of ``[[resource//]]`` wikilinks for -inbound assets landed via ``ingest`` — non-markdown -artefacts the task itself produced are summarized inline). - -Inputs (from RuntimeContext): - messages (list[Msg], required): conversation slice to inspect. - note (str, optional): caller-supplied note hint (task name - or ``daily//.md`` path) to bias slug selection - and disambiguate same-day tasks. - -Output (written to context.response.answer): - { - "skipped": True if the agent reported [SKIP], - "actions": one-line action statement from the agent, - "note": note file path relative to the vault, or None, - "summary": full markdown content of the synced note, - for the calling agent to reload into a - compacted context. None when SKIP / failed. - } - -The agent's toolkit is assembled by ``add_as_tool`` — each entry in -``_NOTE_TOOLS`` is a job name (registered in the active config); -the wrapper turns ``job(**kwargs)`` into a ``ToolResponse``. The job -indirection means the agent sees exactly the same tool surface -(name / description / parameter schema) as the L2 MCP layer. - -Override interface — note shape (single-file layout, frontmatter -fields, section discipline) is opinionated convention, not a core -invariant, so callers can fully replace it without touching reme4: - -* ``prompt_dict`` (inherited from ``BaseStep``) overrides the - ``system_prompt`` / ``user_message`` templates wholesale — this is - how a service layer swaps in its own note schema (e.g. a different - section list, different frontmatter fields). -* ``toolkit`` replaces the tool surface ``_NOTE_TOOLS`` builds. - -What ships here (``synchronizer.yaml``) is just one viable convention; -the service layer (plugin configs, custom callers) is the right place -to pin down the *deployment-specific* shape. -""" - -import datetime -import re -import zoneinfo -from pathlib import Path - -from agentscope.agent import ReActAgent -from agentscope.message import Msg -from agentscope.tool import Toolkit -from pydantic import BaseModel, Field - -from ..base_step import BaseStep - -from ...components import R - - -_NOTE_PATH_RE = re.compile(r"daily/\d{4}-\d{2}-\d{2}/[^/\s]+\.md") - - -_NOTE_TOOLS: tuple[str, ...] = ( - "file_list", - "file_read", - "file_write", - "file_append", - "file_edit", - "file_stat", - "frontmatter_read", - "frontmatter_update", - "frontmatter_delete", - "daily_read", - "daily_write", - "daily_reindex", -) - - -def _coerce_messages(raw) -> list[Msg]: - """Normalize incoming messages to ``Msg`` instances. - - The Python caller hands in ``list[Msg]`` directly; the MCP layer - delivers ``list[dict]`` (each dict shaped roughly ``{name?, role?, - content?}``). Both shapes land here. - """ - if not raw: - return [] - out: list[Msg] = [] - for item in raw: - if isinstance(item, Msg): - out.append(item) - continue - if isinstance(item, dict): - out.append( - Msg( - name=item.get("name") or item.get("role") or "user", - role=item.get("role") or "user", - content=item.get("content", ""), - ), - ) - return out - - -def _format_history(messages: list[Msg]) -> str: - """Render the conversation as a speaker-tagged transcript. - - Skips messages whose text content is empty (tool-only frames - don't help the LLM judge task state). - """ - if not messages: - return "(empty)" - lines: list[str] = [] - for msg in messages: - speaker = msg.name or msg.role or "?" - text = (msg.get_text_content() or "").strip() - if not text: - continue - lines.append(f"[{speaker}]\n{text}") - return "\n\n".join(lines) or "(no text)" - - -class SynchronizerResult(BaseModel): - """Outcome of a single note-sync call. - - Without per-tool audit (the agent's toolkit is the job surface, - which doesn't expose per-call records back to the orchestrator), - the structured outcome is just what the agent reports plus what - we re-read from disk after it returns. - """ - - used_llm: bool = Field(default=False) - skipped: bool = Field(default=False) - actions: str = Field( - default="", - description="One-line action statement from the agent (e.g. " - "'updated daily/2026-05-15/auth-refactor.md' or '[SKIP]').", - ) - note: str | None = Field( - default=None, - description="Note file path relative to the vault, " - "e.g. 'daily/2026-05-15/auth-refactor.md'. None when SKIP / failed.", - ) - summary: str | None = Field( - default=None, - description="Full markdown content of the note file. Lets the calling " - "agent reload the warm summary into a freshly compacted context without an extra read.", - ) - - -@R.register("synchronizer") -class Synchronizer(BaseStep): - """Drive daily-note sync via a ReAct agent.""" - - def __init__( - self, - toolkit: Toolkit | None = None, - console_enabled: bool = False, - timezone: str | None = None, - inherit_window_days: int = 7, - **kwargs, - ): - super().__init__(**kwargs) - self.toolkit = toolkit - self.console_enabled = console_enabled - self.timezone = timezone - self.inherit_window_days = inherit_window_days - - def _now(self) -> datetime.datetime: - if self.timezone: - try: - return datetime.datetime.now(zoneinfo.ZoneInfo(self.timezone)) - except Exception as e: - self.logger.error( - f"Invalid timezone {self.timezone!r}, falling back to local time: {e}", - ) - return datetime.datetime.now() - - def _vault_dir(self) -> Path: - wd = getattr(self.file_store, "vault_path", None) - return Path(wd).resolve() if wd else Path.cwd().resolve() - - def _build_toolkit(self) -> Toolkit: - """Bind every note-relevant job as a tool function. - - Each entry in ``_NOTE_TOOLS`` is a job name registered in the - active config; ``add_as_tool`` wraps ``job(**kwargs)`` into a - ``ToolResponse``. The job indirection means the agent sees exactly - the same tool surface (name / description / parameter schema) as - the L2 MCP layer. - """ - toolkit = self.toolkit or Toolkit() - for job_name in _NOTE_TOOLS: - self.add_as_tool(toolkit, job_name) - return toolkit - - async def execute(self): - assert self.context is not None - messages: list[Msg] = _coerce_messages(self.context.get("messages")) - note_hint: str = self.context.get("note", "") or "" - - if not messages: - result = SynchronizerResult(used_llm=False, skipped=True) - self.context.response.success = True - self.context.response.answer = "Skipped: no messages supplied" - self.context.response.metadata.update(result.model_dump()) - return - - toolkit = self._build_toolkit() - - agent = ReActAgent( - name="reme_synchronizer", - model=self.as_llm, - sys_prompt=self.prompt_format("system_prompt"), - formatter=self.as_llm_formatter, - toolkit=toolkit, - ) - agent.set_console_output_enabled(self.console_enabled) - - user_message: str = self.prompt_format( - "user_message", - today=self._now().strftime("%Y-%m-%d"), - vault_dir=str(self._vault_dir()), - inherit_window_days=self.inherit_window_days, - note=note_hint or "(none)", - history=_format_history(messages), - ) - - final_msg: Msg = await agent.reply( - Msg(name="reme", role="user", content=user_message), - ) - actions = (final_msg.get_text_content() or "").strip() - - result = SynchronizerResult(used_llm=True, actions=actions) - if "[SKIP]" in actions.upper(): - result.skipped = True - - # Reload the freshly written note so the calling agent can drop - # it back into a compacted context without an extra read trip. - if not result.skipped: - self._reload_note(result, actions) - - self.context.response.success = True - self.context.response.answer = actions or "Synchronization completed" - self.context.response.metadata.update(result.model_dump()) - - def _reload_note(self, result: SynchronizerResult, actions: str) -> None: - """Parse the agent's action line for the note path and read the - full file back into ``result.summary``. Best-effort: a parse miss - leaves the context-management fields as None but does not fail - the step (persistence already succeeded).""" - match = _NOTE_PATH_RE.search(actions) - if not match: - return - note_path = match.group(0) - try: - absolute = (Path(self.file_store.vault_path or ".") / note_path).resolve() - text = absolute.read_text(encoding="utf-8") - except Exception as e: - self.logger.warning(f"synchronizer: could not reload note {note_path!r}: {e}") - return - result.note = note_path - result.summary = text diff --git a/reme4/steps/jobs/synchronizer.yaml b/reme4/steps/jobs/synchronizer.yaml deleted file mode 100644 index 137e130f..00000000 --- a/reme4/steps/jobs/synchronizer.yaml +++ /dev/null @@ -1,144 +0,0 @@ -system_prompt: | - You persist an in-progress task as a daily note so the next session - can pick it up. One call writes at most one note. - - ## Note shape - - /daily//.md # the whole note, one file - - One note per task per day; cross-day continuation uses an - `inherits:` frontmatter wikilink. Anything worth preserving - (verbatim user prompt, key tool output, intermediate data) goes - inline inside this file — there are no sibling materials. - - ## Five steps per call - - ### Step 1 — Skip check - - Is the conversation a real multi-step task in progress? Casual Q&A, a - single-shot answer, or idle chat → reply `[SKIP]` (alone, literal) and - stop. When genuinely ambiguous, default to writing — losing work is - worse than an extra note. - - ### Step 2 — Pick the slug - - - Note hint already a kebab-case slug → use verbatim. - - Otherwise mint a stable kebab-case from the task topic (≤60 chars). - - **Reuse the same slug across calls for the same logical thread** — - same slug = same file = upsert. Fragmenting one thread across - multiple slugs is the worst failure mode. - - ### Step 3 — Discover the branch - - Call `file_list path=daily/{today}` (and `daily_read` / - `frontmatter_read` on candidates as needed) to determine one of three - branches: - - - **UPDATE** — `daily/{today}/.md` already exists: - `daily_read slug=` to fetch body + frontmatter, merge per - Step 4, then `daily_write slug= overwrite=true` with the - full new body + frontmatter. - - - **INHERIT** — no file today, but within the last - {inherit_window_days} days an active note with the same name - exists: confirm via `daily_read` on the earlier note, then - `daily_write slug=` (default `overwrite=false`) with a - fresh body that sets `inherits: [[daily//.md]]` - in frontmatter and copies the predecessor's `Objective` + `Plan`. - Progress / Findings / Decisions / Next start empty. **Do not - modify the predecessor.** - - - **CREATE** — neither: `daily_write slug=` (default - `overwrite=false`) with the full body + frontmatter. Idempotent — - no-ops if a same-slug note already exists today (caller falls - back to the UPDATE branch). - - ### Step 4 — Write - - Pick the smallest write for each change: - - | Change | Tool | - |---|---| - | New / replacement full note | `daily_write` (one shot: body + frontmatter + index refresh) | - | Append to a trailing append-only section | `file_append` — cheaper than R-M-W | - | One frontmatter key | `frontmatter_update` (call `daily_reindex` after if `name`/`description` changed) | - | Mid-body restructure | `daily_read` + `daily_write overwrite=true` | - - Suggested body shape (sections are convention — adapt as fits): - - ```markdown - --- - name: - description: <2-3 sentences: what + why> - inherits: [[daily//.md]] # INHERIT only - --- - ## Objective - - - ## Plan - - - ## Progress - - - - ## Findings - - - - ## Decisions - - - - ## Next - - [ ] - - ## References - - [[resource//]] — # external assets the task consumed - ``` - - Section discipline: Progress / Findings / Decisions are append-only - (never delete history). Plan / Next are wholesale-rewritten each call. - Objective is set once. `## References` lists `[[resource//]]` - wikilinks for inbound assets that arrived through an external channel - and were landed by `ingest`; non-markdown content the task - itself produced (raw outputs, screenshots) should be summarized inline - or skipped — there is no per-note sibling folder anymore. - - Wikilink form is fixed: note refs use the full vault-relative - path with `.md` (`[[daily//.md]]`); resource refs use the - canonical resource path (`[[resource//]]`). Short forms - don't resolve. - - ### Step 5 — Emit one line - - Your final reply must be exactly one line in this form: - - daily//.md - - - `` ∈ `created` / `inherited` / `updated` - - Or `[SKIP]` (literal, alone) if Step 1 said skip. This line is parsed - mechanically — match the format exactly. - - ## Boundaries - - - **Never write under `digest/`** — that tier is downstream. - - **Never write the `status` frontmatter** — `status` is reserved for - the downstream distillation pass, which uses absence to find pending - work. Touching it from here makes the note invisible to the - next distill run. - - **`daily_write` defaults to `overwrite=false`** (idempotent skip-if-exists, - mirroring the old `daily_resolve` probe). Pass `overwrite=true` only when - you've already read the file via `daily_read` and intend to replace it - (the UPDATE branch). On a surprise collision (`created: false`) — fall - back to UPDATE rather than blindly overwriting. - -user_message: | - Today: {today} - Vault dir: {vault_dir} - Inherit window: last {inherit_window_days} days - Note hint: {note} - - # Recent conversation - - {history} - - Run the five steps from the system prompt. Final reply = one line. diff --git a/reme4/steps/transfer/ingest.py b/reme4/steps/transfer/ingest.py index 1d9b31e1..4ecf87c1 100644 --- a/reme4/steps/transfer/ingest.py +++ b/reme4/steps/transfer/ingest.py @@ -63,7 +63,7 @@ Parameters: only. * ``description`` (required) — analysis hint for downstream agents: where the asset came from, what kind of content it carries, and how - it should be interpreted. The digester / synchronizer reads this + it should be interpreted. The digester / auto_memory reads this verbatim from ``meta.json`` to decide how to read the asset (skim vs. deep parse, structured extraction vs. summarization, etc.), so callers should write enough detail to drive that decision — not diff --git a/tests/test_base_file_watcher.py b/tests/test_base_file_watcher.py index dcf56abf..60022aa9 100644 --- a/tests/test_base_file_watcher.py +++ b/tests/test_base_file_watcher.py @@ -12,7 +12,7 @@ Usage: pytest tests/test_base_file_watcher.py -v -k "test_existing" """ -# pylint: disable=redefined-outer-name,protected-access,unused-argument +# pylint: disable=redefined-outer-name,protected-access,unused-argument,no-name-in-module import asyncio import tempfile diff --git a/tests/test_reme_light_watch_paths.py b/tests/test_reme_light_watch_paths.py index 4d0a91f9..70afc7e8 100644 --- a/tests/test_reme_light_watch_paths.py +++ b/tests/test_reme_light_watch_paths.py @@ -7,6 +7,7 @@ filesystems (Windows NTFS, macOS APFS/HFS+). See agentscope-ai/ReMe#228. """ # pylint: disable=redefined-outer-name,protected-access,missing-function-docstring,missing-class-docstring +# pylint: disable=no-name-in-module import tempfile from pathlib import Path diff --git a/tests4/integration/__init__.py b/tests4/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests4/integration/test_as_llm.py b/tests4/integration/test_as_llm.py new file mode 100644 index 00000000..909caab6 --- /dev/null +++ b/tests4/integration/test_as_llm.py @@ -0,0 +1,92 @@ +"""Integration tests: drive ReActAgent through LLMDemoStep + Application wiring. + +Requires LLM_API_KEY (and optionally LLM_BASE_URL / LLM_MODEL_NAME) in the +environment or a .env file at the repo root. Hits the real Anthropic API. +""" + +import asyncio +import os +import tempfile + +from reme4 import Application +from reme4.config import resolve_app_config +from reme4.steps.common.llm_demo import LLMDemoStep +from reme4.utils import load_env + +load_env() + + +class _temp_chdir: + """chdir to path for the duration of the block; restore on exit.""" + + def __init__(self, path): + self.path = path + self._old = None + + def __enter__(self): + self._old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self._old) + + +async def _make_app() -> Application: + """Build and start an Application from the default config (LLM wired via env vars).""" + cfg = resolve_app_config(log_to_console=False, log_to_file=False, enable_logo=False) + app = Application(**cfg) + await app.start() + return app + + +def test_llm_demo_step_basic_chat(): + """LLMDemoStep drives ReActAgent through self.as_llm/as_llm_formatter.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + step = LLMDemoStep(app_context=app.context) + response = await step( + query="What is 1 + 1? Reply with just the number.", + ) + text = (response.answer or "").strip() + print(f"\n[basic_chat] response: {text!r}") + assert text, "Empty assistant response" + assert "2" in text, f"Expected '2' in response, got: {text!r}" + print("✓ test_llm_demo_step_basic_chat passed") + finally: + await app.close() + + asyncio.run(run()) + + +def test_llm_demo_step_with_tool(): + """LLMDemoStep registers the add tool and the agent invokes it.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + step = LLMDemoStep(app_context=app.context) + response = await step( + query="Use the add tool to compute 21 + 21 and report the result.", + sys_prompt="Use the `add` tool whenever the user asks to add numbers.", + use_add_tool=True, + ) + text = (response.answer or "").strip() + print(f"\n[with_tool] response: {text!r}") + assert "42" in text, f"Expected '42' in response, got: {text!r}" + print("✓ test_llm_demo_step_with_tool passed") + finally: + await app.close() + + asyncio.run(run()) + + +if __name__ == "__main__": + print("=== LLMDemoStep + ReActAgent integration tests ===") + test_llm_demo_step_basic_chat() + test_llm_demo_step_with_tool() + print("\nAll integration tests passed!") diff --git a/tests4/integration/test_auto_memory.py b/tests4/integration/test_auto_memory.py new file mode 100644 index 00000000..88ec0eb6 --- /dev/null +++ b/tests4/integration/test_auto_memory.py @@ -0,0 +1,355 @@ +"""Integration test for the auto_memory job (planner + writer end-to-end). + +Drives the full ``auto_memory`` orchestrator against a real LLM. The scenario +seeds one existing daily note covering an ongoing event, then feeds a +10-message conversation that: + +1. continues the existing event with new status / facts (expects an UPDATE + that preserves the old facts and appends the new ones), and +2. introduces a brand-new topic (expects a CREATE under today's daily folder + with a stem distinct from the seeded one). + +Requires LLM_API_KEY (and optionally LLM_BASE_URL / LLM_MODEL_NAME) in the +environment or a .env file at the repo root. Hits the real Anthropic API. +""" + +import asyncio +import json +import os +import tempfile +from datetime import date as _date +from pathlib import Path + +from agentscope.agent import ReActAgent + +from reme4 import Application +from reme4.config import resolve_app_config +from reme4.utils import load_env + +load_env() + +# Where agent.memory jsonl dumps land — same directory as this test file. +DUMP_DIR = Path(__file__).resolve().parent + + +SEED_STEM = "auth-middleware-rewrite" +SEED_BODY = """--- +name: auth-middleware-rewrite +description: JWT auth middleware rewrite driven by legal/compliance requirements around session token storage +--- + +# 背景 + +- 项目：JWT auth middleware 重写，替换旧的 session middleware +- 动机：legal/compliance 要求，旧的 session token 存储方式不符合新合规要求 +- 决策：采用 RS256 签名，密钥放在 KMS，refresh token 写 redis 集群 +- 团队：Alice 主导，Bob 协助 + +# 时间线 + +- 2026-05-20 立项 kickoff +- 2026-05-23 设计评审通过 + +# 当下状态 + +- 进度：实现中 +- 卡点：暂无 +- 下一步：完成 refresh token 写入流程 +""" + + +def _today() -> str: + return _date.today().isoformat() + + +class _temp_chdir: + def __init__(self, path): + self.path = path + self._old = None + + def __enter__(self): + self._old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self._old) + + +async def _make_app() -> Application: + cfg = resolve_app_config(log_to_console=False, log_to_file=False, enable_logo=False) + app = Application(**cfg) + await app.start() + return app + + +def _seed_note(vault_root: Path, today: str) -> Path: + """Write the existing auth-middleware-rewrite note under today's daily folder.""" + day_dir = vault_root / "daily" / today + day_dir.mkdir(parents=True, exist_ok=True) + path = day_dir / f"{SEED_STEM}.md" + path.write_text(SEED_BODY, encoding="utf-8") + return path + + +def _make_messages() -> list[dict]: + """A 10-turn conversation: first half continues the auth event, second half opens a new topic.""" + return [ + { + "name": "user", + "role": "user", + "content": "状态更新：PR #432（auth middleware rewrite）今天已经合并到 dev 分支，等待 staging 验收。", + }, + { + "name": "assistant", + "role": "assistant", + "content": "好的，已记录。staging 验收前要先跑回归测试吗？", + }, + { + "name": "user", + "role": "user", + "content": ( + "对。测试时发现 refresh token TTL 设 7d 在 redis 集群挂了——" + "redis maxmemory-policy 默认 allkeys-lru，会随机驱逐 token，导致用户被强制登出。" + ), + }, + { + "name": "assistant", + "role": "assistant", + "content": "理解，要切到 volatile-ttl 才能只驱逐带 TTL 的 key，对吧？", + }, + { + "name": "user", + "role": "user", + "content": ( + "对，下一步：周五 2026-05-29 前把 redis 配置改成 volatile-ttl 并重测，" "blocked 在 SRE @lihua 的排期。" + ), + }, + { + "name": "user", + "role": "user", + "content": ( + "切个话题，最近在调 pytorch 分布式训练。结论：DDP 启动推荐用 torchrun，" "比 mp.spawn 稳很多。" + ), + }, + { + "name": "assistant", + "role": "assistant", + "content": "是因为信号处理的原因吗？", + }, + { + "name": "user", + "role": "user", + "content": ( + "主要是 NCCL backend 初始化更干净。mp.spawn 在 4 卡以上偶尔会卡死握手；" + "复现版本 pytorch 2.5.1 + nccl 2.21.5。" + ), + }, + { + "name": "user", + "role": "user", + "content": ( + "另外 batch size 用 64*world_size，per-rank lr 用 linear scaling rule " + "（lr = base_lr * world_size）。" + ), + }, + { + "name": "user", + "role": "user", + "content": "这两件事都先记一下。", + }, + ] + + +def _read_text(p: Path) -> str: + return p.read_text(encoding="utf-8") + + +class _AgentMemoryRecorder: + """Monkey-patches ReActAgent.__init__ to capture every agent created inside + the ``with`` block, then dumps each agent's memory to a jsonl file in + DUMP_DIR on exit. One file per agent: ``agent_memory__.jsonl``, + one message per line as ``Msg.to_dict()``. + """ + + def __init__(self, dump_dir: Path, prefix: str = "agent_memory"): + """init""" + self.dump_dir = dump_dir + self.prefix = prefix + self.agents: list[ReActAgent] = [] + self._orig_init = None + self.dumped_paths: list[Path] = [] + + def __enter__(self): + """Monkey-patch ReActAgent.__init__.""" + self._orig_init = ReActAgent.__init__ + agents = self.agents + orig = self._orig_init + + def _capturing_init(agent_self, *args, **kwargs): + orig(agent_self, *args, **kwargs) + agents.append(agent_self) + + ReActAgent.__init__ = _capturing_init + return self + + def __exit__(self, *exc): + """Restore the original __init__ and dump all agent memories.""" + ReActAgent.__init__ = self._orig_init + + async def dump(self) -> list[Path]: + """Dump all agent memories.""" + # Wipe any prior dumps from this prefix so reruns don't accumulate stale files. + for stale in self.dump_dir.glob(f"{self.prefix}_*.jsonl"): + stale.unlink() + + for idx, agent in enumerate(self.agents, 1): + messages = await agent.memory.get_memory() + name = getattr(agent, "name", "agent") or "agent" + out_path = self.dump_dir / f"{self.prefix}_{idx:02d}_{name}.jsonl" + with out_path.open("w", encoding="utf-8") as f: + for msg in messages: + f.write(json.dumps(msg.to_dict(), ensure_ascii=False, default=str) + "\n") + self.dumped_paths.append(out_path) + return self.dumped_paths + + +def test_auto_memory_updates_existing_and_creates_new(): + """End-to-end: planner survey + UPDATE one seeded note + CREATE one new note.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + vault_root = Path(app.config.vault_dir).absolute() + today = _today() + seed_path = _seed_note(vault_root, today) + seed_before = _read_text(seed_path) + assert "legal/compliance" in seed_before + + day_dir = vault_root / "daily" / today + files_before = {p.name for p in day_dir.glob("*.md")} + assert files_before == {f"{SEED_STEM}.md"} + + messages = _make_messages() + + print("\n" + "=" * 70) + print("[setup] vault_root =", vault_root) + print("[setup] today =", today) + print("[setup] seed_path =", seed_path) + print(f"[setup] seed body ({len(seed_before)} bytes):\n{seed_before}") + print(f"[setup] feeding {len(messages)} messages to auto_memory:") + for i, m in enumerate(messages, 1): + print(f" {i:2d}. [{m['role']}] {m['content']}") + print("=" * 70) + + with _AgentMemoryRecorder(DUMP_DIR) as recorder: + response = await app.run_job("auto_memory", messages=messages) + dumped = await recorder.dump() + print(f"\n[dump] captured {len(recorder.agents)} agent(s); wrote {len(dumped)} jsonl file(s):") + for p in dumped: + print(f" - {p}") + + # --- response-level assertions ------------------------------- + assert response.success is True, f"job failed: {response.answer!r}" + meta = response.metadata or {} + memory_updates = meta.get("memory_updates") or [] + + print("\n" + "=" * 70) + print(f"[planner] planned {len(memory_updates)} update(s):") + for u in memory_updates: + print(f" - path: {u.get('path')}") + print(f" description: {u.get('description')}") + print(f"\n[writer] written_count = {meta.get('written_count')}") + print(f"[writer] answer:\n{response.answer}") + print("=" * 70) + + assert ( + len(memory_updates) >= 2 + ), f"expected at least 2 planned updates (1 UPDATE + 1 CREATE), got {memory_updates}" + + # Every emitted path must live under today's daily folder. + paths = [u["path"] for u in memory_updates] + for p in paths: + assert p.startswith(f"daily/{today}/") and p.endswith( + ".md", + ), f"path {p!r} violates daily//.md shape" + + # --- UPDATE branch: seeded note ------------------------------ + update_path_str = f"daily/{today}/{SEED_STEM}.md" + assert ( + update_path_str in paths + ), f"planner did not reuse the seeded path {update_path_str!r}; got {paths}" + seed_after = _read_text(seed_path) + + print("\n" + "=" * 70) + print(f"[UPDATE] {seed_path} ({len(seed_before)} → {len(seed_after)} bytes)") + print(f"[UPDATE] body after:\n{seed_after}") + print("=" * 70) + + # Old facts must survive (timeline append + facts merge-and-dedupe). + for old_fact in ("legal/compliance", "RS256", "Alice"): + assert ( + old_fact in seed_after + ), f"UPDATE dropped pre-existing fact {old_fact!r}\n--- AFTER ---\n{seed_after}" + # At least some of the new facts from the conversation must land. + new_hits = [ + needle + for needle in ("PR #432", "432", "volatile-ttl", "maxmemory-policy", "2026-05-29") + if needle in seed_after + ] + print("[UPDATE] preserved old facts: ['legal/compliance', 'RS256', 'Alice']") + print(f"[UPDATE] landed new facts: {new_hits}") + assert ( + len(new_hits) >= 2 + ), f"UPDATE only landed {new_hits!r} of expected new facts\n--- AFTER ---\n{seed_after}" + + # --- CREATE branch: new topic -------------------------------- + files_after = {p.name for p in day_dir.glob("*.md")} + new_files = files_after - files_before + print(f"\n[CREATE] new files under daily/{today}/: {sorted(new_files)}") + assert new_files, f"no new note created under daily/{today}/; planner paths: {paths}" + # Find the file that actually covers the pytorch topic. + pytorch_path: Path | None = None + for fname in new_files: + text = _read_text(day_dir / fname) + if any(kw in text for kw in ("torchrun", "NCCL", "pytorch", "mp.spawn")): + pytorch_path = day_dir / fname + break + assert pytorch_path is not None, f"no created note covers the pytorch topic; new files: {new_files}" + pytorch_text = _read_text(pytorch_path) + + print("=" * 70) + print(f"[CREATE] {pytorch_path} ({len(pytorch_text)} bytes)") + print(f"[CREATE] body:\n{pytorch_text}") + print("=" * 70) + + topic_hits = [ + needle + for needle in ("torchrun", "mp.spawn", "NCCL", "2.5.1", "linear scaling", "world_size") + if needle in pytorch_text + ] + print(f"[CREATE] landed topic facts: {topic_hits}") + assert ( + len(topic_hits) >= 3 + ), f"CREATE only captured {topic_hits!r} of expected new-topic facts\n--- CREATE ---\n{pytorch_text}" + # frontmatter sanity: name should equal the file stem, description non-empty. + stem = pytorch_path.stem + assert ( + f"name: {stem}" in pytorch_text + ), f"frontmatter name does not match stem {stem!r}\n{pytorch_text[:400]}" + print(f"[CREATE] frontmatter name matches stem {stem!r}") + + print("\n" + "=" * 70) + print("✓ test_auto_memory_updates_existing_and_creates_new passed") + print("=" * 70) + finally: + await app.close() + + asyncio.run(run()) + + +if __name__ == "__main__": + print("=== auto_memory integration test ===") + test_auto_memory_updates_existing_and_creates_new() + print("\nAll integration tests passed!") diff --git a/tests4/unit/__init__.py b/tests4/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests4/unittest/test_background_steps.py b/tests4/unit/test_background_steps.py similarity index 100% rename from tests4/unittest/test_background_steps.py rename to tests4/unit/test_background_steps.py diff --git a/tests4/unittest/test_bm25_index_perf.py b/tests4/unit/test_bm25_index_perf.py similarity index 100% rename from tests4/unittest/test_bm25_index_perf.py rename to tests4/unit/test_bm25_index_perf.py diff --git a/tests4/unittest/test_chunked_file_parser.py b/tests4/unit/test_chunked_file_parser.py similarity index 100% rename from tests4/unittest/test_chunked_file_parser.py rename to tests4/unit/test_chunked_file_parser.py diff --git a/tests4/unittest/test_common_steps.py b/tests4/unit/test_common_steps.py similarity index 100% rename from tests4/unittest/test_common_steps.py rename to tests4/unit/test_common_steps.py diff --git a/tests4/unittest/test_crud_steps.py b/tests4/unit/test_crud_steps.py similarity index 100% rename from tests4/unittest/test_crud_steps.py rename to tests4/unit/test_crud_steps.py diff --git a/tests4/unittest/test_daily_steps.py b/tests4/unit/test_daily_steps.py similarity index 96% rename from tests4/unittest/test_daily_steps.py rename to tests4/unit/test_daily_steps.py index 873eb8b5..7909d971 100644 --- a/tests4/unittest/test_daily_steps.py +++ b/tests4/unit/test_daily_steps.py @@ -117,11 +117,10 @@ def test_daily_list_default_date_is_today(): await step() payload = _metadata(step) assert payload["date"] == _today() - paths = sorted(n["path"] for n in payload["notes"]) - assert paths == [ - f"daily/{_today()}/today-a.md", - f"daily/{_today()}/today-b.md", - ] + assert payload["count"] == 2 + answer = step.context.response.answer + assert f"daily/{_today()}/today-a.md" in answer + assert f"daily/{_today()}/today-b.md" in answer await store.close() print("✓ test_daily_list_default_date_is_today passed") @@ -143,8 +142,9 @@ def test_daily_list_filters_by_date(): await step(date="2026-05-18") payload = _metadata(step) assert payload["date"] == "2026-05-18" - paths = [n["path"] for n in payload["notes"]] - assert paths == ["daily/2026-05-18/a.md"] + assert payload["count"] == 1 + answer = step.context.response.answer + assert "daily/2026-05-18/a.md" in answer await store.close() print("✓ test_daily_list_filters_by_date passed") @@ -167,13 +167,11 @@ def test_daily_list_returns_path_slug_metadata(): step = daily_list_step.DailyListStep(file_store=store) await step(date="2026-05-18") payload = _metadata(step) - assert payload["notes"] == [ - { - "path": "daily/2026-05-18/alpha.md", - "slug": "alpha", - "metadata": {"name": "Alpha Project", "description": "JWT auth migration"}, - }, - ] + assert payload["count"] == 1 + answer = step.context.response.answer + assert "daily/2026-05-18/alpha.md" in answer + assert "Alpha Project" in answer + assert "JWT auth migration" in answer await store.close() print("✓ test_daily_list_returns_path_slug_metadata passed") @@ -200,8 +198,9 @@ def test_daily_list_ignores_subdirectories(): step = daily_list_step.DailyListStep(file_store=store) await step(date="2026-05-18") payload = _metadata(step) - paths = [n["path"] for n in payload["notes"]] - assert paths == ["daily/2026-05-18/main.md"] + assert payload["count"] == 1 + answer = step.context.response.answer + assert "daily/2026-05-18/main.md" in answer await store.close() print("✓ test_daily_list_ignores_subdirectories passed") @@ -218,7 +217,7 @@ def test_daily_list_empty_when_no_daily_dir(): step = daily_list_step.DailyListStep(file_store=store) await step(date="2026-05-18") payload = _metadata(step) - assert payload == {"date": "2026-05-18", "notes": []} + assert payload == {"date": "2026-05-18", "count": 0} await store.close() print("✓ test_daily_list_empty_when_no_daily_dir passed") @@ -250,7 +249,7 @@ def test_daily_list_does_not_refresh_index(): def test_daily_list_response_shape(): - """daily_list returns only {date, notes}.""" + """daily_list returns only {date, count}.""" async def run(): with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): @@ -262,7 +261,7 @@ def test_daily_list_response_shape(): step = daily_list_step.DailyListStep(file_store=store) await step(date="2026-05-18") payload = _metadata(step) - assert set(payload.keys()) == {"date", "notes"} + assert set(payload.keys()) == {"date", "count"} await store.close() print("✓ test_daily_list_response_shape passed") diff --git a/tests4/unittest/test_file_catalog.py b/tests4/unit/test_file_catalog.py similarity index 100% rename from tests4/unittest/test_file_catalog.py rename to tests4/unit/test_file_catalog.py diff --git a/tests4/unittest/test_file_graph.py b/tests4/unit/test_file_graph.py similarity index 100% rename from tests4/unittest/test_file_graph.py rename to tests4/unit/test_file_graph.py diff --git a/tests4/unittest/test_file_store.py b/tests4/unit/test_file_store.py similarity index 100% rename from tests4/unittest/test_file_store.py rename to tests4/unit/test_file_store.py diff --git a/tests4/unittest/test_keyword_index.py b/tests4/unit/test_keyword_index.py similarity index 100% rename from tests4/unittest/test_keyword_index.py rename to tests4/unit/test_keyword_index.py diff --git a/tests4/unittest/test_link_expansion.py b/tests4/unit/test_link_expansion.py similarity index 100% rename from tests4/unittest/test_link_expansion.py rename to tests4/unit/test_link_expansion.py diff --git a/tests4/unittest/test_linked_file_parser.py b/tests4/unit/test_linked_file_parser.py similarity index 100% rename from tests4/unittest/test_linked_file_parser.py rename to tests4/unit/test_linked_file_parser.py diff --git a/tests4/unittest/test_neo4j_file_graph.py b/tests4/unit/test_neo4j_file_graph.py similarity index 100% rename from tests4/unittest/test_neo4j_file_graph.py rename to tests4/unit/test_neo4j_file_graph.py diff --git a/tests4/unittest/test_read_image_steps.py b/tests4/unit/test_read_image_steps.py similarity index 100% rename from tests4/unittest/test_read_image_steps.py rename to tests4/unit/test_read_image_steps.py diff --git a/tests4/unittest/test_read_with_neighbors.py b/tests4/unit/test_read_with_neighbors.py similarity index 100% rename from tests4/unittest/test_read_with_neighbors.py rename to tests4/unit/test_read_with_neighbors.py diff --git a/tests4/unittest/test_resource_steps.py b/tests4/unit/test_resource_steps.py similarity index 100% rename from tests4/unittest/test_resource_steps.py rename to tests4/unit/test_resource_steps.py diff --git a/tests4/unittest/test_tokenizer.py b/tests4/unit/test_tokenizer.py similarity index 100% rename from tests4/unittest/test_tokenizer.py rename to tests4/unit/test_tokenizer.py diff --git a/tests4/unittest/test_wikilink_utils.py b/tests4/unit/test_wikilink_utils.py similarity index 100% rename from tests4/unittest/test_wikilink_utils.py rename to tests4/unit/test_wikilink_utils.py diff --git a/tests4/unittest/test_write_metadata_lock.py b/tests4/unit/test_write_metadata_lock.py similarity index 100% rename from tests4/unittest/test_write_metadata_lock.py rename to tests4/unit/test_write_metadata_lock.py