From 3cb2579ff77af40d2b92713b5d2a2f21c8e42403 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Fri, 29 May 2026 12:07:44 +0800 Subject: [PATCH] refactor(steps): update auto-memory (#263) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(steps): update naming conventions in components and configuration Updated naming conventions across multiple files, changing colon-separated names to underscore-separated format, and added new step definitions along with documentation updates. Key changes: - Replaced `Synchronizer` with `AutoMemory` as the counterpart component for cold-write operations - Updated naming conventions in all related configuration files (e.g., `frontmatter:read` → `frontmatter_read`) - Added new step definitions such as `submit_slug_updates` and `auto_memory` - Updated relevant documentation - Modified log output format for improved readability * refactor(evolve): Refactor the auto-memory module and update related configurations - Remove the old slug update commit step file - Add new auto-memory planner and writer steps - Update __init__.py to export the new step classes - Modify the auto_memory configuration structure in default.yaml - Update the slug field description for clearer explanation of its purpose * up * up * refactor(tests): Move unit test directory from `tests4/unittest` to `tests4/unit` Additionally, the assertion logic in test files has been updated: direct comparisons of `payload["notes"]` have been replaced with checks verifying the presence of paths and metadata within the response content. Furthermore, some test expectations have been simplified—for example, using `count` instead of asserting against specific note lists. Specific changes include: - Updating workflow configurations to align with the new test directory structure - Modifying assertions across multiple test methods to make them more flexible and maintainable - Cleaning up and optimizing parts of the test code structure This is a comprehensive test refactoring effort aimed at improving test readability and robustness. * Refactor(steps): Update memory writing logic and optimize JSON schema structure Improved the write strategy description in `auto_memory_writer.yaml` to emphasize using `edit` over `write`. Adjusted the `json_schema` structure in `base_step.py` to support the new function definition format. Also corrected grammatical issues in the related documentation. * Fix: Improve frontend data parsing error handling and update test files Added capture and handling logic for YAML parsing exceptions, providing more detailed error messages when frontend data format issues occur. Also corrected the description text in a test file. --- .github/workflows/unittest.yml | 2 +- docs4/reme_design.md | 8 +- .../file_parser/linked_file_parser.py | 2 +- reme4/config/default.yaml | 50 ++- reme4/config/qwenpaw.yaml | 14 +- reme4/steps/__init__.py | 7 + reme4/steps/base_step.py | 9 +- reme4/steps/common/llm_demo.py | 78 ++++ reme4/steps/evolve/__init__.py | 0 reme4/steps/evolve/_evolve.py | 29 ++ reme4/steps/evolve/auto_memory_planner.py | 120 ++++++ reme4/steps/evolve/auto_memory_planner.yaml | 142 +++++++ reme4/steps/evolve/auto_memory_writer.py | 89 +++++ reme4/steps/evolve/auto_memory_writer.yaml | 167 ++++++++ reme4/steps/file_io/daily_list.py | 20 +- reme4/steps/file_io/edit.py | 7 +- reme4/steps/file_io/frontmatter_read.py | 9 +- reme4/steps/jobs/__init__.py | 9 +- reme4/steps/jobs/digester.py | 4 +- reme4/steps/jobs/synchronizer.py | 276 -------------- reme4/steps/jobs/synchronizer.yaml | 144 ------- reme4/steps/transfer/ingest.py | 2 +- tests/test_base_file_watcher.py | 2 +- tests/test_reme_light_watch_paths.py | 1 + tests4/integration/__init__.py | 0 tests4/integration/test_as_llm.py | 92 +++++ tests4/integration/test_auto_memory.py | 355 ++++++++++++++++++ tests4/unit/__init__.py | 0 .../test_background_steps.py | 0 .../test_bm25_index_perf.py | 0 .../test_chunked_file_parser.py | 0 .../{unittest => unit}/test_common_steps.py | 0 tests4/{unittest => unit}/test_crud_steps.py | 0 tests4/{unittest => unit}/test_daily_steps.py | 37 +- .../{unittest => unit}/test_file_catalog.py | 0 tests4/{unittest => unit}/test_file_graph.py | 0 tests4/{unittest => unit}/test_file_store.py | 0 .../{unittest => unit}/test_keyword_index.py | 0 .../{unittest => unit}/test_link_expansion.py | 0 .../test_linked_file_parser.py | 0 .../test_neo4j_file_graph.py | 0 .../test_read_image_steps.py | 0 .../test_read_with_neighbors.py | 0 .../{unittest => unit}/test_resource_steps.py | 0 tests4/{unittest => unit}/test_tokenizer.py | 0 .../{unittest => unit}/test_wikilink_utils.py | 0 .../test_write_metadata_lock.py | 0 47 files changed, 1199 insertions(+), 476 deletions(-) create mode 100644 reme4/steps/common/llm_demo.py create mode 100644 reme4/steps/evolve/__init__.py create mode 100644 reme4/steps/evolve/_evolve.py create mode 100644 reme4/steps/evolve/auto_memory_planner.py create mode 100644 reme4/steps/evolve/auto_memory_planner.yaml create mode 100644 reme4/steps/evolve/auto_memory_writer.py create mode 100644 reme4/steps/evolve/auto_memory_writer.yaml delete mode 100644 reme4/steps/jobs/synchronizer.py delete mode 100644 reme4/steps/jobs/synchronizer.yaml create mode 100644 tests4/integration/__init__.py create mode 100644 tests4/integration/test_as_llm.py create mode 100644 tests4/integration/test_auto_memory.py create mode 100644 tests4/unit/__init__.py rename tests4/{unittest => unit}/test_background_steps.py (100%) rename tests4/{unittest => unit}/test_bm25_index_perf.py (100%) rename tests4/{unittest => unit}/test_chunked_file_parser.py (100%) rename tests4/{unittest => unit}/test_common_steps.py (100%) rename tests4/{unittest => unit}/test_crud_steps.py (100%) rename tests4/{unittest => unit}/test_daily_steps.py (96%) rename tests4/{unittest => unit}/test_file_catalog.py (100%) rename tests4/{unittest => unit}/test_file_graph.py (100%) rename tests4/{unittest => unit}/test_file_store.py (100%) rename tests4/{unittest => unit}/test_keyword_index.py (100%) rename tests4/{unittest => unit}/test_link_expansion.py (100%) rename tests4/{unittest => unit}/test_linked_file_parser.py (100%) rename tests4/{unittest => unit}/test_neo4j_file_graph.py (100%) rename tests4/{unittest => unit}/test_read_image_steps.py (100%) rename tests4/{unittest => unit}/test_read_with_neighbors.py (100%) rename tests4/{unittest => unit}/test_resource_steps.py (100%) rename tests4/{unittest => unit}/test_tokenizer.py (100%) rename tests4/{unittest => unit}/test_wikilink_utils.py (100%) rename tests4/{unittest => unit}/test_write_metadata_lock.py (100%) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 263bd23a..02f8acdf 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -36,7 +36,7 @@ jobs: - name: Run tests4 unit tests run: | - pytest tests4/unittest \ + pytest tests4/unit \ -v \ --tb=long \ -s \ diff --git a/docs4/reme_design.md b/docs4/reme_design.md index 56d30120..d2e69904 100644 --- a/docs4/reme_design.md +++ b/docs4/reme_design.md @@ -88,11 +88,11 @@ reme4 search query="..." backend=mcp | crud | append | path="My Note" content="New line" | | crud | delete | path="My Note -| daily:crud | daily:xxx | 与 crud 参数保持一致 | +| daily_crud | daily_xxx | 与 crud 参数保持一致 | -- daily:resolve name=xxxx (符合一定规范 win下要求) -- daily:list date=xxxx 返回path -- daily:index +- daily_resolve name=xxxx (符合一定规范 win下要求) +- daily_list date=xxxx 返回path +- daily_index frontmatter read path frontmatter update path metadata={} diff --git a/reme4/components/file_parser/linked_file_parser.py b/reme4/components/file_parser/linked_file_parser.py index 229b9d14..bfd7522a 100644 --- a/reme4/components/file_parser/linked_file_parser.py +++ b/reme4/components/file_parser/linked_file_parser.py @@ -113,7 +113,7 @@ class LinkedFileParser(BaseFileParser): def __init__( self, encoding: str = "utf-8", - chunk_chars: int = 2000, + chunk_chars: int = 10000, embed_toc: bool = True, **kwargs, ): diff --git a/reme4/config/default.yaml b/reme4/config/default.yaml index 42ac251e..09ce7a7f 100644 --- a/reme4/config/default.yaml +++ b/reme4/config/default.yaml @@ -107,7 +107,7 @@ jobs: expand_links: true max_links_per_direction: 10 - daily:create: + daily_create: backend: base description: "Provision a note slug under a daily folder: daily//.md" parameters: @@ -115,7 +115,7 @@ jobs: properties: slug: type: string - description: "the file stem" + description: "the file stem: event name or topic name" date: type: string description: "YYYY-MM-DD; empty = today" @@ -125,7 +125,7 @@ jobs: steps: - backend: daily_create_step - daily:list: + daily_list: backend: base description: "List notes under a single day." parameters: @@ -138,7 +138,7 @@ jobs: steps: - backend: daily_list_step - daily:reindex: + daily_reindex: backend: base description: "Rebuild the day-index page daily/.md." parameters: @@ -151,7 +151,7 @@ jobs: steps: - backend: daily_reindex_step - frontmatter:delete: + frontmatter_delete: backend: base description: "Drop keys from a file's frontmatter." parameters: @@ -171,7 +171,7 @@ jobs: steps: - backend: frontmatter_delete_step - frontmatter:read: + frontmatter_read: backend: base description: "Read a file's frontmatter as a dict." parameters: @@ -185,7 +185,7 @@ jobs: steps: - backend: frontmatter_read_step - frontmatter:update: + frontmatter_update: backend: base description: "Merge key-values into a file's frontmatter." parameters: @@ -369,6 +369,30 @@ jobs: steps: - backend: edit_step + auto_memory: + backend: base + description: "Auto-memory orchestrator" + parameters: + type: object + properties: + messages: + type: array + description: "messages" + items: + type: object + memory_hint: + type: string + description: "optional hint" + timezone: + type: string + description: "IANA timezone, e.g. Asia/Shanghai" + default: "Asia/Shanghai" + required: + - messages + steps: + - backend: auto_memory_planner_step + - backend: auto_memory_writer_step + components: tokenizer: default: @@ -382,6 +406,18 @@ components: model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-v4} dimensions: 1024 + as_llm: + default: + backend: ${LLM_BACKEND:-anthropic} + model_name: ${LLM_MODEL_NAME:-glm-5} + api_key: ${LLM_API_KEY:-} + client_kwargs: + base_url: ${LLM_BASE_URL:-https://dashscope.aliyuncs.com/apps/anthropic} + + as_llm_formatter: + default: + backend: ${LLM_FORMATTER_BACKEND:-anthropic} + file_graph: default: backend: local diff --git a/reme4/config/qwenpaw.yaml b/reme4/config/qwenpaw.yaml index 96d30ad9..5c61b8ba 100644 --- a/reme4/config/qwenpaw.yaml +++ b/reme4/config/qwenpaw.yaml @@ -185,7 +185,7 @@ jobs: steps: - backend: stat_step - frontmatter:read: + frontmatter_read: backend: base description: "Read a file's YAML frontmatter as a dict." parameters: @@ -197,7 +197,7 @@ jobs: required: - path steps: - - backend: frontmatter:read_step + - backend: frontmatter_read_step # ── Write Operations────────────────────────────────────────────────────────── write: @@ -267,7 +267,7 @@ jobs: steps: - backend: append_step - frontmatter:update: + frontmatter_update: backend: base description: "Merge keys into a file's YAML frontmatter." parameters: @@ -286,7 +286,7 @@ jobs: steps: - backend: frontmatter_update_step - frontmatter:delete: + frontmatter_delete: backend: base description: "Drop keys from a file's YAML frontmatter." parameters: @@ -369,7 +369,7 @@ jobs: - backend: download_step # ── Daily Operations (slug provisioning + day-index rollup) ────────── - daily:create: + daily_create: backend: base description: "Provision daily//.md (empty body, frontmatter {name: slug}); idempotent; refreshes the day index." parameters: @@ -387,7 +387,7 @@ jobs: steps: - backend: daily_create_step - daily:list: + daily_list: backend: base description: "List notes under a single day." parameters: @@ -400,7 +400,7 @@ jobs: steps: - backend: daily_list_step - daily:reindex: + daily_reindex: backend: base description: "Rebuild the day-index page daily/.md." parameters: diff --git a/reme4/steps/__init__.py b/reme4/steps/__init__.py index daa59838..afd7bbb0 100644 --- a/reme4/steps/__init__.py +++ b/reme4/steps/__init__.py @@ -4,8 +4,11 @@ from .base_step import BaseStep from .common.demo import DemoEchoStep1, DemoEchoStep2 from .common.health_check import HealthCheckStep from .common.help import HelpStep +from .common.llm_demo import LLMDemoStep from .common.stream_demo import StreamDemoStep1, StreamDemoStep2 from .common.version import VersionStep +from .evolve.auto_memory_planner import AutoMemoryPlannerStep +from .evolve.auto_memory_writer import AutoMemoryWriterStep from .file_io.daily_create import DailyCreateStep from .file_io.daily_list import DailyListStep from .file_io.daily_reindex import DailyReindexStep @@ -38,9 +41,13 @@ __all__ = [ "DemoEchoStep2", "HealthCheckStep", "HelpStep", + "LLMDemoStep", "StreamDemoStep1", "StreamDemoStep2", "VersionStep", + # evolve + "AutoMemoryPlannerStep", + "AutoMemoryWriterStep", # file_io "DeleteStep", "EditStep", diff --git a/reme4/steps/base_step.py b/reme4/steps/base_step.py index d60564e1..0b74620e 100644 --- a/reme4/steps/base_step.py +++ b/reme4/steps/base_step.py @@ -202,5 +202,12 @@ class BaseStep(ABC): tool_func=run_job, func_name=job_name, func_description=job.description, - json_schema=job.parameters, + json_schema={ + "type": "function", + "function": { + "name": job_name, + "description": job.description, + "parameters": job.parameters, + }, + }, ) diff --git a/reme4/steps/common/llm_demo.py b/reme4/steps/common/llm_demo.py new file mode 100644 index 00000000..fc6dda06 --- /dev/null +++ b/reme4/steps/common/llm_demo.py @@ -0,0 +1,78 @@ +"""Demo step that drives a ReActAgent via BaseStep.as_llm/as_llm_formatter.""" + +from agentscope.agent import ReActAgent +from agentscope.message import Msg, TextBlock +from agentscope.tool import Toolkit, ToolResponse + +from ..base_step import BaseStep +from ...components import R + + +def _add(a: float, b: float) -> ToolResponse: + """Add two numbers and return the sum. + + Args: + a: first addend + b: second addend + """ + return ToolResponse(content=[TextBlock(type="text", text=str(a + b))]) + + +@R.register("llm_demo_step") +class LLMDemoStep(BaseStep): + """Drive a ReActAgent powered by ``self.as_llm`` / ``self.as_llm_formatter``. + + Inputs (from RuntimeContext): + query (str, required): user message content. + sys_prompt (str, optional): system prompt for the agent. + use_add_tool (bool, optional): register the ``add`` tool when True. + console_enabled (bool, optional): mirror agent output to stdout. + + Output (written to context.response.answer):fa + The agent's final reply text. + """ + + DEFAULT_SYS_PROMPT = "You are a concise assistant. Reply in one short sentence." + + async def execute(self): + assert self.context is not None + query: str = self.context.get("query", "") + sys_prompt: str = self.context.get("sys_prompt") or self.DEFAULT_SYS_PROMPT + use_add_tool: bool = bool(self.context.get("use_add_tool", False)) + console_enabled: bool = bool(self.context.get("console_enabled", False)) + + if not query: + self.context.response.success = False + self.context.response.answer = "Skipped: empty query" + return self.context.response + + toolkit = Toolkit() + if use_add_tool: + toolkit.register_tool_function(_add) + + agent = ReActAgent( + name=self.name, + sys_prompt=sys_prompt, + model=self.as_llm, + formatter=self.as_llm_formatter, + toolkit=toolkit, + ) + agent.set_console_output_enabled(console_enabled) + + response: Msg = await agent.reply( + Msg(name="user", role="user", content=query), + ) + text = (response.get_text_content() or "").strip() + self.logger.info(f"[{self.name}] response: {text!r}") + + self.context.response.success = True + self.context.response.answer = text + self.context.response.metadata.update( + { + "query": query, + "sys_prompt": sys_prompt, + "use_add_tool": use_add_tool, + "response": text, + }, + ) + return self.context.response diff --git a/reme4/steps/evolve/__init__.py b/reme4/steps/evolve/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reme4/steps/evolve/_evolve.py b/reme4/steps/evolve/_evolve.py new file mode 100644 index 00000000..bac0a0d4 --- /dev/null +++ b/reme4/steps/evolve/_evolve.py @@ -0,0 +1,29 @@ +"""Shared helpers for evolve steps.""" + +import datetime +import zoneinfo + +from agentscope.message import Msg + + +def now(timezone: str | None = None) -> datetime.datetime: + """Return current datetime in the given IANA timezone, falling back to local.""" + if not timezone: + return datetime.datetime.now() + try: + return datetime.datetime.now(zoneinfo.ZoneInfo(timezone)) + except Exception: + return datetime.datetime.now() + + +def format_history(messages: list[Msg], include_timestamp: bool = True) -> str: + """Render a conversation slice as a human-readable transcript.""" + lines: list[str] = [] + for msg in messages: + text = (msg.get_text_content() or "").strip() + if not text: + continue + speaker = msg.name or msg.role or "?" + header = f"[{speaker} @ {msg.timestamp}]" if include_timestamp else f"[{speaker}]" + lines.append(f"{header}\n{text}") + return "\n\n".join(lines) or "(empty)" diff --git a/reme4/steps/evolve/auto_memory_planner.py b/reme4/steps/evolve/auto_memory_planner.py new file mode 100644 index 00000000..d39ff0d7 --- /dev/null +++ b/reme4/steps/evolve/auto_memory_planner.py @@ -0,0 +1,120 @@ +"""``auto_memory_planner`` — planner for the auto-memory system. + +Inspects the recent conversation, surveys today's existing daily notes +via ``daily_list``, reads any candidate that already covers the topic +via ``read``, and emits a list of daily-note upsert tasks +``(path, description)`` through structured output. The result is +exposed under ``response.metadata['memory_updates']`` for the +orchestrator (``auto_memory``) to feed into ``auto_memory_writer`` +one task at a time. + +The planner never writes notes itself — planning only. + +Inputs (from RuntimeContext): + messages (list[Msg], required): conversation slice to inspect. + memory_hint (str, optional): caller-supplied hint to bias filename + stem selection or disambiguate same-day tasks. + +Output (written to context.response): + answer: one-line human summary of what was planned. + metadata['memory_updates']: list of ``{path, description}`` dicts. +""" + +from agentscope.agent import ReActAgent +from agentscope.message import Msg +from agentscope.tool import Toolkit +from pydantic import BaseModel, Field + +from ._evolve import format_history, now +from ..base_step import BaseStep +from ...components import R + + +class MemoryUpdateTask(BaseModel): + """One daily-note upsert task emitted by the planner.""" + + path: str = Field( + description="Vault-relative note path, form `daily//.md`. " + "Reuse an existing path to upsert.", + ) + description: str = Field(description="Flat fact checklist — what to preserve, not how to categorize or format.") + + +class MemoryUpdatesPlan(BaseModel): + """Structured output emitted by the planner's finish-tool.""" + + memory_updates: list[MemoryUpdateTask] = Field( + default_factory=list, + description="List of daily-note upsert tasks; empty means nothing worth persisting.", + ) + + +@R.register("auto_memory_planner_step") +class AutoMemoryPlannerStep(BaseStep): + """Plan daily-note upsert tasks via a ReAct agent with structured output.""" + + def __init__(self, console_enabled: bool = False, **kwargs): + super().__init__(**kwargs) + self.console_enabled = console_enabled + self.planner_tools: list[str] = ["daily_list", "read"] + + async def execute(self): + assert self.context is not None + messages: list[Msg] = [ + item if isinstance(item, Msg) else Msg.from_dict(item) for item in self.context.get("messages", []) + ] + memory_hint: str = self.context.get("memory_hint", "") + current = now(self.context.get("timezone")) + + if not messages: + self.context.response.success = True + self.context.response.answer = "Skipped: no messages supplied" + self.context.response.metadata.update({"memory_updates": []}) + return + + toolkit = Toolkit() + for job_name in self.planner_tools: + self.add_as_tool(toolkit, job_name) + + agent = ReActAgent( + name="auto_memory_planner", + model=self.as_llm, + sys_prompt=self.prompt_format("system_prompt"), + formatter=self.as_llm_formatter, + toolkit=toolkit, + ) + agent.set_console_output_enabled(self.console_enabled) + + user_message: str = self.prompt_format( + "user_message", + today=current.strftime("%Y-%m-%d"), + vault_dir=str(self.file_store.vault_path), + note=memory_hint or "(none)", + history=format_history(messages), + ) + + final_msg: Msg = await agent.reply( + Msg(name="reme", role="user", content=user_message), + structured_model=MemoryUpdatesPlan, + ) + + meta: dict = final_msg.metadata if isinstance(final_msg.metadata, dict) else {} + raw_tasks = meta.get("memory_updates") or [] + cleaned: list[dict] = [] + for item in raw_tasks: + if not isinstance(item, dict): + continue + path = str(item.get("path") or "").strip() + description = str(item.get("description") or "").strip() + if path and description and path.endswith(".md") and ".." not in path.split("/"): + cleaned.append({"path": path, "description": description}) + + self.context.response.success = True + self.context.response.metadata.update({"memory_updates": cleaned, "count": len(cleaned)}) + if not cleaned: + self.context.response.answer = "[SKIP] No memory updates planned" + return + + lines = [f"Planned {len(cleaned)} memory update(s):"] + lines += [f"- {t['path']}: {t['description']}" for t in cleaned] + self.context.response.answer = "\n".join(lines) diff --git a/reme4/steps/evolve/auto_memory_planner.yaml b/reme4/steps/evolve/auto_memory_planner.yaml new file mode 100644 index 00000000..3e6b7284 --- /dev/null +++ b/reme4/steps/evolve/auto_memory_planner.yaml @@ -0,0 +1,142 @@ +system_prompt: | + You are the planner of an auto-memory system. Each invocation's task: inspect the recent conversation, then emit a list of daily-note upsert tasks via the `generate_response` finish tool. You never write notes yourself — a separate writer agent handles that. + +user_message: | + Today: {today} + Vault dir: {vault_dir} + Extra hint: {note} + + # Recent conversation + + {history} + + # Your task + + Inspect the conversation above and emit a list of daily-note upsert tasks. Follow the five-step workflow below, then submit your planned `memory_updates` by calling `generate_response`. + + ## Five steps per invocation + + ### Step 1 — Skip check + Is the conversation a substantive exchange that produced information worth long-term memory — such as user preferences, project decisions, technical facts, workflow knowledge, or status updates? Pure greetings or small talk with no such information → emit an empty `memory_updates` list (the orchestrator treats this as a skip). + When truly ambiguous, default to writing — losing memory is worse than an extra note. + + ### Step 2 — Survey today + Call `daily_list` to see all existing `daily//.md` files along with their `name`, `description`, and other metadata. + + ### Step 3 — Read candidates + For any existing note whose `name` / `description` looks relevant to this conversation, call `read path=daily//.md` to see its body. Skip clearly unrelated notes. + + ### Step 4 — Plan paths + + Decide one or more `(path, description)` pairs. Each `path` is the vault-relative note path of form `daily//.md`, where `` is the filename without the `.md` suffix. + + #### 4a. Filename stem naming conventions + + The stem is the note's filename without `.md` — its job is to **uniquely identify** this event/topic, not to cram in all context. Constraints: + + - **Format**: English kebab-case, composed of nouns / noun phrases, stable as a filename. + - **Length**: Usually 2-5 words, roughly 15-50 characters. **Too short and uninformative** (e.g. `bug`, `chat`, `misc`, `notes-1`) → unusable; **too long with progress/blockers/dates packed in** → unusable, those belong in the body and frontmatter. + - **Information density**: Reading the stem alone should identify which event/topic it refers to; progress, status, and context do not go into the stem. + - **Event type** (specific event / task / outage / debug / release): the stem identifies "which event". E.g.: `auth-middleware-rewrite`, `ingest-pipeline-oom`. + - **Topic type** (ongoing concept / knowledge domain / preference / tool recipe): the stem identifies "which topic". E.g.: `pytorch-distributed-training`, `pr-summary-style`. + + #### 4b. Reuse vs. create + + - When the conversation continues a thread you found in Step 3 (same logical event, same topic), **reuse the same path** (same stem under today's folder). Fragmenting one thread across multiple paths is the worst failure mode. + - When no existing note covers the topic, **create a new path** of form `daily//.md` with the stem following 4a conventions. + - A single conversation may span multiple unrelated topics — emit a separate task for each distinct event/topic rather than merging into one giant task. + + #### 4c. What goes in the description + + The description only answers "what facts to preserve" — how to format the body, structure sections, or merge updates is the writer's responsibility, not yours. + + The description is a flat fact checklist listing everything from the conversation worth preserving. Quote key original wording or numbers verbatim. Do not categorize the facts — categorization into note sections is the writer's job. + + Coverage should be comprehensive, including but not limited to: + - Durable facts about the user (role, project, responsibilities, tools, preferences, constraints, goals) + - Domain knowledge (concepts, decision conclusions and rationale, dependency versions, design constraints, system topology) + - Replayable operations (command sequences, script recipes, workflows, debugging steps) + - Current status (progress, blockers, next steps, open questions) + - Timeline events (events that occurred, decision moments) + + ### Step 5 — Submit + Call `generate_response` with `memory_updates=[{{path, description}}, ...]`. If Step 1 determined a skip, emit `[]`. This is your only way to finish — do not produce free text after Step 4; the structured payload from the finish tool is your entire deliverable. + + ## Boundaries + + - You never write notes — do not call `write`, `edit`, or `frontmatter_update`. Those tools belong to the writer. + - When surveying, stay in today's `daily/` folder — do not traverse the entire vault. + - Every `path` you emit must be of form `daily//.md` — never write outside today's daily folder. + - Emit exactly one `memory_updates` list per invocation. Do not call `generate_response` more than once. + + +system_prompt_zh: | + 你是自动记忆系统的规划者。每次调用的任务：研究最近的对话，然后通过 `generate_response` 完成工具发出一份日记 upsert 任务列表。你自己不写笔记——由独立的写入代理负责。 + +user_message_zh: | + 今天：{today} + Vault 目录：{vault_dir} + 额外提示：{note} + + # 最近的对话 + + {history} + + # 你的任务 + + 研究上面的对话，发出一份日记 upsert 任务列表。按下面的五步流程执行，最后通过调用 `generate_response` 提交计划好的 `memory_updates`。 + + ## 每次调用的五个步骤 + + ### 步骤 1 — 跳过检查 + 对话是否产生了值得长期记忆的信息——如用户偏好、项目决策、技术事实、工作流知识或状态更新？纯粹的寒暄或闲聊，且未透露任何此类信息 → 发出空的 `memory_updates` 列表（编排器会视为跳过）。 + 当真正模棱两可时，默认写入——丢失记忆比多写一条笔记更糟。 + + ### 步骤 2 — 概览今天 + 调用 `daily_list` 查看所有现存的 `daily//.md` 及其 `name`、`description` 和其他 metadata 信息。 + + ### 步骤 3 — 阅读候选 + 对任何 `name` / `description` 看起来与本次对话相关的现有笔记，调用 `read path=daily//.md` 直接阅读正文。明显无关的笔记跳过。 + + ### 步骤 4 — 规划 path + + 决定一个或多个 `(path, description)` 对。每个 `path` 是 vault 相对路径，形如 `daily//.md`；其中 `` 段就是笔记文件名去掉 `.md` 后缀的部分。 + + #### 4a. 文件名 stem 命名规范 + + stem 就是笔记文件名去掉 `.md` 的部分——它的职责是**唯一标识**这条 event/topic，不是塞进所有上下文。约束： + + - **格式**：英文 kebab-case，由名词 / 名词短语组成，稳定可作文件名。 + - **长度**：通常 2-5 个词，约 15-50 字符。**太短无信息**（如 `bug`、`chat`、`misc`、`notes-1`）→ 不可用；**太长把进度/卡点/日期都塞进去** → 不可用，那些属于正文与 frontmatter。 + - **信息含量**：读 stem 即可识别该 event/topic 是什么；进度、状态、上下文不进 stem。 + - **event 类型**（具体事件 / 任务 / 故障 / 调试 / 发布）：stem 标识"是哪件事"。例：`auth-middleware-rewrite`、`ingest-pipeline-oom`。 + - **topic 类型**（持续性的概念 / 知识领域 / 偏好 / 工具配方）：stem 标识"是哪类主题"。例：`pytorch-distributed-training`、`pr-summary-style`。 + + #### 4b. 复用 vs 新建 + + - 当对话延续一条你在步骤 3 中发现的现有线索时（同一逻辑事件、同一主题），**复用相同的 path**（即今天目录下相同的 stem）。把一条线索碎片化到多个 path 是最严重的失败模式。 + - 当没有现存笔记覆盖该主题时，**新建一个形如 `daily//.md` 的 path**，stem 段遵循 4a 规范。 + - 一次对话可能跨越多个无关主题——为每个不同的 event/topic 各发一个任务，而不是合成一个巨型任务。 + + #### 4c. description 的内容 + + description 只负责回答"要保留什么事实"——正文如何分 section、frontmatter 怎么写、UPDATE 如何合并都由写入者处理，不在你的职责内。 + + description 是一份扁平的事实清单，列出对话中全部值得保留的内容——关键处逐字引用原始措辞或数字。不要对事实做分类——分到笔记 section 是写入者的工作。 + + 覆盖范围要全面，包括但不限于： + - 用户身份相关的持久事实（角色、项目、职责、工具、偏好、约束、目标） + - 领域知识（概念、决策结论与理由、依赖版本、设计约束、系统拓扑） + - 可重放的操作（命令序列、脚本配方、工作流、调试步骤） + - 当下状态（进度、卡点、下一步、未决问题） + - 时间线事件（发生的事件、决策时刻） + + ### 步骤 5 — 提交 + 调用 `generate_response`，传入 `memory_updates=[{{path, description}}, ...]`。如果步骤 1 判定跳过，就发出 `[]`。这是你唯一的收尾方式——步骤 4 之后不要再产出自由文本；完成工具的结构化负载就是全部交付物。 + + ## 边界 + + - 你从不写笔记——不调用 `write`、`edit`、`frontmatter_update`。那些工具属于写入者。 + - 概览时只待在今天的 `daily/` 文件夹——不要走遍整个 vault。 + - 发出的每个 `path` 必须形如 `daily//.md`——绝不写到今天 daily 目录之外。 + - 每次调用只产出一份 `memory_updates` 列表。不要多次调用 `generate_response`。 diff --git a/reme4/steps/evolve/auto_memory_writer.py b/reme4/steps/evolve/auto_memory_writer.py new file mode 100644 index 00000000..6c0fbf4b --- /dev/null +++ b/reme4/steps/evolve/auto_memory_writer.py @@ -0,0 +1,89 @@ +"""``auto_memory_writer`` — execute daily-note upserts. + +Reads the ``memory_updates`` list produced by ``auto_memory_planner`` +from ``context.response.metadata['memory_updates']``, then iterates +over each ``{path, description}`` task: decides UPDATE vs CREATE by +probing the vault, and writes the note via ``frontmatter_read`` / +``frontmatter_update`` / ``read`` / ``edit`` / ``write``. + +A fresh ReAct agent is created per task to keep conversations isolated. + +Inputs (from RuntimeContext): + messages (list[Msg], required): conversation slice (context). + memory_hint (str, optional): caller-supplied note hint. + response.metadata['memory_updates'] (list[dict]): planner output. + +Output (written to context.response): + answer: one line per task — `` ``. + metadata['written_count']: number of tasks executed. +""" + +from agentscope.agent import ReActAgent +from agentscope.message import Msg +from agentscope.tool import Toolkit + +from ._evolve import format_history, now +from ..base_step import BaseStep +from ...components import R + + +@R.register("auto_memory_writer_step") +class AutoMemoryWriterStep(BaseStep): + """Execute note upserts from the planner's task list.""" + + def __init__(self, console_enabled: bool = False, **kwargs): + super().__init__(**kwargs) + self.console_enabled = console_enabled + self.writer_tools: list[str] = ["frontmatter_read", "frontmatter_update", "read", "edit", "write"] + + async def execute(self): + assert self.context is not None + memory_updates: list[dict] = self.context.response.metadata.get("memory_updates") or [] + if not memory_updates: + self.context.response.success = True + self.context.response.answer = "[SKIP] No memory updates to write" + return + + current = now(self.context.get("timezone")) + messages: list[Msg] = [ + item if isinstance(item, Msg) else Msg.from_dict(item) for item in self.context.get("messages", []) + ] + memory_hint: str = self.context.get("memory_hint", "") + + toolkit = Toolkit() + for job_name in self.writer_tools: + self.add_as_tool(toolkit, job_name) + + results: list[str] = [] + for task in memory_updates: + note_path = task.get("path", "") + description = task.get("description", "") + if not note_path or not description: + continue + + agent = ReActAgent( + name="auto_memory_writer", + model=self.as_llm, + sys_prompt=self.prompt_format("system_prompt"), + formatter=self.as_llm_formatter, + toolkit=toolkit, + ) + agent.set_console_output_enabled(self.console_enabled) + + user_message: str = self.prompt_format( + "user_message", + today=current.strftime("%Y-%m-%d"), + vault_dir=str(self.file_store.vault_path), + note=memory_hint or "(none)", + note_path=note_path, + writing_hint=description, + history=format_history(messages), + ) + + final_msg: Msg = await agent.reply(Msg(name="reme", role="user", content=user_message)) + result_line = (final_msg.get_text_content() or "").strip() + results.append(result_line) + + self.context.response.success = True + self.context.response.answer = "\n".join(results) if results else "[SKIP] No valid tasks" + self.context.response.metadata.update({"written_count": len(results)}) diff --git a/reme4/steps/evolve/auto_memory_writer.yaml b/reme4/steps/evolve/auto_memory_writer.yaml new file mode 100644 index 00000000..fc0f3912 --- /dev/null +++ b/reme4/steps/evolve/auto_memory_writer.yaml @@ -0,0 +1,167 @@ +system_prompt: | + You are the writer of the auto-memory system. The planner hands you exactly one target path + description per call; you create or update the corresponding daily note at that path. You never plan new paths or filenames yourself. + +user_message: | + Today: {today} + Vault directory: {vault_dir} + Extra hint: {note} + Target path: {note_path} + + # Writing hint to reference + + {writing_hint} + + # Recent conversation + + {history} + + # Your task + + Referring to the hint above, create or update the note at the target path. Follow the four-step process below. + + ## Output format: body + + Your job is to capture the facts from the recent conversation into the body — not a single one may be dropped, nor watered down by paraphrase. + + Coverage must be comprehensive, including but not limited to: + - Persistent facts about the user's identity (role, project, responsibilities, tools, preferences, constraints, goals) + - Domain knowledge (concepts, decisions and their rationale, dependency versions, design constraints, system topology) + - Replayable operations (command sequences, script recipes, workflows, debugging steps) + - Current state (progress, blockers, next steps, open questions) + - Timeline events (things that happened, decision moments) + + The body format is free-form — use whatever structure best fits the content. The only hard rule is **completeness**: every fact in writing_hint must appear in the body. Quote the original wording or numbers verbatim at critical points. + + Merge rules for UPDATE: + - Timeline / history entries: append only — never delete existing entries. + - Current-state entries (progress, blockers, next steps): rewrite the whole section to reflect the latest snapshot. + - Everything else: merge and dedupe (keep all old facts, add new facts, drop exact duplicates). + + ## Output format: frontmatter + + Only two required fields: + + ```yaml + --- + name: + description: + --- + ``` + + Rules: + - `name` must be the filename stem of the target path (the part between the last `/` and `.md`), copied verbatim. Do not Title-Case it; do not rewrite it. + - `description` must be an exhaustive summary: mention every key fact, decision, and state point in the body, so the description itself works as a reliable index entry. Vague descriptions like "notes" / "misc" / "various topics" are unacceptable. + - **Never set `status`** — it is a field reserved for the downstream distill stage; touching it will cause the note to be skipped on the next distill run. + - On UPDATE, refresh `description` to reflect the updated body. + + ## The four steps for each call + + ### Step 1 — Probe + Call `frontmatter_read path=`: + - Returns frontmatter dict → **UPDATE branch** (note already exists) + - Returns error / not-found → **CREATE branch** (does not yet exist) + + ### Step 2 — UPDATE branch + 1. `read path=` to view the current body. + 2. Plan the merge per the rules above (timeline appends, facts merge-and-dedupe, current-state rewritten in full). + 3. Execute the write — **prefer `edit` over `write`**; only fall back to `write` when the change is genuinely too sweeping for `edit` to express cleanly: + - **Default: `edit`** — `edit path= old= new=` for each changed section (frontmatter is unaffected). Use multiple `edit` calls if needed to cover several sections. + - After any body change, you must also refresh frontmatter → `frontmatter_update path= metadata={{"description": ""}}`. + - **Fallback: `write`** — only when the body has changed so extensively that multiple edits would be harder to get right than a full rewrite → `write path= name= description= content=`, resetting body and frontmatter in one shot. + + ### Step 2' — CREATE branch + 1. Write the full body, capturing every fact from writing_hint. + 2. Write the frontmatter with `name` and `description`. + 3. `write path= name= description= content=` — done in one call. + + ### Step 3 — Summarize what changed + State in one sentence what you did (which file you created / what you updated). This sentence is your final text output and must be strictly a single line. + + ## Boundaries + + - Each call targets exactly one path: the one assigned. Even if the conversation mentions other notes, do not touch them. + - `write` unconditionally overwrites both body and frontmatter — use it carefully. + +system_prompt_zh: | + 你是自动记忆系统的写入者。规划者每次调用会交给你恰好一个目标路径 + description；你在该路径创建或更新对应的日记。你从不规划新的路径或文件名。 + +user_message_zh: | + 今天：{today} + Vault 目录：{vault_dir} + 额外提示：{note} + 目标路径：{note_path} + + # 可以参考的提示 + + {writing_hint} + + # 最近的对话 + + {history} + + # 你的任务 + + 参考上面的提示，在目标路径创建或更新日记。按下面的四步流程执行。 + + ## 输出格式要求：正文 + + 你的职责是捕捉最近的对话中的事实写入正文——一条都不能漏，也不能被改写淡化。 + + 覆盖范围要全面，包括但不限于： + - 用户身份相关的持久事实（角色、项目、职责、工具、偏好、约束、目标） + - 领域知识（概念、决策结论与理由、依赖版本、设计约束、系统拓扑） + - 可重放的操作（命令序列、脚本配方、工作流、调试步骤） + - 当下状态（进度、卡点、下一步、未决问题） + - 时间线事件（发生的事件、决策时刻） + + 正文格式自由——用最适合内容的结构。唯一的硬性规则是**完整性**：writing_hint 中的每一条事实都必须出现在正文中。关键处逐字引用原始措辞或数字。 + + UPDATE 时的合并规则： + - 时间线 / 历史条目：仅追加，永远不删除已有条目。 + - 当下状态类条目（进度、卡点、下一步）：整段重写，反映最新快照。 + - 其余内容：合并去重（保留全部旧事实，添加新事实，去除完全重复项）。 + + ## 输出格式要求：frontmatter + + 只有两个必填字段： + + ```yaml + --- + name: <必须等于目标路径的文件名 stem，逐字照抄> + description: <正文的详细总结——具体到仅凭 description 就能传达笔记中的全部核心信息> + --- + ``` + + 规则： + - `name` 必须是目标路径的文件名 stem（最后一个 `/` 与 `.md` 之间的部分），逐字照抄。不要 Title-Case 化，不要改写。 + - `description` 必须是详尽的总结：提及正文中的每一个关键事实、决策和状态要点，使 description 本身就能作为可靠的索引条目。模糊的描述如 "notes" / "misc" / "各种主题" 不可接受。 + - **永远不要设置 `status`**——它是下游蒸馏阶段保留的字段；动了它会让笔记在下一次 distill 运行中被忽略。 + - UPDATE 时，刷新 `description` 以反映更新后的正文内容。 + + ## 每次调用的四个步骤 + + ### 步骤 1 — 探测 + 调用 `frontmatter_read path=<目标路径>`： + - 返回 frontmatter 字典 → **UPDATE 分支**（笔记已存在） + - 返回错误 / not-found → **CREATE 分支**（尚不存在） + + ### 步骤 2 — UPDATE 分支 + 1. `read path=<目标路径>` 查看当前正文。 + 2. 按上述规则规划合并（时间线追加，事实合并去重，当下状态整段重写）。 + 3. 执行写入——**优先使用 `edit`，而非 `write`**；仅当变更范围确实过大、edit 难以清晰表达时才退回到 `write`： + - **默认：`edit`** — `edit path=<目标路径> old=<原文片段> new=<替换片段>`，对每个变更区域分别调用（frontmatter 不受影响）。需要时可多次调用 `edit` 覆盖多个区域。 + - 正文变更后，必须同时刷新 frontmatter → `frontmatter_update path=<目标路径> metadata={{"description": "<更新后的总结>"}}`。 + - **退路：`write`** — 仅当正文改动极大、多次 edit 反而更难准确操作时 → `write path=<目标路径> name= description= content=<完整正文>`，一次性重置正文和 frontmatter。 + + ### 步骤 2' — CREATE 分支 + 1. 编写完整正文，捕捉 writing_hint 中的全部事实。 + 2. 编写 frontmatter，包含 `name` 和 `description`。 + 3. `write path=<目标路径> name= description= content=<正文内容>`——一次性完成。 + + ### 步骤 3 — 总结改动内容 + 用一句话说明你做了什么（创建了哪个文件 / 更新了哪些内容）。这句话是你最后一次文本输出，必须严格只有一行。 + + ## 边界 + + - 每次调用只针对一个目标路径：被分配的那个。即使对话里提到其他笔记，也不要碰。 + - `write` 会无条件覆盖正文和frontmatter，请谨慎使用。 diff --git a/reme4/steps/file_io/daily_list.py b/reme4/steps/file_io/daily_list.py index 0552e975..05d9bedc 100644 --- a/reme4/steps/file_io/daily_list.py +++ b/reme4/steps/file_io/daily_list.py @@ -37,11 +37,27 @@ class DailyListStep(BaseStep): """Keep only the user-facing keys (drop internal scan_notes fields, if any).""" return {"path": note["path"], "slug": note["slug"], "metadata": note["metadata"]} + @staticmethod + def _format_note_line(note: dict) -> str: + """Format a single note as ``- path: ... name: ... description: ... ``.""" + meta: dict = note.get("metadata", {}) + ordered_keys = [k for k in ("name", "description") if k in meta] + ordered_keys += [k for k in meta if k not in ("name", "description")] + parts = [f"- path: {note['path']}"] + for key in ordered_keys: + value = meta[key] + if value is None or value == "": + continue + value_str = str(value).replace("\r\n", " ").replace("\r", " ").replace("\n", " ") + parts.append(f"{key}: {value_str}") + return " ".join(parts) + async def execute(self): """Scan ``//`` and emit one projected record per note.""" assert self.context is not None day, daily_dir, vault_dir = self._collect_params() notes = [self._project(n) for n in scan_notes(vault_dir, day, daily_dir)] self.context.response.success = True - self.context.response.answer = f"Listed {len(notes)} note(s) for {day}" - self.context.response.metadata.update({"date": day, "notes": notes}) + lines = [self._format_note_line(n) for n in notes] + self.context.response.answer = "\n".join(lines) if lines else f"No notes found for {day}" + self.context.response.metadata.update({"date": day, "count": len(notes)}) diff --git a/reme4/steps/file_io/edit.py b/reme4/steps/file_io/edit.py index 4e6809aa..34160e7d 100644 --- a/reme4/steps/file_io/edit.py +++ b/reme4/steps/file_io/edit.py @@ -1,6 +1,7 @@ """Find-and-replace text in a markdown file body (front matter is preserved).""" import frontmatter +import yaml from ._file_io import ( NON_MD_WARNING, @@ -73,7 +74,11 @@ class EditStep(BaseStep): # Markdown: parse frontmatter and operate on body only. Non-markdown: # there's no frontmatter convention, so operate on the full text. if is_md: - post = frontmatter.loads(raw_text) + try: + post = frontmatter.loads(raw_text) + except yaml.YAMLError as exc: + self._fail(f"failed to parse frontmatter in {target}: {exc}", path=str(target)) + return None body = post.content not_found_msg = ( f"text to replace was not found in the body of {target} (front matter is excluded from edit)" diff --git a/reme4/steps/file_io/frontmatter_read.py b/reme4/steps/file_io/frontmatter_read.py index c01bf6f5..c9c28e82 100644 --- a/reme4/steps/file_io/frontmatter_read.py +++ b/reme4/steps/file_io/frontmatter_read.py @@ -11,6 +11,7 @@ doesn't exist; otherwise ``{exists: true, frontmatter: {...}}``. from pathlib import Path import frontmatter +import yaml from ..base_step import BaseStep from ...components import R @@ -37,7 +38,13 @@ class FrontmatterReadStep(BaseStep): self.context.response.metadata.update({"path": path, "error": "not markdown"}) return - meta = dict(frontmatter.loads(target.read_text(encoding="utf-8")).metadata) + try: + meta = dict(frontmatter.loads(target.read_text(encoding="utf-8")).metadata) + except yaml.YAMLError as exc: + self.context.response.success = False + self.context.response.answer = f"Error: failed to parse frontmatter in {path}: {exc}" + self.context.response.metadata.update({"path": path, "exists": True, "error": str(exc)}) + return self.context.response.success = True self.context.response.answer = f"Read frontmatter from {path} ({len(meta)} key(s))" self.context.response.metadata.update({"path": path, "exists": True, "frontmatter": meta}) diff --git a/reme4/steps/jobs/__init__.py b/reme4/steps/jobs/__init__.py index 952c3cd4..ff1927ec 100644 --- a/reme4/steps/jobs/__init__.py +++ b/reme4/steps/jobs/__init__.py @@ -1,10 +1,3 @@ -"""Jobs steps — composite ReAct-agent-driven workflows. - -Two steps: - - digester — cold-write: distill daily notes into digest/ (R-M-W via a ReAct agent). - synchronizer — hot-write: persist in-progress task as a daily note. -""" +"""Jobs steps — composite ReAct-agent-driven workflows.""" from . import digester # noqa: F401 -- @R.register("digester") -from . import synchronizer # noqa: F401 -- @R.register("synchronizer") diff --git a/reme4/steps/jobs/digester.py b/reme4/steps/jobs/digester.py index 641ae39e..f40b7f74 100644 --- a/reme4/steps/jobs/digester.py +++ b/reme4/steps/jobs/digester.py @@ -1,6 +1,6 @@ """Smart Digester — knowledge distillation from daily notes to digest/. -The Digester is the **cold-write** counterpart to Synchronizer (hot-write). +The Digester is the **cold-write** counterpart to AutoMemory (hot-write). It reads completed work in ``daily//.md`` note files, identifies entities / concepts / claims / methods worth preserving long-term, and sinks them into ``digest/`` as canonical-entry nodes so @@ -26,7 +26,7 @@ extra). After processing each daily, the agent must call (or ``metadata={"status": "skipped"}`` when intentionally bypassed). Convention: absent ≡ ``pending``, so the next pass finds residual work via ``file_list path=daily recursive=true`` + per-item ``frontmatter_read`` to filter for absent ``status``. -Only the digester writes ``status``; Synchronizer / hand-edits must +Only the digester writes ``status``; AutoMemory / hand-edits must leave it alone. No degraded path — distillation strictly requires an LLM. When ``as_llm`` diff --git a/reme4/steps/jobs/synchronizer.py b/reme4/steps/jobs/synchronizer.py deleted file mode 100644 index 0eabaede..00000000 --- a/reme4/steps/jobs/synchronizer.py +++ /dev/null @@ -1,276 +0,0 @@ -"""Synchronizer — daily-note sync ReAct agent. - -Watches the agent's recent conversation and persists in-progress -tasks as a daily note inside reme's vault_dir, so future agent -invocations can pick the work back up. Pure sync mechanism — does -**not** compress the agent's context (compression is the agent's -own concern). - -Note layout: a single markdown file ``daily//.md``. -Everything worth preserving (verbatim user prompt, key tool output, -intermediate data) goes inline inside this file — there are no -sibling materials. References to it use the full path relative to -the vault (``[[daily//.md]]``); short or no-extension -forms do not resolve. Frontmatter carries ``name`` / ``description`` -plus an optional ``inherits`` wikilink for cross-day continuation. -Body splits into ``Objective`` / ``Plan`` / ``Progress`` / -``Findings`` / ``Decisions`` / ``Next`` / ``References`` sections -(the last is a list of ``[[resource//]]`` wikilinks for -inbound assets landed via ``ingest`` — non-markdown -artefacts the task itself produced are summarized inline). - -Inputs (from RuntimeContext): - messages (list[Msg], required): conversation slice to inspect. - note (str, optional): caller-supplied note hint (task name - or ``daily//.md`` path) to bias slug selection - and disambiguate same-day tasks. - -Output (written to context.response.answer): - { - "skipped": True if the agent reported [SKIP], - "actions": one-line action statement from the agent, - "note": note file path relative to the vault, or None, - "summary": full markdown content of the synced note, - for the calling agent to reload into a - compacted context. None when SKIP / failed. - } - -The agent's toolkit is assembled by ``add_as_tool`` — each entry in -``_NOTE_TOOLS`` is a job name (registered in the active config); -the wrapper turns ``job(**kwargs)`` into a ``ToolResponse``. The job -indirection means the agent sees exactly the same tool surface -(name / description / parameter schema) as the L2 MCP layer. - -Override interface — note shape (single-file layout, frontmatter -fields, section discipline) is opinionated convention, not a core -invariant, so callers can fully replace it without touching reme4: - -* ``prompt_dict`` (inherited from ``BaseStep``) overrides the - ``system_prompt`` / ``user_message`` templates wholesale — this is - how a service layer swaps in its own note schema (e.g. a different - section list, different frontmatter fields). -* ``toolkit`` replaces the tool surface ``_NOTE_TOOLS`` builds. - -What ships here (``synchronizer.yaml``) is just one viable convention; -the service layer (plugin configs, custom callers) is the right place -to pin down the *deployment-specific* shape. -""" - -import datetime -import re -import zoneinfo -from pathlib import Path - -from agentscope.agent import ReActAgent -from agentscope.message import Msg -from agentscope.tool import Toolkit -from pydantic import BaseModel, Field - -from ..base_step import BaseStep - -from ...components import R - - -_NOTE_PATH_RE = re.compile(r"daily/\d{4}-\d{2}-\d{2}/[^/\s]+\.md") - - -_NOTE_TOOLS: tuple[str, ...] = ( - "file_list", - "file_read", - "file_write", - "file_append", - "file_edit", - "file_stat", - "frontmatter_read", - "frontmatter_update", - "frontmatter_delete", - "daily_read", - "daily_write", - "daily_reindex", -) - - -def _coerce_messages(raw) -> list[Msg]: - """Normalize incoming messages to ``Msg`` instances. - - The Python caller hands in ``list[Msg]`` directly; the MCP layer - delivers ``list[dict]`` (each dict shaped roughly ``{name?, role?, - content?}``). Both shapes land here. - """ - if not raw: - return [] - out: list[Msg] = [] - for item in raw: - if isinstance(item, Msg): - out.append(item) - continue - if isinstance(item, dict): - out.append( - Msg( - name=item.get("name") or item.get("role") or "user", - role=item.get("role") or "user", - content=item.get("content", ""), - ), - ) - return out - - -def _format_history(messages: list[Msg]) -> str: - """Render the conversation as a speaker-tagged transcript. - - Skips messages whose text content is empty (tool-only frames - don't help the LLM judge task state). - """ - if not messages: - return "(empty)" - lines: list[str] = [] - for msg in messages: - speaker = msg.name or msg.role or "?" - text = (msg.get_text_content() or "").strip() - if not text: - continue - lines.append(f"[{speaker}]\n{text}") - return "\n\n".join(lines) or "(no text)" - - -class SynchronizerResult(BaseModel): - """Outcome of a single note-sync call. - - Without per-tool audit (the agent's toolkit is the job surface, - which doesn't expose per-call records back to the orchestrator), - the structured outcome is just what the agent reports plus what - we re-read from disk after it returns. - """ - - used_llm: bool = Field(default=False) - skipped: bool = Field(default=False) - actions: str = Field( - default="", - description="One-line action statement from the agent (e.g. " - "'updated daily/2026-05-15/auth-refactor.md' or '[SKIP]').", - ) - note: str | None = Field( - default=None, - description="Note file path relative to the vault, " - "e.g. 'daily/2026-05-15/auth-refactor.md'. None when SKIP / failed.", - ) - summary: str | None = Field( - default=None, - description="Full markdown content of the note file. Lets the calling " - "agent reload the warm summary into a freshly compacted context without an extra read.", - ) - - -@R.register("synchronizer") -class Synchronizer(BaseStep): - """Drive daily-note sync via a ReAct agent.""" - - def __init__( - self, - toolkit: Toolkit | None = None, - console_enabled: bool = False, - timezone: str | None = None, - inherit_window_days: int = 7, - **kwargs, - ): - super().__init__(**kwargs) - self.toolkit = toolkit - self.console_enabled = console_enabled - self.timezone = timezone - self.inherit_window_days = inherit_window_days - - def _now(self) -> datetime.datetime: - if self.timezone: - try: - return datetime.datetime.now(zoneinfo.ZoneInfo(self.timezone)) - except Exception as e: - self.logger.error( - f"Invalid timezone {self.timezone!r}, falling back to local time: {e}", - ) - return datetime.datetime.now() - - def _vault_dir(self) -> Path: - wd = getattr(self.file_store, "vault_path", None) - return Path(wd).resolve() if wd else Path.cwd().resolve() - - def _build_toolkit(self) -> Toolkit: - """Bind every note-relevant job as a tool function. - - Each entry in ``_NOTE_TOOLS`` is a job name registered in the - active config; ``add_as_tool`` wraps ``job(**kwargs)`` into a - ``ToolResponse``. The job indirection means the agent sees exactly - the same tool surface (name / description / parameter schema) as - the L2 MCP layer. - """ - toolkit = self.toolkit or Toolkit() - for job_name in _NOTE_TOOLS: - self.add_as_tool(toolkit, job_name) - return toolkit - - async def execute(self): - assert self.context is not None - messages: list[Msg] = _coerce_messages(self.context.get("messages")) - note_hint: str = self.context.get("note", "") or "" - - if not messages: - result = SynchronizerResult(used_llm=False, skipped=True) - self.context.response.success = True - self.context.response.answer = "Skipped: no messages supplied" - self.context.response.metadata.update(result.model_dump()) - return - - toolkit = self._build_toolkit() - - agent = ReActAgent( - name="reme_synchronizer", - model=self.as_llm, - sys_prompt=self.prompt_format("system_prompt"), - formatter=self.as_llm_formatter, - toolkit=toolkit, - ) - agent.set_console_output_enabled(self.console_enabled) - - user_message: str = self.prompt_format( - "user_message", - today=self._now().strftime("%Y-%m-%d"), - vault_dir=str(self._vault_dir()), - inherit_window_days=self.inherit_window_days, - note=note_hint or "(none)", - history=_format_history(messages), - ) - - final_msg: Msg = await agent.reply( - Msg(name="reme", role="user", content=user_message), - ) - actions = (final_msg.get_text_content() or "").strip() - - result = SynchronizerResult(used_llm=True, actions=actions) - if "[SKIP]" in actions.upper(): - result.skipped = True - - # Reload the freshly written note so the calling agent can drop - # it back into a compacted context without an extra read trip. - if not result.skipped: - self._reload_note(result, actions) - - self.context.response.success = True - self.context.response.answer = actions or "Synchronization completed" - self.context.response.metadata.update(result.model_dump()) - - def _reload_note(self, result: SynchronizerResult, actions: str) -> None: - """Parse the agent's action line for the note path and read the - full file back into ``result.summary``. Best-effort: a parse miss - leaves the context-management fields as None but does not fail - the step (persistence already succeeded).""" - match = _NOTE_PATH_RE.search(actions) - if not match: - return - note_path = match.group(0) - try: - absolute = (Path(self.file_store.vault_path or ".") / note_path).resolve() - text = absolute.read_text(encoding="utf-8") - except Exception as e: - self.logger.warning(f"synchronizer: could not reload note {note_path!r}: {e}") - return - result.note = note_path - result.summary = text diff --git a/reme4/steps/jobs/synchronizer.yaml b/reme4/steps/jobs/synchronizer.yaml deleted file mode 100644 index 137e130f..00000000 --- a/reme4/steps/jobs/synchronizer.yaml +++ /dev/null @@ -1,144 +0,0 @@ -system_prompt: | - You persist an in-progress task as a daily note so the next session - can pick it up. One call writes at most one note. - - ## Note shape - - /daily//.md # the whole note, one file - - One note per task per day; cross-day continuation uses an - `inherits:` frontmatter wikilink. Anything worth preserving - (verbatim user prompt, key tool output, intermediate data) goes - inline inside this file — there are no sibling materials. - - ## Five steps per call - - ### Step 1 — Skip check - - Is the conversation a real multi-step task in progress? Casual Q&A, a - single-shot answer, or idle chat → reply `[SKIP]` (alone, literal) and - stop. When genuinely ambiguous, default to writing — losing work is - worse than an extra note. - - ### Step 2 — Pick the slug - - - Note hint already a kebab-case slug → use verbatim. - - Otherwise mint a stable kebab-case from the task topic (≤60 chars). - - **Reuse the same slug across calls for the same logical thread** — - same slug = same file = upsert. Fragmenting one thread across - multiple slugs is the worst failure mode. - - ### Step 3 — Discover the branch - - Call `file_list path=daily/{today}` (and `daily_read` / - `frontmatter_read` on candidates as needed) to determine one of three - branches: - - - **UPDATE** — `daily/{today}/.md` already exists: - `daily_read slug=` to fetch body + frontmatter, merge per - Step 4, then `daily_write slug= overwrite=true` with the - full new body + frontmatter. - - - **INHERIT** — no file today, but within the last - {inherit_window_days} days an active note with the same name - exists: confirm via `daily_read` on the earlier note, then - `daily_write slug=` (default `overwrite=false`) with a - fresh body that sets `inherits: [[daily//.md]]` - in frontmatter and copies the predecessor's `Objective` + `Plan`. - Progress / Findings / Decisions / Next start empty. **Do not - modify the predecessor.** - - - **CREATE** — neither: `daily_write slug=` (default - `overwrite=false`) with the full body + frontmatter. Idempotent — - no-ops if a same-slug note already exists today (caller falls - back to the UPDATE branch). - - ### Step 4 — Write - - Pick the smallest write for each change: - - | Change | Tool | - |---|---| - | New / replacement full note | `daily_write` (one shot: body + frontmatter + index refresh) | - | Append to a trailing append-only section | `file_append` — cheaper than R-M-W | - | One frontmatter key | `frontmatter_update` (call `daily_reindex` after if `name`/`description` changed) | - | Mid-body restructure | `daily_read` + `daily_write overwrite=true` | - - Suggested body shape (sections are convention — adapt as fits): - - ```markdown - --- - name: - description: <2-3 sentences: what + why> - inherits: [[daily//.md]] # INHERIT only - --- - ## Objective - - - ## Plan - - - ## Progress - - - - ## Findings - - - - ## Decisions - - - - ## Next - - [ ] - - ## References - - [[resource//]] — # external assets the task consumed - ``` - - Section discipline: Progress / Findings / Decisions are append-only - (never delete history). Plan / Next are wholesale-rewritten each call. - Objective is set once. `## References` lists `[[resource//]]` - wikilinks for inbound assets that arrived through an external channel - and were landed by `ingest`; non-markdown content the task - itself produced (raw outputs, screenshots) should be summarized inline - or skipped — there is no per-note sibling folder anymore. - - Wikilink form is fixed: note refs use the full vault-relative - path with `.md` (`[[daily//.md]]`); resource refs use the - canonical resource path (`[[resource//]]`). Short forms - don't resolve. - - ### Step 5 — Emit one line - - Your final reply must be exactly one line in this form: - - daily//.md - - - `` ∈ `created` / `inherited` / `updated` - - Or `[SKIP]` (literal, alone) if Step 1 said skip. This line is parsed - mechanically — match the format exactly. - - ## Boundaries - - - **Never write under `digest/`** — that tier is downstream. - - **Never write the `status` frontmatter** — `status` is reserved for - the downstream distillation pass, which uses absence to find pending - work. Touching it from here makes the note invisible to the - next distill run. - - **`daily_write` defaults to `overwrite=false`** (idempotent skip-if-exists, - mirroring the old `daily_resolve` probe). Pass `overwrite=true` only when - you've already read the file via `daily_read` and intend to replace it - (the UPDATE branch). On a surprise collision (`created: false`) — fall - back to UPDATE rather than blindly overwriting. - -user_message: | - Today: {today} - Vault dir: {vault_dir} - Inherit window: last {inherit_window_days} days - Note hint: {note} - - # Recent conversation - - {history} - - Run the five steps from the system prompt. Final reply = one line. diff --git a/reme4/steps/transfer/ingest.py b/reme4/steps/transfer/ingest.py index 1d9b31e1..4ecf87c1 100644 --- a/reme4/steps/transfer/ingest.py +++ b/reme4/steps/transfer/ingest.py @@ -63,7 +63,7 @@ Parameters: only. * ``description`` (required) — analysis hint for downstream agents: where the asset came from, what kind of content it carries, and how - it should be interpreted. The digester / synchronizer reads this + it should be interpreted. The digester / auto_memory reads this verbatim from ``meta.json`` to decide how to read the asset (skim vs. deep parse, structured extraction vs. summarization, etc.), so callers should write enough detail to drive that decision — not diff --git a/tests/test_base_file_watcher.py b/tests/test_base_file_watcher.py index dcf56abf..60022aa9 100644 --- a/tests/test_base_file_watcher.py +++ b/tests/test_base_file_watcher.py @@ -12,7 +12,7 @@ Usage: pytest tests/test_base_file_watcher.py -v -k "test_existing" """ -# pylint: disable=redefined-outer-name,protected-access,unused-argument +# pylint: disable=redefined-outer-name,protected-access,unused-argument,no-name-in-module import asyncio import tempfile diff --git a/tests/test_reme_light_watch_paths.py b/tests/test_reme_light_watch_paths.py index 4d0a91f9..70afc7e8 100644 --- a/tests/test_reme_light_watch_paths.py +++ b/tests/test_reme_light_watch_paths.py @@ -7,6 +7,7 @@ filesystems (Windows NTFS, macOS APFS/HFS+). See agentscope-ai/ReMe#228. """ # pylint: disable=redefined-outer-name,protected-access,missing-function-docstring,missing-class-docstring +# pylint: disable=no-name-in-module import tempfile from pathlib import Path diff --git a/tests4/integration/__init__.py b/tests4/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests4/integration/test_as_llm.py b/tests4/integration/test_as_llm.py new file mode 100644 index 00000000..909caab6 --- /dev/null +++ b/tests4/integration/test_as_llm.py @@ -0,0 +1,92 @@ +"""Integration tests: drive ReActAgent through LLMDemoStep + Application wiring. + +Requires LLM_API_KEY (and optionally LLM_BASE_URL / LLM_MODEL_NAME) in the +environment or a .env file at the repo root. Hits the real Anthropic API. +""" + +import asyncio +import os +import tempfile + +from reme4 import Application +from reme4.config import resolve_app_config +from reme4.steps.common.llm_demo import LLMDemoStep +from reme4.utils import load_env + +load_env() + + +class _temp_chdir: + """chdir to path for the duration of the block; restore on exit.""" + + def __init__(self, path): + self.path = path + self._old = None + + def __enter__(self): + self._old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self._old) + + +async def _make_app() -> Application: + """Build and start an Application from the default config (LLM wired via env vars).""" + cfg = resolve_app_config(log_to_console=False, log_to_file=False, enable_logo=False) + app = Application(**cfg) + await app.start() + return app + + +def test_llm_demo_step_basic_chat(): + """LLMDemoStep drives ReActAgent through self.as_llm/as_llm_formatter.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + step = LLMDemoStep(app_context=app.context) + response = await step( + query="What is 1 + 1? Reply with just the number.", + ) + text = (response.answer or "").strip() + print(f"\n[basic_chat] response: {text!r}") + assert text, "Empty assistant response" + assert "2" in text, f"Expected '2' in response, got: {text!r}" + print("✓ test_llm_demo_step_basic_chat passed") + finally: + await app.close() + + asyncio.run(run()) + + +def test_llm_demo_step_with_tool(): + """LLMDemoStep registers the add tool and the agent invokes it.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + step = LLMDemoStep(app_context=app.context) + response = await step( + query="Use the add tool to compute 21 + 21 and report the result.", + sys_prompt="Use the `add` tool whenever the user asks to add numbers.", + use_add_tool=True, + ) + text = (response.answer or "").strip() + print(f"\n[with_tool] response: {text!r}") + assert "42" in text, f"Expected '42' in response, got: {text!r}" + print("✓ test_llm_demo_step_with_tool passed") + finally: + await app.close() + + asyncio.run(run()) + + +if __name__ == "__main__": + print("=== LLMDemoStep + ReActAgent integration tests ===") + test_llm_demo_step_basic_chat() + test_llm_demo_step_with_tool() + print("\nAll integration tests passed!") diff --git a/tests4/integration/test_auto_memory.py b/tests4/integration/test_auto_memory.py new file mode 100644 index 00000000..88ec0eb6 --- /dev/null +++ b/tests4/integration/test_auto_memory.py @@ -0,0 +1,355 @@ +"""Integration test for the auto_memory job (planner + writer end-to-end). + +Drives the full ``auto_memory`` orchestrator against a real LLM. The scenario +seeds one existing daily note covering an ongoing event, then feeds a +10-message conversation that: + +1. continues the existing event with new status / facts (expects an UPDATE + that preserves the old facts and appends the new ones), and +2. introduces a brand-new topic (expects a CREATE under today's daily folder + with a stem distinct from the seeded one). + +Requires LLM_API_KEY (and optionally LLM_BASE_URL / LLM_MODEL_NAME) in the +environment or a .env file at the repo root. Hits the real Anthropic API. +""" + +import asyncio +import json +import os +import tempfile +from datetime import date as _date +from pathlib import Path + +from agentscope.agent import ReActAgent + +from reme4 import Application +from reme4.config import resolve_app_config +from reme4.utils import load_env + +load_env() + +# Where agent.memory jsonl dumps land — same directory as this test file. +DUMP_DIR = Path(__file__).resolve().parent + + +SEED_STEM = "auth-middleware-rewrite" +SEED_BODY = """--- +name: auth-middleware-rewrite +description: JWT auth middleware rewrite driven by legal/compliance requirements around session token storage +--- + +# 背景 + +- 项目：JWT auth middleware 重写，替换旧的 session middleware +- 动机：legal/compliance 要求，旧的 session token 存储方式不符合新合规要求 +- 决策：采用 RS256 签名，密钥放在 KMS，refresh token 写 redis 集群 +- 团队：Alice 主导，Bob 协助 + +# 时间线 + +- 2026-05-20 立项 kickoff +- 2026-05-23 设计评审通过 + +# 当下状态 + +- 进度：实现中 +- 卡点：暂无 +- 下一步：完成 refresh token 写入流程 +""" + + +def _today() -> str: + return _date.today().isoformat() + + +class _temp_chdir: + def __init__(self, path): + self.path = path + self._old = None + + def __enter__(self): + self._old = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self._old) + + +async def _make_app() -> Application: + cfg = resolve_app_config(log_to_console=False, log_to_file=False, enable_logo=False) + app = Application(**cfg) + await app.start() + return app + + +def _seed_note(vault_root: Path, today: str) -> Path: + """Write the existing auth-middleware-rewrite note under today's daily folder.""" + day_dir = vault_root / "daily" / today + day_dir.mkdir(parents=True, exist_ok=True) + path = day_dir / f"{SEED_STEM}.md" + path.write_text(SEED_BODY, encoding="utf-8") + return path + + +def _make_messages() -> list[dict]: + """A 10-turn conversation: first half continues the auth event, second half opens a new topic.""" + return [ + { + "name": "user", + "role": "user", + "content": "状态更新：PR #432（auth middleware rewrite）今天已经合并到 dev 分支，等待 staging 验收。", + }, + { + "name": "assistant", + "role": "assistant", + "content": "好的，已记录。staging 验收前要先跑回归测试吗？", + }, + { + "name": "user", + "role": "user", + "content": ( + "对。测试时发现 refresh token TTL 设 7d 在 redis 集群挂了——" + "redis maxmemory-policy 默认 allkeys-lru，会随机驱逐 token，导致用户被强制登出。" + ), + }, + { + "name": "assistant", + "role": "assistant", + "content": "理解，要切到 volatile-ttl 才能只驱逐带 TTL 的 key，对吧？", + }, + { + "name": "user", + "role": "user", + "content": ( + "对，下一步：周五 2026-05-29 前把 redis 配置改成 volatile-ttl 并重测，" "blocked 在 SRE @lihua 的排期。" + ), + }, + { + "name": "user", + "role": "user", + "content": ( + "切个话题，最近在调 pytorch 分布式训练。结论：DDP 启动推荐用 torchrun，" "比 mp.spawn 稳很多。" + ), + }, + { + "name": "assistant", + "role": "assistant", + "content": "是因为信号处理的原因吗？", + }, + { + "name": "user", + "role": "user", + "content": ( + "主要是 NCCL backend 初始化更干净。mp.spawn 在 4 卡以上偶尔会卡死握手；" + "复现版本 pytorch 2.5.1 + nccl 2.21.5。" + ), + }, + { + "name": "user", + "role": "user", + "content": ( + "另外 batch size 用 64*world_size，per-rank lr 用 linear scaling rule " + "（lr = base_lr * world_size）。" + ), + }, + { + "name": "user", + "role": "user", + "content": "这两件事都先记一下。", + }, + ] + + +def _read_text(p: Path) -> str: + return p.read_text(encoding="utf-8") + + +class _AgentMemoryRecorder: + """Monkey-patches ReActAgent.__init__ to capture every agent created inside + the ``with`` block, then dumps each agent's memory to a jsonl file in + DUMP_DIR on exit. One file per agent: ``agent_memory__.jsonl``, + one message per line as ``Msg.to_dict()``. + """ + + def __init__(self, dump_dir: Path, prefix: str = "agent_memory"): + """init""" + self.dump_dir = dump_dir + self.prefix = prefix + self.agents: list[ReActAgent] = [] + self._orig_init = None + self.dumped_paths: list[Path] = [] + + def __enter__(self): + """Monkey-patch ReActAgent.__init__.""" + self._orig_init = ReActAgent.__init__ + agents = self.agents + orig = self._orig_init + + def _capturing_init(agent_self, *args, **kwargs): + orig(agent_self, *args, **kwargs) + agents.append(agent_self) + + ReActAgent.__init__ = _capturing_init + return self + + def __exit__(self, *exc): + """Restore the original __init__ and dump all agent memories.""" + ReActAgent.__init__ = self._orig_init + + async def dump(self) -> list[Path]: + """Dump all agent memories.""" + # Wipe any prior dumps from this prefix so reruns don't accumulate stale files. + for stale in self.dump_dir.glob(f"{self.prefix}_*.jsonl"): + stale.unlink() + + for idx, agent in enumerate(self.agents, 1): + messages = await agent.memory.get_memory() + name = getattr(agent, "name", "agent") or "agent" + out_path = self.dump_dir / f"{self.prefix}_{idx:02d}_{name}.jsonl" + with out_path.open("w", encoding="utf-8") as f: + for msg in messages: + f.write(json.dumps(msg.to_dict(), ensure_ascii=False, default=str) + "\n") + self.dumped_paths.append(out_path) + return self.dumped_paths + + +def test_auto_memory_updates_existing_and_creates_new(): + """End-to-end: planner survey + UPDATE one seeded note + CREATE one new note.""" + + async def run(): + with tempfile.TemporaryDirectory() as tmp, _temp_chdir(tmp): + app = await _make_app() + try: + vault_root = Path(app.config.vault_dir).absolute() + today = _today() + seed_path = _seed_note(vault_root, today) + seed_before = _read_text(seed_path) + assert "legal/compliance" in seed_before + + day_dir = vault_root / "daily" / today + files_before = {p.name for p in day_dir.glob("*.md")} + assert files_before == {f"{SEED_STEM}.md"} + + messages = _make_messages() + + print("\n" + "=" * 70) + print("[setup] vault_root =", vault_root) + print("[setup] today =", today) + print("[setup] seed_path =", seed_path) + print(f"[setup] seed body ({len(seed_before)} bytes):\n{seed_before}") + print(f"[setup] feeding {len(messages)} messages to auto_memory:") + for i, m in enumerate(messages, 1): + print(f" {i:2d}. [{m['role']}] {m['content']}") + print("=" * 70) + + with _AgentMemoryRecorder(DUMP_DIR) as recorder: + response = await app.run_job("auto_memory", messages=messages) + dumped = await recorder.dump() + print(f"\n[dump] captured {len(recorder.agents)} agent(s); wrote {len(dumped)} jsonl file(s):") + for p in dumped: + print(f" - {p}") + + # --- response-level assertions ------------------------------- + assert response.success is True, f"job failed: {response.answer!r}" + meta = response.metadata or {} + memory_updates = meta.get("memory_updates") or [] + + print("\n" + "=" * 70) + print(f"[planner] planned {len(memory_updates)} update(s):") + for u in memory_updates: + print(f" - path: {u.get('path')}") + print(f" description: {u.get('description')}") + print(f"\n[writer] written_count = {meta.get('written_count')}") + print(f"[writer] answer:\n{response.answer}") + print("=" * 70) + + assert ( + len(memory_updates) >= 2 + ), f"expected at least 2 planned updates (1 UPDATE + 1 CREATE), got {memory_updates}" + + # Every emitted path must live under today's daily folder. + paths = [u["path"] for u in memory_updates] + for p in paths: + assert p.startswith(f"daily/{today}/") and p.endswith( + ".md", + ), f"path {p!r} violates daily//.md shape" + + # --- UPDATE branch: seeded note ------------------------------ + update_path_str = f"daily/{today}/{SEED_STEM}.md" + assert ( + update_path_str in paths + ), f"planner did not reuse the seeded path {update_path_str!r}; got {paths}" + seed_after = _read_text(seed_path) + + print("\n" + "=" * 70) + print(f"[UPDATE] {seed_path} ({len(seed_before)} → {len(seed_after)} bytes)") + print(f"[UPDATE] body after:\n{seed_after}") + print("=" * 70) + + # Old facts must survive (timeline append + facts merge-and-dedupe). + for old_fact in ("legal/compliance", "RS256", "Alice"): + assert ( + old_fact in seed_after + ), f"UPDATE dropped pre-existing fact {old_fact!r}\n--- AFTER ---\n{seed_after}" + # At least some of the new facts from the conversation must land. + new_hits = [ + needle + for needle in ("PR #432", "432", "volatile-ttl", "maxmemory-policy", "2026-05-29") + if needle in seed_after + ] + print("[UPDATE] preserved old facts: ['legal/compliance', 'RS256', 'Alice']") + print(f"[UPDATE] landed new facts: {new_hits}") + assert ( + len(new_hits) >= 2 + ), f"UPDATE only landed {new_hits!r} of expected new facts\n--- AFTER ---\n{seed_after}" + + # --- CREATE branch: new topic -------------------------------- + files_after = {p.name for p in day_dir.glob("*.md")} + new_files = files_after - files_before + print(f"\n[CREATE] new files under daily/{today}/: {sorted(new_files)}") + assert new_files, f"no new note created under daily/{today}/; planner paths: {paths}" + # Find the file that actually covers the pytorch topic. + pytorch_path: Path | None = None + for fname in new_files: + text = _read_text(day_dir / fname) + if any(kw in text for kw in ("torchrun", "NCCL", "pytorch", "mp.spawn")): + pytorch_path = day_dir / fname + break + assert pytorch_path is not None, f"no created note covers the pytorch topic; new files: {new_files}" + pytorch_text = _read_text(pytorch_path) + + print("=" * 70) + print(f"[CREATE] {pytorch_path} ({len(pytorch_text)} bytes)") + print(f"[CREATE] body:\n{pytorch_text}") + print("=" * 70) + + topic_hits = [ + needle + for needle in ("torchrun", "mp.spawn", "NCCL", "2.5.1", "linear scaling", "world_size") + if needle in pytorch_text + ] + print(f"[CREATE] landed topic facts: {topic_hits}") + assert ( + len(topic_hits) >= 3 + ), f"CREATE only captured {topic_hits!r} of expected new-topic facts\n--- CREATE ---\n{pytorch_text}" + # frontmatter sanity: name should equal the file stem, description non-empty. + stem = pytorch_path.stem + assert ( + f"name: {stem}" in pytorch_text + ), f"frontmatter name does not match stem {stem!r}\n{pytorch_text[:400]}" + print(f"[CREATE] frontmatter name matches stem {stem!r}") + + print("\n" + "=" * 70) + print("✓ test_auto_memory_updates_existing_and_creates_new passed") + print("=" * 70) + finally: + await app.close() + + asyncio.run(run()) + + +if __name__ == "__main__": + print("=== auto_memory integration test ===") + test_auto_memory_updates_existing_and_creates_new() + print("\nAll integration tests passed!") diff --git a/tests4/unit/__init__.py b/tests4/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests4/unittest/test_background_steps.py b/tests4/unit/test_background_steps.py similarity index 100% rename from tests4/unittest/test_background_steps.py rename to tests4/unit/test_background_steps.py diff --git a/tests4/unittest/test_bm25_index_perf.py b/tests4/unit/test_bm25_index_perf.py similarity index 100% rename from tests4/unittest/test_bm25_index_perf.py rename to tests4/unit/test_bm25_index_perf.py diff --git a/tests4/unittest/test_chunked_file_parser.py b/tests4/unit/test_chunked_file_parser.py similarity index 100% rename from tests4/unittest/test_chunked_file_parser.py rename to tests4/unit/test_chunked_file_parser.py diff --git a/tests4/unittest/test_common_steps.py b/tests4/unit/test_common_steps.py similarity index 100% rename from tests4/unittest/test_common_steps.py rename to tests4/unit/test_common_steps.py diff --git a/tests4/unittest/test_crud_steps.py b/tests4/unit/test_crud_steps.py similarity index 100% rename from tests4/unittest/test_crud_steps.py rename to tests4/unit/test_crud_steps.py diff --git a/tests4/unittest/test_daily_steps.py b/tests4/unit/test_daily_steps.py similarity index 96% rename from tests4/unittest/test_daily_steps.py rename to tests4/unit/test_daily_steps.py index 873eb8b5..7909d971 100644 --- a/tests4/unittest/test_daily_steps.py +++ b/tests4/unit/test_daily_steps.py @@ -117,11 +117,10 @@ def test_daily_list_default_date_is_today(): await step() payload = _metadata(step) assert payload["date"] == _today() - paths = sorted(n["path"] for n in payload["notes"]) - assert paths == [ - f"daily/{_today()}/today-a.md", - f"daily/{_today()}/today-b.md", - ] + assert payload["count"] == 2 + answer = step.context.response.answer + assert f"daily/{_today()}/today-a.md" in answer + assert f"daily/{_today()}/today-b.md" in answer await store.close() print("✓ test_daily_list_default_date_is_today passed") @@ -143,8 +142,9 @@ def test_daily_list_filters_by_date(): await step(date="2026-05-18") payload = _metadata(step) assert payload["date"] == "2026-05-18" - paths = [n["path"] for n in payload["notes"]] - assert paths == ["daily/2026-05-18/a.md"] + assert payload["count"] == 1 + answer = step.context.response.answer + assert "daily/2026-05-18/a.md" in answer await store.close() print("✓ test_daily_list_filters_by_date passed") @@ -167,13 +167,11 @@ def test_daily_list_returns_path_slug_metadata(): step = daily_list_step.DailyListStep(file_store=store) await step(date="2026-05-18") payload = _metadata(step) - assert payload["notes"] == [ - { - "path": "daily/2026-05-18/alpha.md", - "slug": "alpha", - "metadata": {"name": "Alpha Project", "description": "JWT auth migration"}, - }, - ] + assert payload["count"] == 1 + answer = step.context.response.answer + assert "daily/2026-05-18/alpha.md" in answer + assert "Alpha Project" in answer + assert "JWT auth migration" in answer await store.close() print("✓ test_daily_list_returns_path_slug_metadata passed") @@ -200,8 +198,9 @@ def test_daily_list_ignores_subdirectories(): step = daily_list_step.DailyListStep(file_store=store) await step(date="2026-05-18") payload = _metadata(step) - paths = [n["path"] for n in payload["notes"]] - assert paths == ["daily/2026-05-18/main.md"] + assert payload["count"] == 1 + answer = step.context.response.answer + assert "daily/2026-05-18/main.md" in answer await store.close() print("✓ test_daily_list_ignores_subdirectories passed") @@ -218,7 +217,7 @@ def test_daily_list_empty_when_no_daily_dir(): step = daily_list_step.DailyListStep(file_store=store) await step(date="2026-05-18") payload = _metadata(step) - assert payload == {"date": "2026-05-18", "notes": []} + assert payload == {"date": "2026-05-18", "count": 0} await store.close() print("✓ test_daily_list_empty_when_no_daily_dir passed") @@ -250,7 +249,7 @@ def test_daily_list_does_not_refresh_index(): def test_daily_list_response_shape(): - """daily_list returns only {date, notes}.""" + """daily_list returns only {date, count}.""" async def run(): with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): @@ -262,7 +261,7 @@ def test_daily_list_response_shape(): step = daily_list_step.DailyListStep(file_store=store) await step(date="2026-05-18") payload = _metadata(step) - assert set(payload.keys()) == {"date", "notes"} + assert set(payload.keys()) == {"date", "count"} await store.close() print("✓ test_daily_list_response_shape passed") diff --git a/tests4/unittest/test_file_catalog.py b/tests4/unit/test_file_catalog.py similarity index 100% rename from tests4/unittest/test_file_catalog.py rename to tests4/unit/test_file_catalog.py diff --git a/tests4/unittest/test_file_graph.py b/tests4/unit/test_file_graph.py similarity index 100% rename from tests4/unittest/test_file_graph.py rename to tests4/unit/test_file_graph.py diff --git a/tests4/unittest/test_file_store.py b/tests4/unit/test_file_store.py similarity index 100% rename from tests4/unittest/test_file_store.py rename to tests4/unit/test_file_store.py diff --git a/tests4/unittest/test_keyword_index.py b/tests4/unit/test_keyword_index.py similarity index 100% rename from tests4/unittest/test_keyword_index.py rename to tests4/unit/test_keyword_index.py diff --git a/tests4/unittest/test_link_expansion.py b/tests4/unit/test_link_expansion.py similarity index 100% rename from tests4/unittest/test_link_expansion.py rename to tests4/unit/test_link_expansion.py diff --git a/tests4/unittest/test_linked_file_parser.py b/tests4/unit/test_linked_file_parser.py similarity index 100% rename from tests4/unittest/test_linked_file_parser.py rename to tests4/unit/test_linked_file_parser.py diff --git a/tests4/unittest/test_neo4j_file_graph.py b/tests4/unit/test_neo4j_file_graph.py similarity index 100% rename from tests4/unittest/test_neo4j_file_graph.py rename to tests4/unit/test_neo4j_file_graph.py diff --git a/tests4/unittest/test_read_image_steps.py b/tests4/unit/test_read_image_steps.py similarity index 100% rename from tests4/unittest/test_read_image_steps.py rename to tests4/unit/test_read_image_steps.py diff --git a/tests4/unittest/test_read_with_neighbors.py b/tests4/unit/test_read_with_neighbors.py similarity index 100% rename from tests4/unittest/test_read_with_neighbors.py rename to tests4/unit/test_read_with_neighbors.py diff --git a/tests4/unittest/test_resource_steps.py b/tests4/unit/test_resource_steps.py similarity index 100% rename from tests4/unittest/test_resource_steps.py rename to tests4/unit/test_resource_steps.py diff --git a/tests4/unittest/test_tokenizer.py b/tests4/unit/test_tokenizer.py similarity index 100% rename from tests4/unittest/test_tokenizer.py rename to tests4/unit/test_tokenizer.py diff --git a/tests4/unittest/test_wikilink_utils.py b/tests4/unit/test_wikilink_utils.py similarity index 100% rename from tests4/unittest/test_wikilink_utils.py rename to tests4/unit/test_wikilink_utils.py diff --git a/tests4/unittest/test_write_metadata_lock.py b/tests4/unit/test_write_metadata_lock.py similarity index 100% rename from tests4/unittest/test_write_metadata_lock.py rename to tests4/unit/test_write_metadata_lock.py