diff --git a/reme2/component/file_watcher/base_file_watcher.py b/reme2/component/file_watcher/base_file_watcher.py index 544a8af4..ee3f1bad 100644 --- a/reme2/component/file_watcher/base_file_watcher.py +++ b/reme2/component/file_watcher/base_file_watcher.py @@ -14,6 +14,7 @@ startup recovery (re-parsing files whose mtime drifted while offline). """ import asyncio +import os import time from pathlib import Path @@ -293,10 +294,30 @@ class BaseFileWatcher(BaseComponent): # Cancel any in-flight parse task FIRST so a delayed cancellation # can't race the cleanup writes below. assert self.file_store is not None, "_on_deleted requires file_store" - await self._cancel_parse_task(path) - await self.file_store.delete_chunks(path) - await self.file_store.delete_file_meta(path) - self.logger.info(f"Deleted {path}") + targets = [path, *self._descendant_indexed_paths(path)] + for p in targets: + await self._cancel_parse_task(p) + await self.file_store.delete_chunks(p) + await self.file_store.delete_file_meta(p) + if len(targets) == 1: + self.logger.info(f"Deleted {path}") + else: + self.logger.info(f"Deleted directory {path} ({len(targets) - 1} indexed children)") + + def _descendant_indexed_paths(self, path: str) -> list[str]: + """Indexed file paths that live beneath `path` as a directory. + + watchfiles emits a single Change.deleted for a removed directory + (no per-file events), so we have to find the orphans ourselves. + Resolves both sides to handle symlinks and trailing-separator + drift between the watcher and the file_store keys. + """ + assert self.file_store is not None + try: + prefix = str(Path(path).resolve()) + os.sep + except OSError: + prefix = path.rstrip(os.sep) + os.sep + return [p for p in self.file_store.nodes if p.startswith(prefix)] # -- Parse task pipeline ------------------------------------------------ diff --git a/reme2/component/runtime_response.py b/reme2/component/runtime_response.py index 92b2732f..8eb26d49 100644 --- a/reme2/component/runtime_response.py +++ b/reme2/component/runtime_response.py @@ -2,8 +2,8 @@ Lives next to `runtime_context.py` because both are about the BaseStep interface — the response side, specifically. Used by every Step that -returns a JSON-shaped payload (memory_*, sync, topic_create, the three -memory services). +returns a JSON-shaped payload (memory_*, sync, the three memory +services). Was previously at `reme2/mcp/steps/_common.py`, which leaked an MCP dependency into `reme2/memory/` services that legitimately need to diff --git a/reme2/config/curated.yaml b/reme2/config/curated.yaml deleted file mode 100644 index 37ac2343..00000000 --- a/reme2/config/curated.yaml +++ /dev/null @@ -1,220 +0,0 @@ -app_name: reme-curated -enable_logo: false -log_to_console: true -log_to_file: false - -# Curated profile: three MCP tools surfaced — -# `query` — graph-aware hybrid retrieval -# `sync` — hot-path event sync (idempotent upsert per (date, name)) -# `ingest` — LLM-driven distillation over existing files (cold path) -# -# The agent calls `sync` continuously through the task — picking a -# stable `name` per logical thread so each call extends the same -# event folder rather than fragmenting. At task completion (or -# PreCompact / SessionEnd) it calls `ingest` once to distill the -# active events into topic-level cognition. -# -# For full direct control over every memory_* primitive see ./full.yaml. - -service: - backend: mcp - transport: stdio - sidecar_http: true - sidecar_http_host: "127.0.0.1" - sidecar_http_port: 8765 - sidecar_info_path: "./vault/.reme/sidecar.json" - -jobs: - - backend: base - name: query - description: | - Graph-aware hybrid retrieval: vector + keyword + 1-hop wikilink - BFS fusion. Use for "what do I know about X" / "did I work on Y" / - "what's connected to [[Z]]". Anchor mode: include `[[Target]]` in - the query to seed BFS at that file. Topic-rooted mode: pass `seeds` - explicitly. Returns chunks ranked by combined relevance + graph - proximity, each tagged with `graph_hop`. - parameters: - type: object - properties: - query: { type: string } - max_results: { type: integer, default: 5 } - min_score: { type: number, default: 0.0 } - graph_depth: - type: integer - default: 1 - description: "BFS hops from seeds. 1 covers immediate neighbors." - seeds: - type: array - items: { type: string } - description: "Explicit seed paths (topic-rooted mode)." - paths: { type: array, items: { type: string } } - tags: { type: array, items: { type: string } } - exclude_paths: { type: array, items: { type: string } } - steps: - - backend: memory_graph_search - - - backend: base - name: sync - description: | - Hot-path event sync: idempotent upsert of an event FOLDER under - `events/{date}/{name}/`. The folder contains the index `{name}.md` - (Event schema, frontmatter + narrative + Materials footer) plus - any raw materials you pass — conversation snippets, tool outputs, - data dumps. The watcher indexes everything inside. - - CONTINUITY MODEL: pick a stable `name` per logical thread and - call `sync` repeatedly through the task. Each call extends the - same folder: - * new `content` → appended under a `## Update — {iso}` section - * new `materials` → siblings (auto-suffix on filename collision) - * `topics` + `tags` merged (union) into frontmatter - * Materials footer regenerated to list every artifact in the folder - - First call (folder doesn't exist) → CREATE; subsequent calls with - the same `name` while the event is `status: active` → APPEND. - If the event is `status: distilled` / `archived`, `sync` REFUSES - and returns a `suggested_name` so you start a fresh thread instead - of mutating prior cognition. - - Zero LLM cost. Call CONTINUOUSLY through a task as facts land, - and especially at PreCompact to dump verbose raw text into - `materials` before context truncation loses it. The folder is - the unit `ingest` later reads from. - parameters: - type: object - required: [name] - properties: - name: - type: string - description: "kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder." - description: { type: string, description: "one-line summary for index frontmatter (set on initial create only)" } - content: { type: string, description: "markdown body. Initial create: the body. Subsequent calls: appended as a `## Update — {iso}` section." } - topics: - type: array - items: { type: string } - description: "related topic wikilinks. Unioned into frontmatter on append." - tags: { type: array, items: { type: string }, description: "free-form tags; unioned on append." } - materials: - type: array - description: "Raw artifacts written as siblings of the index. Filenames must be safe (letters/digits/dot/underscore/dash). Filename collision with an existing artifact auto-suffixes (foo.txt → foo-2.txt)." - items: - type: object - required: [filename, content] - properties: - filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt'" } - content: { type: string } - on_date: { type: string, description: "ISO date for events/{date}/ bucket; defaults to today" } - origin_session_id: { type: string, description: "set on initial create only" } - steps: - - backend: sync - - - backend: base - name: ingest - description: | - Cold-path LLM-driven distillation. Run on EXPLICIT HANDOFF only: - task completion / SessionEnd / when the agent decides the working - set is ready. Not a per-turn tool. - - The agent hands off the working set in two interchangeable forms - (use both as appropriate): - * `content` — inline material to distill: a hint / summary, or - raw text the agent is feeding directly. - * `related_paths` — paths the agent points at: event folder - indexes (Ingestor follows `## Materials` to read each - artifact), individual material files, or candidate topics. - - The Ingestor reads the working set + linked topics, decides which - existing topics to update / create, and flips each distilled - event's status to "distilled". Returns an audit trail. - - NOT for raw event logging — that's `sync`'s job. - parameters: - type: object - required: [content] - properties: - content: - type: string - description: "Inline material the agent is feeding the Ingestor: distillation hint, session summary, or raw text. Combined with `related_paths` to form the working set." - hint: - type: string - description: "Caller guidance about target / intent." - target_path: - type: string - description: "Optional suggested path; required for the no-LLM degraded path." - metadata: - type: object - description: "Suggested frontmatter for any new topic." - related_paths: - type: array - items: { type: string } - description: "Pointers the agent is feeding the Ingestor: event folder indexes (Ingestor reads materials from `## Materials`), individual material files, or candidate topics." - steps: - - backend: ingestor - -components: - # Ingestor LLM (opt-in). Without this the Ingestor degrades to a - # direct create from explicit `target_path`; edits/renames/deletes - # require the LLM. Uncomment + provide LLM_API_KEY to enable. - # - # as_llm: - # default: - # backend: openai - # model_name: ${LLM_MODEL_NAME:-gpt-4o-mini} - # api_key: ${LLM_API_KEY} - # client_kwargs: - # base_url: ${LLM_BASE_URL:-https://api.openai.com/v1} - # stream: false - # - # as_llm_formatter: - # default: - # backend: openai - - as_token_counter: - default: - backend: estimated - - # Embedding is opt-in: leave embedding_model="" on file_store to run - # keyword-only; uncomment + flip to "default" to enable hybrid search. - # - # embedding_model: - # default: - # backend: openai - # model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small} - # dimensions: 1536 - # pass_dimensions: false - # enable_cache: true - # max_batch_size: 10 - # max_cache_size: 2000 - # max_input_length: 8192 - - edge_extractor: - default: - backend: regex - - file_parser: - md: - backend: md - edge_extractor: default - default: - backend: text - - file_store: - default: - backend: local - embedding_model: "" - store_name: "reme" - db_path: "./vault/.reme" - fts_enabled: true - - file_watcher: - default: - backend: full - file_store: default - default_parser: md - watch_path: "./vault" - recursive: true - - # Retriever (`hybrid`) is a Step, not a pre-instantiated component — - # the `query` shell builds it on demand. Tune defaults by attaching - # knobs to the `memory_graph_search` step under the `query` job above. diff --git a/reme2/config/full.yaml b/reme2/config/expert.yaml similarity index 77% rename from reme2/config/full.yaml rename to reme2/config/expert.yaml index 7630371e..06fbb2ec 100644 --- a/reme2/config/full.yaml +++ b/reme2/config/expert.yaml @@ -1,16 +1,22 @@ -app_name: reme-full +app_name: reme-expert enable_logo: false log_to_console: true log_to_file: false -# Full-exposure profile: every memory_* read + write primitive + the -# typed `topic_create` + the LLM-driven `ingest` are surfaced as MCP -# tools. The agent picks whatever it needs — finest granularity, no -# opinion enforced beyond the wikilink-uniqueness gate. +# Expert-exposure profile: every memory_* read + write primitive plus +# `sync` (event log) are surfaced as MCP tools. The agent picks whatever +# it needs — finest granularity, no opinion enforced beyond the +# wikilink-uniqueness gate. +# +# **No `ingest` tool.** Cold-path R-M-W in expert mode is owned by the +# host agent (Claude Code, with the `reme-expert` SKILL loaded), not by +# reme2's internal ReActAgent. The reme-expert plugin spawns a +# `reme-distiller` subagent at SessionEnd to handle distillation in its +# own context window, using the same memory_* primitives. # # Use when you want the agent to manage memory directly with full -# control. For an opinionated minimal surface, see ./curated.yaml -# (only `query` + `ingest`). +# control. For an opinionated minimal surface (where reme2's internal +# Ingestor owns R-M-W), see ./service.yaml. service: backend: mcp @@ -78,68 +84,6 @@ jobs: steps: - backend: sync - - backend: base - name: ingest - description: | - Cold-path LLM-driven distillation. Run on EXPLICIT HANDOFF only: - task completion / SessionEnd / when the agent decides the working - set is ready. Not a per-turn tool. - - The agent hands off the working set in two interchangeable forms - (use both as appropriate): - * `content` — inline material to distill: a hint / summary, or - raw text the agent is feeding directly. - * `related_paths` — paths the agent points at: event folder - indexes (the Ingestor will follow `## Materials` and read - each artifact), individual material files, or candidate - topics flagged for update. - - The Ingestor reads the working set + linked topics, decides which - existing topics to update / create, and flips each distilled - event's status to "distilled". - - NOT for raw event logging — use `sync` for that. - parameters: - type: object - required: [content] - properties: - content: - type: string - description: "Inline material the agent is feeding the Ingestor: distillation hint, session summary, or raw text. Combined with `related_paths` to form the working set." - hint: { type: string, description: "Caller guidance about target / intent." } - target_path: { type: string, description: "Optional suggested path; required for the no-LLM degraded path." } - metadata: { type: object, description: "Suggested frontmatter for any new topic." } - related_paths: - type: array - items: { type: string } - description: "Pointers the agent is feeding the Ingestor: event folder indexes (Ingestor will read materials from `## Materials`), individual material files, or candidate topics." - steps: - - backend: ingestor - - - backend: base - name: topic_create - description: | - Create a typed topic at `topics/{folder}/{name}.md` with hard - schema validation (judgment categories require `confidence`) and - wikilink-uniqueness enforcement. - parameters: - type: object - required: [folder, name, category] - properties: - folder: { type: string } - name: { type: string } - category: - type: string - enum: [company, sector, concept, method, tool, profile, thesis, model, questions, fundamentals] - description: { type: string } - content: { type: string } - confidence: { type: string, enum: ["⏳", "✅", "❌"] } - market: { type: string } - ticker: { type: string } - tags: { type: array, items: { type: string } } - steps: - - backend: topic_create - # -- Read tools -------------------------------------------------------- - backend: base @@ -252,14 +196,34 @@ jobs: steps: - backend: memory_count_tokens + - backend: base + name: memory_lint + description: | + Read-only projection of the Maintainer's lint findings. Walks the + indexed files (optionally restricted to a path prefix) and returns + structural issues — broken wikilinks, schema violations, stem + collisions. Never mutates. The agent decides what to do with each + finding using existing memory_* primitives (rename, update, + property_update, archive, delete). + parameters: + type: object + properties: + target_prefix: + type: string + description: "restrict scan to relpaths starting with this prefix (e.g. 'events/2026-05-09/'). Empty string scans the whole vault." + steps: + - backend: memory_lint + # -- Write primitives (raw building blocks) ---------------------------- - backend: base name: memory_create description: | Create a new file (raw primitive — no LLM reasoning). Prefer - `ingest` when you want an LLM curator. Wikilink-uniqueness gate - runs unless `force=true`. + `ingest` when you want an LLM curator. Two gates run unless + `force=true`: (1) path template — must be `topics/{folder}/{name}.md`, + `events/{date}/{name}/...`, or `Archive/...`; (2) wikilink uniqueness + — `[[stem]]` must resolve to ≤1 path post-create. parameters: type: object required: [path] @@ -268,7 +232,10 @@ jobs: metadata: { type: object } content: { type: string } overwrite: { type: boolean, default: false } - force: { type: boolean, default: false } + force: + type: boolean + default: false + description: "bypass BOTH the path-template gate and the wikilink-uniqueness gate. Use only when you intentionally need a non-template path or accept the ambiguity." steps: - backend: memory_create @@ -290,7 +257,11 @@ jobs: - backend: base name: memory_property_update - description: "Update a single YAML frontmatter key. value=null deletes the key." + description: | + Update a single YAML frontmatter key. value=null deletes the key. + When key='status', enforces the active → distilled → archived + single-direction state machine (skip / reverse refused). Pass + `force=true` to bypass. parameters: type: object required: [path, key] @@ -298,6 +269,10 @@ jobs: path: { type: string } key: { type: string } value: {} + force: + type: boolean + default: false + description: "bypass the status state machine (only meaningful when key='status')." steps: - backend: memory_property_update diff --git a/reme2/config/paw.yaml b/reme2/config/paw.yaml deleted file mode 100644 index efc994f9..00000000 --- a/reme2/config/paw.yaml +++ /dev/null @@ -1,79 +0,0 @@ -enable_logo: false -log_to_console: true -log_to_file: false - -service: - backend: http - -jobs: - - backend: base - name: test - description: "test job" - parameters: - type: object - properties: - name: - type: string - description: "name of the user" - steps: - - name: "test1" - backend: xxx - - name: "test2" - backend: xxx - -components: - as_llm: - default: - backend: openai - model_name: qwen3.6-plus - - as_llm_formatter: - default: - backend: openai - - embedding_model: - default: - backend: openai - model_name: text-embedding-v3 - dimensions: 1024 - pass_dimensions: false - enable_cache: true - max_batch_size: 10 - max_cache_size: 2000 - max_input_length: 8192 - - edge_extractor: - default: - backend: regex - llm: - backend: llm - as_llm: default - as_llm_formatter: default - file_store: default - max_input_chars: 8000 - min_confidence: 0.5 - max_iters: 8 - - file_parser: - md: - backend: md - edge_extractor: default - embedding_model: default - default: - backend: text - embedding_model: default - - file_store: - default: - backend: local - embedding_model: default - store_name: "reme" - db_path: ".reme/store" - - file_watcher: - default: - backend: full - file_store: default - default_parser: default - watch_paths: [ "./test_data" ] - recursive: true diff --git a/reme2/config/service.yaml b/reme2/config/service.yaml new file mode 100644 index 00000000..5eb8d074 --- /dev/null +++ b/reme2/config/service.yaml @@ -0,0 +1,276 @@ +app_name: reme-service +enable_logo: false +log_to_console: true +log_to_file: false + +# Service profile: service-aligned MCP surface — three tools, one per +# memory service: +# `retrieve` — graph-aware hybrid retrieval (Retriever projection) +# `remember` — single write entry point (Ingestor projection) +# * mode=log → zero-LLM event-folder upsert +# * mode=distill → LLM R-M-W into topic graph +# `maintain` — vault hygiene sweep (Maintainer projection) +# runs lint + decay; merge/split require an LLM and +# are off by default. +# +# The agent calls `remember(mode=log, name=…, content=…, materials=…)` +# continuously through the task — picking a stable `name` per logical +# thread so each call extends the same event folder rather than +# fragmenting. At task completion (or PreCompact / SessionEnd) it +# calls `remember(mode=distill, content=…, related_paths=…)` once to +# distill the active events into topic-level cognition. `maintain` is +# typically cron-driven, but the agent can invoke it explicitly when +# it suspects vault drift. +# +# For full direct control over every memory_* primitive (raw writes, +# fine-grained reads, separate sync/ingest tools, memory_lint) see +# ./expert.yaml. + +service: + backend: mcp + transport: stdio + sidecar_http: true + sidecar_http_host: "127.0.0.1" + sidecar_http_port: 8765 + sidecar_info_path: "./vault/.reme/sidecar.json" + +jobs: + - backend: base + name: retrieve + description: | + Graph-aware hybrid retrieval: vector + keyword + 1-hop wikilink + BFS fusion. Use for "what do I know about X" / "did I work on Y" / + "what's connected to [[Z]]". Anchor mode: include `[[Target]]` in + the query to seed BFS at that file. Topic-rooted mode: pass `seeds` + explicitly. Returns chunks ranked by combined relevance + graph + proximity, each tagged with `graph_hop`. + parameters: + type: object + properties: + query: { type: string } + max_results: { type: integer, default: 5 } + min_score: { type: number, default: 0.0 } + graph_depth: + type: integer + default: 1 + description: "BFS hops from seeds. 1 covers immediate neighbors." + seeds: + type: array + items: { type: string } + description: "Explicit seed paths (topic-rooted mode)." + paths: { type: array, items: { type: string } } + tags: { type: array, items: { type: string } } + exclude_paths: { type: array, items: { type: string } } + steps: + - backend: memory_graph_search + + - backend: base + name: remember + description: | + Single write entry point — projects the Ingestor service. Two + modes via the `mode` parameter: + + * `mode: log` (zero LLM, hot path) — idempotent upsert of an + event FOLDER under `events/{date}/{name}/`. The folder + contains the index `{name}.md` (Event schema, frontmatter + + narrative + Materials footer) plus any raw materials you + pass — conversation snippets, tool outputs, data dumps. The + watcher indexes everything inside. + + CONTINUITY MODEL: pick a stable `name` per logical thread + and call `remember(mode=log, ...)` repeatedly through the + task. Each call extends the same folder: + - new `content` → appended under a `## Update — {iso}` + section + - new `materials` → siblings (auto-suffix on filename + collision) + - `topics` + `tags` merged (union) into frontmatter + - Materials footer regenerated to list every artifact + + First call (folder doesn't exist) → CREATE; subsequent calls + with the same `name` while the event is `status: active` → + APPEND. If `status: distilled` / `archived`, REFUSES and + returns `suggested_name` so you start a fresh thread instead + of mutating prior cognition. Call CONTINUOUSLY through a task + as facts land, especially at PreCompact to dump verbose raw + text into `materials` before context truncation. + + * `mode: distill` (LLM R-M-W loop, cold path) — DEFAULT. Run + on EXPLICIT HANDOFF only: task completion / SessionEnd / when + you decide the working set is ready. Not a per-turn tool. + + Hand off the working set in two interchangeable forms: + - `content` — inline material to distill (hint, summary, or + raw text) + - `related_paths` — pointers (event folder indexes; the + Ingestor follows `## Materials` to read each artifact, + individual material files, candidate topics). + + The Ingestor reads the working set + linked topics, decides + which existing topics to update / create, and flips each + distilled event's status to "distilled". Returns an audit + trail. + parameters: + type: object + required: [content] + properties: + mode: + type: string + enum: [log, distill] + default: distill + description: "log = zero-LLM event-folder upsert (requires `name`); distill = LLM R-M-W into topic graph (default)." + # mode=log params + name: + type: string + description: "(mode=log) kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder." + description: + type: string + description: "(mode=log) one-line summary for index frontmatter (set on initial create only)." + topics: + type: array + items: { type: string } + description: "(mode=log) related topic wikilinks. Unioned into frontmatter on append." + tags: + type: array + items: { type: string } + description: "(mode=log) free-form tags; unioned on append." + materials: + type: array + description: "(mode=log) raw artifacts written as siblings of the index. Filenames must be safe (letters/digits/dot/underscore/dash). Filename collision auto-suffixes (foo.txt → foo-2.txt)." + items: + type: object + required: [filename, content] + properties: + filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt'" } + content: { type: string } + on_date: + type: string + description: "(mode=log) ISO date for events/{date}/ bucket; defaults to today." + origin_session_id: + type: string + description: "(mode=log) optional source session identifier (set on initial create only)." + # shared / mode=distill params + content: + type: string + description: "Required. mode=log: markdown body for the index (initial create) or appended `## Update — {iso}` section. mode=distill: inline material the Ingestor distills — hint, summary, or raw text." + hint: + type: string + description: "(mode=distill) caller guidance about target / intent." + target_path: + type: string + description: "(mode=distill) optional suggested path; required for the no-LLM degraded path." + metadata: + type: object + description: "(mode=distill) suggested frontmatter for any new topic." + related_paths: + type: array + items: { type: string } + description: "(mode=distill) pointers — event folder indexes (Ingestor reads materials from `## Materials`), individual material files, or candidate topics." + steps: + - backend: ingestor + + - backend: base + name: maintain + description: | + Vault hygiene sweep — projects the Maintainer service. One pass: + scan signals → propose ops → resolve conflicts → apply. Returns + an audit trail of what ran and what changed. + + Default behavior runs `lint` (broken wikilinks, schema violations, + stem collisions — read-only diagnostics) and `decay` (move stale + events past their freshness window under `/Archive/`). + Merge / split require an LLM and are off unless explicitly opted + into via `ops`. + + Typically cron-driven; the agent invokes it on demand when it + suspects vault drift (after a heavy session of edits, after a + bulk rename, etc.). Dry-run by default — flip `dry_run=false` to + apply changes. + parameters: + type: object + properties: + target_prefix: + type: string + description: "restrict scan to relpaths starting with this prefix (e.g. 'events/2026-05-09/'). Empty string scans the whole vault." + dry_run: + type: boolean + default: true + description: "if true (default) returns the plan without mutating; flip to false to actually apply lint+decay (and merge/split if opted in)." + ops: + type: array + items: + type: string + enum: [lint, decay, merge, split] + description: "subset of ops to run. Defaults to ['lint','decay']. Merge/split are LLM-driven and currently scaffolded — enabling them without an LLM is a no-op." + decay_days: + type: integer + description: "freshness window for the decay proposer (default = Maintainer constructor `decay_days`, typically 90)." + steps: + - backend: maintainer + +components: + # Ingestor LLM (opt-in). Without this the Ingestor degrades to a + # direct create from explicit `target_path`; edits/renames/deletes + # require the LLM. Uncomment + provide LLM_API_KEY to enable. + # + # as_llm: + # default: + # backend: openai + # model_name: ${LLM_MODEL_NAME:-gpt-4o-mini} + # api_key: ${LLM_API_KEY} + # client_kwargs: + # base_url: ${LLM_BASE_URL:-https://api.openai.com/v1} + # stream: false + # + # as_llm_formatter: + # default: + # backend: openai + + as_token_counter: + default: + backend: estimated + + # Embedding is opt-in: leave embedding_model="" on file_store to run + # keyword-only; uncomment + flip to "default" to enable hybrid search. + # + # embedding_model: + # default: + # backend: openai + # model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small} + # dimensions: 1536 + # pass_dimensions: false + # enable_cache: true + # max_batch_size: 10 + # max_cache_size: 2000 + # max_input_length: 8192 + + edge_extractor: + default: + backend: regex + + file_parser: + md: + backend: md + edge_extractor: default + default: + backend: text + + file_store: + default: + backend: local + embedding_model: "" + store_name: "reme" + db_path: "./vault/.reme" + fts_enabled: true + + file_watcher: + default: + backend: full + file_store: default + default_parser: md + watch_path: "./vault" + recursive: true + + # Retriever (`hybrid`) is a Step, not a pre-instantiated component — + # the `query` shell builds it on demand. Tune defaults by attaching + # knobs to the `memory_graph_search` step under the `query` job above. diff --git a/reme2/config/smoke_test.py b/reme2/config/smoke_test.py deleted file mode 100644 index b874f2c2..00000000 --- a/reme2/config/smoke_test.py +++ /dev/null @@ -1,308 +0,0 @@ -"""Smoke-test reme2/config/full.yaml + reme2/config/curated.yaml. - -Boots Application with each profile against a temporary vault, lists -the registered jobs, then calls a representative subset to confirm -end-to-end wiring (parser → file_store → memory_io / ingest / -memory_graph_search). The Ingestor degrades gracefully when no LLM is -configured — we exercise the degraded path so the test is hermetic. - -Run: - python reme2/config/smoke_test.py -""" - -import asyncio -import json -import sys -import tempfile -from pathlib import Path - -_REPO_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(_REPO_ROOT)) - -# Eager-import side-effect modules so all @R.register() decorators run. -import reme2 # noqa: E402,F401 -import reme2.mcp.steps.memory_io # noqa: E402,F401 -import reme2.mcp.steps.memory_retriever # noqa: E402,F401 -import reme2.memory.ingestor # noqa: E402,F401 -import reme2.memory.summarizer # noqa: E402,F401 -import reme2.memory.maintainer # noqa: E402,F401 -import reme2.mcp.steps # noqa: E402,F401 - -from reme2.application import Application # noqa: E402 -from reme2.config import parse_args # noqa: E402 - -CONFIG_DIR = Path(__file__).resolve().parent -PROFILES = { - "full": CONFIG_DIR / "full.yaml", - "curated": CONFIG_DIR / "curated.yaml", -} - - -def _seed_vault(vault: Path) -> None: - """Drop a few markdown files so reads return something.""" - (vault / "topics" / "Alice").mkdir(parents=True, exist_ok=True) - (vault / "topics" / "Alice" / "Alice.md").write_text( - "---\ntitle: Alice\ncategory: profile\ntags: [person]\n---\n" - "# Alice\n\nAlice works on [[Project X]] with [[Bob]].\n" - "[author:: [[Alice]]]\n", - encoding="utf-8", - ) - (vault / "topics" / "Bob").mkdir(parents=True, exist_ok=True) - (vault / "topics" / "Bob" / "Bob.md").write_text( - "---\ntitle: Bob\ncategory: profile\ntags: [person]\n---\n" - "# Bob\n\nBob collaborates with [[Alice]] on [[Project X]].\n", - encoding="utf-8", - ) - (vault / "topics" / "Project X").mkdir(parents=True, exist_ok=True) - (vault / "topics" / "Project X" / "Project X.md").write_text( - "---\ntitle: Project X\ncategory: concept\n---\n" - "# Project X\n\nA major initiative led by [[Alice]] with [[Bob]].\n", - encoding="utf-8", - ) - - -async def _wait_for_index(watcher, expected_min: int, timeout_s: float = 15.0) -> None: - last = -1 - stable_for = 0 - for _ in range(int(timeout_s / 0.25)): - now = len(watcher.file_store) - if now == last and now >= expected_min: - stable_for += 1 - if stable_for >= 4: - return - else: - stable_for = 0 - last = now - await asyncio.sleep(0.25) - - -def _decode(resp) -> object: - """Job answers are JSON strings; decode for inspection. Pass through dicts/lists.""" - if isinstance(resp.answer, (dict, list)): - return resp.answer - if isinstance(resp.answer, str): - try: - return json.loads(resp.answer) - except (json.JSONDecodeError, TypeError): - return resp.answer - return resp.answer - - -async def _run_profile(name: str, config_path: Path) -> dict: - """Boot the profile against a fresh temp vault; exercise jobs; return summary.""" - print(f"\n========== profile: {name} ==========") - - with tempfile.TemporaryDirectory() as tmp: - vault = Path(tmp) / "vault" - vault.mkdir() - _seed_vault(vault) - - # Override watch_path / db_path / sidecar info; force HTTP service so - # we never block on stdio MCP (we only call jobs directly). - _, cfg = parse_args( - "start", - f"config={config_path}", - f"components.file_watcher.default.watch_path={vault}", - f"components.file_store.default.db_path={vault}/.reme", - ) - cfg["service"] = {"backend": "http"} - - app = Application(**cfg) - await app.start() - try: - jobs = sorted(app.context.jobs.keys()) - print(f" registered jobs ({len(jobs)}): {jobs}") - - watcher = app.context.components["file_watcher"]["default"] - await _wait_for_index(watcher, expected_min=3) - print(f" file_store nodes after sync: {len(watcher.file_store)}") - assert len(watcher.file_store) >= 3, "watcher did not index seed files" - - results: dict = {"jobs": jobs, "checks": []} - alice_path = str((vault / "topics" / "Alice" / "Alice.md").resolve()) - - if "memory_get" in jobs: - r = _decode(await app.run_job("memory_get", path=alice_path)) - ok = isinstance(r, dict) and r.get("exists") is True - print(f" memory_get(Alice.md) → exists={ok}, edges={len(r.get('link', []))}") - results["checks"].append(("memory_get", ok)) - - if "memory_list" in jobs: - r = _decode(await app.run_job("memory_list", tags=["person"])) - ok = isinstance(r, dict) and r.get("count", 0) >= 2 - print(f" memory_list(tags=[person]) → count={r.get('count') if isinstance(r, dict) else '?'}") - results["checks"].append(("memory_list", ok)) - - if "memory_search" in jobs: - r = _decode(await app.run_job("memory_search", query="collaborates", max_results=3, min_score=0.0)) - hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) - ok = len(hits) > 0 - print(f" memory_search('collaborates') → {len(hits)} hits") - results["checks"].append(("memory_search", ok)) - - if "memory_links" in jobs: - r = _decode(await app.run_job("memory_links", path=alice_path)) - ok = isinstance(r, dict) and len(r.get("links", [])) >= 2 - print(f" memory_links(Alice.md) → {len(r.get('links', []))} resolved") - results["checks"].append(("memory_links", ok)) - - if "query" in jobs: - r = _decode(await app.run_job("query", query="Alice Bob", max_results=3, min_score=0.0)) - hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) - ok = len(hits) > 0 - print(f" query('Alice Bob') → {len(hits)} hits") - results["checks"].append(("query", ok)) - - if "ingest" in jobs: - # Degraded path: no LLM key set in env → Ingestor falls back - # to direct create from `target_path`. That's the hermetic path. - target = str((vault / "topics" / "smoke" / "smoke.md").resolve()) - r = _decode(await app.run_job( - "ingest", - content="# smoke topic\n\nrecorded by config smoke test.\n", - target_path=target, - metadata={"category": "concept", "title": "smoke"}, - )) - ok = isinstance(r, dict) and (r.get("applied") or r.get("skipped")) - print(f" ingest(degraded create) → applied={len(r.get('applied', [])) if isinstance(r, dict) else '?'}, " - f"used_llm={r.get('used_llm') if isinstance(r, dict) else '?'}") - results["checks"].append(("ingest", bool(ok))) - # Confirm the file landed on disk. - assert Path(target).is_file(), "ingest did not produce the target file" - - if "sync" in jobs: - # CREATE call. - r = _decode(await app.run_job( - "sync", - name="smoke-event", - description="smoke test event", - content="## ops\n- ran the smoke test\n", - topics=["[[Alice]]"], - tags=["smoke"], - materials=[ - {"filename": "raw-prompt.md", "content": "# raw user prompt\n\nrun the smoke test\n"}, - {"filename": "tool-output.txt", "content": "tool ran ok\nexit=0\n"}, - ], - )) - ok = ( - isinstance(r, dict) - and r.get("created") is True - and r.get("action") == "created" - and len(r.get("materials", [])) == 2 - ) - materials = r.get("materials", []) if isinstance(r, dict) else [] - print(f" sync(create smoke-event w/ 2 materials) → created={r.get('created') if isinstance(r, dict) else '?'}, " - f"materials={len(materials)}") - results["checks"].append(("sync.create", bool(ok))) - for m in materials: - assert Path(m).is_file(), f"event material missing: {m}" - if materials: - event_dir = Path(materials[0]).parent - index_text = (event_dir / "smoke-event.md").read_text(encoding="utf-8") - assert "## Materials" in index_text, "index .md missing Materials section" - assert "raw-prompt.md" in index_text, "Materials section missing raw-prompt link" - await _wait_for_index(watcher, expected_min=len(watcher.file_store) + 2) - - # APPEND call: same name, new content + new + colliding material. - r2 = _decode(await app.run_job( - "sync", - name="smoke-event", - content="## follow-up\n- second pass facts\n", - topics=["[[Bob]]"], # union with prior [[Alice]] - tags=["follow-up"], # union with prior [smoke] - materials=[ - {"filename": "tool-output.txt", "content": "second tool run\nexit=0\n"}, # collision → auto-suffix - {"filename": "summary.md", "content": "# summary\nsecond pass\n"}, - ], - )) - ok2 = ( - isinstance(r2, dict) - and r2.get("created") is False - and r2.get("action") == "appended" - and len(r2.get("materials", [])) == 2 - ) - appended_paths = r2.get("materials", []) if isinstance(r2, dict) else [] - print(f" sync(append smoke-event w/ collision) → action={r2.get('action') if isinstance(r2, dict) else '?'}, " - f"new_materials={len(appended_paths)}") - results["checks"].append(("sync.append", bool(ok2))) - # Collision should have produced tool-output-2.txt; summary.md untouched. - names_appended = {Path(p).name for p in appended_paths} - assert "tool-output-2.txt" in names_appended, f"collision auto-suffix missing: {names_appended}" - assert "summary.md" in names_appended, f"clean filename missing: {names_appended}" - # Index should now contain BOTH the original ops section and the Update section. - if materials: - index_text2 = (Path(materials[0]).parent / "smoke-event.md").read_text(encoding="utf-8") - assert "## ops" in index_text2, "original content lost on append" - assert "## Update —" in index_text2, "missing Update section after append" - assert "follow-up" in index_text2, "appended content not in index" - # Frontmatter union check - assert "[[Bob]]" in index_text2, "topic union failed" - # Materials footer should now list 4 files (2 original + summary + tool-output-2) - assert "tool-output-2.txt" in index_text2, "Materials footer missing collided file" - assert "summary.md" in index_text2, "Materials footer missing new file" - - # REFUSAL: flip status to distilled, third sync should refuse. - index_path = str(Path(materials[0]).parent / "smoke-event.md") if materials else None - if index_path and "memory_property_update" in jobs: - await app.run_job("memory_property_update", path=index_path, key="status", value="distilled") - r3 = _decode(await app.run_job( - "sync", - name="smoke-event", - content="should refuse", - )) - ok3 = isinstance(r3, dict) and "error" in r3 and r3.get("status") == "distilled" - print(f" sync(refuse on distilled) → error={'error' in (r3 if isinstance(r3, dict) else {})}, " - f"suggested_name={r3.get('suggested_name') if isinstance(r3, dict) else '?'}") - results["checks"].append(("sync.refuse_distilled", bool(ok3))) - - if "topic_create" in jobs: - r = _decode(await app.run_job( - "topic_create", - folder="Carol", - name="Carol", - category="profile", - description="smoke test", - content="# Carol\n", - tags=["person"], - )) - ok = isinstance(r, dict) and (r.get("created") is True or "path" in r) - print(f" topic_create(Carol) → created={r.get('created') if isinstance(r, dict) else '?'}") - results["checks"].append(("topic_create", bool(ok))) - - return results - finally: - await app.close() - - -async def _main() -> int: - summary: dict[str, dict] = {} - for name, path in PROFILES.items(): - try: - summary[name] = await _run_profile(name, path) - except Exception as e: - print(f" ✗ profile '{name}' failed: {type(e).__name__}: {e}") - import traceback - traceback.print_exc() - summary[name] = {"error": str(e)} - - print("\n========== summary ==========") - failed = 0 - for name, result in summary.items(): - if "error" in result: - print(f" {name}: ERROR — {result['error']}") - failed += 1 - continue - checks = result.get("checks", []) - passed = sum(1 for _, ok in checks if ok) - total = len(checks) - marker = "✓" if passed == total else "✗" - print(f" {marker} {name}: {passed}/{total} checks passed; jobs={len(result['jobs'])}") - for label, ok in checks: - if not ok: - print(f" ✗ {label}") - failed += 1 - return 0 if failed == 0 else 1 - - -if __name__ == "__main__": - sys.exit(asyncio.run(_main())) diff --git a/reme2/mcp/README.md b/reme2/mcp/README.md index 7ae4b705..032f29f3 100644 --- a/reme2/mcp/README.md +++ b/reme2/mcp/README.md @@ -9,14 +9,21 @@ Agent-facing MCP interface layer. Exposes the markdown vault under ``` reme2/mcp/ ├── __init__.py -├── server.py MCP server bootstrap; defaults to ../config/full.yaml +├── server.py MCP server bootstrap; defaults to ../config/service.yaml └── steps/ @R.register MCP step shells - ├── memory_io.py memory_create/update/get/list/links/... ├── memory_retriever.py memory_search + memory_graph_search - ├── sync.py hot-path event-folder upsert - └── topic_create.py typed topic creation w/ schema gates + ├── memory_lint.py read-only Maintainer projection (lint findings) + └── sync.py hot-path event-folder upsert ``` +The 12 `memory_*` primitives (create/update/property_update/rename/ +delete/archive/get/list/links/backlinks/resolve_wikilink/count_tokens) +live one layer down in `reme2/memory/memory_toolkit.py` — each is a +single `BaseStep` subclass with two class methods: `execute()` for the +MCP path (this layer) and a same-named method for the agent toolkit +path that the Ingestor's ReActAgent consumes. Importing +`reme2.mcp.steps` triggers all `@R.register` registrations. + The MCP layer's job is to **project** existing primitives as MCP tools — it owns no business logic. Everything else lives outside the transport boundary so memory services don't form an import cycle: @@ -25,49 +32,97 @@ transport boundary so memory services don't form an import cycle: |---|---| | Memory File System primitives | `reme2/component/file_store/`, `file_watcher/`, `file_parser/` | | Three memory services | `reme2/memory/` — Retriever / Ingestor / Maintainer | -| Pure write helpers + agent toolkit | `reme2/memory/memory_io.py` | -| Vault domain models (Event, Topic) | `reme2/schema/vault/` | +| Engine API (pure, schema-free) | `reme2/memory/memory_io.py` | +| Schema-bound tools (BaseStep + agent toolkit) | `reme2/memory/memory_toolkit.py` | +| Memory schema (4 axes + presets + parser) | `reme2/memory/schema/` | | Path templates + name disambiguation | `reme2/utils/vault_paths.py` | | Step response serialization | `reme2/component/runtime_response.py` | Dependency direction is strict: ``` -reme2.mcp → reme2.memory → reme2.component / reme2.schema / reme2.utils +reme2.mcp → reme2.memory (incl. memory.schema) → reme2.component / reme2.utils ``` +The Ingestor (`reme2/memory/ingestor.py`) self-registers its `ingest` +MCP face — there's no shell for it under `steps/`. Topic creation lives +inside the Ingestor's R-M-W loop; there is no separate `topic_create` +tool. + ## Run ```bash -# stdio MCP server, full tool surface +# stdio MCP server, service-tier surface (loads ../config/service.yaml) python -m reme2.mcp.server -# override config or any field -python -m reme2.mcp.server config=reme2/config/curated.yaml +# pick a different profile +python -m reme2.mcp.server config=reme2/config/expert.yaml + +# override any nested key (Hydra-style) python -m reme2.mcp.server components.file_watcher.default.watch_path=/abs/vault ``` -Config profiles (in `reme2/config/`): -- `full.yaml` — every memory_* primitive + sync + topic_create + ingest -- `curated.yaml` — opinionated 3-tool surface (`query`, `sync`, `ingest`) +Env vars (read by `server.py` and applied as overrides): -## Tools exposed (full profile) +| Var | Effect | +|---|---| +| `VAULT_PATH` | Sets `file_watcher.default.watch_path`, `file_store.default.db_path`, and `service.sidecar_info_path` in one shot. Wins over CLI args. | +| `VAULT_HTTP_PORT` | Override `service.sidecar_http_port` (default `8765`). | + +Both shipped profiles register an `mcp` service with `transport: stdio` +plus a sidecar HTTP server on `127.0.0.1:8765`. The sidecar lets local +hooks (e.g. the plugin's `vault_recall.py`) reach the same in-process +FileGraph over HTTP without re-bootstrapping; on startup the server +writes `{host, port}` to `service.sidecar_info_path` +(default `./vault/.reme/sidecar.json`). + +## Profiles + +Configs live in `reme2/config/`: + +- **`expert.yaml`** — every primitive surfaced (16 tools): `sync` + `ingest` + + `memory_search` / `memory_graph_search` + 5 read primitives + 6 raw + write primitives + `memory_count_tokens` + `memory_lint`. For agents + that should manage the vault directly with full control. +- **`service.yaml`** — opinionated 3-tool surface: `retrieve` (graph-aware + hybrid retrieval), `remember` (single write entry — `mode=log` / + `mode=distill`), `maintain` (vault hygiene sweep). One tool per + memory service; the minimum the agent needs to read, log, distill, + and clean up. + +### Tools exposed (expert profile) | Tool | Path | Purpose | |---|---|---| | `sync` | steps/sync.py | Hot-path event-folder upsert (idempotent per `(date, name)`). | -| `ingest` | reme2/memory/ingestor.py | Cold-path LLM-driven distillation. | -| `topic_create` | steps/topic_create.py | Typed topic creation with schema gates. | +| `ingest` | reme2/memory/ingestor.py | Cold-path LLM-driven distillation; owns topic creation. | | `memory_search` / `memory_graph_search` | steps/memory_retriever.py | V+K hybrid + optional graph BFS. | -| `memory_get` / `memory_list` / `memory_links` / `memory_backlinks` / `memory_resolve_wikilink` | steps/memory_io.py | Read primitives. | -| `memory_create` / `memory_update` / `memory_property_update` / `memory_rename` / `memory_delete` / `memory_archive` | steps/memory_io.py | Raw write primitives (prefer `ingest` / `sync`). | -| `memory_count_tokens` | steps/memory_io.py | Token estimation. | +| `memory_lint` | steps/memory_lint.py | Read-only projection of Maintainer's lint findings. | +| `memory_get` / `memory_list` / `memory_links` / `memory_backlinks` / `memory_resolve_wikilink` | reme2/memory/memory_toolkit.py | Read primitives. | +| `memory_create` / `memory_update` / `memory_property_update` / `memory_rename` / `memory_delete` / `memory_archive` | reme2/memory/memory_toolkit.py | Raw write primitives (prefer `ingest` / `sync`). | +| `memory_count_tokens` | reme2/memory/memory_toolkit.py | Token estimation. | -## Smoke test +### Tools exposed (service profile) + +| Tool | Backend | Purpose | +|---|---|---| +| `retrieve` | `memory_graph_search` | Graph-aware hybrid retrieval (vector + keyword + 1-hop wikilink BFS). | +| `remember` | `ingestor` | Single write entry — `mode=log` (zero-LLM event-folder upsert) / `mode=distill` (LLM R-M-W into topic graph). | +| `maintain` | `maintainer` | Vault hygiene sweep — lint + decay (merge/split require LLM, off by default). | + +## End-to-end tests ```bash -python reme2/config/smoke_test.py +python -m reme2.mcp.test ``` -Boots both `full` and `curated` profiles in a temp vault, exercises a -representative subset of jobs end-to-end (in-process — no MCP -transport). +Boots one Application per profile against a temp vault, exercises every +registered MCP job end-to-end (in-process — no MCP transport), and +prints a per-check status line plus a final pass/fail summary. Exit +code = number of failed checks. + +Layout (`reme2/mcp/test/`): + +- `_helpers.py` — shared vault seed, app factory, response decoder +- `test_expert.py` — expert profile (25 checks across all 16 tools incl. schema gates) +- `test_service.py` — service profile (10 checks across `retrieve` / `remember` / `maintain`) +- `__main__.py` — CLI runner diff --git a/reme2/mcp/__init__.py b/reme2/mcp/__init__.py index 2a107aae..8cfebfd7 100644 --- a/reme2/mcp/__init__.py +++ b/reme2/mcp/__init__.py @@ -2,19 +2,19 @@ The Agent-facing surface: server entrypoint + step shells that wrap the three services in `reme2.memory` (Retriever, Ingestor, Maintainer) -plus hot-write primitives (sync, topic_create, memory_*) that bypass -services and land directly on the Memory File System. +plus the hot-write primitive (`sync`) and the raw `memory_*` write/read +tools that bypass services and land directly on the Memory File System. -This package depends on `reme2.memory`, `reme2.schema.vault`, -`reme2.utils`, `reme2.component` — never the reverse. Domain types -(Topic / Event), pure helpers (path builders, naming), and the write -primitives all live outside `mcp/` so memory-layer services can use -them without forming an import cycle through the transport layer. +This package depends on `reme2.memory`, `reme2.utils`, `reme2.component` +— never the reverse. The Memory schema (the typed shape of every +frontmatter) lives under `reme2.memory.schema/` so the services that +own validation can use it without an import cycle through this +transport layer. Sub-packages: - steps/ - all @R.register MCP step shells (memory_io, memory_retriever, - sync, topic_create). - server.py - MCP server bootstrap (defaults to ../config/full.yaml). + steps/ - all @R.register MCP step shells (memory_toolkit, + memory_retriever, memory_lint, sync). + server.py - MCP server bootstrap (defaults to ../config/service.yaml). """ __version__ = "0.1.0" diff --git a/reme2/mcp/server.py b/reme2/mcp/server.py index 2a4f8c74..ce4bb705 100644 --- a/reme2/mcp/server.py +++ b/reme2/mcp/server.py @@ -4,7 +4,7 @@ Usage: python -m reme2.mcp.server [config=path/to/yaml] [service.transport=stdio] python reme2/mcp/server.py [config=path/to/yaml] [service.transport=stdio] -By default loads `reme2/config/full.yaml` and starts the +By default loads `reme2/config/service.yaml` and starts the ReMe2 application with the MCP service registered. Env vars: @@ -42,7 +42,7 @@ from reme2.application import Application # noqa: E402 from reme2.config import parse_args # noqa: E402 -_DEFAULT_CONFIG = str(_REPO_ROOT / "reme2" / "config" / "full.yaml") +_DEFAULT_CONFIG = str(_REPO_ROOT / "reme2" / "config" / "service.yaml") def main() -> None: diff --git a/reme2/mcp/steps/__init__.py b/reme2/mcp/steps/__init__.py index 1b7a0e70..41d8a7c5 100644 --- a/reme2/mcp/steps/__init__.py +++ b/reme2/mcp/steps/__init__.py @@ -1,31 +1,40 @@ """MCP step shells — the @R.register classes the model invokes as MCP tools. -Two groups: +Three groups: Hot-write primitives that bypass services and land on MFS directly: - sync — idempotent event-folder upsert (event log) - topic_create — typed topic creation (hard schema gates) - memory_io — memory_create / update / property_update / rename / - delete / archive / get / list / links / backlinks / - resolve_wikilink / count_tokens + sync — idempotent event-folder upsert (event log) + memory_toolkit — memory_create / update / property_update / + rename / delete / archive / get / list / + links / backlinks / resolve_wikilink / + count_tokens. Each step lives in + `reme2.memory.memory_toolkit` and exposes + BOTH an `execute()` (this MCP path) and a + same-named class method (the agent toolkit + path used by the Ingestor's ReActAgent). - Service delegate: - memory_retriever — memory_search + memory_graph_search; thin - wrappers that delegate to the configured - Retriever component (`reme2.memory.retriever`). + Service delegates (thin shells over the three memory services): + memory_retriever — memory_search + memory_graph_search; delegate + to the Retriever component + (`reme2.memory.retriever`). + memory_lint — read-only projection of the Maintainer's lint + findings (`reme2.memory.maintainer`). + +Topic creation is owned by the Ingestor (`reme2.memory.ingestor`) — its +LLM-driven R-M-W loop decides when a new topic is warranted, applies the +schema preset, and routes the write through `memory_create`. There is no +separate `topic_create` MCP tool. The Ingestor's MCP face (`ingest`) is registered from -`reme2.memory.ingestor`, since it's a service step rather than an MFS -primitive. Importing this package triggers all the step registrations -hosted here. +`reme2.memory.ingestor`. Importing this package triggers all the step +registrations hosted here and in `reme2.memory.memory_toolkit`. """ -from . import memory_io # noqa: F401 -- triggers @R.register for memory_* +from ...memory import memory_toolkit # noqa: F401 -- triggers @R.register for memory_* +from . import memory_lint # noqa: F401 -- triggers @R.register for memory_lint from . import memory_retriever # noqa: F401 -- memory_search / memory_graph_search from .sync import Sync -from .topic_create import TopicCreate __all__ = [ "Sync", - "TopicCreate", ] diff --git a/reme2/mcp/steps/memory_io.py b/reme2/mcp/steps/memory_io.py deleted file mode 100644 index 037930fb..00000000 --- a/reme2/mcp/steps/memory_io.py +++ /dev/null @@ -1,208 +0,0 @@ -"""MCP step shells over the Memory File System engine API. - -Per `structure.md`, .md files are the SSOT and the engine surface lives -in `reme2.memory.memory_io` (CRUD writes + MFS reads + Projections). -This module only hosts the `@R.register("memory_*")` Step shells — -each one translates RuntimeContext ↔ JSON payload and delegates to the -matching engine API function. - -Search ops live in `memory_retriever.py` (Retriever composes V/K/graph -projections with policy). -""" - -from __future__ import annotations - -from pathlib import Path - -from ...component import R -from ...component.base_step import BaseStep -from ...component.runtime_response import _set_answer -from ...enumeration import ComponentEnum -from ...memory import memory_io - - -@R.register("memory_get") -class MemoryGet(BaseStep): - """Read a single memory file (frontmatter + body, optional chunks).""" - - async def execute(self): - assert self.context is not None - path: str = self.context.get("path", "") - include_chunks: bool = bool(self.context.get("include_chunks", False)) - assert path, "path is required" - result = await memory_io.read_file(self.file_store, path, include_chunks=include_chunks) - _set_answer(self.context, result) - - -@R.register("memory_list") -class MemoryList(BaseStep): - """List indexed files filtered by frontmatter fields, tags, or path prefix.""" - - async def execute(self): - assert self.context is not None - result = memory_io.list_files( - self.file_store, - path_prefix=self.context.get("path_prefix"), - tags=self.context.get("tags") or [], - metadata=self.context.get("metadata") or {}, - limit=int(self.context.get("limit", 100)), - ) - _set_answer(self.context, result) - - -@R.register("memory_backlinks") -class MemoryBacklinks(BaseStep): - """Files linking to a given path. Each entry carries the typed-edge predicate.""" - - async def execute(self): - assert self.context is not None - path: str = self.context.get("path", "") - assert path, "path is required" - _set_answer(self.context, memory_io.backlinks_of(self.file_store, path)) - - -@R.register("memory_links") -class MemoryLinks(BaseStep): - """Files a given path links to (resolved). Each entry carries the typed-edge predicate.""" - - async def execute(self): - assert self.context is not None - path: str = self.context.get("path", "") - assert path, "path is required" - _set_answer(self.context, memory_io.links_of(self.file_store, path)) - - -@R.register("memory_resolve_wikilink") -class MemoryResolveWikilink(BaseStep): - """Resolve a `[[target]]` wikilink with full ambiguity context.""" - - async def execute(self): - assert self.context is not None - wikilink: str = self.context.get("wikilink", "") or "" - assert wikilink, "wikilink is required" - payload = memory_io.wikilink_lookup(self.file_store, wikilink) - self.context.response.success = bool(payload.get("exists")) - _set_answer(self.context, payload) - - -@R.register("memory_create") -class MemoryCreate(BaseStep): - """Create a markdown file. Wikilink-uniqueness gate runs unless force=True.""" - - async def execute(self): - assert self.context is not None - path: str = self.context.get("path", "") - metadata: dict = dict(self.context.get("metadata", {}) or {}) - content: str = self.context.get("content", "") or "" - overwrite: bool = bool(self.context.get("overwrite", False)) - force: bool = bool(self.context.get("force", False)) - - assert path, "path is required" - target = Path(path) - - ok, payload = memory_io.write_create( - self.file_store, target, - metadata=metadata, content=content, - overwrite=overwrite, force=force, - ) - self.context.response.success = ok - if ok: - payload = {**payload, "path": str(target.resolve())} - _set_answer(self.context, payload) - - -@R.register("memory_delete") -class MemoryDelete(BaseStep): - """Delete a file. Watcher removes from store + graph.""" - - async def execute(self): - assert self.context is not None - path: str = self.context.get("path", "") - assert path, "path is required" - ok, payload = memory_io.write_delete(path) - self.context.response.success = ok - _set_answer(self.context, payload) - - -@R.register("memory_rename") -class MemoryRename(BaseStep): - """Rename a file and rewrite incoming wikilinks across the vault.""" - - async def execute(self): - assert self.context is not None - old_path: str = self.context.get("old_path", "") - new_path: str = self.context.get("new_path", "") - assert old_path and new_path, "old_path and new_path are required" - - watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[index,union-attr] - vault_root = Path(watcher.watch_path).resolve() # type: ignore[union-attr] - - ok, payload = memory_io.write_rename(self.file_store, vault_root, old_path, new_path) - self.context.response.success = ok - _set_answer(self.context, payload) - - -@R.register("memory_property_update") -class MemoryPropertyUpdate(BaseStep): - """Update a single YAML frontmatter key. value=null deletes the key.""" - - async def execute(self): - assert self.context is not None - path: str = self.context.get("path", "") - key: str = self.context.get("key", "") - value = self.context.get("value") - assert path and key, "path and key are required" - ok, payload = memory_io.write_property_update(path, key, value) - self.context.response.success = ok - _set_answer(self.context, payload) - - -@R.register("memory_update") -class MemoryUpdate(BaseStep): - """Edit-style content update: replace `old_string` with `new_string`.""" - - async def execute(self): - assert self.context is not None - path: str = self.context.get("path", "") - old_string: str = self.context.get("old_string", "") - new_string: str = self.context.get("new_string", "") - replace_all: bool = bool(self.context.get("replace_all", False)) - assert path, "path is required" - ok, payload = memory_io.write_update(path, old_string, new_string, replace_all=replace_all) - self.context.response.success = ok - _set_answer(self.context, payload) - - -@R.register("memory_archive") -class MemoryArchive(BaseStep): - """Archive a file: flip `status: archived` then move to `/Archive/`.""" - - async def execute(self): - assert self.context is not None - path: str = self.context.get("path", "") or "" - archive_dir_name: str = self.context.get("archive_dir", "Archive") or "Archive" - assert path, "path is required" - - watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default") - vault_root = Path(getattr(watcher, "watch_path", ".")).resolve() if watcher else Path.cwd() - - ok, payload = memory_io.write_archive(vault_root, path, archive_dir_name) - self.context.response.success = ok - _set_answer(self.context, payload) - - -@R.register("memory_count_tokens") -class MemoryCountTokens(BaseStep): - """Estimate tokens for a file body or raw text. One of `path`/`text` required.""" - - async def execute(self): - assert self.context is not None - path: str = self.context.get("path", "") or "" - text: str = self.context.get("text", "") or "" - result = await memory_io.count_tokens( - self.as_token_counter, - path=path or None, - text=text or None, - ) - self.context.response.success = "error" not in result - _set_answer(self.context, result) diff --git a/reme2/mcp/steps/memory_lint.py b/reme2/mcp/steps/memory_lint.py new file mode 100644 index 00000000..29b5007b --- /dev/null +++ b/reme2/mcp/steps/memory_lint.py @@ -0,0 +1,70 @@ +"""memory_lint — read-only projection of the Maintainer's lint findings. + +Tier A surface for the Maintainer service. The host agent calls this +to discover what's wrong with the vault (broken wikilinks, schema +violations, stem collisions) and decides what to do with each finding +using the existing memory_* write primitives. + +Equivalent to invoking `Maintainer.execute(ops=["lint"], dry_run=True)` +but with a focused response shape and a tighter parameter surface — the +agent doesn't need to know about decay/merge/split knobs. +""" + +from __future__ import annotations + +from ...component import R +from ...component.base_step import BaseStep +from ...component.runtime_response import _set_answer +from ...enumeration import ComponentEnum +from ...memory.maintainer import Maintainer + + +@R.register("memory_lint") +class MemoryLint(BaseStep): + """Run the Maintainer's lint pass and surface findings only. + + Inputs (RuntimeContext, all optional): + target_prefix (str): restrict scan to relpaths starting with + this prefix (e.g. "events/2026-05-09/"). + + Output (context.response.answer): + { + "scanned": int, # files inspected + "findings": [LintFinding, ...], # each {path, kind, detail} + "target_prefix": str, + "ran_at": iso, + } + """ + + async def execute(self): + assert self.context is not None + target_prefix = str(self.context.get("target_prefix") or "") + + # Delegate to a Maintainer step. Force ops=["lint"] + dry_run so + # we never mutate. The Maintainer reads these from the context + # and produces a full audit; we narrow the response shape below. + if getattr(self, "_maintainer", None) is None: + self._maintainer = R.get(ComponentEnum.STEP, "maintainer")( + app_context=self.app_context, + ) + self.context["ops"] = ["lint"] + self.context["dry_run"] = True + self.context["target_prefix"] = target_prefix + await self._maintainer(self.context) + + # The Maintainer wrote a full audit to context.response.answer + # (proposed/plan/applied/skipped/failed/...). For lint, all the + # action lives in `proposed` (LintFindings never get applied or + # dropped). Reshape into a focused response. + import json + raw = self.context.response.answer + audit = json.loads(raw) if isinstance(raw, str) else (raw or {}) + findings = audit.get("proposed") or [] + + _set_answer(self.context, { + "scanned": audit.get("scanned", 0), + "findings": findings, + "target_prefix": target_prefix, + "ran_at": audit.get("ran_at", ""), + }) + self.context.response.success = True diff --git a/reme2/mcp/steps/memory_retriever.py b/reme2/mcp/steps/memory_retriever.py index d4046c6d..60294da7 100644 --- a/reme2/mcp/steps/memory_retriever.py +++ b/reme2/mcp/steps/memory_retriever.py @@ -114,7 +114,7 @@ class MemorySearch(BaseStep): isinstance(max_results, int) and max_results > 0 ), f"max_results must be a positive integer, got {max_results}" - chunk_filter = memory_io.make_chunk_filter( + chunk_filter = memory_io.make_filter( self.file_store, paths=self.context.get("paths") or None, tags=self.context.get("tags") or None, @@ -168,7 +168,7 @@ class MemoryGraphSearch(BaseStep): assert query or explicit_seeds, "query or seeds must be provided" assert max_results > 0 - chunk_filter = memory_io.make_chunk_filter( + chunk_filter = memory_io.make_filter( self.file_store, paths=ctx.get("paths") or None, tags=ctx.get("tags") or None, diff --git a/reme2/mcp/steps/sync.py b/reme2/mcp/steps/sync.py index 09eb81d7..b2ce30d1 100644 --- a/reme2/mcp/steps/sync.py +++ b/reme2/mcp/steps/sync.py @@ -41,8 +41,8 @@ from pydantic import ValidationError from reme2.component import R from reme2.component.base_step import BaseStep -from reme2.memory.memory_io import write_create -from reme2.schema.vault import Event +from reme2.memory.memory_io import create_file +from reme2.memory.schema import EVENT_PRESET, Memory from reme2.utils.vault_paths import event_path, next_suffixed_stem @@ -238,11 +238,12 @@ class Sync(BaseStep): on_date.isoformat() if isinstance(on_date, date_type) else (on_date or today) ) + # Start from EVENT_PRESET (4 axes + status + legacy `category`), + # layer caller-supplied identity fields on top. metadata: dict = { + **EVENT_PRESET, "title": name, "description": description, - "category": "event", - "status": "active", "tags": tags, "topics": topics, "created": on_date_str, @@ -252,10 +253,10 @@ class Sync(BaseStep): metadata["originSessionId"] = origin_session_id try: - Event.model_validate(metadata) + Memory.model_validate(metadata) except ValidationError as e: self._set_error({ - "error": "Event schema validation failed", + "error": "Memory schema validation failed", "details": e.errors(include_context=False, include_url=False), }) return @@ -281,7 +282,7 @@ class Sync(BaseStep): material_filenames = [m["filename"] for m in materials] index_body = self._emit_body(content, material_filenames) - ok, payload = write_create( + ok, payload = create_file( self.file_store, target, metadata=metadata, content=index_body, ) @@ -391,11 +392,11 @@ class Sync(BaseStep): meta["updated"] = date_type.today().isoformat() try: - Event.model_validate(meta) + Memory.model_validate(meta) except ValidationError as e: self._set_error({ "path": str(target.resolve()), - "error": "Event schema validation failed on append", + "error": "Memory schema validation failed on append", "details": e.errors(include_context=False, include_url=False), }) return diff --git a/reme2/mcp/steps/topic_create.py b/reme2/mcp/steps/topic_create.py deleted file mode 100644 index 50e34baa..00000000 --- a/reme2/mcp/steps/topic_create.py +++ /dev/null @@ -1,142 +0,0 @@ -"""topic_create — create a new topic under topics/{folder}/{name}.md. - -The only file-creation entry point for topics. Enforces: - - path template (topics/{folder}/{name}.md) - - folder topic identification (folder == name) - - judgment-category strong-confidence (via Topic.model_validator) - - wikilink uniqueness (creating this file must not make `[[name]]` ambiguous) - -Writes via `write_create` — the canonical L1 invariant gate that also -performs the wikilink-uniqueness check. The Ingestor's R-M-W loop is -reserved for content-driven flows; topic creation is path-driven, so -the direct call is sufficient. -""" - -import json -from datetime import date -from pathlib import Path - -from pydantic import ValidationError - -from reme2.component import R -from reme2.component.base_step import BaseStep -from reme2.memory.memory_io import write_create -from reme2.schema.vault import Topic -from reme2.utils.vault_paths import next_suffixed_stem, topic_path - - -@R.register("topic_create") -class TopicCreate(BaseStep): - """Create a topic file with the standard frontmatter template.""" - - def __init__(self, vault_root: str = "", topics_dir: str = "topics", **kwargs): - super().__init__(**kwargs) - self.vault_root = vault_root - self.topics_dir = topics_dir - - def _root(self) -> Path: - if self.vault_root: - return Path(self.vault_root) - watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[union-attr] - return Path(watcher.watch_path) - - async def execute(self): - assert self.context is not None - folder: str = self.context.get("folder", "") - name: str = self.context.get("name", "") - category: str = self.context.get("category", "") - description: str = self.context.get("description", "") or "" - content: str = self.context.get("content", "") or "" - confidence = self.context.get("confidence") - market = self.context.get("market") - ticker = self.context.get("ticker") - tags: list[str] = self.context.get("tags") or [] - - assert folder, "folder is required" - assert name, "name is required" - assert category, "category is required" - - target = topic_path(self._root(), folder, name, self.topics_dir) - if target.exists(): - self.context.response.success = False - self.context.response.answer = json.dumps({ - "path": str(target.resolve()), - "error": "topic already exists; use memory_update / memory_property_update to modify", - }, ensure_ascii=False) - return - - today = date.today().isoformat() - metadata: dict = { - "title": name, - "description": description, - "category": category, - "tags": tags, - "created": today, - "updated": today, - } - if confidence is not None: - metadata["confidence"] = confidence - if market is not None: - metadata["market"] = market - if ticker is not None: - metadata["ticker"] = ticker - - # Topic schema validation (judgment categories require confidence). - try: - Topic.model_validate(metadata) - except ValidationError as e: - self.context.response.success = False - self.context.response.answer = json.dumps({ - "error": "Topic schema validation failed", - "details": e.errors(include_context=False, include_url=False), - }, ensure_ascii=False) - return - - # Pre-check uniqueness so we can surface a `suggested_name` to the - # agent. The actual write also routes through the same gate inside - # write_create, so this is a UX nicety, not a correctness check. - graph = self.file_store - conflicts = graph.collisions_after_create(target) - if conflicts: - taken = {Path(p).stem for p in graph.nodes} - suggested_name = next_suffixed_stem(taken, name) - self.context.response.success = False - self.context.response.answer = json.dumps({ - "error": ( - f"stem `[[{name}]]` would resolve ambiguously " - f"to {len(conflicts) + 1} paths after this create" - ), - "conflicts": conflicts, - "suggested_name": suggested_name, - "hint": ( - f"retry with name='{suggested_name}' (numeric suffix), " - f"OR use a domain-specific qualifier (e.g. '{name}-Inc' / " - f"'{name}-v2') — semantic names beat numeric. If you " - f"actually meant the existing topic, call memory_get on " - f"one of `conflicts` instead." - ), - }, ensure_ascii=False) - return - - ok, payload = write_create( - self.file_store, target, - metadata=metadata, content=content, - ) - - if not ok: - self.context.response.success = False - self.context.response.answer = json.dumps({ - "path": str(target.resolve()), - "error": payload.get("error", "create failed"), - "details": payload, - }, ensure_ascii=False) - return - - is_folder_topic = folder == name - self.context.response.success = True - self.context.response.answer = json.dumps({ - "path": str(target.resolve()), - "category": category, - "is_folder_topic": is_folder_topic, - "created": True, - }, ensure_ascii=False) diff --git a/reme2/mcp/test/__init__.py b/reme2/mcp/test/__init__.py new file mode 100644 index 00000000..4f0cf2aa --- /dev/null +++ b/reme2/mcp/test/__init__.py @@ -0,0 +1,19 @@ +"""End-to-end tests for the reme2 MCP profiles. + +Two suites — one per shipped profile (`expert`, `service`) — boot +`reme2.application.Application` against a temp vault and exercise every +MCP-exposed job through `app.run_job(...)`. We force `service.backend = +http` so the suites never block on stdio MCP transport; jobs are +called in-process. + +Layout: + + _helpers.py shared vault seed, app factory, response decoder + test_expert.py expert profile (every memory_* primitive surfaced) + test_service.py service profile (retrieve / remember / maintain) + __main__.py CLI: `python -m reme2.mcp.test` + +The test functions are plain `async def check_(app, ctx)` returning +True / raising. The CLI runner reuses one app per profile to keep the +boot cost out of the per-test budget. +""" diff --git a/reme2/mcp/test/__main__.py b/reme2/mcp/test/__main__.py new file mode 100644 index 00000000..a5f56302 --- /dev/null +++ b/reme2/mcp/test/__main__.py @@ -0,0 +1,77 @@ +"""CLI runner: `python -m reme2.mcp.test` + +Boots one Application per profile against a fresh temp vault, runs the +profile's CHECKS list in order, prints a per-check status line plus a +final summary. Exit code is the number of failed checks across both +profiles. + +Per-profile suites are intentionally sequential — later checks rely on +side-effects from earlier ones (e.g. `sync.append` after `sync.create`). +Cross-profile work happens in independent vaults. +""" + +from __future__ import annotations + +import asyncio +import shutil +import sys +import traceback + +from . import test_expert, test_service +from ._helpers import AppContext, make_context + + +SUITES = { + "expert": test_expert.CHECKS, + "service": test_service.CHECKS, +} + + +async def _run_suite(profile: str, checks: list[tuple[str, callable]]) -> tuple[int, int]: + """Run one suite. Returns (passed, total).""" + print(f"\n========== profile: {profile} ==========") + ctx: AppContext | None = None + tmp = None + passed = 0 + try: + ctx, tmp = await make_context(profile) + print(f" bootstrapped: vault={ctx.vault}, jobs={len(ctx.jobs)}, " + f"file_store={len(ctx.file_store)}") + for label, fn in checks: + try: + summary = await fn(ctx) + print(f" ✓ {label:32s} {summary}") + passed += 1 + except Exception as e: + print(f" ✗ {label:32s} {type(e).__name__}: {e}") + traceback.print_exc() + except Exception as e: + print(f" ✗ bootstrap failed: {type(e).__name__}: {e}") + traceback.print_exc() + finally: + if ctx is not None: + try: + await ctx.app.close() + except Exception as e: # noqa: BLE001 + print(f" (warn) app.close failed: {e}") + if tmp is not None: + shutil.rmtree(tmp, ignore_errors=True) + return passed, len(checks) + + +async def _main() -> int: + summary: dict[str, tuple[int, int]] = {} + for profile, checks in SUITES.items(): + summary[profile] = await _run_suite(profile, checks) + + print("\n========== summary ==========") + failed = 0 + for profile, (passed, total) in summary.items(): + marker = "✓" if passed == total else "✗" + print(f" {marker} {profile}: {passed}/{total}") + failed += (total - passed) + return 0 if failed == 0 else failed + + +if __name__ == "__main__": + sys.exit(asyncio.run(_main())) diff --git a/reme2/mcp/test/_helpers.py b/reme2/mcp/test/_helpers.py new file mode 100644 index 00000000..f01a43c4 --- /dev/null +++ b/reme2/mcp/test/_helpers.py @@ -0,0 +1,162 @@ +"""Shared fixtures for the MCP profile tests. + +Each suite uses one app per profile (boot cost ~2s) and a fresh temp +vault. The app is configured with `service.backend = http` so the +stdio MCP listener never starts — jobs are invoked directly via +`app.run_job`. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from dataclasses import dataclass +from pathlib import Path + +# Make the repo importable when this module is loaded standalone +# (`python -m reme2.mcp.test` already has the path; direct imports +# from a fresh interpreter need this guard). +_REPO_ROOT = Path(__file__).resolve().parents[3] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +# Eager-import side-effect modules so all @R.register() decorators run +# before Application introspects the registry. +import reme2 # noqa: E402,F401 +import reme2.mcp.steps # noqa: E402,F401 +import reme2.memory # noqa: E402,F401 -- registers retriever + maintainer +import reme2.memory.ingestor # noqa: E402,F401 +import reme2.memory.summarizer # noqa: E402,F401 + +from reme2.application import Application # noqa: E402 +from reme2.config import parse_args # noqa: E402 + + +CONFIG_DIR = _REPO_ROOT / "reme2" / "config" +PROFILES = { + "expert": CONFIG_DIR / "expert.yaml", + "service": CONFIG_DIR / "service.yaml", +} + +# How many seed files `seed_vault` writes — checks that need to wait +# for indexing budget against this baseline. +SEED_FILE_COUNT = 3 + + +@dataclass +class AppContext: + """Bundle of everything a test function needs.""" + + app: Application + vault: Path + jobs: list[str] + + @property + def watcher(self): + return self.app.context.components["file_watcher"]["default"] + + @property + def file_store(self): + return self.watcher.file_store + + def abs_path(self, *parts: str) -> str: + return str((self.vault.joinpath(*parts)).resolve()) + + +def seed_vault(vault: Path) -> None: + """Drop a small connected topic graph (Alice ↔ Bob ↔ Project X).""" + (vault / "topics" / "Alice").mkdir(parents=True, exist_ok=True) + (vault / "topics" / "Alice" / "Alice.md").write_text( + "---\ntitle: Alice\ncategory: profile\ntags: [person]\n---\n" + "# Alice\n\nAlice works on [[Project X]] with [[Bob]].\n" + "[author:: [[Alice]]]\n", + encoding="utf-8", + ) + (vault / "topics" / "Bob").mkdir(parents=True, exist_ok=True) + (vault / "topics" / "Bob" / "Bob.md").write_text( + "---\ntitle: Bob\ncategory: profile\ntags: [person]\n---\n" + "# Bob\n\nBob collaborates with [[Alice]] on [[Project X]].\n", + encoding="utf-8", + ) + (vault / "topics" / "Project X").mkdir(parents=True, exist_ok=True) + (vault / "topics" / "Project X" / "Project X.md").write_text( + "---\ntitle: Project X\ncategory: concept\n---\n" + "# Project X\n\nA major initiative led by [[Alice]] with [[Bob]].\n", + encoding="utf-8", + ) + + +async def wait_for_index(watcher, expected_min: int, timeout_s: float = 15.0) -> None: + """Poll `len(file_store)` until it reaches `expected_min` and stays + stable across 4 ticks. Raises if the budget runs out — a watcher + that never indexed seed files is a hard wiring failure, not a + soft check.""" + deadline = asyncio.get_event_loop().time() + timeout_s + last = -1 + stable = 0 + while asyncio.get_event_loop().time() < deadline: + now = len(watcher.file_store) + if now == last and now >= expected_min: + stable += 1 + if stable >= 4: + return + else: + stable = 0 + last = now + await asyncio.sleep(0.25) + raise RuntimeError( + f"watcher did not reach >= {expected_min} files within {timeout_s}s " + f"(last seen: {last})" + ) + + +def decode(resp) -> object: + """Job answers come back as JSON strings — decode for inspection. + Pass through dicts / lists if the step already returned native types.""" + if isinstance(resp.answer, (dict, list)): + return resp.answer + if isinstance(resp.answer, str): + try: + return json.loads(resp.answer) + except (json.JSONDecodeError, TypeError): + return resp.answer + return resp.answer + + +async def build_app(profile_path: Path, vault: Path) -> Application: + """Boot an Application against `vault` using the given profile. + + Forces `service.backend = http` so the stdio MCP listener never + starts (we only call jobs directly), and rewrites the watcher / + file_store paths to point at the temp vault. + """ + _, cfg = parse_args( + "start", + f"config={profile_path}", + f"components.file_watcher.default.watch_path={vault}", + f"components.file_store.default.db_path={vault}/.reme", + ) + cfg["service"] = {"backend": "http"} + app = Application(**cfg) + await app.start() + return app + + +async def make_context(profile: str) -> tuple[AppContext, Path]: + """Build a fresh temp vault + app for `profile`, wait for indexing. + + The caller owns the temp dir lifecycle — returns its Path so it can + be `shutil.rmtree`d after `app.close()`. + """ + import tempfile + + tmp = Path(tempfile.mkdtemp(prefix=f"reme2-mcp-{profile}-")) + vault = tmp / "vault" + vault.mkdir() + seed_vault(vault) + app = await build_app(PROFILES[profile], vault) + await wait_for_index(app.context.components["file_watcher"]["default"], + expected_min=SEED_FILE_COUNT) + jobs = sorted(app.context.jobs.keys()) + return AppContext(app=app, vault=vault, jobs=jobs), tmp diff --git a/reme2/mcp/test/test_expert.py b/reme2/mcp/test/test_expert.py new file mode 100644 index 00000000..6dcfdad0 --- /dev/null +++ b/reme2/mcp/test/test_expert.py @@ -0,0 +1,425 @@ +"""Expert-profile MCP tests. + +Covers all 16 jobs registered by `reme2/config/expert.yaml`: + + Hot-write sync + Read memory_search, memory_graph_search, + memory_get, memory_list, memory_links, + memory_backlinks, memory_resolve_wikilink, + memory_count_tokens, memory_lint + Raw write memory_create, memory_update, memory_property_update, + memory_rename, memory_delete, memory_archive + +Each `check_*` is an `async def` that takes a populated `AppContext`, +runs one MCP job (or a small sequence), asserts on the response, and +returns a one-line summary string. The orchestrator runs them in the +listed order and collects pass/fail. +""" + +from __future__ import annotations + +from pathlib import Path + +from ._helpers import AppContext, decode, wait_for_index + + +# Manifest of every tool the expert profile must expose. Used by +# `check_registry` and as the upper bound for `wait_for_index` budgets. +EXPECTED_JOBS: tuple[str, ...] = ( + "sync", + "memory_search", + "memory_graph_search", + "memory_get", + "memory_list", + "memory_backlinks", + "memory_links", + "memory_resolve_wikilink", + "memory_count_tokens", + "memory_lint", + "memory_create", + "memory_update", + "memory_property_update", + "memory_rename", + "memory_delete", + "memory_archive", +) + + +# ---------- registry --------------------------------------------------- + + +async def check_registry(ctx: AppContext) -> str: + missing = [j for j in EXPECTED_JOBS if j not in ctx.jobs] + assert not missing, f"missing jobs: {missing}" + extras = [j for j in ctx.jobs if j not in EXPECTED_JOBS] + return f"{len(ctx.jobs)} jobs registered (extras: {extras or 'none'})" + + +# ---------- read primitives ------------------------------------------- + + +async def check_memory_get(ctx: AppContext) -> str: + alice = ctx.abs_path("topics", "Alice", "Alice.md") + r = decode(await ctx.app.run_job("memory_get", path=alice)) + assert isinstance(r, dict) and r.get("exists") is True, r + assert "metadata" in r, "missing metadata block" + return f"exists=True, edges={len(r.get('link', []))}" + + +async def check_memory_list(ctx: AppContext) -> str: + r = decode(await ctx.app.run_job("memory_list", tags=["person"])) + assert isinstance(r, dict) and r.get("count", 0) >= 2, r + paths = {item.get("path") for item in r.get("items", [])} + assert any("Alice.md" in p for p in paths), paths + return f"count={r['count']}" + + +async def check_memory_search(ctx: AppContext) -> str: + r = decode(await ctx.app.run_job( + "memory_search", query="collaborates", max_results=3, min_score=0.0, + )) + hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) + assert len(hits) > 0, r + return f"{len(hits)} hits" + + +async def check_memory_graph_search(ctx: AppContext) -> str: + r = decode(await ctx.app.run_job( + "memory_graph_search", query="Alice", max_results=5, min_score=0.0, + graph_depth=1, + )) + hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) + assert len(hits) > 0, r + # At least one result should carry a graph_hop annotation + hops = {h.get("graph_hop") for h in hits if isinstance(h, dict)} + return f"{len(hits)} hits, hops seen={sorted(h for h in hops if h is not None)}" + + +async def check_memory_links(ctx: AppContext) -> str: + alice = ctx.abs_path("topics", "Alice", "Alice.md") + r = decode(await ctx.app.run_job("memory_links", path=alice)) + assert isinstance(r, dict) and len(r.get("links", [])) >= 2, r + return f"{len(r['links'])} resolved outgoing links" + + +async def check_memory_backlinks(ctx: AppContext) -> str: + alice = ctx.abs_path("topics", "Alice", "Alice.md") + r = decode(await ctx.app.run_job("memory_backlinks", path=alice)) + # Bob.md and Project X.md both link to [[Alice]] + assert isinstance(r, dict) and len(r.get("backlinks", [])) >= 2, r + return f"{len(r['backlinks'])} incoming backlinks" + + +async def check_memory_resolve_wikilink(ctx: AppContext) -> str: + r = decode(await ctx.app.run_job("memory_resolve_wikilink", wikilink="Alice")) + assert isinstance(r, dict) and r.get("exists") is True, r + assert "Alice.md" in (r.get("path") or ""), r + return "stem 'Alice' → Alice.md" + + +async def check_memory_count_tokens(ctx: AppContext) -> str: + r = decode(await ctx.app.run_job( + "memory_count_tokens", text="hello world from the smoke test", + )) + assert isinstance(r, dict) and isinstance(r.get("tokens"), int), r + assert r["tokens"] > 0, r + return f"text → {r['tokens']} tokens" + + +async def check_memory_lint(ctx: AppContext) -> str: + """Maintainer lint pass — read-only. The seeded vault is healthy + (no broken wikilinks / schema violations), so we just verify the + shell wires through and returns the expected envelope.""" + r = decode(await ctx.app.run_job("memory_lint")) + assert isinstance(r, dict), r + assert "scanned" in r and "findings" in r, r + assert isinstance(r["findings"], list), r + assert r["scanned"] >= len(ctx.file_store), r + return f"scanned={r['scanned']}, findings={len(r['findings'])}" + + +# ---------- hot-write: sync (create / append / refusal) --------------- + + +async def check_sync_create(ctx: AppContext) -> str: + r = decode(await ctx.app.run_job( + "sync", + name="suite-event", + description="full-profile suite", + content="## ops\n- ran the suite\n", + topics=["[[Alice]]"], + tags=["suite"], + materials=[ + {"filename": "raw-prompt.md", "content": "# user prompt\n\nrun suite\n"}, + {"filename": "tool-output.txt", "content": "exit=0\n"}, + ], + )) + assert isinstance(r, dict), r + assert r.get("created") is True and r.get("action") == "created", r + materials = r.get("materials", []) + assert len(materials) == 2, materials + for m in materials: + assert Path(m).is_file(), f"material missing on disk: {m}" + event_dir = Path(materials[0]).parent + index_text = (event_dir / "suite-event.md").read_text(encoding="utf-8") + assert "## Materials" in index_text, "Materials footer absent" + assert "raw-prompt.md" in index_text, "Materials footer missing raw-prompt link" + # 4-axis schema must be on disk + assert "lifecycle: streaming" in index_text, "schema axis 'lifecycle' missing" + assert "role: observation" in index_text, "schema axis 'role' missing" + # Stash for downstream checks via the context (small mutation pattern). + ctx.abs_path("__suite_event_dir__") # noop; readable side-effect below + setattr(ctx, "_suite_event_dir", event_dir) + setattr(ctx, "_suite_event_index", event_dir / "suite-event.md") + await wait_for_index(ctx.watcher, expected_min=len(ctx.file_store)) + return f"created folder w/ {len(materials)} materials" + + +async def check_sync_append(ctx: AppContext) -> str: + r = decode(await ctx.app.run_job( + "sync", + name="suite-event", + content="## follow-up\n- second pass\n", + topics=["[[Bob]]"], # union with [[Alice]] + tags=["follow-up"], + materials=[ + {"filename": "tool-output.txt", "content": "second run\n"}, # collision + {"filename": "summary.md", "content": "# summary\n"}, + ], + )) + assert isinstance(r, dict), r + assert r.get("created") is False and r.get("action") == "appended", r + appended = r.get("materials", []) + names = {Path(p).name for p in appended} + assert "tool-output-2.txt" in names, f"collision auto-suffix failed: {names}" + assert "summary.md" in names, f"new material missing: {names}" + index_text = getattr(ctx, "_suite_event_index").read_text(encoding="utf-8") + assert "## ops" in index_text, "original body lost" + assert "## Update —" in index_text, "Update section missing" + assert "[[Bob]]" in index_text, "topic union failed" + assert "tool-output-2.txt" in index_text, "Materials footer lost the collided file" + return f"appended w/ collision → {sorted(names)}" + + +async def check_sync_refuse_distilled(ctx: AppContext) -> str: + index_path = str(getattr(ctx, "_suite_event_index")) + await ctx.app.run_job( + "memory_property_update", path=index_path, key="status", value="distilled", + ) + r = decode(await ctx.app.run_job( + "sync", name="suite-event", content="should be refused", + )) + assert isinstance(r, dict) and "error" in r, r + assert r.get("status") == "distilled", r + assert r.get("suggested_name"), r + return f"refused → suggested={r['suggested_name']!r}" + + +# ---------- raw memory_* writes --------------------------------------- + + +async def check_memory_create(ctx: AppContext) -> str: + target = ctx.abs_path("topics", "Carol", "Carol.md") + r = decode(await ctx.app.run_job( + "memory_create", + path=target, + metadata={ + "title": "Carol", + "lifecycle": "evolving", + "scope": "class", + "source": "curated", + "role": "profile", + "category": "profile", + "tags": ["person"], + }, + content="# Carol\n\nKnows [[Alice]].\n", + )) + assert isinstance(r, dict) and r.get("created") is True, r + assert "error" not in r, r + assert Path(target).is_file(), target + await wait_for_index(ctx.watcher, expected_min=len(ctx.file_store)) + return f"created {Path(target).name}" + + +async def check_memory_update(ctx: AppContext) -> str: + carol = ctx.abs_path("topics", "Carol", "Carol.md") + r = decode(await ctx.app.run_job( + "memory_update", + path=carol, + old_string="Knows [[Alice]].", + new_string="Knows [[Alice]] and [[Bob]].", + )) + assert isinstance(r, dict) and r.get("replaced", 0) >= 1, r + assert "Bob" in Path(carol).read_text(encoding="utf-8"), "edit not on disk" + return f"body edit applied (replaced={r['replaced']})" + + +async def check_memory_property_update(ctx: AppContext) -> str: + carol = ctx.abs_path("topics", "Carol", "Carol.md") + r = decode(await ctx.app.run_job( + "memory_property_update", path=carol, key="confidence", value="✅", + )) + assert isinstance(r, dict) and "error" not in r, r + assert r.get("key") == "confidence" and r.get("value") == "✅", r + assert "confidence: ✅" in Path(carol).read_text(encoding="utf-8"), \ + "property write not on disk" + return "set confidence=✅" + + +async def check_memory_rename(ctx: AppContext) -> str: + src = ctx.abs_path("topics", "Carol", "Carol.md") + dst = ctx.abs_path("topics", "Carol", "Carol-renamed.md") + r = decode(await ctx.app.run_job( + "memory_rename", old_path=src, new_path=dst, + )) + assert isinstance(r, dict) and "error" not in r, r + assert r.get("new_path") and Path(r["new_path"]).is_file(), r + assert not Path(src).exists(), f"old path still on disk: {src}" + setattr(ctx, "_carol_path", r["new_path"]) + return "Carol.md → Carol-renamed.md" + + +async def check_memory_archive(ctx: AppContext) -> str: + carol = getattr(ctx, "_carol_path", ctx.abs_path("topics", "Carol", "Carol-renamed.md")) + r = decode(await ctx.app.run_job("memory_archive", path=carol)) + assert isinstance(r, dict) and r.get("archived") is True, r + archived_path = r.get("new_path") + assert archived_path and "Archive" in archived_path, r + assert Path(archived_path).is_file(), archived_path + setattr(ctx, "_carol_archived", archived_path) + return f"→ {Path(archived_path).resolve().relative_to(ctx.vault.resolve())}" + + +async def check_memory_delete(ctx: AppContext) -> str: + target = getattr(ctx, "_carol_archived", None) \ + or ctx.abs_path("topics", "Carol", "Carol-renamed.md") + r = decode(await ctx.app.run_job("memory_delete", path=target)) + assert isinstance(r, dict) and r.get("deleted") is True, r + assert not Path(target).exists(), target + return "removed" + + +# ---------- schema gates (P4) ----------------------------------------- + + +async def check_schema_path_template_refuses(ctx: AppContext) -> str: + """memory_create rejects paths outside topics/{X}/{Y}.md, + events/{date}/{name}/..., or Archive/...""" + target = ctx.abs_path("notes", "freeform.md") # outside any template + r = decode(await ctx.app.run_job( + "memory_create", + path=target, + metadata={"title": "freeform", + "lifecycle": "evolving", "scope": "class", + "source": "curated", "role": "concept"}, + content="should be refused", + )) + assert isinstance(r, dict), r + assert "error" in r and "template" in r["error"].lower(), r + assert not Path(target).exists(), "file shouldn't have been written" + return "refused notes/freeform.md (no template)" + + +async def check_schema_path_template_force_bypass(ctx: AppContext) -> str: + """force=True bypasses the template gate.""" + target = ctx.abs_path("notes", "forced.md") + r = decode(await ctx.app.run_job( + "memory_create", + path=target, + metadata={"title": "forced", + "lifecycle": "evolving", "scope": "class", + "source": "curated", "role": "concept"}, + content="forced through", + force=True, + )) + assert isinstance(r, dict) and r.get("created") is True, r + assert Path(target).is_file(), target + return "force=True bypassed" + + +async def check_schema_status_skip_refused(ctx: AppContext) -> str: + """Suite-event ended distilled (sync.refuse_distilled flipped it). + Trying distilled → active is reverse — must refuse.""" + event_index = getattr(ctx, "_suite_event_index", None) + assert event_index is not None, "suite-event index not staged" + r = decode(await ctx.app.run_job( + "memory_property_update", + path=str(event_index), + key="status", + value="active", + )) + assert isinstance(r, dict), r + assert "error" in r and "transition" in r["error"].lower(), r + assert r.get("prior") == "distilled", r + return f"refused {r.get('prior')!r} → {r.get('requested')!r}" + + +async def check_schema_status_invalid_value(ctx: AppContext) -> str: + """Random string for status is refused before any state-machine check.""" + event_index = getattr(ctx, "_suite_event_index", None) + assert event_index is not None, "suite-event index not staged" + r = decode(await ctx.app.run_job( + "memory_property_update", + path=str(event_index), + key="status", + value="bogus", + )) + assert isinstance(r, dict), r + assert "error" in r and "invalid" in r["error"].lower(), r + return "refused status='bogus'" + + +async def check_schema_status_force_bypass(ctx: AppContext) -> str: + """force=True bypasses the state machine — useful when the agent + intentionally needs to step outside conventions.""" + event_index = getattr(ctx, "_suite_event_index", None) + assert event_index is not None, "suite-event index not staged" + r = decode(await ctx.app.run_job( + "memory_property_update", + path=str(event_index), + key="status", + value="active", + force=True, + )) + assert isinstance(r, dict), r + assert "error" not in r, r + # restore for any downstream checks + decode(await ctx.app.run_job( + "memory_property_update", + path=str(event_index), key="status", value="distilled", force=True, + )) + return "force=True bypassed" + + +# ---------- ordered manifest ----------------------------------------- + + +# (label, async fn) — runner executes top-to-bottom; later checks may +# rely on side effects from earlier ones (e.g. sync.append needs +# sync.create to have run). +CHECKS: list[tuple[str, callable]] = [ + ("registry", check_registry), + ("memory_get", check_memory_get), + ("memory_list", check_memory_list), + ("memory_search", check_memory_search), + ("memory_graph_search", check_memory_graph_search), + ("memory_links", check_memory_links), + ("memory_backlinks", check_memory_backlinks), + ("memory_resolve_wikilink", check_memory_resolve_wikilink), + ("memory_count_tokens", check_memory_count_tokens), + ("memory_lint", check_memory_lint), + ("sync.create", check_sync_create), + ("sync.append", check_sync_append), + ("sync.refuse_distilled", check_sync_refuse_distilled), + ("memory_create", check_memory_create), + ("memory_update", check_memory_update), + ("memory_property_update", check_memory_property_update), + ("memory_rename", check_memory_rename), + ("memory_archive", check_memory_archive), + ("memory_delete", check_memory_delete), + ("schema.path_template_refuses", check_schema_path_template_refuses), + ("schema.path_template_force", check_schema_path_template_force_bypass), + ("schema.status_skip_refused", check_schema_status_skip_refused), + ("schema.status_invalid_value", check_schema_status_invalid_value), + ("schema.status_force", check_schema_status_force_bypass), +] diff --git a/reme2/mcp/test/test_service.py b/reme2/mcp/test/test_service.py new file mode 100644 index 00000000..66241814 --- /dev/null +++ b/reme2/mcp/test/test_service.py @@ -0,0 +1,231 @@ +"""Service-profile MCP tests. + +Covers the 3-tool service surface (`reme2/config/service.yaml`): + + retrieve — graph-aware hybrid retrieval (memory_graph_search backend) + remember — single write entry point (Ingestor projection): + mode=log → zero-LLM event-folder upsert + mode=distill → LLM R-M-W (degraded path here, no LLM) + maintain — vault hygiene sweep (Maintainer projection) + +Same shape as `test_expert.py`: each `check_*` is a one-job-or-sequence +async function returning a one-line summary. Ordering matters because +`remember.log_append` and `remember.log_refuse_distilled` rely on side +effects from `remember.log_create`. +""" + +from __future__ import annotations + +from pathlib import Path + +from ._helpers import AppContext, decode, wait_for_index + + +EXPECTED_JOBS: tuple[str, ...] = ("retrieve", "remember", "maintain") + + +# ---------- registry --------------------------------------------------- + + +async def check_registry(ctx: AppContext) -> str: + missing = [j for j in EXPECTED_JOBS if j not in ctx.jobs] + assert not missing, f"missing jobs: {missing}" + extras = [j for j in ctx.jobs if j not in EXPECTED_JOBS] + assert not extras, ( + f"curated profile leaked extra jobs: {extras} — keep the surface tight" + ) + return f"{len(ctx.jobs)} jobs registered" + + +# ---------- read: retrieve (graph-aware hybrid) ----------------------- + + +async def check_retrieve_basic(ctx: AppContext) -> str: + r = decode(await ctx.app.run_job( + "retrieve", query="Alice Bob", max_results=5, min_score=0.0, + )) + hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) + assert len(hits) > 0, r + return f"{len(hits)} hits" + + +async def check_retrieve_anchored(ctx: AppContext) -> str: + """Wikilink-anchored mode — `[[Project X]]` in the query seeds BFS.""" + r = decode(await ctx.app.run_job( + "retrieve", + query="What touches [[Project X]]?", + max_results=5, + min_score=0.0, + graph_depth=1, + )) + hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) + assert len(hits) > 0, r + paths = {h.get("path") for h in hits if isinstance(h, dict)} + # Project X's neighbors (Alice, Bob) should surface via 1-hop BFS + has_neighbor = any(p and ("Alice" in p or "Bob" in p) for p in paths) + assert has_neighbor, f"BFS didn't pull in [[Project X]] neighbors: {paths}" + return f"{len(hits)} hits incl. project-x neighbors" + + +async def check_retrieve_topic_seeded(ctx: AppContext) -> str: + """Topic-rooted mode — explicit `seeds=[...]` instead of inline wikilink.""" + project_x = ctx.abs_path("topics", "Project X", "Project X.md") + r = decode(await ctx.app.run_job( + "retrieve", + query="collaborates", + max_results=5, + min_score=0.0, + seeds=[project_x], + graph_depth=1, + )) + hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) + assert len(hits) > 0, r + return f"{len(hits)} hits (seeded at Project X)" + + +# ---------- write: remember (mode=log: create / append / refusal) ----- + + +async def check_remember_log_create(ctx: AppContext) -> str: + r = decode(await ctx.app.run_job( + "remember", + mode="log", + name="curated-event", + description="curated-profile suite", + content="## ops\n- ran the curated suite\n", + topics=["[[Alice]]"], + tags=["curated"], + materials=[ + {"filename": "raw-prompt.md", "content": "# user prompt\n\nrun curated suite\n"}, + {"filename": "tool-output.txt", "content": "exit=0\n"}, + ], + )) + assert isinstance(r, dict), r + assert r.get("created") is True and r.get("action") == "created", r + materials = r.get("materials", []) + assert len(materials) == 2, materials + for m in materials: + assert Path(m).is_file(), f"material missing on disk: {m}" + event_dir = Path(materials[0]).parent + index_text = (event_dir / "curated-event.md").read_text(encoding="utf-8") + assert "## Materials" in index_text, "Materials footer absent" + assert "lifecycle: streaming" in index_text, "schema axis 'lifecycle' missing" + assert "role: observation" in index_text, "schema axis 'role' missing" + setattr(ctx, "_event_dir", event_dir) + setattr(ctx, "_event_index", event_dir / "curated-event.md") + await wait_for_index(ctx.watcher, expected_min=len(ctx.file_store)) + return f"created folder w/ {len(materials)} materials" + + +async def check_remember_log_append(ctx: AppContext) -> str: + r = decode(await ctx.app.run_job( + "remember", + mode="log", + name="curated-event", + content="## follow-up\n- second pass\n", + topics=["[[Bob]]"], + tags=["follow-up"], + materials=[ + {"filename": "tool-output.txt", "content": "second run\n"}, # collision + ], + )) + assert isinstance(r, dict), r + assert r.get("created") is False and r.get("action") == "appended", r + appended = r.get("materials", []) + names = {Path(p).name for p in appended} + assert "tool-output-2.txt" in names, f"collision auto-suffix failed: {names}" + index_text = getattr(ctx, "_event_index").read_text(encoding="utf-8") + assert "## Update —" in index_text, "Update section missing" + assert "[[Bob]]" in index_text, "topic union failed" + return f"appended w/ collision → {sorted(names)}" + + +async def check_remember_log_refuse_distilled(ctx: AppContext) -> str: + """Curated profile lacks `memory_property_update`, so we flip status by + rewriting the file directly — same observable effect on remember(mode=log).""" + index_path = Path(getattr(ctx, "_event_index")) + text = index_path.read_text(encoding="utf-8") + text = text.replace("status: active", "status: distilled", 1) + index_path.write_text(text, encoding="utf-8") + # Give the watcher a moment to re-parse before sync re-reads frontmatter. + await wait_for_index(ctx.watcher, expected_min=len(ctx.file_store)) + + r = decode(await ctx.app.run_job( + "remember", mode="log", name="curated-event", content="should be refused", + )) + assert isinstance(r, dict) and "error" in r, r + assert r.get("status") == "distilled", r + assert r.get("suggested_name"), r + return f"refused → suggested={r['suggested_name']!r}" + + +# ---------- write: remember (mode=distill, degraded path) ------------- + + +async def check_remember_distill_degraded(ctx: AppContext) -> str: + target = ctx.abs_path("topics", "curated-ingested", "curated-ingested.md") + r = decode(await ctx.app.run_job( + "remember", + # mode defaults to "distill" + content="# curated-ingested\n\nproduced by the curated test suite.\n", + target_path=target, + metadata={ + "title": "curated-ingested", + "lifecycle": "evolving", + "scope": "class", + "source": "curated", + "role": "concept", + "category": "concept", + }, + )) + assert isinstance(r, dict), r + applied = r.get("applied") or [] + assert len(applied) == 1 and applied[0].get("ok") is True, r + assert r.get("used_llm") is False, "expected degraded path (no LLM)" + assert Path(target).is_file(), target + return f"applied=1, used_llm=False" + + +# ---------- maintain (lint + decay sweep) ----------------------------- + + +async def check_maintain_dry_run(ctx: AppContext) -> str: + """Default ops=[lint,decay] with dry_run=true (the parameter default) + surfaces the plan without mutating. The seeded vault has nothing + stale and no broken wikilinks, so we just verify the envelope.""" + r = decode(await ctx.app.run_job("maintain")) + assert isinstance(r, dict), r + for key in ("ops_run", "scanned", "proposed", "plan", "applied", "dry_run"): + assert key in r, f"missing key {key!r} in audit: {r}" + assert r["dry_run"] is True, r + assert r["ops_run"] == ["lint", "decay"], r + assert r["applied"] == [], "dry_run should never apply" + return f"scanned={r['scanned']}, proposed={len(r['proposed'])}" + + +async def check_maintain_targeted(ctx: AppContext) -> str: + """target_prefix narrows scan; events/ subtree should yield the + one event folder created earlier in this suite.""" + r = decode(await ctx.app.run_job( + "maintain", target_prefix="events/", dry_run=True, + )) + assert isinstance(r, dict), r + assert r["scanned"] >= 1, r # at least the suite's own event folder + return f"scanned={r['scanned']} under events/" + + +# ---------- ordered manifest ----------------------------------------- + + +CHECKS: list[tuple[str, callable]] = [ + ("registry", check_registry), + ("retrieve.basic", check_retrieve_basic), + ("retrieve.anchored", check_retrieve_anchored), + ("retrieve.topic_seeded", check_retrieve_topic_seeded), + ("remember.log_create", check_remember_log_create), + ("remember.log_append", check_remember_log_append), + ("remember.log_refuse_distilled", check_remember_log_refuse_distilled), + ("remember.distill_degraded", check_remember_distill_degraded), + ("maintain.dry_run", check_maintain_dry_run), + ("maintain.targeted", check_maintain_targeted), +] diff --git a/reme2/memory/__init__.py b/reme2/memory/__init__.py index f1d48797..86f14c23 100644 --- a/reme2/memory/__init__.py +++ b/reme2/memory/__init__.py @@ -12,7 +12,7 @@ Per the architecture blueprint: Lint, woken by cron or thresholds. - summarizer.py Auxiliary used by the services. -Hot-write MCP step shells (sync, topic_create, memory_*) live in +Hot-write MCP step shells (sync, memory_*) live in `reme2.mcp.steps`, NOT here — they bypass services and write MFS directly. Importing this package triggers @R.register on the three services so configs that name them resolve at boot. diff --git a/reme2/memory/ingestor.py b/reme2/memory/ingestor.py index 9dfc71a5..70381f99 100644 --- a/reme2/memory/ingestor.py +++ b/reme2/memory/ingestor.py @@ -5,10 +5,10 @@ write entry point** to the markdown vault. Every mutation (create, body edit, frontmatter flip, rename, delete, archive) flows through here. Mirrors `Summarizer`'s pattern — drives a `ReActAgent` whose toolkit is -built by `memory_io.build_memory_toolkit`. The agent runs its own R-M-W -loop: read related files via tools, decide which ones to mutate, call -the right write tool. Every write tool records into an audit list, so -the caller gets a deterministic mutation trail regardless of how the +built by `memory_toolkit.build_memory_toolkit`. The agent runs its own +R-M-W loop: read related files via tools, decide which ones to mutate, +call the right write tool. Every write tool records into an audit list, +so the caller gets a deterministic mutation trail regardless of how the agent's reasoning unfolded. When no LLM is configured, falls back to a direct create from @@ -30,7 +30,8 @@ from pydantic import BaseModel, Field from ..component.runtime_response import _set_answer, _to_jsonable from . import memory_io -from .memory_io import MemoryIO, write_create +from .memory_io import create_file +from .memory_toolkit import build_memory_toolkit from ..component import R from ..component.base_step import BaseStep from ..enumeration import ComponentEnum @@ -84,6 +85,7 @@ class Ingestor(BaseStep): self.toolkit = toolkit self.console_enabled = console_enabled self.timezone = timezone + self._protocol = (Path(__file__).parent / "protocol.md").read_text(encoding="utf-8") def _now(self) -> datetime.datetime: if self.timezone: @@ -102,6 +104,17 @@ class Ingestor(BaseStep): async def execute(self): assert self.context is not None + mode: str = (self.context.get("mode") or "distill").lower() + if mode == "log": + await self._delegate_to_sync() + return + if mode != "distill": + self.context.response.success = False + _set_answer(self.context, { + "error": f"unknown mode {mode!r}; expected 'log' or 'distill'", + }) + return + content: str = self.context.get("content", "") or "" hint: str = self.context.get("hint", "") or "" target_path: str = self.context.get("target_path") or "" @@ -112,7 +125,7 @@ class Ingestor(BaseStep): # Auto-discover wikilink targets in content as a hint for the agent. for link in extract_wikilinks(content): - hit = memory_io.wikilink_lookup(self.file_store, link)["path"] + hit = memory_io.resolve_wikilink(self.file_store, link)["path"] if hit and hit not in related_paths: related_paths.append(hit) @@ -124,13 +137,17 @@ class Ingestor(BaseStep): return vault_root = self._vault_root() - mio = MemoryIO(self.file_store, vault_root) - toolkit = mio.register_all(self.toolkit) + audit: list[dict] = [] + toolkit = build_memory_toolkit(self.app_context, audit=audit, toolkit=self.toolkit) agent = ReActAgent( name="reme_ingestor", model=self.as_llm, - sys_prompt=self.prompt_format("system_prompt", vault_root=str(vault_root)), + sys_prompt=self.prompt_format( + "system_prompt", + vault_root=str(vault_root), + protocol=self._protocol, + ), formatter=self.as_llm_formatter, toolkit=toolkit, ) @@ -153,9 +170,9 @@ class Ingestor(BaseStep): summary = final_msg.get_text_content() or "" result = IngestResult(used_llm=True) - for entry in mio.audit: + for entry in audit: (result.applied if entry.get("ok") else result.failed).append(entry) - if not mio.audit and summary.strip().upper().startswith("SKIP"): + if not audit and summary.strip().upper().startswith("SKIP"): result.skipped = True self.context.response.success = result.success @@ -177,7 +194,7 @@ class Ingestor(BaseStep): path = Path(target_path) if not path.is_absolute(): path = self._vault_root() / path - ok, payload = write_create( + ok, payload = create_file( self.file_store, path, metadata=metadata, content=content, ) @@ -185,3 +202,13 @@ class Ingestor(BaseStep): bucket.append({"op": "create", "ok": ok, "path": str(path), "result": payload}) return result + async def _delegate_to_sync(self) -> None: + """Hot-path event-folder upsert — same code path as the standalone + `sync` step. Lazy-instantiated so we don't pay the construction + cost on every distill call.""" + from ..mcp.steps.sync import Sync + + if getattr(self, "_sync_step", None) is None: + self._sync_step = Sync(app_context=self.app_context) + await self._sync_step(self.context) + diff --git a/reme2/memory/ingestor.yaml b/reme2/memory/ingestor.yaml index 33f37c12..29e6e67a 100644 --- a/reme2/memory/ingestor.yaml +++ b/reme2/memory/ingestor.yaml @@ -24,68 +24,19 @@ system_prompt: | indexes, individual material files, candidate topics it flagged for update). Both forms can appear together; treat them as a single working set. - - An event is a FOLDER containing the index `{name}.md` plus + - An event is a FOLDER containing the index `{{name}}.md` plus materials (raw conversation snippets, tool outputs, data dumps). Whenever a path in `related_paths` is an event index, `memory_get` the index first, then `memory_get` any materials whose content you need (the index lists them under `## Materials`). - # Vault conventions - - Topics live under `topics/{{folder}}/{{name}}.md`. A folder topic - has folder == name. - - Events live under `events/{{YYYY-MM-DD}}/{{name}}/{{name}}.md` — - this is an INDEX inside a folder; sibling files are materials. - - Frontmatter is YAML; `category`, `created`, `updated`, `tags`, - `status` are common. - - Cross-file references use `[[wikilink]]` syntax (stem-form `[[X]]` - or path-form `[[topics/X/X]]`). + # Memory protocol (single source — schema, tools, decision rules) - # Available tools - Read tools (use these to gather context BEFORE writing): - - memory_get(path, include_chunks=False): full file content + frontmatter. - Call this on each event index AND on the materials it lists. - - memory_list(path_prefix=None, tags=None, metadata=None, limit=100): - list indexed files filtered by prefix / tags / frontmatter. - - memory_resolve_wikilink(wikilink): resolve `[[X]]` to a path. - - memory_backlinks(path): files linking to a given path. - - memory_links(path): files a given path links to. + {protocol} - Write tools (each returns success + payload; mutations are SSOT-routed): - - memory_update(path, old_string, new_string, replace_all=False): - body edit by exact-string substitution. Use a tail snippet to append. - - memory_property_update(path, key, value): change one frontmatter - key (value=null deletes it). After distilling an event into one or - more topics, flip that event's status to "distilled" with this. - - memory_create(path, metadata, content, overwrite=False, force=False): - new file. Reserve for genuinely NEW topics — do NOT use this to log - events; `sync` (deterministic) owns events. ALL paths must - be ABSOLUTE under vault_root. - - memory_rename(old_path, new_path): move file + rewrite cross-vault - wikilinks. - - memory_delete(path): remove a file. - - memory_archive(path): flip `status: archived` and move under - `/Archive/`. - - # Decision rules - 1. If material is ALREADY covered by existing topics → reply with a - single line `SKIP: ` and call no tools. - 2. If material CONTRADICTS an existing block → memory_update with a - unique snippet of the outdated text and the corrected replacement. - 3. If material EXTENDS an existing topic → memory_update using a - unique TAIL snippet of the existing body, with new_string = - tail + blank line + new content. - 4. If material warrants a GENUINELY NEW topic → memory_create at - `topics/{{folder}}/{{name}}.md`. Do NOT memory_create under events/. - 5. After integrating an event's content into a topic, flip that event's - status to "distilled" with memory_property_update. - 6. Never delete unless the material explicitly asks for deletion. - 7. Keep edits minimal — read related files first, edit only what - must change. - 8. Always include reasonable frontmatter on memory_create — at minimum - `title`, `category`, `created`, `updated`. Use today's date for - `created` / `updated`. - 9. After all writes, end with a one-paragraph summary of what you did - and why. + # Final step + After all writes, end with a one-paragraph summary of what you did + and why. user_message: | # CONTEXT diff --git a/reme2/memory/maintainer.py b/reme2/memory/maintainer.py index 4776d225..62f8fdbf 100644 --- a/reme2/memory/maintainer.py +++ b/reme2/memory/maintainer.py @@ -63,8 +63,8 @@ from ..component import R from ..component.base_step import BaseStep from ..component.runtime_response import _set_answer from ..enumeration import ComponentEnum -from ..schema.vault.registry import schema_for from . import memory_io +from .schema import parse_frontmatter # --------------------------------------------------------------------------- @@ -122,11 +122,19 @@ class FileSignal(BaseModel): from `file_store.nodes` + edge index — no body reads, no LLM calls. Heavier signals (token counts, embeddings) are pulled lazily inside the proposers that actually need them. + + The 4 schema axes (`lifecycle / scope / source / role`) are the + primary drivers of decay / merge / split heuristics; legacy + `category` is preserved for back-compat reads only. """ path: str relpath: str = "" # path relative to vault_root, "" if outside - category: str = "" + lifecycle: str = "" # streaming / evolving / frozen + scope: str = "" # instance / class + source: str = "" # auto / curated / derived + role: str = "" # observation / claim / question / ... + category: str = "" # legacy field, kept for migration windows status: str = "" age_days: int = 0 metadata: dict = Field(default_factory=dict) @@ -262,7 +270,14 @@ class Maintainer(BaseStep): return Path(getattr(watcher, "watch_path", ".")).resolve() def _scan_signals(self, *, target_prefix: str = "") -> list[FileSignal]: - """One walk over the indexed files. Cheap signals only.""" + """One walk over the indexed files. Cheap signals only. + + Each frontmatter is run through `parse_frontmatter` so the legacy + `category` field is auto-translated to the 4 axes (see + `LEGACY_AXES_FROM_CATEGORY` in `memory.schema.memory`). Files that + fail to parse still get a signal — proposers can use the empty + axes to decide whether to ignore or surface them. + """ vault_root = self._vault_root() now = datetime.datetime.now().timestamp() signals: list[FileSignal] = [] @@ -277,9 +292,14 @@ class Maintainer(BaseStep): continue fm = meta.metadata or {} age_seconds = max(0.0, now - (meta.st_mtime or now)) + parsed, _ = parse_frontmatter(fm) signals.append(FileSignal( path=path, relpath=relpath, + lifecycle=str(parsed.lifecycle.value) if parsed else "", + scope=str(parsed.scope.value) if parsed else "", + source=str(parsed.source.value) if parsed else "", + role=str(parsed.role.value) if parsed else "", category=str(fm.get("category") or ""), status=str(fm.get("status") or ""), age_days=int(age_seconds // 86400), @@ -295,22 +315,21 @@ class Maintainer(BaseStep): out: list[LintFinding] = [] for sig in signals: for link in sig.declared_topics: - if not memory_io.wikilink_lookup(self.file_store, link)["exists"]: + if not memory_io.resolve_wikilink(self.file_store, link)["exists"]: out.append(LintFinding( path=sig.path, kind="broken_wikilink", detail=f"unresolved wikilink {link!r}", )) - cls = schema_for(sig.category) - if cls is not None: - try: - cls(**sig.metadata) - except Exception as e: - out.append(LintFinding( - path=sig.path, kind="schema_violation", - detail=f"{cls.__name__}: {type(e).__name__}: {e}"[:240], - )) + # Memory schema check: tolerant parse, surface every error + # the parser collected. Empty `errors` ↔ valid frontmatter. + _, errors = parse_frontmatter(sig.metadata) + for err in errors: + out.append(LintFinding( + path=sig.path, kind="schema_violation", + detail=f"Memory: {err}"[:240], + )) # Stem collisions: the engine API exposes the ambiguous-stem map. - ambig = memory_io.all_ambiguous_wikilinks(self.file_store) + ambig = memory_io.find_collisions(self.file_store) for stem, paths in ambig.items(): for p in paths: out.append(LintFinding( @@ -322,10 +341,16 @@ class Maintainer(BaseStep): def _propose_decay( self, signals: list[FileSignal], decay_days: int, ) -> list[DecayOp]: - """Distilled events past the freshness window.""" + """Distilled streaming memories past the freshness window. + + Schema-driven: a memory decays when its `lifecycle` is `streaming` + (write-once, decays after a freshness window) AND its `status` has + already been flipped to the configured terminal target (default: + `distilled`). Evolving / frozen memories never decay. + """ out: list[DecayOp] = [] for sig in signals: - if sig.category != "event": + if sig.lifecycle != "streaming": continue if sig.status != self.target_status: continue @@ -333,7 +358,7 @@ class Maintainer(BaseStep): continue out.append(DecayOp( path=sig.path, age_days=sig.age_days, - reason=f"event {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)", + reason=f"streaming {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)", )) return out diff --git a/reme2/memory/memory_io.py b/reme2/memory/memory_io.py index ed89b019..ab14329e 100644 --- a/reme2/memory/memory_io.py +++ b/reme2/memory/memory_io.py @@ -8,24 +8,38 @@ is layered: This module is the **single public API surface** over that engine. Every consumer — MCP step shells, the three memory services (Retriever, -Ingestor, Maintainer), and the agent toolkit — talks to the engine -through these functions, not by reaching into `BaseFileStore` directly. -That keeps `file_store` an implementation detail (could be local sqlite, -remote, etc.) and gives the layering one place to evolve. +Ingestor, Maintainer), and the agent toolkit (`memory_toolkit`) — talks +to the engine through these functions, not by reaching into +`BaseFileStore` directly. That keeps `file_store` an implementation +detail (could be local sqlite, remote, etc.) and gives the layering one +place to evolve. -Four sections: +Naming convention: + - Verb-first: `get_file`, `create_file`, `search_vector`. + - `file_store` is always the first positional argument when the + function needs the engine handle; remaining args are keyword-only. + - Pure-disk writes (`update_body`, `update_meta`, `delete_file`, + `archive_file`) don't take `file_store` — they hit the filesystem + and the watcher picks them up. The asymmetry is honest. - 1. CRUD writes — write_create / delete / update / property_update - / rename / archive. The MFS write entry. - 2. MFS reads — read_file / list_files / links_of / backlinks_of - / wikilink_lookup / count_tokens / iter_files. - Primary-key lookups against the file_store cache. - 3. Projections — vector_search / keyword_search / expand_neighbors - / extract_anchors / chunks_by_paths / make_chunk_filter - / all_ambiguous_wikilinks. The read entry to derived - indexes (composed by Retriever into V+K+graph fusion). - 4. Toolkit — `MemoryIO` class wrapping the read/write helpers as - `agentscope.tool` callables for ReActAgent (Ingestor). +Three sections: + + 1. MFS Reads — get_file / list_files / get_links / get_backlinks + / resolve_wikilink / iter_files / count_tokens. + Primary-key lookups against the file_store cache + (with disk fallthrough for the file body). + 2. MFS Writes — create_file / update_body / update_meta + / rename_file / delete_file / archive_file. + The MFS write entry. All return (ok, payload). + 3. Projection Queries — search_vector / search_keyword / expand_neighbors + / extract_anchors / get_chunks / make_filter + / find_collisions. Read-only — projections + are derived by the Watcher; callers don't + write them directly. + +Schema policy and the agent toolkit projection live one layer up in +`reme2/memory/memory_toolkit.py` — this file is policy-free and remains +the engine's outward API. Lives in `reme2/memory/` (not `reme2/mcp/`) so memory services and the MCP transport layer can both consume it without forming an import cycle @@ -34,29 +48,224 @@ through the transport layer. from __future__ import annotations -import json import re import shutil from collections.abc import Iterable, Iterator from pathlib import Path -from typing import Any import frontmatter -from agentscope.message import TextBlock -from agentscope.tool import Toolkit, ToolResponse -from ..component.runtime_response import _to_jsonable from ..schema import ChunkFilter, FileChunk, FileMetadata from ..utils.wikilink import WIKILINK_RE # =========================================================================== -# Section 1 — CRUD writes +# Section 1 — MFS Reads +# =========================================================================== +# +# Primary-key lookups against the file_store's in-memory cache (file +# meta + edges + stem index). `get_file` falls through to disk for the +# body so callers see the latest text even if the watcher hasn't picked +# up a write yet. + + +async def get_file( + file_store, + path: str, + *, + include_chunks: bool = False, +) -> dict: + """Read frontmatter + body for one path. Optionally include parsed chunks. + + On-disk frontmatter is the source of truth — the file_store cache may + lag a write that hasn't been picked up by the watcher yet. + """ + meta = file_store.get_file_meta(path) + result: dict = {"path": path, "exists": False} + if meta is not None: + edges = file_store.get_edges(path) + result.update({ + "exists": True, + "metadata": meta.metadata, + "link": [e.model_dump(exclude_none=True) for e in edges], + }) + + file_path = Path(path) + if file_path.is_file(): + raw = file_path.read_text(encoding="utf-8") + post = frontmatter.loads(raw) + result["exists"] = True + result["content"] = post.content + result["metadata"] = dict(post.metadata) + + if include_chunks: + chunks = await file_store.get_chunks(path) + result["chunks"] = [c.model_dump(exclude_none=True) for c in chunks] + return result + + +def list_files( + file_store, + *, + path_prefix: str | None = None, + tags: list[str] | None = None, + metadata: dict | None = None, + limit: int = 100, +) -> dict: + """List indexed files filtered by frontmatter exact-match, tags, and prefix. + + Returns {items: [{path, metadata}], count}. + """ + metadata_filter = metadata or {} + tag_filter = tags or [] + items: list[dict] = [] + for path, meta in file_store.nodes.items(): + if path_prefix and not path.startswith(path_prefix): + continue + md = meta.metadata or {} + if metadata_filter and any(md.get(k) != v for k, v in metadata_filter.items()): + continue + if tag_filter: + file_tags = set(md.get("tags", []) or []) + if not all(t in file_tags for t in tag_filter): + continue + items.append({"path": path, "metadata": md}) + if len(items) >= limit: + break + return {"items": items, "count": len(items)} + + +def _edge_to_dict(file_meta, edge) -> dict: + return { + "path": file_meta.path, + "metadata": file_meta.metadata, + "predicate": edge.predicate, + "anchor": edge.anchor, + "alias": edge.alias, + "embed": edge.embed, + "source": edge.source, + "confidence": edge.confidence, + } + + +def get_links(file_store, path: str) -> dict: + """Files that `path` links TO (resolved). Each entry carries the typed-edge predicate.""" + return { + "path": path, + "links": [_edge_to_dict(m, e) for m, e in file_store.get_links(path)], + } + + +def get_backlinks(file_store, path: str) -> dict: + """Files that link TO `path`. Each entry carries the typed-edge predicate.""" + return { + "path": path, + "backlinks": [_edge_to_dict(m, e) for m, e in file_store.get_backlinks(path)], + } + + +def resolve_wikilink(file_store, wikilink: str) -> dict: + """Resolve a `[[target]]` wikilink with full ambiguity context. + + Distinct from `file_store.resolve_wikilink(target)` (which returns + just the path or None) — this surface returns the rich payload + callers need to disambiguate: + + unique resolution → {wikilink, path, exists: True, + ambiguous: False, candidates: [path]} + ambiguous → {wikilink, path: None, exists: False, + ambiguous: True, candidates: [...]} + dangling → {wikilink, path: None, exists: False, + ambiguous: False, candidates: []} + """ + # Path-form (`a/b` or `a/b.md`): file_store already returns + # exactly the one path that exists, or None. + if "/" in wikilink or wikilink.endswith(".md"): + hit = file_store.resolve_wikilink(wikilink) + return { + "wikilink": wikilink, + "path": hit, + "exists": hit is not None, + "ambiguous": False, + "candidates": [hit] if hit else [], + } + + # Stem-form: candidates list reveals 0/1/N resolution. + candidates = file_store.wikilink_candidates(wikilink) + if len(candidates) == 1: + return { + "wikilink": wikilink, + "path": candidates[0], + "exists": True, + "ambiguous": False, + "candidates": candidates, + } + return { + "wikilink": wikilink, + "path": None, + "exists": False, + "ambiguous": len(candidates) > 1, + "candidates": candidates, + } + + +def iter_files(file_store) -> Iterator[tuple[str, FileMetadata]]: + """Walk every indexed (path, FileMetadata). Used by Maintainer scans. + + Equivalent to `file_store.nodes.items()`, exposed here so consumers + don't have to know about the underlying cache attribute name. + """ + return iter(file_store.nodes.items()) + + +async def count_tokens( + token_counter, + *, + path: str | None = None, + text: str | None = None, +) -> dict: + """Estimate tokens for a file body (frontmatter excluded) or raw text. + + Powers the Maintainer's split-trigger. Exactly one of `path` / `text` + must be provided. Takes a `token_counter` rather than `file_store` + because the engine's tokenization is a separate component — this + function lives here to keep the engine's outward surface in one place. + """ + if path: + target = Path(path) + if not target.is_file(): + return {"path": str(target), "error": "file not found"} + raw = target.read_text(encoding="utf-8") + post = frontmatter.loads(raw) + body = post.content + tokens = await token_counter.count(messages=[], text=body) + return { + "source": "file", + "path": str(target.resolve()), + "tokens": tokens, + "body_chars": len(body), + } + if text: + tokens = await token_counter.count(messages=[], text=text) + return { + "source": "text", + "tokens": tokens, + "body_chars": len(text), + } + return {"error": "one of `path` or `text` is required"} + + +# =========================================================================== +# Section 2 — MFS Writes # =========================================================================== # # Every mutation in the system funnels through these. Hot-write MCP shells -# (sync, topic_create, memory_*) call them directly; cold-write services +# (sync, memory_*) call them directly; cold-write services # (Ingestor R-M-W, Maintainer decay) compose them. +# +# `create_file` and `rename_file` take `file_store` because they need the +# wikilink-uniqueness gate / backlinks index. The other writes don't — +# they just touch disk and let the Watcher catch up. def _replace_wikilink_targets(text: str, mapping: dict[str, str]) -> str: @@ -79,9 +288,10 @@ def _replace_wikilink_targets(text: str, mapping: dict[str, str]) -> str: return WIKILINK_RE.sub(sub, text) -def write_create( +def create_file( file_store, path: Path, + *, metadata: dict, content: str, overwrite: bool = False, @@ -124,17 +334,9 @@ def write_create( return True, {"path": str(path), "created": True} -def write_delete(path: Path | str) -> tuple[bool, dict]: - """Delete a file. Watcher removes from store + graph.""" - target = Path(path) - if not target.exists(): - return False, {"path": str(target), "error": "not found"} - target.unlink() - return True, {"path": str(target), "deleted": True} - - -def write_update( +def update_body( path: Path | str, + *, old_string: str, new_string: str, replace_all: bool = False, @@ -146,7 +348,7 @@ def write_update( if not old_string: return False, { "path": str(target), - "error": "old_string is required (use write_create to write a new file)", + "error": "old_string is required (use create_file to write a new file)", } raw = target.read_text(encoding="utf-8") occurrences = raw.count(old_string) @@ -169,7 +371,7 @@ def write_update( } -def write_property_update(path: Path | str, key: str, value) -> tuple[bool, dict]: +def update_meta(path: Path | str, *, key: str, value) -> tuple[bool, dict]: """Update a single YAML frontmatter key. value=None deletes the key.""" target = Path(path) if not target.is_file(): @@ -184,9 +386,10 @@ def write_property_update(path: Path | str, key: str, value) -> tuple[bool, dict return True, {"path": str(target), "key": key, "value": value} -def write_rename( +def rename_file( file_store, vault_root: Path | str, + *, old_path: Path | str, new_path: Path | str, ) -> tuple[bool, dict]: @@ -275,10 +478,20 @@ def write_rename( } -def write_archive( +def delete_file(path: Path | str) -> tuple[bool, dict]: + """Delete a file. Watcher removes from store + graph.""" + target = Path(path) + if not target.exists(): + return False, {"path": str(target), "error": "not found"} + target.unlink() + return True, {"path": str(target), "deleted": True} + + +def archive_file( vault_root: Path | str, path: Path | str, - archive_dir_name: str = "Archive", + *, + archive_dir: str = "Archive", ) -> tuple[bool, dict]: """Archive a file: flip `status: archived`, then move under `//`. @@ -299,16 +512,16 @@ def write_archive( "error": f"path is outside vault_root {vault}", } - dst = vault / archive_dir_name / rel + dst = vault / archive_dir / rel if dst.exists(): return False, { "path": str(src), "error": f"archive destination already exists: {dst}", } - ok, prop_payload = write_property_update(src, "status", "archived") + ok, prop_payload = update_meta(src, key="status", value="archived") if not ok: - return False, {**prop_payload, "stage": "property_update"} + return False, {**prop_payload, "stage": "update_meta"} dst.parent.mkdir(parents=True, exist_ok=True) try: @@ -327,213 +540,24 @@ def write_archive( # =========================================================================== -# Section 2 — MFS reads (primary-key lookups against the file_store cache) -# =========================================================================== -# -# These don't touch the projection indexes — they hit the in-memory file -# meta + edge cache that the file_store maintains as a mirror of disk. -# For body / chunk content reads, `read_file` does fall through to the -# disk to get the latest text (file_store cache may lag a write). - - -async def read_file( - file_store, - path: str, - *, - include_chunks: bool = False, -) -> dict: - """Read frontmatter + body for one path. Optionally include parsed chunks. - - On-disk frontmatter is the source of truth — the file_store cache may - lag a write that hasn't been picked up by the watcher yet. - """ - meta = file_store.get_file_meta(path) - result: dict = {"path": path, "exists": False} - if meta is not None: - edges = file_store.get_edges(path) - result.update({ - "exists": True, - "metadata": meta.metadata, - "link": [e.model_dump(exclude_none=True) for e in edges], - }) - - file_path = Path(path) - if file_path.is_file(): - raw = file_path.read_text(encoding="utf-8") - post = frontmatter.loads(raw) - result["exists"] = True - result["content"] = post.content - result["metadata"] = dict(post.metadata) - - if include_chunks: - chunks = await file_store.get_chunks(path) - result["chunks"] = [c.model_dump(exclude_none=True) for c in chunks] - return result - - -def list_files( - file_store, - *, - path_prefix: str | None = None, - tags: list[str] | None = None, - metadata: dict | None = None, - limit: int = 100, -) -> dict: - """List indexed files filtered by frontmatter exact-match, tags, and prefix. - - Returns {items: [{path, metadata}], count}. - """ - metadata_filter = metadata or {} - tag_filter = tags or [] - items: list[dict] = [] - for path, meta in file_store.nodes.items(): - if path_prefix and not path.startswith(path_prefix): - continue - md = meta.metadata or {} - if metadata_filter and any(md.get(k) != v for k, v in metadata_filter.items()): - continue - if tag_filter: - file_tags = set(md.get("tags", []) or []) - if not all(t in file_tags for t in tag_filter): - continue - items.append({"path": path, "metadata": md}) - if len(items) >= limit: - break - return {"items": items, "count": len(items)} - - -def _edge_to_dict(file_meta, edge) -> dict: - return { - "path": file_meta.path, - "metadata": file_meta.metadata, - "predicate": edge.predicate, - "anchor": edge.anchor, - "alias": edge.alias, - "embed": edge.embed, - "source": edge.source, - "confidence": edge.confidence, - } - - -def links_of(file_store, path: str) -> dict: - """Files that `path` links TO (resolved). Each entry carries the typed-edge predicate.""" - return { - "path": path, - "links": [_edge_to_dict(m, e) for m, e in file_store.get_links(path)], - } - - -def backlinks_of(file_store, path: str) -> dict: - """Files that link TO `path`. Each entry carries the typed-edge predicate.""" - return { - "path": path, - "backlinks": [_edge_to_dict(m, e) for m, e in file_store.get_backlinks(path)], - } - - -def wikilink_lookup(file_store, wikilink: str) -> dict: - """Resolve a `[[target]]` wikilink with full ambiguity context. - - Distinct from `file_store.resolve_wikilink(target)` (which returns - the single resolved path or None) — this surface returns the rich - payload callers need to disambiguate: - - unique resolution → {wikilink, path, exists: True, - ambiguous: False, candidates: [path]} - ambiguous → {wikilink, path: None, exists: False, - ambiguous: True, candidates: [...]} - dangling → {wikilink, path: None, exists: False, - ambiguous: False, candidates: []} - """ - # Path-form (`a/b` or `a/b.md`): file_store already returns - # exactly the one path that exists, or None. - if "/" in wikilink or wikilink.endswith(".md"): - hit = file_store.resolve_wikilink(wikilink) - return { - "wikilink": wikilink, - "path": hit, - "exists": hit is not None, - "ambiguous": False, - "candidates": [hit] if hit else [], - } - - # Stem-form: candidates list reveals 0/1/N resolution. - candidates = file_store.wikilink_candidates(wikilink) - if len(candidates) == 1: - return { - "wikilink": wikilink, - "path": candidates[0], - "exists": True, - "ambiguous": False, - "candidates": candidates, - } - return { - "wikilink": wikilink, - "path": None, - "exists": False, - "ambiguous": len(candidates) > 1, - "candidates": candidates, - } - - -async def count_tokens( - token_counter, - *, - path: str | None = None, - text: str | None = None, -) -> dict: - """Estimate tokens for a file body (frontmatter excluded) or raw text. - - Powers the Maintainer's split-trigger. Exactly one of `path` / `text` - must be provided. - """ - if path: - target = Path(path) - if not target.is_file(): - return {"path": str(target), "error": "file not found"} - raw = target.read_text(encoding="utf-8") - post = frontmatter.loads(raw) - body = post.content - tokens = await token_counter.count(messages=[], text=body) - return { - "source": "file", - "path": str(target.resolve()), - "tokens": tokens, - "body_chars": len(body), - } - if text: - tokens = await token_counter.count(messages=[], text=text) - return { - "source": "text", - "tokens": tokens, - "body_chars": len(text), - } - return {"error": "one of `path` or `text` is required"} - - -def iter_files(file_store) -> Iterator[tuple[str, FileMetadata]]: - """Walk every indexed (path, FileMetadata). Used by Maintainer scans. - - Equivalent to `file_store.nodes.items()`, exposed here so consumers - don't have to know about the underlying cache attribute name. - """ - return iter(file_store.nodes.items()) - - -# =========================================================================== -# Section 3 — Projection queries (Vector / FTS / File Graph) +# Section 3 — Projection Queries # =========================================================================== # # Per `structure.md` §"核心引擎", the Vector index, FTS5 index, and File # Graph are downstream **projections** of the MFS — they can be wholly # rebuilt from disk. These functions are the read entry to those -# projections; `Retriever` composes them with policy (V+K weighting, +# projections; the Retriever composes them with policy (V+K weighting, # graph BFS, intent routing) for ranked retrieval. +# +# Projections are write-only-via-Watcher: the API surface deliberately +# exposes no setters. To "update" a projection, write to the MFS (Section +# 2) and let the Watcher rebuild. -async def vector_search( +async def search_vector( file_store, query: str, + *, limit: int, chunk_filter: ChunkFilter | None = None, ) -> list[FileChunk]: @@ -541,9 +565,10 @@ async def vector_search( return await file_store.vector_search(query, limit, chunk_filter) -async def keyword_search( +async def search_keyword( file_store, query: str, + *, limit: int, chunk_filter: ChunkFilter | None = None, ) -> list[FileChunk]: @@ -567,12 +592,12 @@ def extract_anchors(file_store, text: str) -> list[str]: return file_store.extract_anchor_paths(text) -async def chunks_by_paths(file_store, paths: Iterable[str]) -> list[FileChunk]: +async def get_chunks(file_store, paths: Iterable[str]) -> list[FileChunk]: """Batch fetch chunks across many paths (used by graph-walk retrieval).""" return await file_store.get_chunks_by_paths(paths) -def make_chunk_filter( +def make_filter( file_store, *, paths: list[str] | None = None, @@ -583,267 +608,7 @@ def make_chunk_filter( return file_store.filter(paths=paths, tags=tags, exclude_paths=exclude_paths) -def all_ambiguous_wikilinks(file_store) -> dict[str, list[str]]: +def find_collisions(file_store) -> dict[str, list[str]]: """Every stem that resolves to >1 path. Used by Maintainer.lint.""" return file_store.all_ambiguous_wikilinks() - -# =========================================================================== -# Section 4 — Agent toolkit -# =========================================================================== -# -# `MemoryIO` adapts the read/write helpers above as `agentscope.tool` -# callables, with a vault-root containment check on writes and an audit -# trail every consumer can inspect after the agent's run. Used by the -# Ingestor's ReActAgent. - - -def _text_response(payload: Any) -> ToolResponse: - text = json.dumps(_to_jsonable(payload), ensure_ascii=False, indent=2) - return ToolResponse(content=[TextBlock(type="text", text=text)]) - - -class MemoryIO: - """Agent-facing tool surface over the Memory File System. - - Args: - file_store: The vault's FileStore (provides graph + index). - vault_root: Containment boundary — write tools refuse paths - that escape it. - - Attributes: - audit: Every successful or failed write is appended here. Read - this after the agent's run to reconstruct the mutation trail. - """ - - _TOOL_NAMES = ( - "memory_get", - "memory_list", - "memory_resolve_wikilink", - "memory_backlinks", - "memory_links", - "memory_create", - "memory_update", - "memory_property_update", - "memory_rename", - "memory_delete", - "memory_archive", - ) - - def __init__(self, file_store, vault_root: str | Path): - self.file_store = file_store - self.vault_root = Path(vault_root).resolve() - self.audit: list[dict] = [] - - def register_all(self, toolkit: Toolkit | None = None) -> Toolkit: - """Register every memory_* method on `toolkit` (or a fresh one).""" - toolkit = toolkit or Toolkit() - for name in self._TOOL_NAMES: - toolkit.register_tool_function( - getattr(self, name), namesake_strategy="override", - ) - return toolkit - - # -- Internals -------------------------------------------------------- - - def _resolve(self, p: str) -> Path: - pp = Path(p) - if not pp.is_absolute(): - pp = self.vault_root / pp - return pp.resolve() - - def _under_vault(self, p: Path) -> bool: - try: - p.relative_to(self.vault_root) - return True - except (ValueError, OSError): - return False - - def _record(self, op: str, ok: bool, **fields) -> dict: - entry = {"op": op, "ok": ok, **fields} - self.audit.append(entry) - return entry - - # -- Read tools (delegate to module-level helpers) -------------------- - - async def memory_get(self, path: str, include_chunks: bool = False) -> ToolResponse: - """Read a memory file (frontmatter + body, optional chunks). - - Args: - path (str): Absolute path to the file. - include_chunks (bool): Include parsed chunk metadata. - """ - target = self._resolve(path) - result = await read_file(self.file_store, str(target), include_chunks=include_chunks) - return _text_response(result) - - async def memory_list( - self, - path_prefix: str | None = None, - tags: list[str] | None = None, - metadata: dict | None = None, - limit: int = 100, - ) -> ToolResponse: - """List indexed vault files filtered by prefix, tags, and frontmatter. - - Args: - path_prefix (str | None): Restrict to paths starting with this prefix. - tags (list[str] | None): All tags must be present on a file. - metadata (dict | None): Exact-match filter on frontmatter keys. - limit (int): Cap on returned items. - """ - return _text_response(list_files( - self.file_store, - path_prefix=path_prefix, tags=tags, - metadata=metadata, limit=limit, - )) - - async def memory_resolve_wikilink(self, wikilink: str) -> ToolResponse: - """Resolve a `[[wikilink]]` to an absolute path. - - Args: - wikilink (str): The wikilink target, e.g. `Topic` or `topics/Topic`. - """ - return _text_response(wikilink_lookup(self.file_store, wikilink)) - - async def memory_backlinks(self, path: str) -> ToolResponse: - """List files linking TO a given path. - - Args: - path (str): Absolute path to inspect. - """ - target = self._resolve(path) - return _text_response(backlinks_of(self.file_store, str(target))) - - async def memory_links(self, path: str) -> ToolResponse: - """List files a given path links to. - - Args: - path (str): Absolute path to inspect. - """ - target = self._resolve(path) - return _text_response(links_of(self.file_store, str(target))) - - # -- Write tools (delegate to write_*; record audit) ------------------ - - async def memory_create( - self, - path: str, - metadata: dict | None = None, - content: str = "", - overwrite: bool = False, - force: bool = False, - ) -> ToolResponse: - """Create a new markdown file in the vault. - - Args: - path (str): Target path (resolved against vault_root if relative). - metadata (dict | None): YAML frontmatter for the new file. - content (str): Body markdown. - overwrite (bool): Allow overwriting an existing file. - force (bool): Allow creates that introduce stem-form wikilink ambiguity. - """ - target = self._resolve(path) - if not self._under_vault(target): - entry = self._record("create", False, path=str(target), - error=f"path is outside vault_root {self.vault_root}") - return _text_response(entry) - ok, payload = write_create( - self.file_store, target, - metadata=dict(metadata or {}), content=content, - overwrite=overwrite, force=force, - ) - entry = self._record("create", ok, path=str(target), result=payload) - return _text_response(entry) - - async def memory_update( - self, - path: str, - old_string: str, - new_string: str, - replace_all: bool = False, - ) -> ToolResponse: - """Edit a file body by exact-string substitution. - - Use a unique snippet for `old_string`. To append, pass the file's - tail as `old_string` and `tail + new_content` as `new_string`. - - Args: - path (str): Absolute path to the file. - old_string (str): Exact text to replace (must be unique unless replace_all). - new_string (str): Replacement text. - replace_all (bool): Replace every occurrence instead of just one. - """ - target = self._resolve(path) - if not self._under_vault(target): - entry = self._record("update", False, path=str(target), - error=f"path is outside vault_root {self.vault_root}") - return _text_response(entry) - ok, payload = write_update(target, old_string, new_string, replace_all=replace_all) - entry = self._record("update", ok, path=str(target), result=payload) - return _text_response(entry) - - async def memory_property_update(self, path: str, key: str, value: Any = None) -> ToolResponse: - """Update one YAML frontmatter key on a file (value=null deletes it). - - Args: - path (str): Absolute path to the file. - key (str): Frontmatter key. - value: New value, or null to delete the key. - """ - target = self._resolve(path) - if not self._under_vault(target): - entry = self._record("property_update", False, path=str(target), - error=f"path is outside vault_root {self.vault_root}") - return _text_response(entry) - ok, payload = write_property_update(target, key, value) - entry = self._record("property_update", ok, path=str(target), result=payload) - return _text_response(entry) - - async def memory_rename(self, old_path: str, new_path: str) -> ToolResponse: - """Rename a file and rewrite cross-vault wikilinks. - - Args: - old_path (str): Current absolute path. - new_path (str): Target absolute path. - """ - old_p = self._resolve(old_path) - new_p = self._resolve(new_path) - if not self._under_vault(old_p) or not self._under_vault(new_p): - entry = self._record("rename", False, old_path=str(old_p), new_path=str(new_p), - error=f"path is outside vault_root {self.vault_root}") - return _text_response(entry) - ok, payload = write_rename(self.file_store, self.vault_root, old_p, new_p) - entry = self._record("rename", ok, old_path=str(old_p), new_path=str(new_p), result=payload) - return _text_response(entry) - - async def memory_delete(self, path: str) -> ToolResponse: - """Delete a file from the vault. - - Args: - path (str): Absolute path to the file. - """ - target = self._resolve(path) - if not self._under_vault(target): - entry = self._record("delete", False, path=str(target), - error=f"path is outside vault_root {self.vault_root}") - return _text_response(entry) - ok, payload = write_delete(target) - entry = self._record("delete", ok, path=str(target), result=payload) - return _text_response(entry) - - async def memory_archive(self, path: str, archive_dir: str = "Archive") -> ToolResponse: - """Flip `status: archived` and move file under `//`. - - Args: - path (str): Absolute path to the file. - archive_dir (str): Subdirectory name under vault_root for archives. - """ - target = self._resolve(path) - if not self._under_vault(target): - entry = self._record("archive", False, path=str(target), - error=f"path is outside vault_root {self.vault_root}") - return _text_response(entry) - ok, payload = write_archive(self.vault_root, target, archive_dir) - entry = self._record("archive", ok, path=str(target), result=payload) - return _text_response(entry) diff --git a/reme2/memory/memory_toolkit.py b/reme2/memory/memory_toolkit.py new file mode 100644 index 00000000..772aecf0 --- /dev/null +++ b/reme2/memory/memory_toolkit.py @@ -0,0 +1,598 @@ +"""Memory toolkit — schema-bound projection of the Memory File System. + +Layered on top of `reme2.memory.memory_io` (the pure core engine). One +`BaseStep` subclass per memory_* tool, each exposing TWO class methods: + + * `execute()` — the MCP surface. Reads parameters from + `RuntimeContext` and writes the result through `_set_answer`. + * a method named after the tool (e.g. `memory_get`) — the agent + toolkit surface. Takes explicit parameters, returns a + `ToolResponse`. agentscope's `Toolkit.register_tool_function` + introspects the signature directly — no hand-authored JSON schema. + +`build_memory_toolkit(app_context, audit, toolkit)` instantiates every +registered memory_* step and binds its tool method to a `Toolkit`. +Each instance carries an `audit` list so the host can surface what the +agent actually called. + +Schema policy (status state machine, path templates) lives at the top +of this file as pure helpers; the relevant write tools (`memory_create` +/ `memory_property_update`) call them. `force=True` is the single +escape hatch — bypasses BOTH the policy gates here and the +wikilink-uniqueness gate downstream in `memory_io.create_file`. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import frontmatter +from agentscope.message import TextBlock +from agentscope.tool import Toolkit, ToolResponse + +from . import memory_io +from ..component import R +from ..component.base_step import BaseStep +from ..component.runtime_response import _set_answer, _to_jsonable +from ..enumeration import ComponentEnum + + +# =========================================================================== +# Section 1 — Schema policy +# =========================================================================== + + +_STATUS_STATES = ("active", "distilled", "archived") +_STATUS_TRANSITIONS: dict[str, set[str]] = { + "active": {"active", "distilled"}, + "distilled": {"distilled", "archived"}, + "archived": {"archived"}, +} + + +def validate_status_transition(prior, requested) -> str | None: + """Return error string if the requested status transition is invalid, + else None. Files without a prior status accept any initial value + (so first-write doesn't get blocked).""" + if requested is None: + return None # delete operation + if requested not in _STATUS_STATES: + return ( + f"invalid status {requested!r}; must be one of " + f"{list(_STATUS_STATES)}" + ) + if prior in _STATUS_STATES and requested not in _STATUS_TRANSITIONS[prior]: + return ( + f"status transition {prior!r} → {requested!r} not allowed; " + f"state machine is single-direction " + f"active → distilled → archived" + ) + return None + + +def validate_path_template(path: Path, vault_root: Path | None) -> str | None: + """Return error string if `path` doesn't match a known agent-facing + template, else None. When `vault_root` is unknown, skip the check. + + Allowed templates (relative to vault_root): + topics/{folder}/{name}.md — topic file + events/{date}/{name}/{filename} — event index OR sibling material + Archive/... — archive moves can land anywhere + """ + if vault_root is None: + return None + try: + rel = path.resolve().relative_to(vault_root) + except ValueError: + return f"path {path} is outside vault_root {vault_root}" + parts = rel.parts + if not parts: + return "path has no components relative to vault_root" + head = parts[0] + if head == "Archive": + return None + if head == "topics" and len(parts) >= 3: + return None + if head == "events" and len(parts) >= 4: + return None + return ( + f"path {rel} doesn't match a known template — expected one of: " + f"topics/{{folder}}/{{name}}.md, " + f"events/{{date}}/{{name}}/{{filename}}, or Archive/..." + ) + + +def update_status( + path: Path | str, + *, + value, + force: bool = False, +) -> tuple[bool, dict]: + """Schema-aware status flip. Reads current status from disk, validates + the transition, then delegates to `memory_io.update_meta`.""" + target = Path(path) + if not force: + prior = None + if target.is_file(): + try: + prior = frontmatter.loads( + target.read_text(encoding="utf-8"), + ).metadata.get("status") + except Exception: + prior = None + err = validate_status_transition(prior, value) + if err is not None: + return False, { + "path": str(target), + "key": "status", + "error": err, + "prior": prior, + "requested": value, + } + return memory_io.update_meta(target, key="status", value=value) + + +def create_file_with_schema( + file_store, + path: Path, + *, + metadata: dict, + content: str, + overwrite: bool = False, + force: bool = False, +) -> tuple[bool, dict]: + """Schema-aware file create. Validates the path template (unless + `force=True`), then delegates to `memory_io.create_file` (which + still enforces the wikilink-uniqueness graph invariant — that gate + lives in the engine because it's structural, not business policy). + + `force=True` bypasses BOTH the template gate here and the wikilink + gate downstream — it's the single escape hatch for any caller that + intentionally needs to step outside conventions. + """ + if not force: + vault_root = getattr(file_store, "vault_root", None) + template_err = validate_path_template(path, vault_root) + if template_err is not None: + return False, { + "path": str(path), + "error": template_err, + "hint": ( + "place topics under topics/{folder}/{name}.md and " + "events under events/{date}/{name}/...; pass " + "force=true only if you intentionally need a " + "non-template path" + ), + } + return memory_io.create_file( + file_store, path, + metadata=metadata, content=content, + overwrite=overwrite, force=force, + ) + + +# =========================================================================== +# Section 2 — Tool response helper +# =========================================================================== + + +def _tool_response( + op: str, + ok: bool, + payload: Any, + audit: list[dict] | None = None, +) -> ToolResponse: + """Wrap a tool-method result as a `ToolResponse` and optionally + append an audit row.""" + if audit is not None: + entry = {"op": op, "ok": ok} + if isinstance(payload, dict): + entry.update(payload) + else: + entry["result"] = payload + audit.append(entry) + text = json.dumps(_to_jsonable(payload), ensure_ascii=False, indent=2) + return ToolResponse(content=[TextBlock(type="text", text=text)]) + + +# =========================================================================== +# Section 3 — Memory steps (one BaseStep per tool, two surfaces each) +# =========================================================================== + + +@R.register("memory_get") +class MemoryGet(BaseStep): + """Read a single memory file (frontmatter + body, optional chunks).""" + + audit: list[dict] | None = None # set by build_memory_toolkit + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path", "") or "" + include_chunks: bool = bool(self.context.get("include_chunks", False)) + assert path, "path is required" + result = await memory_io.get_file(self.file_store, path, include_chunks=include_chunks) + _set_answer(self.context, result) + + async def memory_get(self, path: str, include_chunks: bool = False) -> ToolResponse: + """Read a single memory file (frontmatter + body, optional chunks).""" + result = await memory_io.get_file(self.file_store, path, include_chunks=include_chunks) + return _tool_response("memory_get", True, result, audit=self.audit) + + +@R.register("memory_list") +class MemoryList(BaseStep): + """List indexed files filtered by frontmatter fields, tags, or path prefix.""" + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + result = memory_io.list_files( + self.file_store, + path_prefix=self.context.get("path_prefix"), + tags=self.context.get("tags") or [], + metadata=self.context.get("metadata") or {}, + limit=int(self.context.get("limit") or 100), + ) + _set_answer(self.context, result) + + async def memory_list( + self, + path_prefix: str | None = None, + tags: list[str] | None = None, + metadata: dict | None = None, + limit: int = 100, + ) -> ToolResponse: + """List indexed files filtered by frontmatter fields, tags, or path prefix.""" + result = memory_io.list_files( + self.file_store, + path_prefix=path_prefix, + tags=tags or [], + metadata=metadata or {}, + limit=limit, + ) + return _tool_response("memory_list", True, result, audit=self.audit) + + +@R.register("memory_backlinks") +class MemoryBacklinks(BaseStep): + """Files linking TO a given path. Each entry carries the typed-edge predicate.""" + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path", "") or "" + assert path, "path is required" + _set_answer(self.context, memory_io.get_backlinks(self.file_store, path)) + + async def memory_backlinks(self, path: str) -> ToolResponse: + """Files linking TO a given path. Each entry carries the typed-edge predicate.""" + result = memory_io.get_backlinks(self.file_store, path) + return _tool_response("memory_backlinks", True, result, audit=self.audit) + + +@R.register("memory_links") +class MemoryLinks(BaseStep): + """Files a given path links to (resolved). Each entry carries the typed-edge predicate.""" + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path", "") or "" + assert path, "path is required" + _set_answer(self.context, memory_io.get_links(self.file_store, path)) + + async def memory_links(self, path: str) -> ToolResponse: + """Files a given path links to (resolved). Each entry carries the typed-edge predicate.""" + result = memory_io.get_links(self.file_store, path) + return _tool_response("memory_links", True, result, audit=self.audit) + + +@R.register("memory_resolve_wikilink") +class MemoryResolveWikilink(BaseStep): + """Resolve a `[[target]]` wikilink with full ambiguity context.""" + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + wikilink: str = self.context.get("wikilink", "") or "" + assert wikilink, "wikilink is required" + payload = memory_io.resolve_wikilink(self.file_store, wikilink) + self.context.response.success = bool(payload.get("exists")) + _set_answer(self.context, payload) + + async def memory_resolve_wikilink(self, wikilink: str) -> ToolResponse: + """Resolve a `[[target]]` wikilink with full ambiguity context.""" + payload = memory_io.resolve_wikilink(self.file_store, wikilink) + return _tool_response( + "memory_resolve_wikilink", + bool(payload.get("exists")), + payload, + audit=self.audit, + ) + + +@R.register("memory_create") +class MemoryCreate(BaseStep): + """Create a markdown file. Path-template gate + wikilink-uniqueness + gate both fire unless `force=True`.""" + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path", "") or "" + metadata: dict = dict(self.context.get("metadata") or {}) + content: str = self.context.get("content", "") or "" + overwrite: bool = bool(self.context.get("overwrite", False)) + force: bool = bool(self.context.get("force", False)) + assert path, "path is required" + + target = Path(path) + ok, payload = create_file_with_schema( + self.file_store, target, + metadata=metadata, content=content, + overwrite=overwrite, force=force, + ) + self.context.response.success = ok + if ok: + payload = {**payload, "path": str(target.resolve())} + _set_answer(self.context, payload) + + async def memory_create( + self, + path: str, + metadata: dict | None = None, + content: str = "", + overwrite: bool = False, + force: bool = False, + ) -> ToolResponse: + """Create a markdown file. Path-template gate + wikilink-uniqueness + gate both fire unless `force=True`.""" + target = Path(path) + ok, payload = create_file_with_schema( + self.file_store, target, + metadata=dict(metadata or {}), + content=content, + overwrite=overwrite, + force=force, + ) + if ok: + payload = {**payload, "path": str(target.resolve())} + return _tool_response("memory_create", ok, payload, audit=self.audit) + + +@R.register("memory_delete") +class MemoryDelete(BaseStep): + """Delete a file. Watcher removes from store + graph.""" + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path", "") or "" + assert path, "path is required" + ok, payload = memory_io.delete_file(path) + self.context.response.success = ok + _set_answer(self.context, payload) + + async def memory_delete(self, path: str) -> ToolResponse: + """Delete a file. Watcher removes from store + graph.""" + ok, payload = memory_io.delete_file(path) + return _tool_response("memory_delete", ok, payload, audit=self.audit) + + +@R.register("memory_rename") +class MemoryRename(BaseStep): + """Rename a file and rewrite incoming wikilinks across the vault.""" + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + old_path: str = self.context.get("old_path", "") or "" + new_path: str = self.context.get("new_path", "") or "" + assert old_path and new_path, "old_path and new_path are required" + + watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[index,union-attr] + vault_root = Path(watcher.watch_path).resolve() # type: ignore[union-attr] + + ok, payload = memory_io.rename_file( + self.file_store, vault_root, + old_path=old_path, new_path=new_path, + ) + self.context.response.success = ok + _set_answer(self.context, payload) + + async def memory_rename(self, old_path: str, new_path: str) -> ToolResponse: + """Rename a file and rewrite incoming wikilinks across the vault.""" + watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default") + vault_root = ( + Path(getattr(watcher, "watch_path", ".")).resolve() if watcher else Path.cwd() + ) + ok, payload = memory_io.rename_file( + self.file_store, vault_root, + old_path=old_path, new_path=new_path, + ) + return _tool_response("memory_rename", ok, payload, audit=self.audit) + + +@R.register("memory_property_update") +class MemoryPropertyUpdate(BaseStep): + """Update a single YAML frontmatter key. value=null deletes the key. + + When key=='status', enforces the active → distilled → archived + state machine via `update_status`. Pass `force=True` to bypass. + Other keys go through the bare engine.""" + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path", "") or "" + key: str = self.context.get("key", "") or "" + value = self.context.get("value") + force: bool = bool(self.context.get("force", False)) + assert path and key, "path and key are required" + + if key == "status": + ok, payload = update_status(path, value=value, force=force) + else: + ok, payload = memory_io.update_meta(path, key=key, value=value) + self.context.response.success = ok + _set_answer(self.context, payload) + + async def memory_property_update( + self, + path: str, + key: str, + value: Any = None, + force: bool = False, + ) -> ToolResponse: + """Update a single YAML frontmatter key. value=null deletes the key. + + When key=='status', enforces the active → distilled → archived + state machine. Pass `force=True` to bypass.""" + if key == "status": + ok, payload = update_status(path, value=value, force=force) + else: + ok, payload = memory_io.update_meta(path, key=key, value=value) + return _tool_response("memory_property_update", ok, payload, audit=self.audit) + + +@R.register("memory_update") +class MemoryUpdate(BaseStep): + """Edit-style content update: replace `old_string` with `new_string`.""" + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path", "") or "" + old_string: str = self.context.get("old_string", "") or "" + new_string: str = self.context.get("new_string", "") or "" + replace_all: bool = bool(self.context.get("replace_all", False)) + assert path, "path is required" + ok, payload = memory_io.update_body( + path, old_string=old_string, new_string=new_string, replace_all=replace_all, + ) + self.context.response.success = ok + _set_answer(self.context, payload) + + async def memory_update( + self, + path: str, + old_string: str, + new_string: str, + replace_all: bool = False, + ) -> ToolResponse: + """Edit-style content update: replace `old_string` with `new_string`.""" + ok, payload = memory_io.update_body( + path, old_string=old_string, new_string=new_string, replace_all=replace_all, + ) + return _tool_response("memory_update", ok, payload, audit=self.audit) + + +@R.register("memory_archive") +class MemoryArchive(BaseStep): + """Archive a file: flip `status: archived` then move to `/Archive/`.""" + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path", "") or "" + archive_dir_name: str = self.context.get("archive_dir", "Archive") or "Archive" + assert path, "path is required" + + watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default") + vault_root = Path(getattr(watcher, "watch_path", ".")).resolve() if watcher else Path.cwd() + + ok, payload = memory_io.archive_file(vault_root, path, archive_dir=archive_dir_name) + self.context.response.success = ok + _set_answer(self.context, payload) + + async def memory_archive(self, path: str, archive_dir: str = "Archive") -> ToolResponse: + """Archive a file: flip `status: archived` then move to `//`.""" + watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default") + vault_root = ( + Path(getattr(watcher, "watch_path", ".")).resolve() if watcher else Path.cwd() + ) + ok, payload = memory_io.archive_file(vault_root, path, archive_dir=archive_dir) + return _tool_response("memory_archive", ok, payload, audit=self.audit) + + +@R.register("memory_count_tokens") +class MemoryCountTokens(BaseStep): + """Estimate tokens for a file body or raw text. One of `path`/`text` required. + + MCP-only — token counting is an editor concern, not part of the + R-M-W loop. No agent toolkit projection.""" + + async def execute(self): + assert self.context is not None + path: str = self.context.get("path", "") or "" + text: str = self.context.get("text", "") or "" + result = await memory_io.count_tokens( + self.as_token_counter, + path=path or None, + text=text or None, + ) + self.context.response.success = "error" not in result + _set_answer(self.context, result) + + +# =========================================================================== +# Section 4 — Toolkit factory +# =========================================================================== + + +# The 11 memory_* tools the Ingestor's ReActAgent gets bound to. +# Each name doubles as the BaseStep registration key AND the tool method +# name on that step. +MEMORY_TOOL_NAMES: tuple[str, ...] = ( + "memory_get", + "memory_list", + "memory_resolve_wikilink", + "memory_backlinks", + "memory_links", + "memory_create", + "memory_update", + "memory_property_update", + "memory_rename", + "memory_delete", + "memory_archive", +) + + +def build_memory_toolkit( + app_context, + audit: list[dict] | None = None, + toolkit: Toolkit | None = None, +) -> Toolkit: + """Bind every memory_* step's tool method to an agentscope `Toolkit`. + + For each name in `MEMORY_TOOL_NAMES`, instantiates the registered + BaseStep against `app_context`, attaches the shared `audit` list, + and registers the same-named class method as a tool function. + agentscope introspects the method signature directly — there is no + separate JSON schema layer. + """ + toolkit = toolkit or Toolkit() + for name in MEMORY_TOOL_NAMES: + step_cls = R.get(ComponentEnum.STEP, name) + if step_cls is None: + continue + instance = step_cls(app_context=app_context) + instance.audit = audit # type: ignore[attr-defined] + toolkit.register_tool_function( + getattr(instance, name), + namesake_strategy="override", + ) + return toolkit diff --git a/reme2/memory/protocol.md b/reme2/memory/protocol.md new file mode 100644 index 00000000..f3df6ce7 --- /dev/null +++ b/reme2/memory/protocol.md @@ -0,0 +1,147 @@ +# Memory Protocol + +Single source of truth for vault schema, conventions, and the R-M-W +write loop. Consumed by: + +- **Ingestor's embedded ReAct prompt** (`reme2/memory/ingestor.yaml` — + injected as `{protocol}` at load time). +- **Strong-agent SKILL** (`reme-plugin/skills/reme/SKILL.md` — + transcluded so the host agent sees the same rules). + +Anything that defines schema invariants, path templates, write tool +semantics, or the R-M-W decision tree belongs here. Anything role- +specific (caller framing, audit trail expectations, summary +requirements) stays in the consumer. + +## Vault layout + +- **Topics** — long-lived cognitive memory at `topics/{folder}/{name}.md`. + A **folder topic** has `folder == name`; it's the cluster's index head. + Short wikilink `[[X]]` resolves to the folder topic if one exists, + else falls back to a unique same-stem file. +- **Events** — fact log of one session at + `events/{YYYY-MM-DD}/{name}/{name}.md`. The `.md` is the **index** + inside a folder; sibling files are **materials** (raw conversation, + tool outputs, data dumps). The index lists them under `## Materials`. + +## Frontmatter — 4 schema axes + +Every memory declares 4 orthogonal axes. The legacy `category` field is +auto-translated to these axes for back-compat reads, but new writes +should set the axes directly. + +| Axis | Values | Meaning | +|---|---|---| +| `lifecycle` | `streaming` / `evolving` / `frozen` | streaming = events (decay/archive); evolving = topics (long-lived, edited); frozen = materials (immutable references) | +| `scope` | `instance` / `class` | instance = a specific moment / object; class = abstract concept / role / pattern | +| `source` | `auto` / `curated` / `derived` | auto = system-captured; curated = human/LLM intent; derived = computed from other memories | +| `role` | `observation` / `claim` / `question` / `profile` / `concept` / `method` / `reference` / `fundamentals` | cognitive role — drives ranking + role-specific validation | + +### Conditional fields + +- `confidence` ∈ {⏳, ✅, ❌} — **REQUIRED** when `role: claim` (legacy + categories `thesis` / `model`). Same gate applies to `role: question` + (legacy `questions`). +- `status` ∈ {`active`, `distilled`, `archived`} — meaningful only for + `lifecycle: streaming`. Topic-style memories ignore it. +- `originSessionId` — should be set when `source: auto`. + +### Standard identity fields + +`title`, `description`, `tags`, `created`, `updated`, `topics`, +`parent`. Use today's date for `created` / `updated` on new writes. + +## Cross-file references + +`[[wikilink]]` syntax. Two forms: + +- **Stem form** `[[X]]` — resolved against the file_store's stem index; + prefers the folder topic if one exists. +- **Path form** `[[topics/X/X]]` or `[[topics/X/X.md]]` — anchored at + the vault root. + +## Status state machine + +`active → distilled → archived` (single direction, no skip). A reverse +or skip transition will be flagged by the Maintainer. + +## Wikilink uniqueness + +Every create path routes through `MemoryCreate.write`, which refuses to +introduce ambiguity (existing `[[X]]` would resolve to ≥2 paths). When +rejected, the response includes a `suggested_name`. Retry with that, or +pick a domain-specific qualifier (`Apple-Inc` beats `Apple-2`). Never +bypass with `force=true` unless you fully understand the ambiguity. + +## Available tools + +### Read tools (gather context BEFORE writing) + +- `memory_get(path, include_chunks=False)` — full file content + + frontmatter. On an event index, follow `## Materials` and read each + artifact whose content you need. +- `memory_list(path_prefix=None, tags=None, metadata=None, limit=100)` + — list indexed files filtered by prefix / tags / frontmatter. +- `memory_resolve_wikilink(wikilink)` — resolve `[[X]]` to a path; + flags ambiguity / dangling. +- `memory_backlinks(path)` — files linking TO the given path. +- `memory_links(path)` — files the given path links to. +- `memory_search(query, …)` — hybrid (vector + keyword) chunk search. +- `memory_graph_search(query, seeds, graph_depth, …)` — vector + + keyword + graph BFS fusion. + +### Write tools (mutations are SSOT-routed; each returns success + +payload + records to audit) + +- `memory_update(path, old_string, new_string, replace_all=False)` — + body edit by exact-string substitution. Use a tail snippet to append. +- `memory_property_update(path, key, value)` — change one frontmatter + key (`value=null` deletes). Use this to flip status. +- `memory_create(path, metadata, content, overwrite=False, force=False)` + — new file. Reserve for genuinely NEW topics. Do NOT use for events + (`sync` owns events). All paths must be ABSOLUTE under vault_root. +- `memory_rename(old_path, new_path)` — move file + rewrite cross-vault + wikilinks. Refuses on destination conflict or stem ambiguity. +- `memory_delete(path)` — remove a file. +- `memory_archive(path)` — flip `status: archived` and move under + `/Archive/`. + +### Hot-write helper (deterministic, no LLM) + +- `sync(name, description?, content?, topics?, tags?, materials?, + on_date?)` — idempotent upsert of an event FOLDER per `(date, name)`. + Reuse the same `name` across calls in one thread to keep extending + the same folder. Refuses on `status: distilled` / `archived` and + returns a `suggested_name`. + +## R-M-W decision rules + +Apply in order. Stop at the first match. + +1. **SKIP** — if material is ALREADY covered by existing topics, reply + with a single line `SKIP: ` and call no tools. +2. **CONTRADICT** — if material CONTRADICTS an existing block, use + `memory_update` with a unique snippet of the outdated text and the + corrected replacement. +3. **EXTEND** — if material EXTENDS an existing topic, use + `memory_update` with a unique TAIL snippet of the existing body, and + `new_string = tail + blank line + new content`. +4. **CREATE** — if material warrants a GENUINELY NEW topic, use + `memory_create` at `topics/{folder}/{name}.md`. Do NOT + `memory_create` under `events/` — `sync` owns that path. +5. **STATUS FLIP** — after integrating an event's content into a + topic, flip that event's status to `distilled` with + `memory_property_update`. + +## Operating principles + +- **Read before write.** Always inspect related topics before deciding + CONTRADICT vs EXTEND vs CREATE. The wikilink-uniqueness gate refuses + blind creates; reading first prevents wasted attempts. +- **Minimal edits.** Edit only what must change. Don't restructure + while updating content. +- **Frontmatter on create.** Always include reasonable frontmatter: + the 4 axes, `title`, `created`, `updated`, plus `confidence` when + `role: claim` or `role: question`. +- **Never delete unless asked.** Distillation flips status; it does + not remove events. diff --git a/reme2/memory/retriever.py b/reme2/memory/retriever.py index 4641ea80..416730f9 100644 --- a/reme2/memory/retriever.py +++ b/reme2/memory/retriever.py @@ -95,7 +95,7 @@ class BaseRetriever(BaseStep): """ assert self.context is not None ctx = self.context - chunk_filter = memory_io.make_chunk_filter( + chunk_filter = memory_io.make_filter( self.file_store, paths=ctx.get("paths") or None, tags=ctx.get("tags") or None, @@ -116,9 +116,9 @@ class BaseRetriever(BaseStep): class HybridRetriever(BaseRetriever): """V + K (+ optional graph BFS) fusion retriever. - Composes the file_store's single-channel primitives (`vector_search`, - `keyword_search`, `expand_neighbors`, `extract_anchor_paths`, - `get_chunks_by_paths`). Knobs (`vector_weight`, `graph_weight`, + Composes the engine API's projection primitives (`search_vector`, + `search_keyword`, `expand_neighbors`, `extract_anchors`, + `get_chunks`). Knobs (`vector_weight`, `graph_weight`, `graph_depth`, `graph_decay`, `graph_direction`, `graph_mode`, `graph_per_path_cap`, `anchor_expand`, `candidate_multiplier`) are constructor defaults; `graph_search` accepts per-call overrides for @@ -173,8 +173,8 @@ class HybridRetriever(BaseRetriever): text_weight = 1.0 - self.vector_weight if fs.vector_enabled and fs.fts_enabled: - v_task = memory_io.vector_search(fs, query, candidates, chunk_filter) - k_task = memory_io.keyword_search(fs, query, candidates, chunk_filter) + v_task = memory_io.search_vector(fs, query, limit=candidates, chunk_filter=chunk_filter) + k_task = memory_io.search_keyword(fs, query, limit=candidates, chunk_filter=chunk_filter) v_results, k_results = await asyncio.gather(v_task, k_task) if not k_results: @@ -186,9 +186,9 @@ class HybridRetriever(BaseRetriever): v_results, k_results, self.vector_weight, text_weight, )[:max_results] elif fs.vector_enabled: - results = await memory_io.vector_search(fs, query, max_results, chunk_filter) + results = await memory_io.search_vector(fs, query, limit=max_results, chunk_filter=chunk_filter) elif fs.fts_enabled: - results = await memory_io.keyword_search(fs, query, max_results, chunk_filter) + results = await memory_io.search_keyword(fs, query, limit=max_results, chunk_filter=chunk_filter) else: results = [] @@ -264,8 +264,8 @@ class HybridRetriever(BaseRetriever): # 1. V + K in parallel (each is a no-op when its backend is disabled). if query: - v_task = memory_io.vector_search(fs, query, candidate_count, chunk_filter) - k_task = memory_io.keyword_search(fs, query, candidate_count, chunk_filter) + v_task = memory_io.search_vector(fs, query, limit=candidate_count, chunk_filter=chunk_filter) + k_task = memory_io.search_keyword(fs, query, limit=candidate_count, chunk_filter=chunk_filter) v_results, k_results = await asyncio.gather(v_task, k_task) else: v_results, k_results = [], [] @@ -296,7 +296,7 @@ class HybridRetriever(BaseRetriever): extra_paths = set(hops) - vk_paths if chunk_filter is not None and chunk_filter.resolved_paths is not None: extra_paths &= chunk_filter.resolved_paths - extra_chunks = await memory_io.chunks_by_paths(fs, extra_paths) + extra_chunks = await memory_io.get_chunks(fs, extra_paths) # Per-path cap so a hub topic with N chunks doesn't flood results. by_path: dict[str, list] = defaultdict(list) for c in extra_chunks: diff --git a/reme2/memory/schema/__init__.py b/reme2/memory/schema/__init__.py new file mode 100644 index 00000000..c6cbda74 --- /dev/null +++ b/reme2/memory/schema/__init__.py @@ -0,0 +1,79 @@ +"""Memory Schema — vault domain "constitution". + +Per `structure.md` §"全局架构蓝图", the Schema layer sits between the +domain-agnostic Core Engine and the three Memory Services. Engines store +arbitrary markdown + open frontmatter dicts; Services read this Schema +to know what a "well-formed memory" is and shape the vault accordingly. + +Public API: + + Memory — single typed class for every vault frontmatter + Lifecycle / Scope / Source / Role — the four behavioral axes (StrEnum) + Status — streaming-only state (active / distilled / archived) + Confidence — claim-only confidence (PENDING / VERIFIED / REJECTED) + + parse_frontmatter(raw) → (Memory | None, errors) tolerant parser + + *_PRESET dicts — common axis combos for hot-write paths + preset_for_category(name) — legacy category → preset lookup + + LEGACY_AXES_FROM_CATEGORY — back-fill table; auto-applied by Memory's + pre-validator so legacy vaults parse free. + +Lives in `reme2/memory/` (not `reme2/schema/` and not `reme2/component/`) +because the Schema is owned by the memory services that consume it. The +engine never imports from here. +""" + +from .memory import ( + LEGACY_AXES_FROM_CATEGORY, + Confidence, + Lifecycle, + Memory, + Role, + Scope, + Source, + Status, +) +from .parser import parse_frontmatter +from .presets import ( + CONCEPT_PRESET, + EVENT_PRESET, + FUNDAMENTALS_PRESET, + MATERIAL_PRESET, + METHOD_PRESET, + MODEL_PRESET, + PRESETS_BY_CATEGORY, + PROFILE_PRESET, + QUESTIONS_PRESET, + THESIS_PRESET, + TOOL_PRESET, + preset_for_category, +) + +__all__ = [ + # core + "Memory", + "Lifecycle", + "Scope", + "Source", + "Role", + "Status", + "Confidence", + "LEGACY_AXES_FROM_CATEGORY", + # parser + "parse_frontmatter", + # presets + "EVENT_PRESET", + "PROFILE_PRESET", + "CONCEPT_PRESET", + "THESIS_PRESET", + "MODEL_PRESET", + "QUESTIONS_PRESET", + "METHOD_PRESET", + "TOOL_PRESET", + "FUNDAMENTALS_PRESET", + "MATERIAL_PRESET", + "PRESETS_BY_CATEGORY", + "preset_for_category", +] diff --git a/reme2/memory/schema/memory.py b/reme2/memory/schema/memory.py new file mode 100644 index 00000000..ce219c66 --- /dev/null +++ b/reme2/memory/schema/memory.py @@ -0,0 +1,317 @@ +"""Memory — the typed shape of a vault frontmatter. + +Single class covers every memory in the vault. The four orthogonal axes +(`lifecycle / scope / source / role`) define the *behavioral shape* — +services read off them to decide decay rules, retrieval ranking, merge +eligibility, etc., instead of switching on a `category` string. + +Role-conditional fields (`confidence`, `status`, `origin_session_id`) +are validated declaratively via `model_validator(mode='after')`. + +Legacy migration: a `model_validator(mode='before')` recognizes the old +`category` field and back-fills the 4 axes via `LEGACY_AXES_FROM_CATEGORY`, +so vault files written before this schema continue to parse without a +migration script. + +Lives in `reme2/memory/schema/` (with the services that consume it), +NOT in `reme2/component/` — the engine is domain-agnostic and never +imports this module. +""" + +from __future__ import annotations + +from datetime import date, datetime +from enum import StrEnum + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +# --------------------------------------------------------------------------- +# The four behavioral axes +# --------------------------------------------------------------------------- + + +class Lifecycle(StrEnum): + """Mutability + decay disposition of a memory. + + streaming — write-once events that decay/archive after a freshness window. + evolving — long-lived memories that get continuously edited; never decay. + frozen — immutable references / artifacts; never edited, never decay. + """ + + STREAMING = "streaming" + EVOLVING = "evolving" + FROZEN = "frozen" + + +class Scope(StrEnum): + """Referential range of a memory. + + instance — refers to a specific moment / object / occurrence. + class — abstracts over many instances; a concept / role / pattern. + """ + + INSTANCE = "instance" + CLASS = "class" + + +class Source(StrEnum): + """Provenance of the memory. + + auto — captured by the system from a session / tool output. + curated — written or shaped by a human (or LLM curator) on purpose. + derived — computed from other memories (Maintainer products, summaries). + """ + + AUTO = "auto" + CURATED = "curated" + DERIVED = "derived" + + +class Role(StrEnum): + """Cognitive role the memory plays. + + Drives schema-aware ranking + role-specific field validation. + Adding a new role is a one-line change here plus (optionally) a preset. + """ + + OBSERVATION = "observation" # what happened (events live here) + CLAIM = "claim" # an assertion needing confidence (thesis, model) + QUESTION = "question" # an open inquiry needing an answer + PROFILE = "profile" # entity description (person, org, system) + CONCEPT = "concept" # abstract idea / definition (company, sector, concept) + METHOD = "method" # procedure / how-to + REFERENCE = "reference" # pointer to external thing (tool, paper, code) + FUNDAMENTALS = "fundamentals" # foundational data / baseline facts + + +class Status(StrEnum): + """Lifecycle state — meaningful only for `lifecycle == streaming`. + + Topic-style memories (evolving / frozen) ignore this field. + """ + + ACTIVE = "active" + DISTILLED = "distilled" + ARCHIVED = "archived" + + +class Confidence(StrEnum): + """Required for `role == claim`. Wire format stays emoji. + + PENDING — claim under investigation; outcome not yet decided. + VERIFIED — claim supported by evidence and currently held. + REJECTED — claim was investigated and disconfirmed; kept for history. + """ + + PENDING = "⏳" + VERIFIED = "✅" + REJECTED = "❌" + + +# --------------------------------------------------------------------------- +# Legacy `category` → 4-axis mapping (auto-migration) +# --------------------------------------------------------------------------- + + +LEGACY_AXES_FROM_CATEGORY: dict[str, dict] = { + # Events: streaming-instance-auto-observation + "event": { + "lifecycle": Lifecycle.STREAMING, + "scope": Scope.INSTANCE, + "source": Source.AUTO, + "role": Role.OBSERVATION, + }, + # Topic categories — all evolving / class / curated, role varies + "company": { + "lifecycle": Lifecycle.EVOLVING, + "scope": Scope.CLASS, + "source": Source.CURATED, + "role": Role.CONCEPT, + }, + "sector": { + "lifecycle": Lifecycle.EVOLVING, + "scope": Scope.CLASS, + "source": Source.CURATED, + "role": Role.CONCEPT, + }, + "concept": { + "lifecycle": Lifecycle.EVOLVING, + "scope": Scope.CLASS, + "source": Source.CURATED, + "role": Role.CONCEPT, + }, + "method": { + "lifecycle": Lifecycle.EVOLVING, + "scope": Scope.CLASS, + "source": Source.CURATED, + "role": Role.METHOD, + }, + "tool": { + "lifecycle": Lifecycle.EVOLVING, + "scope": Scope.CLASS, + "source": Source.CURATED, + "role": Role.REFERENCE, + }, + "profile": { + "lifecycle": Lifecycle.EVOLVING, + "scope": Scope.CLASS, + "source": Source.CURATED, + "role": Role.PROFILE, + }, + "thesis": { + "lifecycle": Lifecycle.EVOLVING, + "scope": Scope.CLASS, + "source": Source.CURATED, + "role": Role.CLAIM, + }, + "model": { + "lifecycle": Lifecycle.EVOLVING, + "scope": Scope.CLASS, + "source": Source.CURATED, + "role": Role.CLAIM, + }, + "questions": { + "lifecycle": Lifecycle.EVOLVING, + "scope": Scope.CLASS, + "source": Source.CURATED, + "role": Role.QUESTION, + }, + "fundamentals": { + "lifecycle": Lifecycle.EVOLVING, + "scope": Scope.CLASS, + "source": Source.CURATED, + "role": Role.FUNDAMENTALS, + }, + # Materials: frozen-instance-auto-reference (siblings of an event index) + "material": { + "lifecycle": Lifecycle.FROZEN, + "scope": Scope.INSTANCE, + "source": Source.AUTO, + "role": Role.REFERENCE, + }, +} + + +# --------------------------------------------------------------------------- +# Memory — the single typed shape +# --------------------------------------------------------------------------- + + +class Memory(BaseModel): + """The typed view of a vault frontmatter. + + `extra="allow"` keeps domain-specific fields (market, ticker, etc.) in + the parsed object without polluting this base schema. `populate_by_name` + lets `originSessionId` (legacy camelCase) populate `origin_session_id`. + """ + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + # -- Identity / common metadata ---------------------------------------- + + title: str = Field(default="") + description: str = Field(default="") + tags: list[str] = Field(default_factory=list) + created: date | None = None + updated: date | None = None + + # -- The four behavioral axes ------------------------------------------ + + lifecycle: Lifecycle + scope: Scope + source: Source + role: Role + + # -- Cross-cutting graph fields ---------------------------------------- + + topics: list[str] = Field(default_factory=list, description="Outbound wikilinks to class memories.") + parent: str | None = Field(default=None, description="Wikilink to owning memory (e.g. material → event).") + + # -- Role / lifecycle / source-conditional fields ---------------------- + + confidence: Confidence | None = Field( + default=None, + description="Required when role == claim. Use ⏳ / ✅ / ❌ as wire form.", + ) + status: Status | None = Field( + default=None, + description="Lifecycle state. Meaningful only when lifecycle == streaming.", + ) + origin_session_id: str | None = Field( + default=None, + alias="originSessionId", + description="Capture session id. Set when source == auto.", + ) + + # -- Migration: pre-validate hook -------------------------------------- + + @model_validator(mode="before") + @classmethod + def _migrate_legacy_category(cls, data): + """Back-fill 4 axes from old `category` field if present. + + Idempotent: if axes are already populated, do nothing. The legacy + `category` field is preserved (via `extra="allow"`) so downstream + code that still reads it keeps working until we delete it. + """ + if not isinstance(data, dict): + return data + if "role" in data and "lifecycle" in data: + return data + cat = data.get("category") + if cat in LEGACY_AXES_FROM_CATEGORY: + data = {**LEGACY_AXES_FROM_CATEGORY[cat], **data} + return data + + # -- Field coercion / cleanup ------------------------------------------ + + @field_validator("created", "updated", mode="before") + @classmethod + def _coerce_date(cls, v): + if v is None or isinstance(v, date): + return v + if isinstance(v, datetime): + return v.date() + if isinstance(v, str): + return datetime.fromisoformat(v).date() + return v + + @field_validator("tags", "topics", mode="before") + @classmethod + def _strip_dedup(cls, v): + """Strip whitespace + dedup string lists. Preserves order.""" + if not isinstance(v, list): + return v + seen: dict[str, None] = {} + for item in v: + if isinstance(item, str): + stripped = item.strip() + if stripped: + seen.setdefault(stripped, None) + return list(seen.keys()) + + # -- Role-conditional checks ------------------------------------------- + + @model_validator(mode="after") + def _enforce_role_conditionals(self): + if self.role is Role.CLAIM and self.confidence is None: + raise ValueError( + "role='claim' requires explicit confidence " + "(one of ⏳ / ✅ / ❌)" + ) + return self + + # -- Convenience properties -------------------------------------------- + + @property + def is_active(self) -> bool: + """True for streaming memories that are still mutable.""" + return self.lifecycle is Lifecycle.STREAMING and self.status is Status.ACTIVE + + @property + def is_terminal(self) -> bool: + """True for memories no longer accepting writes (distilled / archived / frozen).""" + if self.lifecycle is Lifecycle.FROZEN: + return True + return self.status in (Status.DISTILLED, Status.ARCHIVED) diff --git a/reme2/memory/schema/parser.py b/reme2/memory/schema/parser.py new file mode 100644 index 00000000..e47744d8 --- /dev/null +++ b/reme2/memory/schema/parser.py @@ -0,0 +1,39 @@ +"""Tolerant parser — never raises, always returns (memory_or_None, errors). + +Used by: + - Maintainer.lint : surfaces schema violations on existing files + - any read path : turn raw frontmatter into a typed view when possible + +For *write* paths (`sync`, Ingestor) call `Memory.model_validate` +directly so validation errors propagate as exceptions and stop the write. +""" + +from __future__ import annotations + +from pydantic import ValidationError + +from .memory import Memory + + +def parse_frontmatter(raw: dict) -> tuple[Memory | None, list[str]]: + """Tolerant parse. Returns (parsed, errors). + + Success → (Memory(...), []) + Failure → (None, ["loc: msg", ...]) + + Migration is automatic via Memory's `model_validator(mode='before')`, + so frontmatter that only carries the legacy `category` field still + parses successfully. + """ + if not isinstance(raw, dict): + return None, [f"frontmatter must be a dict, got {type(raw).__name__}"] + try: + return Memory.model_validate(raw), [] + except ValidationError as e: + msgs: list[str] = [] + for err in e.errors(include_context=False, include_url=False): + loc = ".".join(str(x) for x in err.get("loc", ())) + msgs.append(f"{loc}: {err.get('msg', '')}".strip(": ")) + return None, msgs + except Exception as e: # noqa: BLE001 — defensive: never raise from a tolerant parser + return None, [f"{type(e).__name__}: {e}"] diff --git a/reme2/memory/schema/presets.py b/reme2/memory/schema/presets.py new file mode 100644 index 00000000..2abb45b2 --- /dev/null +++ b/reme2/memory/schema/presets.py @@ -0,0 +1,134 @@ +"""Memory presets — common axis combinations for hot-write paths. + +Hot-write tools (`sync`) and cold-write services (Ingestor R-M-W) start +with a preset, layer the agent's input on top, then validate the merged +dict against `Memory`. Adding a new memory shape = adding one preset +entry here + (optionally) a `LEGACY_AXES_FROM_CATEGORY` row in memory.py +for migration. + +Presets carry only the 4 axes + optional default `status` for streaming +memories. Identity fields (title, description, tags, created, updated) +come from the caller. + +Values are stored as **plain strings** (not StrEnum members) so they +flow cleanly through `frontmatter.dumps` → `yaml.dump`, which doesn't +know how to represent enum subclasses. Pydantic still coerces them +back into StrEnum members when `Memory.model_validate` runs. +""" + +from __future__ import annotations + +from .memory import Lifecycle, Role, Scope, Source, Status + + +EVENT_PRESET: dict = { + "lifecycle": Lifecycle.STREAMING.value, + "scope": Scope.INSTANCE.value, + "source": Source.AUTO.value, + "role": Role.OBSERVATION.value, + "status": Status.ACTIVE.value, + # Legacy compat: the `category` field is preserved by extra="allow", + # but emit it explicitly so old tooling that still reads `category` + # (hooks, scripts, downstream consumers) keeps working. + "category": "event", +} + +PROFILE_PRESET: dict = { + "lifecycle": Lifecycle.EVOLVING.value, + "scope": Scope.CLASS.value, + "source": Source.CURATED.value, + "role": Role.PROFILE.value, + "category": "profile", +} + +CONCEPT_PRESET: dict = { + "lifecycle": Lifecycle.EVOLVING.value, + "scope": Scope.CLASS.value, + "source": Source.CURATED.value, + "role": Role.CONCEPT.value, + "category": "concept", +} + +THESIS_PRESET: dict = { + "lifecycle": Lifecycle.EVOLVING.value, + "scope": Scope.CLASS.value, + "source": Source.CURATED.value, + "role": Role.CLAIM.value, + "category": "thesis", +} + +MODEL_PRESET: dict = { + "lifecycle": Lifecycle.EVOLVING.value, + "scope": Scope.CLASS.value, + "source": Source.CURATED.value, + "role": Role.CLAIM.value, + "category": "model", +} + +QUESTIONS_PRESET: dict = { + "lifecycle": Lifecycle.EVOLVING.value, + "scope": Scope.CLASS.value, + "source": Source.CURATED.value, + "role": Role.QUESTION.value, + "category": "questions", +} + +METHOD_PRESET: dict = { + "lifecycle": Lifecycle.EVOLVING.value, + "scope": Scope.CLASS.value, + "source": Source.CURATED.value, + "role": Role.METHOD.value, + "category": "method", +} + +TOOL_PRESET: dict = { + "lifecycle": Lifecycle.EVOLVING.value, + "scope": Scope.CLASS.value, + "source": Source.CURATED.value, + "role": Role.REFERENCE.value, + "category": "tool", +} + +FUNDAMENTALS_PRESET: dict = { + "lifecycle": Lifecycle.EVOLVING.value, + "scope": Scope.CLASS.value, + "source": Source.CURATED.value, + "role": Role.FUNDAMENTALS.value, + "category": "fundamentals", +} + +MATERIAL_PRESET: dict = { + "lifecycle": Lifecycle.FROZEN.value, + "scope": Scope.INSTANCE.value, + "source": Source.AUTO.value, + "role": Role.REFERENCE.value, + "category": "material", +} + + +# Old `category` → preset, for migration / lookup use. +PRESETS_BY_CATEGORY: dict[str, dict] = { + "event": EVENT_PRESET, + "profile": PROFILE_PRESET, + "company": CONCEPT_PRESET, + "sector": CONCEPT_PRESET, + "concept": CONCEPT_PRESET, + "thesis": THESIS_PRESET, + "model": MODEL_PRESET, + "questions": QUESTIONS_PRESET, + "method": METHOD_PRESET, + "tool": TOOL_PRESET, + "fundamentals": FUNDAMENTALS_PRESET, + "material": MATERIAL_PRESET, +} + + +def preset_for_category(category: str) -> dict | None: + """Look up the preset bound to a legacy category name. + + Returns a fresh dict each call (callers may mutate it). Returns + None for unknown categories — caller decides whether to refuse or + fall through to a generic shape. + """ + p = PRESETS_BY_CATEGORY.get(category) + return dict(p) if p is not None else None diff --git a/reme2/schema/vault/__init__.py b/reme2/schema/vault/__init__.py deleted file mode 100644 index 1bf89733..00000000 --- a/reme2/schema/vault/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Vault business schemas — domain models for the markdown vault. - -These are the *business objects* the user stores in the vault (Topic, -Event), distinct from the engine schemas in `reme2/schema/` (FileMetadata, -FileChunk, ChunkFilter — internal data types). - -Single Topic class covers all categories under topics/ (no satellite / -tentacle subclassing); folder-topic is identified by path convention. - -Lives under `reme2/schema/vault/` (not in `reme2/mcp/`) so memory -services (Maintainer.lint, Ingestor) can validate frontmatter without -importing the MCP transport layer — that's what was creating the -mcp ↔ memory dependency cycle. -""" - -from .event import Event, EventStatus -from .frontmatter import VaultBaseFrontmatter, parse_frontmatter -from .registry import ALL_KNOWN_CATEGORIES, schema_for -from .topic import ( - INDEX_CATEGORIES, - JUDGMENT_CATEGORIES, - CONTENT_CATEGORIES, - Confidence, - Topic, - TopicCategory, -) - -__all__ = [ - "ALL_KNOWN_CATEGORIES", - "CONTENT_CATEGORIES", - "Confidence", - "Event", - "EventStatus", - "INDEX_CATEGORIES", - "JUDGMENT_CATEGORIES", - "Topic", - "TopicCategory", - "VaultBaseFrontmatter", - "parse_frontmatter", - "schema_for", -] diff --git a/reme2/schema/vault/event.py b/reme2/schema/vault/event.py deleted file mode 100644 index 0ec580f7..00000000 --- a/reme2/schema/vault/event.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Event schema — task process records under events/{date}/{name}/.""" - -from typing import Literal - -from .frontmatter import VaultBaseFrontmatter - -EventStatus = Literal["active", "distilled", "archived"] - - -class Event(VaultBaseFrontmatter): - """events/{YYYY-MM-DD}/{name}/{name}.md.""" - - category: Literal["event"] = "event" # type: ignore[assignment] - status: EventStatus = "active" - topics: list[str] = [] - originSessionId: str | None = None diff --git a/reme2/schema/vault/frontmatter.py b/reme2/schema/vault/frontmatter.py deleted file mode 100644 index 4b5a3c62..00000000 --- a/reme2/schema/vault/frontmatter.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Common frontmatter schema shared by all vault files.""" - -from datetime import date, datetime -from typing import Any - -from pydantic import BaseModel, ConfigDict, Field, field_validator - - -class VaultBaseFrontmatter(BaseModel): - """Fields present on every vault file.""" - - model_config = ConfigDict(extra="allow") - - title: str = Field(...) - description: str = Field(default="") - tags: list[str] = Field(default_factory=list) - category: str = Field(...) - created: date | None = None - updated: date | None = None - - @field_validator("created", "updated", mode="before") - @classmethod - def _coerce_date(cls, v): - if v is None or isinstance(v, date): - return v - if isinstance(v, datetime): - return v.date() - if isinstance(v, str): - return datetime.fromisoformat(v).date() - return v - - -def parse_frontmatter(raw: dict[str, Any]) -> VaultBaseFrontmatter: - """Tolerant parse — never raises; missing required fields fall back to defaults.""" - safe = dict(raw or {}) - safe.setdefault("title", safe.get("name") or "") - safe.setdefault("category", "unknown") - return VaultBaseFrontmatter.model_validate(safe) diff --git a/reme2/schema/vault/registry.py b/reme2/schema/vault/registry.py deleted file mode 100644 index 8d539087..00000000 --- a/reme2/schema/vault/registry.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Map vault category → strict pydantic model for validation. - -Used by the Maintainer to derive lint rules from schema instead of -hardcoding. Two business object schemas: - - event → Event - - everything else → Topic (judgment subset enforced via model_validator) -""" - -from typing import get_args - -from pydantic import BaseModel - -from .event import Event -from .topic import Topic, TopicCategory - -_REGISTRY: dict[str, type[BaseModel]] = {"event": Event} -for _cat in get_args(TopicCategory): - _REGISTRY[_cat] = Topic - -ALL_KNOWN_CATEGORIES: frozenset[str] = frozenset(_REGISTRY.keys()) - - -def schema_for(category: str | None) -> type[BaseModel] | None: - """Return the pydantic schema bound to a category, or None for unknown.""" - if not category: - return None - return _REGISTRY.get(category) diff --git a/reme2/schema/vault/topic.py b/reme2/schema/vault/topic.py deleted file mode 100644 index c54f6d49..00000000 --- a/reme2/schema/vault/topic.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Topic schema — every .md under topics/ is a Topic. - -No satellite/tentacle/folder note subclassing. Folder topic = Topic whose -filename equals its parent directory name (path convention; not a schema field). -""" - -from typing import Literal - -from pydantic import model_validator - -from .frontmatter import VaultBaseFrontmatter - -# Topic categories grouped by usage: -# - Index categories: usually appear as folder topics; may have ticker/market -# - Content categories: cluster siblings; thesis/model/questions need confidence -INDEX_CATEGORIES = {"company", "sector", "concept", "method", "tool", "profile"} -JUDGMENT_CATEGORIES = {"thesis", "model", "questions"} -CONTENT_CATEGORIES = JUDGMENT_CATEGORIES | {"fundamentals"} - -TopicCategory = Literal[ - "company", "sector", "concept", "method", "tool", "profile", - "thesis", "model", "questions", "fundamentals", -] -Confidence = Literal["⏳", "✅", "❌"] - - -class Topic(VaultBaseFrontmatter): - """topics/{folder}/{name}.md — long-lived cognitive memory node. - - Folder topic is identified by path convention: filename stem == parent - folder name. Not represented in this schema directly; checked at runtime - via `Path(p).stem == Path(p).parent.name`. - """ - - category: TopicCategory # type: ignore[assignment] - market: str | None = None - ticker: str | None = None - confidence: Confidence | None = None - - @model_validator(mode="after") - def _confidence_required_for_judgments(self): - if self.category in JUDGMENT_CATEGORIES and self.confidence is None: - raise ValueError( - f"category={self.category} requires explicit confidence " - f"(one of ⏳ / ✅ / ❌)" - ) - return self diff --git a/reme2/utils/vault_paths.py b/reme2/utils/vault_paths.py index 52394bc7..18019d29 100644 --- a/reme2/utils/vault_paths.py +++ b/reme2/utils/vault_paths.py @@ -1,8 +1,8 @@ """Vault path generation + naming disambiguation. -Pure-Python helpers consumed by the MCP step shells (`sync`, -`topic_create`) and any future flow that needs to materialize a path -under the vault layout. No async, no MCP awareness — easy to unit-test. +Pure-Python helpers consumed by the `sync` MCP step shell and any +future flow that needs to materialize a path under the vault layout. +No async, no MCP awareness — easy to unit-test. Wikilink uniqueness is a *graph* property, not a path-builder concern — see `reme2.component.file_store.BaseFileStore.collisions_after_create` @@ -31,16 +31,6 @@ def event_path( return Path(vault_root) / events_dir / on_date / name / f"{name}.md" -def topic_path( - vault_root: str | Path, - folder: str, - name: str, - topics_dir: str = "topics", -) -> Path: - """topics/{folder}/{name}.md under the given vault root.""" - return Path(vault_root) / topics_dir / folder / f"{name}.md" - - def is_folder_topic(path: str | Path) -> bool: """True if filename stem == parent directory name (folder note convention).""" p = Path(path) @@ -50,11 +40,10 @@ def is_folder_topic(path: str | Path) -> bool: def next_suffixed_stem(taken: Iterable[str], base: str) -> str: """Lowest unused `-N` (N≥2). Returns `base` itself if not taken. - Used by topic_create / sync when a same-stem (topic) or - same-name-on-same-day (event) collision is detected — the suggested - suffix is *advisory*; the create still rejects so the agent can pick - a domain-specific qualifier (e.g., `Apple-Inc` vs `Apple-Fruit`) - that carries more meaning than a numeric suffix. + Used by `sync` when a same-name-on-same-day collision is detected — + the suggested suffix is *advisory*; the create still rejects so the + agent can pick a domain-specific qualifier (e.g. `Apple-Inc` vs + `Apple-Fruit`) that carries more meaning than a numeric suffix. Examples (with taken={"BABA", "BABA-2"}): next_suffixed_stem(taken, "BABA") == "BABA-3"