scriptoriumadmin/ReMe
1
0
Fork 0
mirror of https://github.com/agentscope-ai/ReMe.git synced 2026-08-28 05:25:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

```

Browse source 
feat(file_watcher): add directory deletion support with descendant indexing

Add support for deleting entire directories and their indexed descendants
in the file watcher. Previously only individual file deletions were
handled properly. Now when a directory is deleted, the system finds all
indexed files beneath that directory path and removes them along with
their metadata and chunks.

The implementation includes:
- New `_descendant_indexed_paths` method to find all indexed files
  under a given directory path
- Updated `_on_deleted` method to process both the target path and
  all its indexed descendants
- Proper handling of symlinks and path resolution differences
- Enhanced logging to show directory deletion with child count

Also adds necessary os import for path operations.

refactor(config): restructure configuration profiles for clarity

Rename curated.yaml to remove outdated configuration file and
rename full.yaml to expert.yaml with updated documentation.
Add new service.yaml configuration profile that provides a
service-aligned MCP surface with three main tools:
- retrieve: graph-aware hybrid retrieval
- remember: single write entry point with log/distill modes
- maintain: vault hygiene sweep

The expert configuration now excludes the ingest tool since
cold-path operations are handled by external agents, and adds
memory_lint tool for structural issue detection.

Updated documentation to clarify the different configuration
profiles and their intended usage patterns.
```
This commit is contained in:
huangsen 2026-05-11 10:52:25 +08:00
parent 514bf35050
commit dd2de16481
39 changed files with 3151 additions and 1885 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
reme2/component/file_watcher/base_file_watcher.py
View file

@ -14,6 +14,7 @@ startup recovery (re-parsing files whose mtime drifted while offline).
"""
import asyncio
import os
import time
from pathlib import Path
@ -293,10 +294,30 @@ class BaseFileWatcher(BaseComponent):
# Cancel any in-flight parse task FIRST so a delayed cancellation
# can't race the cleanup writes below.
assert self.file_store is not None, "_on_deleted requires file_store"
await self._cancel_parse_task(path)
await self.file_store.delete_chunks(path)
await self.file_store.delete_file_meta(path)
self.logger.info(f"Deleted {path}")
targets = [path, *self._descendant_indexed_paths(path)]
for p in targets:
await self._cancel_parse_task(p)
await self.file_store.delete_chunks(p)
await self.file_store.delete_file_meta(p)
if len(targets) == 1:
self.logger.info(f"Deleted {path}")
else:
self.logger.info(f"Deleted directory {path} ({len(targets) - 1} indexed children)")
def _descendant_indexed_paths(self, path: str) -> list[str]:
"""Indexed file paths that live beneath `path` as a directory.
watchfiles emits a single Change.deleted for a removed directory
(no per-file events), so we have to find the orphans ourselves.
Resolves both sides to handle symlinks and trailing-separator
drift between the watcher and the file_store keys.
"""
assert self.file_store is not None
try:
prefix = str(Path(path).resolve()) + os.sep
except OSError:
prefix = path.rstrip(os.sep) + os.sep
return [p for p in self.file_store.nodes if p.startswith(prefix)]
# -- Parse task pipeline ------------------------------------------------

4
reme2/component/runtime_response.py
View file

@ -2,8 +2,8 @@
Lives next to `runtime_context.py` because both are about the BaseStep
interface — the response side, specifically. Used by every Step that
returns a JSON-shaped payload (memory_*, sync, topic_create, the three
memory services).
returns a JSON-shaped payload (memory_*, sync, the three memory
services).
Was previously at `reme2/mcp/steps/_common.py`, which leaked an MCP
dependency into `reme2/memory/` services that legitimately need to

220
reme2/config/curated.yaml
View file

@ -1,220 +0,0 @@
app_name: reme-curated
enable_logo: false
log_to_console: true
log_to_file: false
# Curated profile: three MCP tools surfaced —
# `query` — graph-aware hybrid retrieval
# `sync` — hot-path event sync (idempotent upsert per (date, name))
# `ingest` — LLM-driven distillation over existing files (cold path)
#
# The agent calls `sync` continuously through the task — picking a
# stable `name` per logical thread so each call extends the same
# event folder rather than fragmenting. At task completion (or
# PreCompact / SessionEnd) it calls `ingest` once to distill the
# active events into topic-level cognition.
#
# For full direct control over every memory_* primitive see ./full.yaml.
service:
backend: mcp
transport: stdio
sidecar_http: true
sidecar_http_host: "127.0.0.1"
sidecar_http_port: 8765
sidecar_info_path: "./vault/.reme/sidecar.json"
jobs:
- backend: base
name: query
description: |
Graph-aware hybrid retrieval: vector + keyword + 1-hop wikilink
BFS fusion. Use for "what do I know about X" / "did I work on Y" /
"what's connected to [[Z]]". Anchor mode: include `[[Target]]` in
the query to seed BFS at that file. Topic-rooted mode: pass `seeds`
explicitly. Returns chunks ranked by combined relevance + graph
proximity, each tagged with `graph_hop`.
parameters:
type: object
properties:
query: { type: string }
max_results: { type: integer, default: 5 }
min_score: { type: number, default: 0.0 }
graph_depth:
type: integer
default: 1
description: "BFS hops from seeds. 1 covers immediate neighbors."
seeds:
type: array
items: { type: string }
description: "Explicit seed paths (topic-rooted mode)."
paths: { type: array, items: { type: string } }
tags: { type: array, items: { type: string } }
exclude_paths: { type: array, items: { type: string } }
steps:
- backend: memory_graph_search
- backend: base
name: sync
description: |
Hot-path event sync: idempotent upsert of an event FOLDER under
`events/{date}/{name}/`. The folder contains the index `{name}.md`
(Event schema, frontmatter + narrative + Materials footer) plus
any raw materials you pass — conversation snippets, tool outputs,
data dumps. The watcher indexes everything inside.
CONTINUITY MODEL: pick a stable `name` per logical thread and
call `sync` repeatedly through the task. Each call extends the
same folder:
* new `content` → appended under a `## Update — {iso}` section
* new `materials` → siblings (auto-suffix on filename collision)
* `topics` + `tags` merged (union) into frontmatter
* Materials footer regenerated to list every artifact in the folder
First call (folder doesn't exist) → CREATE; subsequent calls with
the same `name` while the event is `status: active` → APPEND.
If the event is `status: distilled` / `archived`, `sync` REFUSES
and returns a `suggested_name` so you start a fresh thread instead
of mutating prior cognition.
Zero LLM cost. Call CONTINUOUSLY through a task as facts land,
and especially at PreCompact to dump verbose raw text into
`materials` before context truncation loses it. The folder is
the unit `ingest` later reads from.
parameters:
type: object
required: [name]
properties:
name:
type: string
description: "kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder."
description: { type: string, description: "one-line summary for index frontmatter (set on initial create only)" }
content: { type: string, description: "markdown body. Initial create: the body. Subsequent calls: appended as a `## Update — {iso}` section." }
topics:
type: array
items: { type: string }
description: "related topic wikilinks. Unioned into frontmatter on append."
tags: { type: array, items: { type: string }, description: "free-form tags; unioned on append." }
materials:
type: array
description: "Raw artifacts written as siblings of the index. Filenames must be safe (letters/digits/dot/underscore/dash). Filename collision with an existing artifact auto-suffixes (foo.txt → foo-2.txt)."
items:
type: object
required: [filename, content]
properties:
filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt'" }
content: { type: string }
on_date: { type: string, description: "ISO date for events/{date}/ bucket; defaults to today" }
origin_session_id: { type: string, description: "set on initial create only" }
steps:
- backend: sync
- backend: base
name: ingest
description: |
Cold-path LLM-driven distillation. Run on EXPLICIT HANDOFF only:
task completion / SessionEnd / when the agent decides the working
set is ready. Not a per-turn tool.
The agent hands off the working set in two interchangeable forms
(use both as appropriate):
* `content` — inline material to distill: a hint / summary, or
raw text the agent is feeding directly.
* `related_paths` — paths the agent points at: event folder
indexes (Ingestor follows `## Materials` to read each
artifact), individual material files, or candidate topics.
The Ingestor reads the working set + linked topics, decides which
existing topics to update / create, and flips each distilled
event's status to "distilled". Returns an audit trail.
NOT for raw event logging — that's `sync`'s job.
parameters:
type: object
required: [content]
properties:
content:
type: string
description: "Inline material the agent is feeding the Ingestor: distillation hint, session summary, or raw text. Combined with `related_paths` to form the working set."
hint:
type: string
description: "Caller guidance about target / intent."
target_path:
type: string
description: "Optional suggested path; required for the no-LLM degraded path."
metadata:
type: object
description: "Suggested frontmatter for any new topic."
related_paths:
type: array
items: { type: string }
description: "Pointers the agent is feeding the Ingestor: event folder indexes (Ingestor reads materials from `## Materials`), individual material files, or candidate topics."
steps:
- backend: ingestor
components:
# Ingestor LLM (opt-in). Without this the Ingestor degrades to a
# direct create from explicit `target_path`; edits/renames/deletes
# require the LLM. Uncomment + provide LLM_API_KEY to enable.
#
# as_llm:
# default:
# backend: openai
# model_name: ${LLM_MODEL_NAME:-gpt-4o-mini}
# api_key: ${LLM_API_KEY}
# client_kwargs:
# base_url: ${LLM_BASE_URL:-https://api.openai.com/v1}
# stream: false
#
# as_llm_formatter:
# default:
# backend: openai
as_token_counter:
default:
backend: estimated
# Embedding is opt-in: leave embedding_model="" on file_store to run
# keyword-only; uncomment + flip to "default" to enable hybrid search.
#
# embedding_model:
# default:
# backend: openai
# model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small}
# dimensions: 1536
# pass_dimensions: false
# enable_cache: true
# max_batch_size: 10
# max_cache_size: 2000
# max_input_length: 8192
edge_extractor:
default:
backend: regex
file_parser:
md:
backend: md
edge_extractor: default
default:
backend: text
file_store:
default:
backend: local
embedding_model: ""
store_name: "reme"
db_path: "./vault/.reme"
fts_enabled: true
file_watcher:
default:
backend: full
file_store: default
default_parser: md
watch_path: "./vault"
recursive: true
# Retriever (`hybrid`) is a Step, not a pre-instantiated component —
# the `query` shell builds it on demand. Tune defaults by attaching
# knobs to the `memory_graph_search` step under the `query` job above.

121
reme2/config/full.yaml → reme2/config/expert.yaml
View file

@ -1,16 +1,22 @@
app_name: reme-full
app_name: reme-expert
enable_logo: false
log_to_console: true
log_to_file: false
# Full-exposure profile: every memory_* read + write primitive + the
# typed `topic_create` + the LLM-driven `ingest` are surfaced as MCP
# tools. The agent picks whatever it needs — finest granularity, no
# opinion enforced beyond the wikilink-uniqueness gate.
# Expert-exposure profile: every memory_* read + write primitive plus
# `sync` (event log) are surfaced as MCP tools. The agent picks whatever
# it needs — finest granularity, no opinion enforced beyond the
# wikilink-uniqueness gate.
#
# **No `ingest` tool.** Cold-path R-M-W in expert mode is owned by the
# host agent (Claude Code, with the `reme-expert` SKILL loaded), not by
# reme2's internal ReActAgent. The reme-expert plugin spawns a
# `reme-distiller` subagent at SessionEnd to handle distillation in its
# own context window, using the same memory_* primitives.
#
# Use when you want the agent to manage memory directly with full
# control. For an opinionated minimal surface, see ./curated.yaml
# (only `query` + `ingest`).
# control. For an opinionated minimal surface (where reme2's internal
# Ingestor owns R-M-W), see ./service.yaml.
service:
backend: mcp
@ -78,68 +84,6 @@ jobs:
steps:
- backend: sync
- backend: base
name: ingest
description: |
Cold-path LLM-driven distillation. Run on EXPLICIT HANDOFF only:
task completion / SessionEnd / when the agent decides the working
set is ready. Not a per-turn tool.
The agent hands off the working set in two interchangeable forms
(use both as appropriate):
* `content` — inline material to distill: a hint / summary, or
raw text the agent is feeding directly.
* `related_paths` — paths the agent points at: event folder
indexes (the Ingestor will follow `## Materials` and read
each artifact), individual material files, or candidate
topics flagged for update.
The Ingestor reads the working set + linked topics, decides which
existing topics to update / create, and flips each distilled
event's status to "distilled".
NOT for raw event logging — use `sync` for that.
parameters:
type: object
required: [content]
properties:
content:
type: string
description: "Inline material the agent is feeding the Ingestor: distillation hint, session summary, or raw text. Combined with `related_paths` to form the working set."
hint: { type: string, description: "Caller guidance about target / intent." }
target_path: { type: string, description: "Optional suggested path; required for the no-LLM degraded path." }
metadata: { type: object, description: "Suggested frontmatter for any new topic." }
related_paths:
type: array
items: { type: string }
description: "Pointers the agent is feeding the Ingestor: event folder indexes (Ingestor will read materials from `## Materials`), individual material files, or candidate topics."
steps:
- backend: ingestor
- backend: base
name: topic_create
description: |
Create a typed topic at `topics/{folder}/{name}.md` with hard
schema validation (judgment categories require `confidence`) and
wikilink-uniqueness enforcement.
parameters:
type: object
required: [folder, name, category]
properties:
folder: { type: string }
name: { type: string }
category:
type: string
enum: [company, sector, concept, method, tool, profile, thesis, model, questions, fundamentals]
description: { type: string }
content: { type: string }
confidence: { type: string, enum: ["⏳", "✅", "❌"] }
market: { type: string }
ticker: { type: string }
tags: { type: array, items: { type: string } }
steps:
- backend: topic_create
# -- Read tools --------------------------------------------------------
- backend: base
@ -252,14 +196,34 @@ jobs:
steps:
- backend: memory_count_tokens
- backend: base
name: memory_lint
description: |
Read-only projection of the Maintainer's lint findings. Walks the
indexed files (optionally restricted to a path prefix) and returns
structural issues — broken wikilinks, schema violations, stem
collisions. Never mutates. The agent decides what to do with each
finding using existing memory_* primitives (rename, update,
property_update, archive, delete).
parameters:
type: object
properties:
target_prefix:
type: string
description: "restrict scan to relpaths starting with this prefix (e.g. 'events/2026-05-09/'). Empty string scans the whole vault."
steps:
- backend: memory_lint
# -- Write primitives (raw building blocks) ----------------------------
- backend: base
name: memory_create
description: |
Create a new file (raw primitive — no LLM reasoning). Prefer
`ingest` when you want an LLM curator. Wikilink-uniqueness gate
runs unless `force=true`.
`ingest` when you want an LLM curator. Two gates run unless
`force=true`: (1) path template — must be `topics/{folder}/{name}.md`,
`events/{date}/{name}/...`, or `Archive/...`; (2) wikilink uniqueness
— `[[stem]]` must resolve to ≤1 path post-create.
parameters:
type: object
required: [path]
@ -268,7 +232,10 @@ jobs:
metadata: { type: object }
content: { type: string }
overwrite: { type: boolean, default: false }
force: { type: boolean, default: false }
force:
type: boolean
default: false
description: "bypass BOTH the path-template gate and the wikilink-uniqueness gate. Use only when you intentionally need a non-template path or accept the ambiguity."
steps:
- backend: memory_create
@ -290,7 +257,11 @@ jobs:
- backend: base
name: memory_property_update
description: "Update a single YAML frontmatter key. value=null deletes the key."
description: |
Update a single YAML frontmatter key. value=null deletes the key.
When key='status', enforces the active → distilled → archived
single-direction state machine (skip / reverse refused). Pass
`force=true` to bypass.
parameters:
type: object
required: [path, key]
@ -298,6 +269,10 @@ jobs:
path: { type: string }
key: { type: string }
value: {}
force:
type: boolean
default: false
description: "bypass the status state machine (only meaningful when key='status')."
steps:
- backend: memory_property_update

79
reme2/config/paw.yaml
View file

@ -1,79 +0,0 @@
enable_logo: false
log_to_console: true
log_to_file: false
service:
backend: http
jobs:
- backend: base
name: test
description: "test job"
parameters:
type: object
properties:
name:
type: string
description: "name of the user"
steps:
- name: "test1"
backend: xxx
- name: "test2"
backend: xxx
components:
as_llm:
default:
backend: openai
model_name: qwen3.6-plus
as_llm_formatter:
default:
backend: openai
embedding_model:
default:
backend: openai
model_name: text-embedding-v3
dimensions: 1024
pass_dimensions: false
enable_cache: true
max_batch_size: 10
max_cache_size: 2000
max_input_length: 8192
edge_extractor:
default:
backend: regex
llm:
backend: llm
as_llm: default
as_llm_formatter: default
file_store: default
max_input_chars: 8000
min_confidence: 0.5
max_iters: 8
file_parser:
md:
backend: md
edge_extractor: default
embedding_model: default
default:
backend: text
embedding_model: default
file_store:
default:
backend: local
embedding_model: default
store_name: "reme"
db_path: ".reme/store"
file_watcher:
default:
backend: full
file_store: default
default_parser: default
watch_paths: [ "./test_data" ]
recursive: true

276
reme2/config/service.yaml Normal file
View file

@ -0,0 +1,276 @@
app_name: reme-service
enable_logo: false
log_to_console: true
log_to_file: false
# Service profile: service-aligned MCP surface — three tools, one per
# memory service:
# `retrieve` — graph-aware hybrid retrieval (Retriever projection)
# `remember` — single write entry point (Ingestor projection)
# * mode=log → zero-LLM event-folder upsert
# * mode=distill → LLM R-M-W into topic graph
# `maintain` — vault hygiene sweep (Maintainer projection)
# runs lint + decay; merge/split require an LLM and
# are off by default.
#
# The agent calls `remember(mode=log, name=…, content=…, materials=…)`
# continuously through the task — picking a stable `name` per logical
# thread so each call extends the same event folder rather than
# fragmenting. At task completion (or PreCompact / SessionEnd) it
# calls `remember(mode=distill, content=…, related_paths=…)` once to
# distill the active events into topic-level cognition. `maintain` is
# typically cron-driven, but the agent can invoke it explicitly when
# it suspects vault drift.
#
# For full direct control over every memory_* primitive (raw writes,
# fine-grained reads, separate sync/ingest tools, memory_lint) see
# ./expert.yaml.
service:
backend: mcp
transport: stdio
sidecar_http: true
sidecar_http_host: "127.0.0.1"
sidecar_http_port: 8765
sidecar_info_path: "./vault/.reme/sidecar.json"
jobs:
- backend: base
name: retrieve
description: |
Graph-aware hybrid retrieval: vector + keyword + 1-hop wikilink
BFS fusion. Use for "what do I know about X" / "did I work on Y" /
"what's connected to [[Z]]". Anchor mode: include `[[Target]]` in
the query to seed BFS at that file. Topic-rooted mode: pass `seeds`
explicitly. Returns chunks ranked by combined relevance + graph
proximity, each tagged with `graph_hop`.
parameters:
type: object
properties:
query: { type: string }
max_results: { type: integer, default: 5 }
min_score: { type: number, default: 0.0 }
graph_depth:
type: integer
default: 1
description: "BFS hops from seeds. 1 covers immediate neighbors."
seeds:
type: array
items: { type: string }
description: "Explicit seed paths (topic-rooted mode)."
paths: { type: array, items: { type: string } }
tags: { type: array, items: { type: string } }
exclude_paths: { type: array, items: { type: string } }
steps:
- backend: memory_graph_search
- backend: base
name: remember
description: |
Single write entry point — projects the Ingestor service. Two
modes via the `mode` parameter:
* `mode: log` (zero LLM, hot path) — idempotent upsert of an
event FOLDER under `events/{date}/{name}/`. The folder
contains the index `{name}.md` (Event schema, frontmatter +
narrative + Materials footer) plus any raw materials you
pass — conversation snippets, tool outputs, data dumps. The
watcher indexes everything inside.
CONTINUITY MODEL: pick a stable `name` per logical thread
and call `remember(mode=log, ...)` repeatedly through the
task. Each call extends the same folder:
- new `content` → appended under a `## Update — {iso}`
section
- new `materials` → siblings (auto-suffix on filename
collision)
- `topics` + `tags` merged (union) into frontmatter
- Materials footer regenerated to list every artifact
First call (folder doesn't exist) → CREATE; subsequent calls
with the same `name` while the event is `status: active` →
APPEND. If `status: distilled` / `archived`, REFUSES and
returns `suggested_name` so you start a fresh thread instead
of mutating prior cognition. Call CONTINUOUSLY through a task
as facts land, especially at PreCompact to dump verbose raw
text into `materials` before context truncation.
* `mode: distill` (LLM R-M-W loop, cold path) — DEFAULT. Run
on EXPLICIT HANDOFF only: task completion / SessionEnd / when
you decide the working set is ready. Not a per-turn tool.
Hand off the working set in two interchangeable forms:
- `content` — inline material to distill (hint, summary, or
raw text)
- `related_paths` — pointers (event folder indexes; the
Ingestor follows `## Materials` to read each artifact,
individual material files, candidate topics).
The Ingestor reads the working set + linked topics, decides
which existing topics to update / create, and flips each
distilled event's status to "distilled". Returns an audit
trail.
parameters:
type: object
required: [content]
properties:
mode:
type: string
enum: [log, distill]
default: distill
description: "log = zero-LLM event-folder upsert (requires `name`); distill = LLM R-M-W into topic graph (default)."
# mode=log params
name:
type: string
description: "(mode=log) kebab-case event identifier (folder + index stem). Reuse the same name across calls in one thread to keep extending the same folder."
description:
type: string
description: "(mode=log) one-line summary for index frontmatter (set on initial create only)."
topics:
type: array
items: { type: string }
description: "(mode=log) related topic wikilinks. Unioned into frontmatter on append."
tags:
type: array
items: { type: string }
description: "(mode=log) free-form tags; unioned on append."
materials:
type: array
description: "(mode=log) raw artifacts written as siblings of the index. Filenames must be safe (letters/digits/dot/underscore/dash). Filename collision auto-suffixes (foo.txt → foo-2.txt)."
items:
type: object
required: [filename, content]
properties:
filename: { type: string, description: "e.g. 'raw-prompt.md', 'tool-output.txt'" }
content: { type: string }
on_date:
type: string
description: "(mode=log) ISO date for events/{date}/ bucket; defaults to today."
origin_session_id:
type: string
description: "(mode=log) optional source session identifier (set on initial create only)."
# shared / mode=distill params
content:
type: string
description: "Required. mode=log: markdown body for the index (initial create) or appended `## Update — {iso}` section. mode=distill: inline material the Ingestor distills — hint, summary, or raw text."
hint:
type: string
description: "(mode=distill) caller guidance about target / intent."
target_path:
type: string
description: "(mode=distill) optional suggested path; required for the no-LLM degraded path."
metadata:
type: object
description: "(mode=distill) suggested frontmatter for any new topic."
related_paths:
type: array
items: { type: string }
description: "(mode=distill) pointers — event folder indexes (Ingestor reads materials from `## Materials`), individual material files, or candidate topics."
steps:
- backend: ingestor
- backend: base
name: maintain
description: |
Vault hygiene sweep — projects the Maintainer service. One pass:
scan signals → propose ops → resolve conflicts → apply. Returns
an audit trail of what ran and what changed.
Default behavior runs `lint` (broken wikilinks, schema violations,
stem collisions — read-only diagnostics) and `decay` (move stale
events past their freshness window under `<vault>/Archive/`).
Merge / split require an LLM and are off unless explicitly opted
into via `ops`.
Typically cron-driven; the agent invokes it on demand when it
suspects vault drift (after a heavy session of edits, after a
bulk rename, etc.). Dry-run by default — flip `dry_run=false` to
apply changes.
parameters:
type: object
properties:
target_prefix:
type: string
description: "restrict scan to relpaths starting with this prefix (e.g. 'events/2026-05-09/'). Empty string scans the whole vault."
dry_run:
type: boolean
default: true
description: "if true (default) returns the plan without mutating; flip to false to actually apply lint+decay (and merge/split if opted in)."
ops:
type: array
items:
type: string
enum: [lint, decay, merge, split]
description: "subset of ops to run. Defaults to ['lint','decay']. Merge/split are LLM-driven and currently scaffolded — enabling them without an LLM is a no-op."
decay_days:
type: integer
description: "freshness window for the decay proposer (default = Maintainer constructor `decay_days`, typically 90)."
steps:
- backend: maintainer
components:
# Ingestor LLM (opt-in). Without this the Ingestor degrades to a
# direct create from explicit `target_path`; edits/renames/deletes
# require the LLM. Uncomment + provide LLM_API_KEY to enable.
#
# as_llm:
# default:
# backend: openai
# model_name: ${LLM_MODEL_NAME:-gpt-4o-mini}
# api_key: ${LLM_API_KEY}
# client_kwargs:
# base_url: ${LLM_BASE_URL:-https://api.openai.com/v1}
# stream: false
#
# as_llm_formatter:
# default:
# backend: openai
as_token_counter:
default:
backend: estimated
# Embedding is opt-in: leave embedding_model="" on file_store to run
# keyword-only; uncomment + flip to "default" to enable hybrid search.
#
# embedding_model:
# default:
# backend: openai
# model_name: ${EMBEDDING_MODEL_NAME:-text-embedding-3-small}
# dimensions: 1536
# pass_dimensions: false
# enable_cache: true
# max_batch_size: 10
# max_cache_size: 2000
# max_input_length: 8192
edge_extractor:
default:
backend: regex
file_parser:
md:
backend: md
edge_extractor: default
default:
backend: text
file_store:
default:
backend: local
embedding_model: ""
store_name: "reme"
db_path: "./vault/.reme"
fts_enabled: true
file_watcher:
default:
backend: full
file_store: default
default_parser: md
watch_path: "./vault"
recursive: true
# Retriever (`hybrid`) is a Step, not a pre-instantiated component —
# the `query` shell builds it on demand. Tune defaults by attaching
# knobs to the `memory_graph_search` step under the `query` job above.

308
reme2/config/smoke_test.py
View file

@ -1,308 +0,0 @@
"""Smoke-test reme2/config/full.yaml + reme2/config/curated.yaml.
Boots Application with each profile against a temporary vault, lists
the registered jobs, then calls a representative subset to confirm
end-to-end wiring (parser → file_store → memory_io / ingest /
memory_graph_search). The Ingestor degrades gracefully when no LLM is
configured — we exercise the degraded path so the test is hermetic.
Run:
python reme2/config/smoke_test.py
"""
import asyncio
import json
import sys
import tempfile
from pathlib import Path
_REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(_REPO_ROOT))
# Eager-import side-effect modules so all @R.register() decorators run.
import reme2 # noqa: E402,F401
import reme2.mcp.steps.memory_io # noqa: E402,F401
import reme2.mcp.steps.memory_retriever # noqa: E402,F401
import reme2.memory.ingestor # noqa: E402,F401
import reme2.memory.summarizer # noqa: E402,F401
import reme2.memory.maintainer # noqa: E402,F401
import reme2.mcp.steps # noqa: E402,F401
from reme2.application import Application # noqa: E402
from reme2.config import parse_args # noqa: E402
CONFIG_DIR = Path(__file__).resolve().parent
PROFILES = {
"full": CONFIG_DIR / "full.yaml",
"curated": CONFIG_DIR / "curated.yaml",
}
def _seed_vault(vault: Path) -> None:
"""Drop a few markdown files so reads return something."""
(vault / "topics" / "Alice").mkdir(parents=True, exist_ok=True)
(vault / "topics" / "Alice" / "Alice.md").write_text(
"---\ntitle: Alice\ncategory: profile\ntags: [person]\n---\n"
"# Alice\n\nAlice works on [[Project X]] with [[Bob]].\n"
"[author:: [[Alice]]]\n",
encoding="utf-8",
)
(vault / "topics" / "Bob").mkdir(parents=True, exist_ok=True)
(vault / "topics" / "Bob" / "Bob.md").write_text(
"---\ntitle: Bob\ncategory: profile\ntags: [person]\n---\n"
"# Bob\n\nBob collaborates with [[Alice]] on [[Project X]].\n",
encoding="utf-8",
)
(vault / "topics" / "Project X").mkdir(parents=True, exist_ok=True)
(vault / "topics" / "Project X" / "Project X.md").write_text(
"---\ntitle: Project X\ncategory: concept\n---\n"
"# Project X\n\nA major initiative led by [[Alice]] with [[Bob]].\n",
encoding="utf-8",
)
async def _wait_for_index(watcher, expected_min: int, timeout_s: float = 15.0) -> None:
last = -1
stable_for = 0
for _ in range(int(timeout_s / 0.25)):
now = len(watcher.file_store)
if now == last and now >= expected_min:
stable_for += 1
if stable_for >= 4:
return
else:
stable_for = 0
last = now
await asyncio.sleep(0.25)
def _decode(resp) -> object:
"""Job answers are JSON strings; decode for inspection. Pass through dicts/lists."""
if isinstance(resp.answer, (dict, list)):
return resp.answer
if isinstance(resp.answer, str):
try:
return json.loads(resp.answer)
except (json.JSONDecodeError, TypeError):
return resp.answer
return resp.answer
async def _run_profile(name: str, config_path: Path) -> dict:
"""Boot the profile against a fresh temp vault; exercise jobs; return summary."""
print(f"\n========== profile: {name} ==========")
with tempfile.TemporaryDirectory() as tmp:
vault = Path(tmp) / "vault"
vault.mkdir()
_seed_vault(vault)
# Override watch_path / db_path / sidecar info; force HTTP service so
# we never block on stdio MCP (we only call jobs directly).
_, cfg = parse_args(
"start",
f"config={config_path}",
f"components.file_watcher.default.watch_path={vault}",
f"components.file_store.default.db_path={vault}/.reme",
)
cfg["service"] = {"backend": "http"}
app = Application(**cfg)
await app.start()
try:
jobs = sorted(app.context.jobs.keys())
print(f" registered jobs ({len(jobs)}): {jobs}")
watcher = app.context.components["file_watcher"]["default"]
await _wait_for_index(watcher, expected_min=3)
print(f" file_store nodes after sync: {len(watcher.file_store)}")
assert len(watcher.file_store) >= 3, "watcher did not index seed files"
results: dict = {"jobs": jobs, "checks": []}
alice_path = str((vault / "topics" / "Alice" / "Alice.md").resolve())
if "memory_get" in jobs:
r = _decode(await app.run_job("memory_get", path=alice_path))
ok = isinstance(r, dict) and r.get("exists") is True
print(f" memory_get(Alice.md) → exists={ok}, edges={len(r.get('link', []))}")
results["checks"].append(("memory_get", ok))
if "memory_list" in jobs:
r = _decode(await app.run_job("memory_list", tags=["person"]))
ok = isinstance(r, dict) and r.get("count", 0) >= 2
print(f" memory_list(tags=[person]) → count={r.get('count') if isinstance(r, dict) else '?'}")
results["checks"].append(("memory_list", ok))
if "memory_search" in jobs:
r = _decode(await app.run_job("memory_search", query="collaborates", max_results=3, min_score=0.0))
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
ok = len(hits) > 0
print(f" memory_search('collaborates') → {len(hits)} hits")
results["checks"].append(("memory_search", ok))
if "memory_links" in jobs:
r = _decode(await app.run_job("memory_links", path=alice_path))
ok = isinstance(r, dict) and len(r.get("links", [])) >= 2
print(f" memory_links(Alice.md) → {len(r.get('links', []))} resolved")
results["checks"].append(("memory_links", ok))
if "query" in jobs:
r = _decode(await app.run_job("query", query="Alice Bob", max_results=3, min_score=0.0))
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
ok = len(hits) > 0
print(f" query('Alice Bob') → {len(hits)} hits")
results["checks"].append(("query", ok))
if "ingest" in jobs:
# Degraded path: no LLM key set in env → Ingestor falls back
# to direct create from `target_path`. That's the hermetic path.
target = str((vault / "topics" / "smoke" / "smoke.md").resolve())
r = _decode(await app.run_job(
"ingest",
content="# smoke topic\n\nrecorded by config smoke test.\n",
target_path=target,
metadata={"category": "concept", "title": "smoke"},
))
ok = isinstance(r, dict) and (r.get("applied") or r.get("skipped"))
print(f" ingest(degraded create) → applied={len(r.get('applied', [])) if isinstance(r, dict) else '?'}, "
f"used_llm={r.get('used_llm') if isinstance(r, dict) else '?'}")
results["checks"].append(("ingest", bool(ok)))
# Confirm the file landed on disk.
assert Path(target).is_file(), "ingest did not produce the target file"
if "sync" in jobs:
# CREATE call.
r = _decode(await app.run_job(
"sync",
name="smoke-event",
description="smoke test event",
content="## ops\n- ran the smoke test\n",
topics=["[[Alice]]"],
tags=["smoke"],
materials=[
{"filename": "raw-prompt.md", "content": "# raw user prompt\n\nrun the smoke test\n"},
{"filename": "tool-output.txt", "content": "tool ran ok\nexit=0\n"},
],
))
ok = (
isinstance(r, dict)
and r.get("created") is True
and r.get("action") == "created"
and len(r.get("materials", [])) == 2
)
materials = r.get("materials", []) if isinstance(r, dict) else []
print(f" sync(create smoke-event w/ 2 materials) → created={r.get('created') if isinstance(r, dict) else '?'}, "
f"materials={len(materials)}")
results["checks"].append(("sync.create", bool(ok)))
for m in materials:
assert Path(m).is_file(), f"event material missing: {m}"
if materials:
event_dir = Path(materials[0]).parent
index_text = (event_dir / "smoke-event.md").read_text(encoding="utf-8")
assert "## Materials" in index_text, "index .md missing Materials section"
assert "raw-prompt.md" in index_text, "Materials section missing raw-prompt link"
await _wait_for_index(watcher, expected_min=len(watcher.file_store) + 2)
# APPEND call: same name, new content + new + colliding material.
r2 = _decode(await app.run_job(
"sync",
name="smoke-event",
content="## follow-up\n- second pass facts\n",
topics=["[[Bob]]"], # union with prior [[Alice]]
tags=["follow-up"], # union with prior [smoke]
materials=[
{"filename": "tool-output.txt", "content": "second tool run\nexit=0\n"}, # collision → auto-suffix
{"filename": "summary.md", "content": "# summary\nsecond pass\n"},
],
))
ok2 = (
isinstance(r2, dict)
and r2.get("created") is False
and r2.get("action") == "appended"
and len(r2.get("materials", [])) == 2
)
appended_paths = r2.get("materials", []) if isinstance(r2, dict) else []
print(f" sync(append smoke-event w/ collision) → action={r2.get('action') if isinstance(r2, dict) else '?'}, "
f"new_materials={len(appended_paths)}")
results["checks"].append(("sync.append", bool(ok2)))
# Collision should have produced tool-output-2.txt; summary.md untouched.
names_appended = {Path(p).name for p in appended_paths}
assert "tool-output-2.txt" in names_appended, f"collision auto-suffix missing: {names_appended}"
assert "summary.md" in names_appended, f"clean filename missing: {names_appended}"
# Index should now contain BOTH the original ops section and the Update section.
if materials:
index_text2 = (Path(materials[0]).parent / "smoke-event.md").read_text(encoding="utf-8")
assert "## ops" in index_text2, "original content lost on append"
assert "## Update —" in index_text2, "missing Update section after append"
assert "follow-up" in index_text2, "appended content not in index"
# Frontmatter union check
assert "[[Bob]]" in index_text2, "topic union failed"
# Materials footer should now list 4 files (2 original + summary + tool-output-2)
assert "tool-output-2.txt" in index_text2, "Materials footer missing collided file"
assert "summary.md" in index_text2, "Materials footer missing new file"
# REFUSAL: flip status to distilled, third sync should refuse.
index_path = str(Path(materials[0]).parent / "smoke-event.md") if materials else None
if index_path and "memory_property_update" in jobs:
await app.run_job("memory_property_update", path=index_path, key="status", value="distilled")
r3 = _decode(await app.run_job(
"sync",
name="smoke-event",
content="should refuse",
))
ok3 = isinstance(r3, dict) and "error" in r3 and r3.get("status") == "distilled"
print(f" sync(refuse on distilled) → error={'error' in (r3 if isinstance(r3, dict) else {})}, "
f"suggested_name={r3.get('suggested_name') if isinstance(r3, dict) else '?'}")
results["checks"].append(("sync.refuse_distilled", bool(ok3)))
if "topic_create" in jobs:
r = _decode(await app.run_job(
"topic_create",
folder="Carol",
name="Carol",
category="profile",
description="smoke test",
content="# Carol\n",
tags=["person"],
))
ok = isinstance(r, dict) and (r.get("created") is True or "path" in r)
print(f" topic_create(Carol) → created={r.get('created') if isinstance(r, dict) else '?'}")
results["checks"].append(("topic_create", bool(ok)))
return results
finally:
await app.close()
async def _main() -> int:
summary: dict[str, dict] = {}
for name, path in PROFILES.items():
try:
summary[name] = await _run_profile(name, path)
except Exception as e:
print(f" ✗ profile '{name}' failed: {type(e).__name__}: {e}")
import traceback
traceback.print_exc()
summary[name] = {"error": str(e)}
print("\n========== summary ==========")
failed = 0
for name, result in summary.items():
if "error" in result:
print(f" {name}: ERROR — {result['error']}")
failed += 1
continue
checks = result.get("checks", [])
passed = sum(1 for _, ok in checks if ok)
total = len(checks)
marker = "✓" if passed == total else "✗"
print(f" {marker} {name}: {passed}/{total} checks passed; jobs={len(result['jobs'])}")
for label, ok in checks:
if not ok:
print(f" ✗ {label}")
failed += 1
return 0 if failed == 0 else 1
if __name__ == "__main__":
sys.exit(asyncio.run(_main()))

103
reme2/mcp/README.md
View file

@ -9,14 +9,21 @@ Agent-facing MCP interface layer. Exposes the markdown vault under
```
reme2/mcp/
├── __init__.py
├── server.py MCP server bootstrap; defaults to ../config/full.yaml
├── server.py MCP server bootstrap; defaults to ../config/service.yaml
└── steps/ @R.register MCP step shells
├── memory_io.py memory_create/update/get/list/links/...
├── memory_retriever.py memory_search + memory_graph_search
├── sync.py hot-path event-folder upsert
└── topic_create.py typed topic creation w/ schema gates
├── memory_lint.py read-only Maintainer projection (lint findings)
└── sync.py hot-path event-folder upsert
```
The 12 `memory_*` primitives (create/update/property_update/rename/
delete/archive/get/list/links/backlinks/resolve_wikilink/count_tokens)
live one layer down in `reme2/memory/memory_toolkit.py` — each is a
single `BaseStep` subclass with two class methods: `execute()` for the
MCP path (this layer) and a same-named method for the agent toolkit
path that the Ingestor's ReActAgent consumes. Importing
`reme2.mcp.steps` triggers all `@R.register` registrations.
The MCP layer's job is to **project** existing primitives as MCP tools
— it owns no business logic. Everything else lives outside the
transport boundary so memory services don't form an import cycle:
@ -25,49 +32,97 @@ transport boundary so memory services don't form an import cycle:
|---|---|
| Memory File System primitives | `reme2/component/file_store/`, `file_watcher/`, `file_parser/` |
| Three memory services | `reme2/memory/` — Retriever / Ingestor / Maintainer |
| Pure write helpers + agent toolkit | `reme2/memory/memory_io.py` |
| Vault domain models (Event, Topic) | `reme2/schema/vault/` |
| Engine API (pure, schema-free) | `reme2/memory/memory_io.py` |
| Schema-bound tools (BaseStep + agent toolkit) | `reme2/memory/memory_toolkit.py` |
| Memory schema (4 axes + presets + parser) | `reme2/memory/schema/` |
| Path templates + name disambiguation | `reme2/utils/vault_paths.py` |
| Step response serialization | `reme2/component/runtime_response.py` |
Dependency direction is strict:
```
reme2.mcp → reme2.memory → reme2.component / reme2.schema / reme2.utils
reme2.mcp → reme2.memory (incl. memory.schema) → reme2.component / reme2.utils
```
The Ingestor (`reme2/memory/ingestor.py`) self-registers its `ingest`
MCP face — there's no shell for it under `steps/`. Topic creation lives
inside the Ingestor's R-M-W loop; there is no separate `topic_create`
tool.
## Run
```bash
# stdio MCP server, full tool surface
# stdio MCP server, service-tier surface (loads ../config/service.yaml)
python -m reme2.mcp.server
# override config or any field
python -m reme2.mcp.server config=reme2/config/curated.yaml
# pick a different profile
python -m reme2.mcp.server config=reme2/config/expert.yaml
# override any nested key (Hydra-style)
python -m reme2.mcp.server components.file_watcher.default.watch_path=/abs/vault
```
Config profiles (in `reme2/config/`):
- `full.yaml` — every memory_* primitive + sync + topic_create + ingest
- `curated.yaml` — opinionated 3-tool surface (`query`, `sync`, `ingest`)
Env vars (read by `server.py` and applied as overrides):
## Tools exposed (full profile)
| Var | Effect |
|---|---|
| `VAULT_PATH` | Sets `file_watcher.default.watch_path`, `file_store.default.db_path`, and `service.sidecar_info_path` in one shot. Wins over CLI args. |
| `VAULT_HTTP_PORT` | Override `service.sidecar_http_port` (default `8765`). |
Both shipped profiles register an `mcp` service with `transport: stdio`
plus a sidecar HTTP server on `127.0.0.1:8765`. The sidecar lets local
hooks (e.g. the plugin's `vault_recall.py`) reach the same in-process
FileGraph over HTTP without re-bootstrapping; on startup the server
writes `{host, port}` to `service.sidecar_info_path`
(default `./vault/.reme/sidecar.json`).
## Profiles
Configs live in `reme2/config/`:
- **`expert.yaml`** — every primitive surfaced (16 tools): `sync` + `ingest`
+ `memory_search` / `memory_graph_search` + 5 read primitives + 6 raw
write primitives + `memory_count_tokens` + `memory_lint`. For agents
that should manage the vault directly with full control.
- **`service.yaml`** — opinionated 3-tool surface: `retrieve` (graph-aware
hybrid retrieval), `remember` (single write entry — `mode=log` /
`mode=distill`), `maintain` (vault hygiene sweep). One tool per
memory service; the minimum the agent needs to read, log, distill,
and clean up.
### Tools exposed (expert profile)
| Tool | Path | Purpose |
|---|---|---|
| `sync` | steps/sync.py | Hot-path event-folder upsert (idempotent per `(date, name)`). |
| `ingest` | reme2/memory/ingestor.py | Cold-path LLM-driven distillation. |
| `topic_create` | steps/topic_create.py | Typed topic creation with schema gates. |
| `ingest` | reme2/memory/ingestor.py | Cold-path LLM-driven distillation; owns topic creation. |
| `memory_search` / `memory_graph_search` | steps/memory_retriever.py | V+K hybrid + optional graph BFS. |
| `memory_get` / `memory_list` / `memory_links` / `memory_backlinks` / `memory_resolve_wikilink` | steps/memory_io.py | Read primitives. |
| `memory_create` / `memory_update` / `memory_property_update` / `memory_rename` / `memory_delete` / `memory_archive` | steps/memory_io.py | Raw write primitives (prefer `ingest` / `sync`). |
| `memory_count_tokens` | steps/memory_io.py | Token estimation. |
| `memory_lint` | steps/memory_lint.py | Read-only projection of Maintainer's lint findings. |
| `memory_get` / `memory_list` / `memory_links` / `memory_backlinks` / `memory_resolve_wikilink` | reme2/memory/memory_toolkit.py | Read primitives. |
| `memory_create` / `memory_update` / `memory_property_update` / `memory_rename` / `memory_delete` / `memory_archive` | reme2/memory/memory_toolkit.py | Raw write primitives (prefer `ingest` / `sync`). |
| `memory_count_tokens` | reme2/memory/memory_toolkit.py | Token estimation. |
## Smoke test
### Tools exposed (service profile)
| Tool | Backend | Purpose |
|---|---|---|
| `retrieve` | `memory_graph_search` | Graph-aware hybrid retrieval (vector + keyword + 1-hop wikilink BFS). |
| `remember` | `ingestor` | Single write entry — `mode=log` (zero-LLM event-folder upsert) / `mode=distill` (LLM R-M-W into topic graph). |
| `maintain` | `maintainer` | Vault hygiene sweep — lint + decay (merge/split require LLM, off by default). |
## End-to-end tests
```bash
python reme2/config/smoke_test.py
python -m reme2.mcp.test
```
Boots both `full` and `curated` profiles in a temp vault, exercises a
representative subset of jobs end-to-end (in-process — no MCP
transport).
Boots one Application per profile against a temp vault, exercises every
registered MCP job end-to-end (in-process — no MCP transport), and
prints a per-check status line plus a final pass/fail summary. Exit
code = number of failed checks.
Layout (`reme2/mcp/test/`):
- `_helpers.py` — shared vault seed, app factory, response decoder
- `test_expert.py` — expert profile (25 checks across all 16 tools incl. schema gates)
- `test_service.py` — service profile (10 checks across `retrieve` / `remember` / `maintain`)
- `__main__.py` — CLI runner

20
reme2/mcp/__init__.py
View file

@ -2,19 +2,19 @@
The Agent-facing surface: server entrypoint + step shells that wrap
the three services in `reme2.memory` (Retriever, Ingestor, Maintainer)
plus hot-write primitives (sync, topic_create, memory_*) that bypass
services and land directly on the Memory File System.
plus the hot-write primitive (`sync`) and the raw `memory_*` write/read
tools that bypass services and land directly on the Memory File System.
This package depends on `reme2.memory`, `reme2.schema.vault`,
`reme2.utils`, `reme2.component` — never the reverse. Domain types
(Topic / Event), pure helpers (path builders, naming), and the write
primitives all live outside `mcp/` so memory-layer services can use
them without forming an import cycle through the transport layer.
This package depends on `reme2.memory`, `reme2.utils`, `reme2.component`
— never the reverse. The Memory schema (the typed shape of every
frontmatter) lives under `reme2.memory.schema/` so the services that
own validation can use it without an import cycle through this
transport layer.
Sub-packages:
steps/ - all @R.register MCP step shells (memory_io, memory_retriever,
sync, topic_create).
server.py - MCP server bootstrap (defaults to ../config/full.yaml).
steps/ - all @R.register MCP step shells (memory_toolkit,
memory_retriever, memory_lint, sync).
server.py - MCP server bootstrap (defaults to ../config/service.yaml).
"""
__version__ = "0.1.0"

4
reme2/mcp/server.py
View file

@ -4,7 +4,7 @@ Usage:
python -m reme2.mcp.server [config=path/to/yaml] [service.transport=stdio]
python reme2/mcp/server.py [config=path/to/yaml] [service.transport=stdio]
By default loads `reme2/config/full.yaml` and starts the
By default loads `reme2/config/service.yaml` and starts the
ReMe2 application with the MCP service registered.
Env vars:
@ -42,7 +42,7 @@ from reme2.application import Application # noqa: E402
from reme2.config import parse_args # noqa: E402
_DEFAULT_CONFIG = str(_REPO_ROOT / "reme2" / "config" / "full.yaml")
_DEFAULT_CONFIG = str(_REPO_ROOT / "reme2" / "config" / "service.yaml")
def main() -> None:

41
reme2/mcp/steps/__init__.py
View file

@ -1,31 +1,40 @@
"""MCP step shells — the @R.register classes the model invokes as MCP tools.
Two groups:
Three groups:
Hot-write primitives that bypass services and land on MFS directly:
sync — idempotent event-folder upsert (event log)
topic_create — typed topic creation (hard schema gates)
memory_io — memory_create / update / property_update / rename /
delete / archive / get / list / links / backlinks /
resolve_wikilink / count_tokens
sync — idempotent event-folder upsert (event log)
memory_toolkit — memory_create / update / property_update /
rename / delete / archive / get / list /
links / backlinks / resolve_wikilink /
count_tokens. Each step lives in
`reme2.memory.memory_toolkit` and exposes
BOTH an `execute()` (this MCP path) and a
same-named class method (the agent toolkit
path used by the Ingestor's ReActAgent).
Service delegate:
memory_retriever — memory_search + memory_graph_search; thin
wrappers that delegate to the configured
Retriever component (`reme2.memory.retriever`).
Service delegates (thin shells over the three memory services):
memory_retriever — memory_search + memory_graph_search; delegate
to the Retriever component
(`reme2.memory.retriever`).
memory_lint — read-only projection of the Maintainer's lint
findings (`reme2.memory.maintainer`).
Topic creation is owned by the Ingestor (`reme2.memory.ingestor`) — its
LLM-driven R-M-W loop decides when a new topic is warranted, applies the
schema preset, and routes the write through `memory_create`. There is no
separate `topic_create` MCP tool.
The Ingestor's MCP face (`ingest`) is registered from
`reme2.memory.ingestor`, since it's a service step rather than an MFS
primitive. Importing this package triggers all the step registrations
hosted here.
`reme2.memory.ingestor`. Importing this package triggers all the step
registrations hosted here and in `reme2.memory.memory_toolkit`.
"""
from . import memory_io # noqa: F401 -- triggers @R.register for memory_*
from ...memory import memory_toolkit # noqa: F401 -- triggers @R.register for memory_*
from . import memory_lint # noqa: F401 -- triggers @R.register for memory_lint
from . import memory_retriever # noqa: F401 -- memory_search / memory_graph_search
from .sync import Sync
from .topic_create import TopicCreate
__all__ = [
"Sync",
"TopicCreate",
]

208
reme2/mcp/steps/memory_io.py
View file

@ -1,208 +0,0 @@
"""MCP step shells over the Memory File System engine API.
Per `structure.md`, .md files are the SSOT and the engine surface lives
in `reme2.memory.memory_io` (CRUD writes + MFS reads + Projections).
This module only hosts the `@R.register("memory_*")` Step shells —
each one translates RuntimeContext ↔ JSON payload and delegates to the
matching engine API function.
Search ops live in `memory_retriever.py` (Retriever composes V/K/graph
projections with policy).
"""
from __future__ import annotations
from pathlib import Path
from ...component import R
from ...component.base_step import BaseStep
from ...component.runtime_response import _set_answer
from ...enumeration import ComponentEnum
from ...memory import memory_io
@R.register("memory_get")
class MemoryGet(BaseStep):
"""Read a single memory file (frontmatter + body, optional chunks)."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
include_chunks: bool = bool(self.context.get("include_chunks", False))
assert path, "path is required"
result = await memory_io.read_file(self.file_store, path, include_chunks=include_chunks)
_set_answer(self.context, result)
@R.register("memory_list")
class MemoryList(BaseStep):
"""List indexed files filtered by frontmatter fields, tags, or path prefix."""
async def execute(self):
assert self.context is not None
result = memory_io.list_files(
self.file_store,
path_prefix=self.context.get("path_prefix"),
tags=self.context.get("tags") or [],
metadata=self.context.get("metadata") or {},
limit=int(self.context.get("limit", 100)),
)
_set_answer(self.context, result)
@R.register("memory_backlinks")
class MemoryBacklinks(BaseStep):
"""Files linking to a given path. Each entry carries the typed-edge predicate."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
assert path, "path is required"
_set_answer(self.context, memory_io.backlinks_of(self.file_store, path))
@R.register("memory_links")
class MemoryLinks(BaseStep):
"""Files a given path links to (resolved). Each entry carries the typed-edge predicate."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
assert path, "path is required"
_set_answer(self.context, memory_io.links_of(self.file_store, path))
@R.register("memory_resolve_wikilink")
class MemoryResolveWikilink(BaseStep):
"""Resolve a `[[target]]` wikilink with full ambiguity context."""
async def execute(self):
assert self.context is not None
wikilink: str = self.context.get("wikilink", "") or ""
assert wikilink, "wikilink is required"
payload = memory_io.wikilink_lookup(self.file_store, wikilink)
self.context.response.success = bool(payload.get("exists"))
_set_answer(self.context, payload)
@R.register("memory_create")
class MemoryCreate(BaseStep):
"""Create a markdown file. Wikilink-uniqueness gate runs unless force=True."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
metadata: dict = dict(self.context.get("metadata", {}) or {})
content: str = self.context.get("content", "") or ""
overwrite: bool = bool(self.context.get("overwrite", False))
force: bool = bool(self.context.get("force", False))
assert path, "path is required"
target = Path(path)
ok, payload = memory_io.write_create(
self.file_store, target,
metadata=metadata, content=content,
overwrite=overwrite, force=force,
)
self.context.response.success = ok
if ok:
payload = {**payload, "path": str(target.resolve())}
_set_answer(self.context, payload)
@R.register("memory_delete")
class MemoryDelete(BaseStep):
"""Delete a file. Watcher removes from store + graph."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
assert path, "path is required"
ok, payload = memory_io.write_delete(path)
self.context.response.success = ok
_set_answer(self.context, payload)
@R.register("memory_rename")
class MemoryRename(BaseStep):
"""Rename a file and rewrite incoming wikilinks across the vault."""
async def execute(self):
assert self.context is not None
old_path: str = self.context.get("old_path", "")
new_path: str = self.context.get("new_path", "")
assert old_path and new_path, "old_path and new_path are required"
watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[index,union-attr]
vault_root = Path(watcher.watch_path).resolve() # type: ignore[union-attr]
ok, payload = memory_io.write_rename(self.file_store, vault_root, old_path, new_path)
self.context.response.success = ok
_set_answer(self.context, payload)
@R.register("memory_property_update")
class MemoryPropertyUpdate(BaseStep):
"""Update a single YAML frontmatter key. value=null deletes the key."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
key: str = self.context.get("key", "")
value = self.context.get("value")
assert path and key, "path and key are required"
ok, payload = memory_io.write_property_update(path, key, value)
self.context.response.success = ok
_set_answer(self.context, payload)
@R.register("memory_update")
class MemoryUpdate(BaseStep):
"""Edit-style content update: replace `old_string` with `new_string`."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "")
old_string: str = self.context.get("old_string", "")
new_string: str = self.context.get("new_string", "")
replace_all: bool = bool(self.context.get("replace_all", False))
assert path, "path is required"
ok, payload = memory_io.write_update(path, old_string, new_string, replace_all=replace_all)
self.context.response.success = ok
_set_answer(self.context, payload)
@R.register("memory_archive")
class MemoryArchive(BaseStep):
"""Archive a file: flip `status: archived` then move to `<vault>/Archive/`."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
archive_dir_name: str = self.context.get("archive_dir", "Archive") or "Archive"
assert path, "path is required"
watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default")
vault_root = Path(getattr(watcher, "watch_path", ".")).resolve() if watcher else Path.cwd()
ok, payload = memory_io.write_archive(vault_root, path, archive_dir_name)
self.context.response.success = ok
_set_answer(self.context, payload)
@R.register("memory_count_tokens")
class MemoryCountTokens(BaseStep):
"""Estimate tokens for a file body or raw text. One of `path`/`text` required."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
text: str = self.context.get("text", "") or ""
result = await memory_io.count_tokens(
self.as_token_counter,
path=path or None,
text=text or None,
)
self.context.response.success = "error" not in result
_set_answer(self.context, result)

70
reme2/mcp/steps/memory_lint.py Normal file
View file

@ -0,0 +1,70 @@
"""memory_lint — read-only projection of the Maintainer's lint findings.
Tier A surface for the Maintainer service. The host agent calls this
to discover what's wrong with the vault (broken wikilinks, schema
violations, stem collisions) and decides what to do with each finding
using the existing memory_* write primitives.
Equivalent to invoking `Maintainer.execute(ops=["lint"], dry_run=True)`
but with a focused response shape and a tighter parameter surface — the
agent doesn't need to know about decay/merge/split knobs.
"""
from __future__ import annotations
from ...component import R
from ...component.base_step import BaseStep
from ...component.runtime_response import _set_answer
from ...enumeration import ComponentEnum
from ...memory.maintainer import Maintainer
@R.register("memory_lint")
class MemoryLint(BaseStep):
"""Run the Maintainer's lint pass and surface findings only.
Inputs (RuntimeContext, all optional):
target_prefix (str): restrict scan to relpaths starting with
this prefix (e.g. "events/2026-05-09/").
Output (context.response.answer):
{
"scanned": int, # files inspected
"findings": [LintFinding, ...], # each {path, kind, detail}
"target_prefix": str,
"ran_at": iso,
}
"""
async def execute(self):
assert self.context is not None
target_prefix = str(self.context.get("target_prefix") or "")
# Delegate to a Maintainer step. Force ops=["lint"] + dry_run so
# we never mutate. The Maintainer reads these from the context
# and produces a full audit; we narrow the response shape below.
if getattr(self, "_maintainer", None) is None:
self._maintainer = R.get(ComponentEnum.STEP, "maintainer")(
app_context=self.app_context,
)
self.context["ops"] = ["lint"]
self.context["dry_run"] = True
self.context["target_prefix"] = target_prefix
await self._maintainer(self.context)
# The Maintainer wrote a full audit to context.response.answer
# (proposed/plan/applied/skipped/failed/...). For lint, all the
# action lives in `proposed` (LintFindings never get applied or
# dropped). Reshape into a focused response.
import json
raw = self.context.response.answer
audit = json.loads(raw) if isinstance(raw, str) else (raw or {})
findings = audit.get("proposed") or []
_set_answer(self.context, {
"scanned": audit.get("scanned", 0),
"findings": findings,
"target_prefix": target_prefix,
"ran_at": audit.get("ran_at", ""),
})
self.context.response.success = True

4
reme2/mcp/steps/memory_retriever.py
View file

@ -114,7 +114,7 @@ class MemorySearch(BaseStep):
isinstance(max_results, int) and max_results > 0
), f"max_results must be a positive integer, got {max_results}"
chunk_filter = memory_io.make_chunk_filter(
chunk_filter = memory_io.make_filter(
self.file_store,
paths=self.context.get("paths") or None,
tags=self.context.get("tags") or None,
@ -168,7 +168,7 @@ class MemoryGraphSearch(BaseStep):
assert query or explicit_seeds, "query or seeds must be provided"
assert max_results > 0
chunk_filter = memory_io.make_chunk_filter(
chunk_filter = memory_io.make_filter(
self.file_store,
paths=ctx.get("paths") or None,
tags=ctx.get("tags") or None,

19
reme2/mcp/steps/sync.py
View file

@ -41,8 +41,8 @@ from pydantic import ValidationError
from reme2.component import R
from reme2.component.base_step import BaseStep
from reme2.memory.memory_io import write_create
from reme2.schema.vault import Event
from reme2.memory.memory_io import create_file
from reme2.memory.schema import EVENT_PRESET, Memory
from reme2.utils.vault_paths import event_path, next_suffixed_stem
@ -238,11 +238,12 @@ class Sync(BaseStep):
on_date.isoformat() if isinstance(on_date, date_type)
else (on_date or today)
)
# Start from EVENT_PRESET (4 axes + status + legacy `category`),
# layer caller-supplied identity fields on top.
metadata: dict = {
**EVENT_PRESET,
"title": name,
"description": description,
"category": "event",
"status": "active",
"tags": tags,
"topics": topics,
"created": on_date_str,
@ -252,10 +253,10 @@ class Sync(BaseStep):
metadata["originSessionId"] = origin_session_id
try:
Event.model_validate(metadata)
Memory.model_validate(metadata)
except ValidationError as e:
self._set_error({
"error": "Event schema validation failed",
"error": "Memory schema validation failed",
"details": e.errors(include_context=False, include_url=False),
})
return
@ -281,7 +282,7 @@ class Sync(BaseStep):
material_filenames = [m["filename"] for m in materials]
index_body = self._emit_body(content, material_filenames)
ok, payload = write_create(
ok, payload = create_file(
self.file_store, target,
metadata=metadata, content=index_body,
)
@ -391,11 +392,11 @@ class Sync(BaseStep):
meta["updated"] = date_type.today().isoformat()
try:
Event.model_validate(meta)
Memory.model_validate(meta)
except ValidationError as e:
self._set_error({
"path": str(target.resolve()),
"error": "Event schema validation failed on append",
"error": "Memory schema validation failed on append",
"details": e.errors(include_context=False, include_url=False),
})
return

142
reme2/mcp/steps/topic_create.py
View file

@ -1,142 +0,0 @@
"""topic_create — create a new topic under topics/{folder}/{name}.md.
The only file-creation entry point for topics. Enforces:
- path template (topics/{folder}/{name}.md)
- folder topic identification (folder == name)
- judgment-category strong-confidence (via Topic.model_validator)
- wikilink uniqueness (creating this file must not make `[[name]]` ambiguous)
Writes via `write_create` — the canonical L1 invariant gate that also
performs the wikilink-uniqueness check. The Ingestor's R-M-W loop is
reserved for content-driven flows; topic creation is path-driven, so
the direct call is sufficient.
"""
import json
from datetime import date
from pathlib import Path
from pydantic import ValidationError
from reme2.component import R
from reme2.component.base_step import BaseStep
from reme2.memory.memory_io import write_create
from reme2.schema.vault import Topic
from reme2.utils.vault_paths import next_suffixed_stem, topic_path
@R.register("topic_create")
class TopicCreate(BaseStep):
"""Create a topic file with the standard frontmatter template."""
def __init__(self, vault_root: str = "", topics_dir: str = "topics", **kwargs):
super().__init__(**kwargs)
self.vault_root = vault_root
self.topics_dir = topics_dir
def _root(self) -> Path:
if self.vault_root:
return Path(self.vault_root)
watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[union-attr]
return Path(watcher.watch_path)
async def execute(self):
assert self.context is not None
folder: str = self.context.get("folder", "")
name: str = self.context.get("name", "")
category: str = self.context.get("category", "")
description: str = self.context.get("description", "") or ""
content: str = self.context.get("content", "") or ""
confidence = self.context.get("confidence")
market = self.context.get("market")
ticker = self.context.get("ticker")
tags: list[str] = self.context.get("tags") or []
assert folder, "folder is required"
assert name, "name is required"
assert category, "category is required"
target = topic_path(self._root(), folder, name, self.topics_dir)
if target.exists():
self.context.response.success = False
self.context.response.answer = json.dumps({
"path": str(target.resolve()),
"error": "topic already exists; use memory_update / memory_property_update to modify",
}, ensure_ascii=False)
return
today = date.today().isoformat()
metadata: dict = {
"title": name,
"description": description,
"category": category,
"tags": tags,
"created": today,
"updated": today,
}
if confidence is not None:
metadata["confidence"] = confidence
if market is not None:
metadata["market"] = market
if ticker is not None:
metadata["ticker"] = ticker
# Topic schema validation (judgment categories require confidence).
try:
Topic.model_validate(metadata)
except ValidationError as e:
self.context.response.success = False
self.context.response.answer = json.dumps({
"error": "Topic schema validation failed",
"details": e.errors(include_context=False, include_url=False),
}, ensure_ascii=False)
return
# Pre-check uniqueness so we can surface a `suggested_name` to the
# agent. The actual write also routes through the same gate inside
# write_create, so this is a UX nicety, not a correctness check.
graph = self.file_store
conflicts = graph.collisions_after_create(target)
if conflicts:
taken = {Path(p).stem for p in graph.nodes}
suggested_name = next_suffixed_stem(taken, name)
self.context.response.success = False
self.context.response.answer = json.dumps({
"error": (
f"stem `[[{name}]]` would resolve ambiguously "
f"to {len(conflicts) + 1} paths after this create"
),
"conflicts": conflicts,
"suggested_name": suggested_name,
"hint": (
f"retry with name='{suggested_name}' (numeric suffix), "
f"OR use a domain-specific qualifier (e.g. '{name}-Inc' / "
f"'{name}-v2') — semantic names beat numeric. If you "
f"actually meant the existing topic, call memory_get on "
f"one of `conflicts` instead."
),
}, ensure_ascii=False)
return
ok, payload = write_create(
self.file_store, target,
metadata=metadata, content=content,
)
if not ok:
self.context.response.success = False
self.context.response.answer = json.dumps({
"path": str(target.resolve()),
"error": payload.get("error", "create failed"),
"details": payload,
}, ensure_ascii=False)
return
is_folder_topic = folder == name
self.context.response.success = True
self.context.response.answer = json.dumps({
"path": str(target.resolve()),
"category": category,
"is_folder_topic": is_folder_topic,
"created": True,
}, ensure_ascii=False)

19
reme2/mcp/test/__init__.py Normal file
View file

@ -0,0 +1,19 @@
"""End-to-end tests for the reme2 MCP profiles.
Two suites — one per shipped profile (`expert`, `service`) — boot
`reme2.application.Application` against a temp vault and exercise every
MCP-exposed job through `app.run_job(...)`. We force `service.backend =
http` so the suites never block on stdio MCP transport; jobs are
called in-process.
Layout:
_helpers.py shared vault seed, app factory, response decoder
test_expert.py expert profile (every memory_* primitive surfaced)
test_service.py service profile (retrieve / remember / maintain)
__main__.py CLI: `python -m reme2.mcp.test`
The test functions are plain `async def check_<name>(app, ctx)` returning
True / raising. The CLI runner reuses one app per profile to keep the
boot cost out of the per-test budget.
"""

77
reme2/mcp/test/__main__.py Normal file
View file

@ -0,0 +1,77 @@
"""CLI runner: `python -m reme2.mcp.test`
Boots one Application per profile against a fresh temp vault, runs the
profile's CHECKS list in order, prints a per-check status line plus a
final summary. Exit code is the number of failed checks across both
profiles.
Per-profile suites are intentionally sequential — later checks rely on
side-effects from earlier ones (e.g. `sync.append` after `sync.create`).
Cross-profile work happens in independent vaults.
"""
from __future__ import annotations
import asyncio
import shutil
import sys
import traceback
from . import test_expert, test_service
from ._helpers import AppContext, make_context
SUITES = {
"expert": test_expert.CHECKS,
"service": test_service.CHECKS,
}
async def _run_suite(profile: str, checks: list[tuple[str, callable]]) -> tuple[int, int]:
"""Run one suite. Returns (passed, total)."""
print(f"\n========== profile: {profile} ==========")
ctx: AppContext | None = None
tmp = None
passed = 0
try:
ctx, tmp = await make_context(profile)
print(f" bootstrapped: vault={ctx.vault}, jobs={len(ctx.jobs)}, "
f"file_store={len(ctx.file_store)}")
for label, fn in checks:
try:
summary = await fn(ctx)
print(f" ✓ {label:32s} {summary}")
passed += 1
except Exception as e:
print(f" ✗ {label:32s} {type(e).__name__}: {e}")
traceback.print_exc()
except Exception as e:
print(f" ✗ bootstrap failed: {type(e).__name__}: {e}")
traceback.print_exc()
finally:
if ctx is not None:
try:
await ctx.app.close()
except Exception as e: # noqa: BLE001
print(f" (warn) app.close failed: {e}")
if tmp is not None:
shutil.rmtree(tmp, ignore_errors=True)
return passed, len(checks)
async def _main() -> int:
summary: dict[str, tuple[int, int]] = {}
for profile, checks in SUITES.items():
summary[profile] = await _run_suite(profile, checks)
print("\n========== summary ==========")
failed = 0
for profile, (passed, total) in summary.items():
marker = "✓" if passed == total else "✗"
print(f" {marker} {profile}: {passed}/{total}")
failed += (total - passed)
return 0 if failed == 0 else failed
if __name__ == "__main__":
sys.exit(asyncio.run(_main()))

162
reme2/mcp/test/_helpers.py Normal file
View file

@ -0,0 +1,162 @@
"""Shared fixtures for the MCP profile tests.
Each suite uses one app per profile (boot cost ~2s) and a fresh temp
vault. The app is configured with `service.backend = http` so the
stdio MCP listener never starts — jobs are invoked directly via
`app.run_job`.
"""
from __future__ import annotations
import asyncio
import json
import sys
from dataclasses import dataclass
from pathlib import Path
# Make the repo importable when this module is loaded standalone
# (`python -m reme2.mcp.test` already has the path; direct imports
# from a fresh interpreter need this guard).
_REPO_ROOT = Path(__file__).resolve().parents[3]
if str(_REPO_ROOT) not in sys.path:
sys.path.insert(0, str(_REPO_ROOT))
# Eager-import side-effect modules so all @R.register() decorators run
# before Application introspects the registry.
import reme2 # noqa: E402,F401
import reme2.mcp.steps # noqa: E402,F401
import reme2.memory # noqa: E402,F401 -- registers retriever + maintainer
import reme2.memory.ingestor # noqa: E402,F401
import reme2.memory.summarizer # noqa: E402,F401
from reme2.application import Application # noqa: E402
from reme2.config import parse_args # noqa: E402
CONFIG_DIR = _REPO_ROOT / "reme2" / "config"
PROFILES = {
"expert": CONFIG_DIR / "expert.yaml",
"service": CONFIG_DIR / "service.yaml",
}
# How many seed files `seed_vault` writes — checks that need to wait
# for indexing budget against this baseline.
SEED_FILE_COUNT = 3
@dataclass
class AppContext:
"""Bundle of everything a test function needs."""
app: Application
vault: Path
jobs: list[str]
@property
def watcher(self):
return self.app.context.components["file_watcher"]["default"]
@property
def file_store(self):
return self.watcher.file_store
def abs_path(self, *parts: str) -> str:
return str((self.vault.joinpath(*parts)).resolve())
def seed_vault(vault: Path) -> None:
"""Drop a small connected topic graph (Alice ↔ Bob ↔ Project X)."""
(vault / "topics" / "Alice").mkdir(parents=True, exist_ok=True)
(vault / "topics" / "Alice" / "Alice.md").write_text(
"---\ntitle: Alice\ncategory: profile\ntags: [person]\n---\n"
"# Alice\n\nAlice works on [[Project X]] with [[Bob]].\n"
"[author:: [[Alice]]]\n",
encoding="utf-8",
)
(vault / "topics" / "Bob").mkdir(parents=True, exist_ok=True)
(vault / "topics" / "Bob" / "Bob.md").write_text(
"---\ntitle: Bob\ncategory: profile\ntags: [person]\n---\n"
"# Bob\n\nBob collaborates with [[Alice]] on [[Project X]].\n",
encoding="utf-8",
)
(vault / "topics" / "Project X").mkdir(parents=True, exist_ok=True)
(vault / "topics" / "Project X" / "Project X.md").write_text(
"---\ntitle: Project X\ncategory: concept\n---\n"
"# Project X\n\nA major initiative led by [[Alice]] with [[Bob]].\n",
encoding="utf-8",
)
async def wait_for_index(watcher, expected_min: int, timeout_s: float = 15.0) -> None:
"""Poll `len(file_store)` until it reaches `expected_min` and stays
stable across 4 ticks. Raises if the budget runs out — a watcher
that never indexed seed files is a hard wiring failure, not a
soft check."""
deadline = asyncio.get_event_loop().time() + timeout_s
last = -1
stable = 0
while asyncio.get_event_loop().time() < deadline:
now = len(watcher.file_store)
if now == last and now >= expected_min:
stable += 1
if stable >= 4:
return
else:
stable = 0
last = now
await asyncio.sleep(0.25)
raise RuntimeError(
f"watcher did not reach >= {expected_min} files within {timeout_s}s "
f"(last seen: {last})"
)
def decode(resp) -> object:
"""Job answers come back as JSON strings — decode for inspection.
Pass through dicts / lists if the step already returned native types."""
if isinstance(resp.answer, (dict, list)):
return resp.answer
if isinstance(resp.answer, str):
try:
return json.loads(resp.answer)
except (json.JSONDecodeError, TypeError):
return resp.answer
return resp.answer
async def build_app(profile_path: Path, vault: Path) -> Application:
"""Boot an Application against `vault` using the given profile.
Forces `service.backend = http` so the stdio MCP listener never
starts (we only call jobs directly), and rewrites the watcher /
file_store paths to point at the temp vault.
"""
_, cfg = parse_args(
"start",
f"config={profile_path}",
f"components.file_watcher.default.watch_path={vault}",
f"components.file_store.default.db_path={vault}/.reme",
)
cfg["service"] = {"backend": "http"}
app = Application(**cfg)
await app.start()
return app
async def make_context(profile: str) -> tuple[AppContext, Path]:
"""Build a fresh temp vault + app for `profile`, wait for indexing.
The caller owns the temp dir lifecycle — returns its Path so it can
be `shutil.rmtree`d after `app.close()`.
"""
import tempfile
tmp = Path(tempfile.mkdtemp(prefix=f"reme2-mcp-{profile}-"))
vault = tmp / "vault"
vault.mkdir()
seed_vault(vault)
app = await build_app(PROFILES[profile], vault)
await wait_for_index(app.context.components["file_watcher"]["default"],
expected_min=SEED_FILE_COUNT)
jobs = sorted(app.context.jobs.keys())
return AppContext(app=app, vault=vault, jobs=jobs), tmp

425
reme2/mcp/test/test_expert.py Normal file
View file

@ -0,0 +1,425 @@
"""Expert-profile MCP tests.
Covers all 16 jobs registered by `reme2/config/expert.yaml`:
Hot-write sync
Read memory_search, memory_graph_search,
memory_get, memory_list, memory_links,
memory_backlinks, memory_resolve_wikilink,
memory_count_tokens, memory_lint
Raw write memory_create, memory_update, memory_property_update,
memory_rename, memory_delete, memory_archive
Each `check_*` is an `async def` that takes a populated `AppContext`,
runs one MCP job (or a small sequence), asserts on the response, and
returns a one-line summary string. The orchestrator runs them in the
listed order and collects pass/fail.
"""
from __future__ import annotations
from pathlib import Path
from ._helpers import AppContext, decode, wait_for_index
# Manifest of every tool the expert profile must expose. Used by
# `check_registry` and as the upper bound for `wait_for_index` budgets.
EXPECTED_JOBS: tuple[str, ...] = (
"sync",
"memory_search",
"memory_graph_search",
"memory_get",
"memory_list",
"memory_backlinks",
"memory_links",
"memory_resolve_wikilink",
"memory_count_tokens",
"memory_lint",
"memory_create",
"memory_update",
"memory_property_update",
"memory_rename",
"memory_delete",
"memory_archive",
)
# ---------- registry ---------------------------------------------------
async def check_registry(ctx: AppContext) -> str:
missing = [j for j in EXPECTED_JOBS if j not in ctx.jobs]
assert not missing, f"missing jobs: {missing}"
extras = [j for j in ctx.jobs if j not in EXPECTED_JOBS]
return f"{len(ctx.jobs)} jobs registered (extras: {extras or 'none'})"
# ---------- read primitives -------------------------------------------
async def check_memory_get(ctx: AppContext) -> str:
alice = ctx.abs_path("topics", "Alice", "Alice.md")
r = decode(await ctx.app.run_job("memory_get", path=alice))
assert isinstance(r, dict) and r.get("exists") is True, r
assert "metadata" in r, "missing metadata block"
return f"exists=True, edges={len(r.get('link', []))}"
async def check_memory_list(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job("memory_list", tags=["person"]))
assert isinstance(r, dict) and r.get("count", 0) >= 2, r
paths = {item.get("path") for item in r.get("items", [])}
assert any("Alice.md" in p for p in paths), paths
return f"count={r['count']}"
async def check_memory_search(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"memory_search", query="collaborates", max_results=3, min_score=0.0,
))
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
assert len(hits) > 0, r
return f"{len(hits)} hits"
async def check_memory_graph_search(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"memory_graph_search", query="Alice", max_results=5, min_score=0.0,
graph_depth=1,
))
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
assert len(hits) > 0, r
# At least one result should carry a graph_hop annotation
hops = {h.get("graph_hop") for h in hits if isinstance(h, dict)}
return f"{len(hits)} hits, hops seen={sorted(h for h in hops if h is not None)}"
async def check_memory_links(ctx: AppContext) -> str:
alice = ctx.abs_path("topics", "Alice", "Alice.md")
r = decode(await ctx.app.run_job("memory_links", path=alice))
assert isinstance(r, dict) and len(r.get("links", [])) >= 2, r
return f"{len(r['links'])} resolved outgoing links"
async def check_memory_backlinks(ctx: AppContext) -> str:
alice = ctx.abs_path("topics", "Alice", "Alice.md")
r = decode(await ctx.app.run_job("memory_backlinks", path=alice))
# Bob.md and Project X.md both link to [[Alice]]
assert isinstance(r, dict) and len(r.get("backlinks", [])) >= 2, r
return f"{len(r['backlinks'])} incoming backlinks"
async def check_memory_resolve_wikilink(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job("memory_resolve_wikilink", wikilink="Alice"))
assert isinstance(r, dict) and r.get("exists") is True, r
assert "Alice.md" in (r.get("path") or ""), r
return "stem 'Alice' → Alice.md"
async def check_memory_count_tokens(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"memory_count_tokens", text="hello world from the smoke test",
))
assert isinstance(r, dict) and isinstance(r.get("tokens"), int), r
assert r["tokens"] > 0, r
return f"text → {r['tokens']} tokens"
async def check_memory_lint(ctx: AppContext) -> str:
"""Maintainer lint pass — read-only. The seeded vault is healthy
(no broken wikilinks / schema violations), so we just verify the
shell wires through and returns the expected envelope."""
r = decode(await ctx.app.run_job("memory_lint"))
assert isinstance(r, dict), r
assert "scanned" in r and "findings" in r, r
assert isinstance(r["findings"], list), r
assert r["scanned"] >= len(ctx.file_store), r
return f"scanned={r['scanned']}, findings={len(r['findings'])}"
# ---------- hot-write: sync (create / append / refusal) ---------------
async def check_sync_create(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"sync",
name="suite-event",
description="full-profile suite",
content="## ops\n- ran the suite\n",
topics=["[[Alice]]"],
tags=["suite"],
materials=[
{"filename": "raw-prompt.md", "content": "# user prompt\n\nrun suite\n"},
{"filename": "tool-output.txt", "content": "exit=0\n"},
],
))
assert isinstance(r, dict), r
assert r.get("created") is True and r.get("action") == "created", r
materials = r.get("materials", [])
assert len(materials) == 2, materials
for m in materials:
assert Path(m).is_file(), f"material missing on disk: {m}"
event_dir = Path(materials[0]).parent
index_text = (event_dir / "suite-event.md").read_text(encoding="utf-8")
assert "## Materials" in index_text, "Materials footer absent"
assert "raw-prompt.md" in index_text, "Materials footer missing raw-prompt link"
# 4-axis schema must be on disk
assert "lifecycle: streaming" in index_text, "schema axis 'lifecycle' missing"
assert "role: observation" in index_text, "schema axis 'role' missing"
# Stash for downstream checks via the context (small mutation pattern).
ctx.abs_path("__suite_event_dir__") # noop; readable side-effect below
setattr(ctx, "_suite_event_dir", event_dir)
setattr(ctx, "_suite_event_index", event_dir / "suite-event.md")
await wait_for_index(ctx.watcher, expected_min=len(ctx.file_store))
return f"created folder w/ {len(materials)} materials"
async def check_sync_append(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"sync",
name="suite-event",
content="## follow-up\n- second pass\n",
topics=["[[Bob]]"], # union with [[Alice]]
tags=["follow-up"],
materials=[
{"filename": "tool-output.txt", "content": "second run\n"}, # collision
{"filename": "summary.md", "content": "# summary\n"},
],
))
assert isinstance(r, dict), r
assert r.get("created") is False and r.get("action") == "appended", r
appended = r.get("materials", [])
names = {Path(p).name for p in appended}
assert "tool-output-2.txt" in names, f"collision auto-suffix failed: {names}"
assert "summary.md" in names, f"new material missing: {names}"
index_text = getattr(ctx, "_suite_event_index").read_text(encoding="utf-8")
assert "## ops" in index_text, "original body lost"
assert "## Update —" in index_text, "Update section missing"
assert "[[Bob]]" in index_text, "topic union failed"
assert "tool-output-2.txt" in index_text, "Materials footer lost the collided file"
return f"appended w/ collision → {sorted(names)}"
async def check_sync_refuse_distilled(ctx: AppContext) -> str:
index_path = str(getattr(ctx, "_suite_event_index"))
await ctx.app.run_job(
"memory_property_update", path=index_path, key="status", value="distilled",
)
r = decode(await ctx.app.run_job(
"sync", name="suite-event", content="should be refused",
))
assert isinstance(r, dict) and "error" in r, r
assert r.get("status") == "distilled", r
assert r.get("suggested_name"), r
return f"refused → suggested={r['suggested_name']!r}"
# ---------- raw memory_* writes ---------------------------------------
async def check_memory_create(ctx: AppContext) -> str:
target = ctx.abs_path("topics", "Carol", "Carol.md")
r = decode(await ctx.app.run_job(
"memory_create",
path=target,
metadata={
"title": "Carol",
"lifecycle": "evolving",
"scope": "class",
"source": "curated",
"role": "profile",
"category": "profile",
"tags": ["person"],
},
content="# Carol\n\nKnows [[Alice]].\n",
))
assert isinstance(r, dict) and r.get("created") is True, r
assert "error" not in r, r
assert Path(target).is_file(), target
await wait_for_index(ctx.watcher, expected_min=len(ctx.file_store))
return f"created {Path(target).name}"
async def check_memory_update(ctx: AppContext) -> str:
carol = ctx.abs_path("topics", "Carol", "Carol.md")
r = decode(await ctx.app.run_job(
"memory_update",
path=carol,
old_string="Knows [[Alice]].",
new_string="Knows [[Alice]] and [[Bob]].",
))
assert isinstance(r, dict) and r.get("replaced", 0) >= 1, r
assert "Bob" in Path(carol).read_text(encoding="utf-8"), "edit not on disk"
return f"body edit applied (replaced={r['replaced']})"
async def check_memory_property_update(ctx: AppContext) -> str:
carol = ctx.abs_path("topics", "Carol", "Carol.md")
r = decode(await ctx.app.run_job(
"memory_property_update", path=carol, key="confidence", value="✅",
))
assert isinstance(r, dict) and "error" not in r, r
assert r.get("key") == "confidence" and r.get("value") == "✅", r
assert "confidence: ✅" in Path(carol).read_text(encoding="utf-8"), \
"property write not on disk"
return "set confidence=✅"
async def check_memory_rename(ctx: AppContext) -> str:
src = ctx.abs_path("topics", "Carol", "Carol.md")
dst = ctx.abs_path("topics", "Carol", "Carol-renamed.md")
r = decode(await ctx.app.run_job(
"memory_rename", old_path=src, new_path=dst,
))
assert isinstance(r, dict) and "error" not in r, r
assert r.get("new_path") and Path(r["new_path"]).is_file(), r
assert not Path(src).exists(), f"old path still on disk: {src}"
setattr(ctx, "_carol_path", r["new_path"])
return "Carol.md → Carol-renamed.md"
async def check_memory_archive(ctx: AppContext) -> str:
carol = getattr(ctx, "_carol_path", ctx.abs_path("topics", "Carol", "Carol-renamed.md"))
r = decode(await ctx.app.run_job("memory_archive", path=carol))
assert isinstance(r, dict) and r.get("archived") is True, r
archived_path = r.get("new_path")
assert archived_path and "Archive" in archived_path, r
assert Path(archived_path).is_file(), archived_path
setattr(ctx, "_carol_archived", archived_path)
return f"→ {Path(archived_path).resolve().relative_to(ctx.vault.resolve())}"
async def check_memory_delete(ctx: AppContext) -> str:
target = getattr(ctx, "_carol_archived", None) \
or ctx.abs_path("topics", "Carol", "Carol-renamed.md")
r = decode(await ctx.app.run_job("memory_delete", path=target))
assert isinstance(r, dict) and r.get("deleted") is True, r
assert not Path(target).exists(), target
return "removed"
# ---------- schema gates (P4) -----------------------------------------
async def check_schema_path_template_refuses(ctx: AppContext) -> str:
"""memory_create rejects paths outside topics/{X}/{Y}.md,
events/{date}/{name}/..., or Archive/..."""
target = ctx.abs_path("notes", "freeform.md") # outside any template
r = decode(await ctx.app.run_job(
"memory_create",
path=target,
metadata={"title": "freeform",
"lifecycle": "evolving", "scope": "class",
"source": "curated", "role": "concept"},
content="should be refused",
))
assert isinstance(r, dict), r
assert "error" in r and "template" in r["error"].lower(), r
assert not Path(target).exists(), "file shouldn't have been written"
return "refused notes/freeform.md (no template)"
async def check_schema_path_template_force_bypass(ctx: AppContext) -> str:
"""force=True bypasses the template gate."""
target = ctx.abs_path("notes", "forced.md")
r = decode(await ctx.app.run_job(
"memory_create",
path=target,
metadata={"title": "forced",
"lifecycle": "evolving", "scope": "class",
"source": "curated", "role": "concept"},
content="forced through",
force=True,
))
assert isinstance(r, dict) and r.get("created") is True, r
assert Path(target).is_file(), target
return "force=True bypassed"
async def check_schema_status_skip_refused(ctx: AppContext) -> str:
"""Suite-event ended distilled (sync.refuse_distilled flipped it).
Trying distilled → active is reverse — must refuse."""
event_index = getattr(ctx, "_suite_event_index", None)
assert event_index is not None, "suite-event index not staged"
r = decode(await ctx.app.run_job(
"memory_property_update",
path=str(event_index),
key="status",
value="active",
))
assert isinstance(r, dict), r
assert "error" in r and "transition" in r["error"].lower(), r
assert r.get("prior") == "distilled", r
return f"refused {r.get('prior')!r} → {r.get('requested')!r}"
async def check_schema_status_invalid_value(ctx: AppContext) -> str:
"""Random string for status is refused before any state-machine check."""
event_index = getattr(ctx, "_suite_event_index", None)
assert event_index is not None, "suite-event index not staged"
r = decode(await ctx.app.run_job(
"memory_property_update",
path=str(event_index),
key="status",
value="bogus",
))
assert isinstance(r, dict), r
assert "error" in r and "invalid" in r["error"].lower(), r
return "refused status='bogus'"
async def check_schema_status_force_bypass(ctx: AppContext) -> str:
"""force=True bypasses the state machine — useful when the agent
intentionally needs to step outside conventions."""
event_index = getattr(ctx, "_suite_event_index", None)
assert event_index is not None, "suite-event index not staged"
r = decode(await ctx.app.run_job(
"memory_property_update",
path=str(event_index),
key="status",
value="active",
force=True,
))
assert isinstance(r, dict), r
assert "error" not in r, r
# restore for any downstream checks
decode(await ctx.app.run_job(
"memory_property_update",
path=str(event_index), key="status", value="distilled", force=True,
))
return "force=True bypassed"
# ---------- ordered manifest -----------------------------------------
# (label, async fn) — runner executes top-to-bottom; later checks may
# rely on side effects from earlier ones (e.g. sync.append needs
# sync.create to have run).
CHECKS: list[tuple[str, callable]] = [
("registry", check_registry),
("memory_get", check_memory_get),
("memory_list", check_memory_list),
("memory_search", check_memory_search),
("memory_graph_search", check_memory_graph_search),
("memory_links", check_memory_links),
("memory_backlinks", check_memory_backlinks),
("memory_resolve_wikilink", check_memory_resolve_wikilink),
("memory_count_tokens", check_memory_count_tokens),
("memory_lint", check_memory_lint),
("sync.create", check_sync_create),
("sync.append", check_sync_append),
("sync.refuse_distilled", check_sync_refuse_distilled),
("memory_create", check_memory_create),
("memory_update", check_memory_update),
("memory_property_update", check_memory_property_update),
("memory_rename", check_memory_rename),
("memory_archive", check_memory_archive),
("memory_delete", check_memory_delete),
("schema.path_template_refuses", check_schema_path_template_refuses),
("schema.path_template_force", check_schema_path_template_force_bypass),
("schema.status_skip_refused", check_schema_status_skip_refused),
("schema.status_invalid_value", check_schema_status_invalid_value),
("schema.status_force", check_schema_status_force_bypass),
]

231
reme2/mcp/test/test_service.py Normal file
View file

@ -0,0 +1,231 @@
"""Service-profile MCP tests.
Covers the 3-tool service surface (`reme2/config/service.yaml`):
retrieve — graph-aware hybrid retrieval (memory_graph_search backend)
remember — single write entry point (Ingestor projection):
mode=log → zero-LLM event-folder upsert
mode=distill → LLM R-M-W (degraded path here, no LLM)
maintain — vault hygiene sweep (Maintainer projection)
Same shape as `test_expert.py`: each `check_*` is a one-job-or-sequence
async function returning a one-line summary. Ordering matters because
`remember.log_append` and `remember.log_refuse_distilled` rely on side
effects from `remember.log_create`.
"""
from __future__ import annotations
from pathlib import Path
from ._helpers import AppContext, decode, wait_for_index
EXPECTED_JOBS: tuple[str, ...] = ("retrieve", "remember", "maintain")
# ---------- registry ---------------------------------------------------
async def check_registry(ctx: AppContext) -> str:
missing = [j for j in EXPECTED_JOBS if j not in ctx.jobs]
assert not missing, f"missing jobs: {missing}"
extras = [j for j in ctx.jobs if j not in EXPECTED_JOBS]
assert not extras, (
f"curated profile leaked extra jobs: {extras} — keep the surface tight"
)
return f"{len(ctx.jobs)} jobs registered"
# ---------- read: retrieve (graph-aware hybrid) -----------------------
async def check_retrieve_basic(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"retrieve", query="Alice Bob", max_results=5, min_score=0.0,
))
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
assert len(hits) > 0, r
return f"{len(hits)} hits"
async def check_retrieve_anchored(ctx: AppContext) -> str:
"""Wikilink-anchored mode — `[[Project X]]` in the query seeds BFS."""
r = decode(await ctx.app.run_job(
"retrieve",
query="What touches [[Project X]]?",
max_results=5,
min_score=0.0,
graph_depth=1,
))
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
assert len(hits) > 0, r
paths = {h.get("path") for h in hits if isinstance(h, dict)}
# Project X's neighbors (Alice, Bob) should surface via 1-hop BFS
has_neighbor = any(p and ("Alice" in p or "Bob" in p) for p in paths)
assert has_neighbor, f"BFS didn't pull in [[Project X]] neighbors: {paths}"
return f"{len(hits)} hits incl. project-x neighbors"
async def check_retrieve_topic_seeded(ctx: AppContext) -> str:
"""Topic-rooted mode — explicit `seeds=[...]` instead of inline wikilink."""
project_x = ctx.abs_path("topics", "Project X", "Project X.md")
r = decode(await ctx.app.run_job(
"retrieve",
query="collaborates",
max_results=5,
min_score=0.0,
seeds=[project_x],
graph_depth=1,
))
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
assert len(hits) > 0, r
return f"{len(hits)} hits (seeded at Project X)"
# ---------- write: remember (mode=log: create / append / refusal) -----
async def check_remember_log_create(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"remember",
mode="log",
name="curated-event",
description="curated-profile suite",
content="## ops\n- ran the curated suite\n",
topics=["[[Alice]]"],
tags=["curated"],
materials=[
{"filename": "raw-prompt.md", "content": "# user prompt\n\nrun curated suite\n"},
{"filename": "tool-output.txt", "content": "exit=0\n"},
],
))
assert isinstance(r, dict), r
assert r.get("created") is True and r.get("action") == "created", r
materials = r.get("materials", [])
assert len(materials) == 2, materials
for m in materials:
assert Path(m).is_file(), f"material missing on disk: {m}"
event_dir = Path(materials[0]).parent
index_text = (event_dir / "curated-event.md").read_text(encoding="utf-8")
assert "## Materials" in index_text, "Materials footer absent"
assert "lifecycle: streaming" in index_text, "schema axis 'lifecycle' missing"
assert "role: observation" in index_text, "schema axis 'role' missing"
setattr(ctx, "_event_dir", event_dir)
setattr(ctx, "_event_index", event_dir / "curated-event.md")
await wait_for_index(ctx.watcher, expected_min=len(ctx.file_store))
return f"created folder w/ {len(materials)} materials"
async def check_remember_log_append(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"remember",
mode="log",
name="curated-event",
content="## follow-up\n- second pass\n",
topics=["[[Bob]]"],
tags=["follow-up"],
materials=[
{"filename": "tool-output.txt", "content": "second run\n"}, # collision
],
))
assert isinstance(r, dict), r
assert r.get("created") is False and r.get("action") == "appended", r
appended = r.get("materials", [])
names = {Path(p).name for p in appended}
assert "tool-output-2.txt" in names, f"collision auto-suffix failed: {names}"
index_text = getattr(ctx, "_event_index").read_text(encoding="utf-8")
assert "## Update —" in index_text, "Update section missing"
assert "[[Bob]]" in index_text, "topic union failed"
return f"appended w/ collision → {sorted(names)}"
async def check_remember_log_refuse_distilled(ctx: AppContext) -> str:
"""Curated profile lacks `memory_property_update`, so we flip status by
rewriting the file directly — same observable effect on remember(mode=log)."""
index_path = Path(getattr(ctx, "_event_index"))
text = index_path.read_text(encoding="utf-8")
text = text.replace("status: active", "status: distilled", 1)
index_path.write_text(text, encoding="utf-8")
# Give the watcher a moment to re-parse before sync re-reads frontmatter.
await wait_for_index(ctx.watcher, expected_min=len(ctx.file_store))
r = decode(await ctx.app.run_job(
"remember", mode="log", name="curated-event", content="should be refused",
))
assert isinstance(r, dict) and "error" in r, r
assert r.get("status") == "distilled", r
assert r.get("suggested_name"), r
return f"refused → suggested={r['suggested_name']!r}"
# ---------- write: remember (mode=distill, degraded path) -------------
async def check_remember_distill_degraded(ctx: AppContext) -> str:
target = ctx.abs_path("topics", "curated-ingested", "curated-ingested.md")
r = decode(await ctx.app.run_job(
"remember",
# mode defaults to "distill"
content="# curated-ingested\n\nproduced by the curated test suite.\n",
target_path=target,
metadata={
"title": "curated-ingested",
"lifecycle": "evolving",
"scope": "class",
"source": "curated",
"role": "concept",
"category": "concept",
},
))
assert isinstance(r, dict), r
applied = r.get("applied") or []
assert len(applied) == 1 and applied[0].get("ok") is True, r
assert r.get("used_llm") is False, "expected degraded path (no LLM)"
assert Path(target).is_file(), target
return f"applied=1, used_llm=False"
# ---------- maintain (lint + decay sweep) -----------------------------
async def check_maintain_dry_run(ctx: AppContext) -> str:
"""Default ops=[lint,decay] with dry_run=true (the parameter default)
surfaces the plan without mutating. The seeded vault has nothing
stale and no broken wikilinks, so we just verify the envelope."""
r = decode(await ctx.app.run_job("maintain"))
assert isinstance(r, dict), r
for key in ("ops_run", "scanned", "proposed", "plan", "applied", "dry_run"):
assert key in r, f"missing key {key!r} in audit: {r}"
assert r["dry_run"] is True, r
assert r["ops_run"] == ["lint", "decay"], r
assert r["applied"] == [], "dry_run should never apply"
return f"scanned={r['scanned']}, proposed={len(r['proposed'])}"
async def check_maintain_targeted(ctx: AppContext) -> str:
"""target_prefix narrows scan; events/ subtree should yield the
one event folder created earlier in this suite."""
r = decode(await ctx.app.run_job(
"maintain", target_prefix="events/", dry_run=True,
))
assert isinstance(r, dict), r
assert r["scanned"] >= 1, r # at least the suite's own event folder
return f"scanned={r['scanned']} under events/"
# ---------- ordered manifest -----------------------------------------
CHECKS: list[tuple[str, callable]] = [
("registry", check_registry),
("retrieve.basic", check_retrieve_basic),
("retrieve.anchored", check_retrieve_anchored),
("retrieve.topic_seeded", check_retrieve_topic_seeded),
("remember.log_create", check_remember_log_create),
("remember.log_append", check_remember_log_append),
("remember.log_refuse_distilled", check_remember_log_refuse_distilled),
("remember.distill_degraded", check_remember_distill_degraded),
("maintain.dry_run", check_maintain_dry_run),
("maintain.targeted", check_maintain_targeted),
]

2
reme2/memory/__init__.py
View file

@ -12,7 +12,7 @@ Per the architecture blueprint:
Lint, woken by cron or thresholds.
- summarizer.py Auxiliary used by the services.
Hot-write MCP step shells (sync, topic_create, memory_*) live in
Hot-write MCP step shells (sync, memory_*) live in
`reme2.mcp.steps`, NOT here — they bypass services and write MFS
directly. Importing this package triggers @R.register on the three
services so configs that name them resolve at boot.

51
reme2/memory/ingestor.py
View file

@ -5,10 +5,10 @@ write entry point** to the markdown vault. Every mutation (create, body
edit, frontmatter flip, rename, delete, archive) flows through here.
Mirrors `Summarizer`'s pattern — drives a `ReActAgent` whose toolkit is
built by `memory_io.build_memory_toolkit`. The agent runs its own R-M-W
loop: read related files via tools, decide which ones to mutate, call
the right write tool. Every write tool records into an audit list, so
the caller gets a deterministic mutation trail regardless of how the
built by `memory_toolkit.build_memory_toolkit`. The agent runs its own
R-M-W loop: read related files via tools, decide which ones to mutate,
call the right write tool. Every write tool records into an audit list,
so the caller gets a deterministic mutation trail regardless of how the
agent's reasoning unfolded.
When no LLM is configured, falls back to a direct create from
@ -30,7 +30,8 @@ from pydantic import BaseModel, Field
from ..component.runtime_response import _set_answer, _to_jsonable
from . import memory_io
from .memory_io import MemoryIO, write_create
from .memory_io import create_file
from .memory_toolkit import build_memory_toolkit
from ..component import R
from ..component.base_step import BaseStep
from ..enumeration import ComponentEnum
@ -84,6 +85,7 @@ class Ingestor(BaseStep):
self.toolkit = toolkit
self.console_enabled = console_enabled
self.timezone = timezone
self._protocol = (Path(__file__).parent / "protocol.md").read_text(encoding="utf-8")
def _now(self) -> datetime.datetime:
if self.timezone:
@ -102,6 +104,17 @@ class Ingestor(BaseStep):
async def execute(self):
assert self.context is not None
mode: str = (self.context.get("mode") or "distill").lower()
if mode == "log":
await self._delegate_to_sync()
return
if mode != "distill":
self.context.response.success = False
_set_answer(self.context, {
"error": f"unknown mode {mode!r}; expected 'log' or 'distill'",
})
return
content: str = self.context.get("content", "") or ""
hint: str = self.context.get("hint", "") or ""
target_path: str = self.context.get("target_path") or ""
@ -112,7 +125,7 @@ class Ingestor(BaseStep):
# Auto-discover wikilink targets in content as a hint for the agent.
for link in extract_wikilinks(content):
hit = memory_io.wikilink_lookup(self.file_store, link)["path"]
hit = memory_io.resolve_wikilink(self.file_store, link)["path"]
if hit and hit not in related_paths:
related_paths.append(hit)
@ -124,13 +137,17 @@ class Ingestor(BaseStep):
return
vault_root = self._vault_root()
mio = MemoryIO(self.file_store, vault_root)
toolkit = mio.register_all(self.toolkit)
audit: list[dict] = []
toolkit = build_memory_toolkit(self.app_context, audit=audit, toolkit=self.toolkit)
agent = ReActAgent(
name="reme_ingestor",
model=self.as_llm,
sys_prompt=self.prompt_format("system_prompt", vault_root=str(vault_root)),
sys_prompt=self.prompt_format(
"system_prompt",
vault_root=str(vault_root),
protocol=self._protocol,
),
formatter=self.as_llm_formatter,
toolkit=toolkit,
)
@ -153,9 +170,9 @@ class Ingestor(BaseStep):
summary = final_msg.get_text_content() or ""
result = IngestResult(used_llm=True)
for entry in mio.audit:
for entry in audit:
(result.applied if entry.get("ok") else result.failed).append(entry)
if not mio.audit and summary.strip().upper().startswith("SKIP"):
if not audit and summary.strip().upper().startswith("SKIP"):
result.skipped = True
self.context.response.success = result.success
@ -177,7 +194,7 @@ class Ingestor(BaseStep):
path = Path(target_path)
if not path.is_absolute():
path = self._vault_root() / path
ok, payload = write_create(
ok, payload = create_file(
self.file_store, path,
metadata=metadata, content=content,
)
@ -185,3 +202,13 @@ class Ingestor(BaseStep):
bucket.append({"op": "create", "ok": ok, "path": str(path), "result": payload})
return result
async def _delegate_to_sync(self) -> None:
"""Hot-path event-folder upsert — same code path as the standalone
`sync` step. Lazy-instantiated so we don't pay the construction
cost on every distill call."""
from ..mcp.steps.sync import Sync
if getattr(self, "_sync_step", None) is None:
self._sync_step = Sync(app_context=self.app_context)
await self._sync_step(self.context)

61
reme2/memory/ingestor.yaml
View file

@ -24,68 +24,19 @@ system_prompt: |
indexes, individual material files, candidate topics it flagged
for update). Both forms can appear together; treat them as a
single working set.
- An event is a FOLDER containing the index `{name}.md` plus
- An event is a FOLDER containing the index `{{name}}.md` plus
materials (raw conversation snippets, tool outputs, data dumps).
Whenever a path in `related_paths` is an event index, `memory_get`
the index first, then `memory_get` any materials whose content you
need (the index lists them under `## Materials`).
# Vault conventions
- Topics live under `topics/{{folder}}/{{name}}.md`. A folder topic
has folder == name.
- Events live under `events/{{YYYY-MM-DD}}/{{name}}/{{name}}.md` —
this is an INDEX inside a folder; sibling files are materials.
- Frontmatter is YAML; `category`, `created`, `updated`, `tags`,
`status` are common.
- Cross-file references use `[[wikilink]]` syntax (stem-form `[[X]]`
or path-form `[[topics/X/X]]`).
# Memory protocol (single source — schema, tools, decision rules)
# Available tools
Read tools (use these to gather context BEFORE writing):
- memory_get(path, include_chunks=False): full file content + frontmatter.
Call this on each event index AND on the materials it lists.
- memory_list(path_prefix=None, tags=None, metadata=None, limit=100):
list indexed files filtered by prefix / tags / frontmatter.
- memory_resolve_wikilink(wikilink): resolve `[[X]]` to a path.
- memory_backlinks(path): files linking to a given path.
- memory_links(path): files a given path links to.
{protocol}
Write tools (each returns success + payload; mutations are SSOT-routed):
- memory_update(path, old_string, new_string, replace_all=False):
body edit by exact-string substitution. Use a tail snippet to append.
- memory_property_update(path, key, value): change one frontmatter
key (value=null deletes it). After distilling an event into one or
more topics, flip that event's status to "distilled" with this.
- memory_create(path, metadata, content, overwrite=False, force=False):
new file. Reserve for genuinely NEW topics — do NOT use this to log
events; `sync` (deterministic) owns events. ALL paths must
be ABSOLUTE under vault_root.
- memory_rename(old_path, new_path): move file + rewrite cross-vault
wikilinks.
- memory_delete(path): remove a file.
- memory_archive(path): flip `status: archived` and move under
`<vault>/Archive/`.
# Decision rules
1. If material is ALREADY covered by existing topics → reply with a
single line `SKIP: <one-line reason>` and call no tools.
2. If material CONTRADICTS an existing block → memory_update with a
unique snippet of the outdated text and the corrected replacement.
3. If material EXTENDS an existing topic → memory_update using a
unique TAIL snippet of the existing body, with new_string =
tail + blank line + new content.
4. If material warrants a GENUINELY NEW topic → memory_create at
`topics/{{folder}}/{{name}}.md`. Do NOT memory_create under events/.
5. After integrating an event's content into a topic, flip that event's
status to "distilled" with memory_property_update.
6. Never delete unless the material explicitly asks for deletion.
7. Keep edits minimal — read related files first, edit only what
must change.
8. Always include reasonable frontmatter on memory_create — at minimum
`title`, `category`, `created`, `updated`. Use today's date for
`created` / `updated`.
9. After all writes, end with a one-paragraph summary of what you did
and why.
# Final step
After all writes, end with a one-paragraph summary of what you did
and why.
user_message: |
# CONTEXT

59
reme2/memory/maintainer.py
View file

@ -63,8 +63,8 @@ from ..component import R
from ..component.base_step import BaseStep
from ..component.runtime_response import _set_answer
from ..enumeration import ComponentEnum
from ..schema.vault.registry import schema_for
from . import memory_io
from .schema import parse_frontmatter
# ---------------------------------------------------------------------------
@ -122,11 +122,19 @@ class FileSignal(BaseModel):
from `file_store.nodes` + edge index — no body reads, no LLM calls.
Heavier signals (token counts, embeddings) are pulled lazily inside
the proposers that actually need them.
The 4 schema axes (`lifecycle / scope / source / role`) are the
primary drivers of decay / merge / split heuristics; legacy
`category` is preserved for back-compat reads only.
"""
path: str
relpath: str = "" # path relative to vault_root, "" if outside
category: str = ""
lifecycle: str = "" # streaming / evolving / frozen
scope: str = "" # instance / class
source: str = "" # auto / curated / derived
role: str = "" # observation / claim / question / ...
category: str = "" # legacy field, kept for migration windows
status: str = ""
age_days: int = 0
metadata: dict = Field(default_factory=dict)
@ -262,7 +270,14 @@ class Maintainer(BaseStep):
return Path(getattr(watcher, "watch_path", ".")).resolve()
def _scan_signals(self, *, target_prefix: str = "") -> list[FileSignal]:
"""One walk over the indexed files. Cheap signals only."""
"""One walk over the indexed files. Cheap signals only.
Each frontmatter is run through `parse_frontmatter` so the legacy
`category` field is auto-translated to the 4 axes (see
`LEGACY_AXES_FROM_CATEGORY` in `memory.schema.memory`). Files that
fail to parse still get a signal — proposers can use the empty
axes to decide whether to ignore or surface them.
"""
vault_root = self._vault_root()
now = datetime.datetime.now().timestamp()
signals: list[FileSignal] = []
@ -277,9 +292,14 @@ class Maintainer(BaseStep):
continue
fm = meta.metadata or {}
age_seconds = max(0.0, now - (meta.st_mtime or now))
parsed, _ = parse_frontmatter(fm)
signals.append(FileSignal(
path=path,
relpath=relpath,
lifecycle=str(parsed.lifecycle.value) if parsed else "",
scope=str(parsed.scope.value) if parsed else "",
source=str(parsed.source.value) if parsed else "",
role=str(parsed.role.value) if parsed else "",
category=str(fm.get("category") or ""),
status=str(fm.get("status") or ""),
age_days=int(age_seconds // 86400),
@ -295,22 +315,21 @@ class Maintainer(BaseStep):
out: list[LintFinding] = []
for sig in signals:
for link in sig.declared_topics:
if not memory_io.wikilink_lookup(self.file_store, link)["exists"]:
if not memory_io.resolve_wikilink(self.file_store, link)["exists"]:
out.append(LintFinding(
path=sig.path, kind="broken_wikilink",
detail=f"unresolved wikilink {link!r}",
))
cls = schema_for(sig.category)
if cls is not None:
try:
cls(**sig.metadata)
except Exception as e:
out.append(LintFinding(
path=sig.path, kind="schema_violation",
detail=f"{cls.__name__}: {type(e).__name__}: {e}"[:240],
))
# Memory schema check: tolerant parse, surface every error
# the parser collected. Empty `errors` ↔ valid frontmatter.
_, errors = parse_frontmatter(sig.metadata)
for err in errors:
out.append(LintFinding(
path=sig.path, kind="schema_violation",
detail=f"Memory: {err}"[:240],
))
# Stem collisions: the engine API exposes the ambiguous-stem map.
ambig = memory_io.all_ambiguous_wikilinks(self.file_store)
ambig = memory_io.find_collisions(self.file_store)
for stem, paths in ambig.items():
for p in paths:
out.append(LintFinding(
@ -322,10 +341,16 @@ class Maintainer(BaseStep):
def _propose_decay(
self, signals: list[FileSignal], decay_days: int,
) -> list[DecayOp]:
"""Distilled events past the freshness window."""
"""Distilled streaming memories past the freshness window.
Schema-driven: a memory decays when its `lifecycle` is `streaming`
(write-once, decays after a freshness window) AND its `status` has
already been flipped to the configured terminal target (default:
`distilled`). Evolving / frozen memories never decay.
"""
out: list[DecayOp] = []
for sig in signals:
if sig.category != "event":
if sig.lifecycle != "streaming":
continue
if sig.status != self.target_status:
continue
@ -333,7 +358,7 @@ class Maintainer(BaseStep):
continue
out.append(DecayOp(
path=sig.path, age_days=sig.age_days,
reason=f"event {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)",
reason=f"streaming {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)",
))
return out

771
reme2/memory/memory_io.py
View file

@ -8,24 +8,38 @@ is layered:
This module is the **single public API surface** over that engine. Every
consumer — MCP step shells, the three memory services (Retriever,
Ingestor, Maintainer), and the agent toolkit — talks to the engine
through these functions, not by reaching into `BaseFileStore` directly.
That keeps `file_store` an implementation detail (could be local sqlite,
remote, etc.) and gives the layering one place to evolve.
Ingestor, Maintainer), and the agent toolkit (`memory_toolkit`) — talks
to the engine through these functions, not by reaching into
`BaseFileStore` directly. That keeps `file_store` an implementation
detail (could be local sqlite, remote, etc.) and gives the layering one
place to evolve.
Four sections:
Naming convention:
- Verb-first: `get_file`, `create_file`, `search_vector`.
- `file_store` is always the first positional argument when the
function needs the engine handle; remaining args are keyword-only.
- Pure-disk writes (`update_body`, `update_meta`, `delete_file`,
`archive_file`) don't take `file_store` — they hit the filesystem
and the watcher picks them up. The asymmetry is honest.
1. CRUD writes — write_create / delete / update / property_update
/ rename / archive. The MFS write entry.
2. MFS reads — read_file / list_files / links_of / backlinks_of
/ wikilink_lookup / count_tokens / iter_files.
Primary-key lookups against the file_store cache.
3. Projections — vector_search / keyword_search / expand_neighbors
/ extract_anchors / chunks_by_paths / make_chunk_filter
/ all_ambiguous_wikilinks. The read entry to derived
indexes (composed by Retriever into V+K+graph fusion).
4. Toolkit — `MemoryIO` class wrapping the read/write helpers as
`agentscope.tool` callables for ReActAgent (Ingestor).
Three sections:
1. MFS Reads — get_file / list_files / get_links / get_backlinks
/ resolve_wikilink / iter_files / count_tokens.
Primary-key lookups against the file_store cache
(with disk fallthrough for the file body).
2. MFS Writes — create_file / update_body / update_meta
/ rename_file / delete_file / archive_file.
The MFS write entry. All return (ok, payload).
3. Projection Queries — search_vector / search_keyword / expand_neighbors
/ extract_anchors / get_chunks / make_filter
/ find_collisions. Read-only — projections
are derived by the Watcher; callers don't
write them directly.
Schema policy and the agent toolkit projection live one layer up in
`reme2/memory/memory_toolkit.py` — this file is policy-free and remains
the engine's outward API.
Lives in `reme2/memory/` (not `reme2/mcp/`) so memory services and the
MCP transport layer can both consume it without forming an import cycle
@ -34,29 +48,224 @@ through the transport layer.
from __future__ import annotations
import json
import re
import shutil
from collections.abc import Iterable, Iterator
from pathlib import Path
from typing import Any
import frontmatter
from agentscope.message import TextBlock
from agentscope.tool import Toolkit, ToolResponse
from ..component.runtime_response import _to_jsonable
from ..schema import ChunkFilter, FileChunk, FileMetadata
from ..utils.wikilink import WIKILINK_RE
# ===========================================================================
# Section 1 — CRUD writes
# Section 1 — MFS Reads
# ===========================================================================
#
# Primary-key lookups against the file_store's in-memory cache (file
# meta + edges + stem index). `get_file` falls through to disk for the
# body so callers see the latest text even if the watcher hasn't picked
# up a write yet.
async def get_file(
file_store,
path: str,
*,
include_chunks: bool = False,
) -> dict:
"""Read frontmatter + body for one path. Optionally include parsed chunks.
On-disk frontmatter is the source of truth — the file_store cache may
lag a write that hasn't been picked up by the watcher yet.
"""
meta = file_store.get_file_meta(path)
result: dict = {"path": path, "exists": False}
if meta is not None:
edges = file_store.get_edges(path)
result.update({
"exists": True,
"metadata": meta.metadata,
"link": [e.model_dump(exclude_none=True) for e in edges],
})
file_path = Path(path)
if file_path.is_file():
raw = file_path.read_text(encoding="utf-8")
post = frontmatter.loads(raw)
result["exists"] = True
result["content"] = post.content
result["metadata"] = dict(post.metadata)
if include_chunks:
chunks = await file_store.get_chunks(path)
result["chunks"] = [c.model_dump(exclude_none=True) for c in chunks]
return result
def list_files(
file_store,
*,
path_prefix: str | None = None,
tags: list[str] | None = None,
metadata: dict | None = None,
limit: int = 100,
) -> dict:
"""List indexed files filtered by frontmatter exact-match, tags, and prefix.
Returns {items: [{path, metadata}], count}.
"""
metadata_filter = metadata or {}
tag_filter = tags or []
items: list[dict] = []
for path, meta in file_store.nodes.items():
if path_prefix and not path.startswith(path_prefix):
continue
md = meta.metadata or {}
if metadata_filter and any(md.get(k) != v for k, v in metadata_filter.items()):
continue
if tag_filter:
file_tags = set(md.get("tags", []) or [])
if not all(t in file_tags for t in tag_filter):
continue
items.append({"path": path, "metadata": md})
if len(items) >= limit:
break
return {"items": items, "count": len(items)}
def _edge_to_dict(file_meta, edge) -> dict:
return {
"path": file_meta.path,
"metadata": file_meta.metadata,
"predicate": edge.predicate,
"anchor": edge.anchor,
"alias": edge.alias,
"embed": edge.embed,
"source": edge.source,
"confidence": edge.confidence,
}
def get_links(file_store, path: str) -> dict:
"""Files that `path` links TO (resolved). Each entry carries the typed-edge predicate."""
return {
"path": path,
"links": [_edge_to_dict(m, e) for m, e in file_store.get_links(path)],
}
def get_backlinks(file_store, path: str) -> dict:
"""Files that link TO `path`. Each entry carries the typed-edge predicate."""
return {
"path": path,
"backlinks": [_edge_to_dict(m, e) for m, e in file_store.get_backlinks(path)],
}
def resolve_wikilink(file_store, wikilink: str) -> dict:
"""Resolve a `[[target]]` wikilink with full ambiguity context.
Distinct from `file_store.resolve_wikilink(target)` (which returns
just the path or None) — this surface returns the rich payload
callers need to disambiguate:
unique resolution → {wikilink, path, exists: True,
ambiguous: False, candidates: [path]}
ambiguous → {wikilink, path: None, exists: False,
ambiguous: True, candidates: [...]}
dangling → {wikilink, path: None, exists: False,
ambiguous: False, candidates: []}
"""
# Path-form (`a/b` or `a/b.md`): file_store already returns
# exactly the one path that exists, or None.
if "/" in wikilink or wikilink.endswith(".md"):
hit = file_store.resolve_wikilink(wikilink)
return {
"wikilink": wikilink,
"path": hit,
"exists": hit is not None,
"ambiguous": False,
"candidates": [hit] if hit else [],
}
# Stem-form: candidates list reveals 0/1/N resolution.
candidates = file_store.wikilink_candidates(wikilink)
if len(candidates) == 1:
return {
"wikilink": wikilink,
"path": candidates[0],
"exists": True,
"ambiguous": False,
"candidates": candidates,
}
return {
"wikilink": wikilink,
"path": None,
"exists": False,
"ambiguous": len(candidates) > 1,
"candidates": candidates,
}
def iter_files(file_store) -> Iterator[tuple[str, FileMetadata]]:
"""Walk every indexed (path, FileMetadata). Used by Maintainer scans.
Equivalent to `file_store.nodes.items()`, exposed here so consumers
don't have to know about the underlying cache attribute name.
"""
return iter(file_store.nodes.items())
async def count_tokens(
token_counter,
*,
path: str | None = None,
text: str | None = None,
) -> dict:
"""Estimate tokens for a file body (frontmatter excluded) or raw text.
Powers the Maintainer's split-trigger. Exactly one of `path` / `text`
must be provided. Takes a `token_counter` rather than `file_store`
because the engine's tokenization is a separate component — this
function lives here to keep the engine's outward surface in one place.
"""
if path:
target = Path(path)
if not target.is_file():
return {"path": str(target), "error": "file not found"}
raw = target.read_text(encoding="utf-8")
post = frontmatter.loads(raw)
body = post.content
tokens = await token_counter.count(messages=[], text=body)
return {
"source": "file",
"path": str(target.resolve()),
"tokens": tokens,
"body_chars": len(body),
}
if text:
tokens = await token_counter.count(messages=[], text=text)
return {
"source": "text",
"tokens": tokens,
"body_chars": len(text),
}
return {"error": "one of `path` or `text` is required"}
# ===========================================================================
# Section 2 — MFS Writes
# ===========================================================================
#
# Every mutation in the system funnels through these. Hot-write MCP shells
# (sync, topic_create, memory_*) call them directly; cold-write services
# (sync, memory_*) call them directly; cold-write services
# (Ingestor R-M-W, Maintainer decay) compose them.
#
# `create_file` and `rename_file` take `file_store` because they need the
# wikilink-uniqueness gate / backlinks index. The other writes don't —
# they just touch disk and let the Watcher catch up.
def _replace_wikilink_targets(text: str, mapping: dict[str, str]) -> str:
@ -79,9 +288,10 @@ def _replace_wikilink_targets(text: str, mapping: dict[str, str]) -> str:
return WIKILINK_RE.sub(sub, text)
def write_create(
def create_file(
file_store,
path: Path,
*,
metadata: dict,
content: str,
overwrite: bool = False,
@ -124,17 +334,9 @@ def write_create(
return True, {"path": str(path), "created": True}
def write_delete(path: Path | str) -> tuple[bool, dict]:
"""Delete a file. Watcher removes from store + graph."""
target = Path(path)
if not target.exists():
return False, {"path": str(target), "error": "not found"}
target.unlink()
return True, {"path": str(target), "deleted": True}
def write_update(
def update_body(
path: Path | str,
*,
old_string: str,
new_string: str,
replace_all: bool = False,
@ -146,7 +348,7 @@ def write_update(
if not old_string:
return False, {
"path": str(target),
"error": "old_string is required (use write_create to write a new file)",
"error": "old_string is required (use create_file to write a new file)",
}
raw = target.read_text(encoding="utf-8")
occurrences = raw.count(old_string)
@ -169,7 +371,7 @@ def write_update(
}
def write_property_update(path: Path | str, key: str, value) -> tuple[bool, dict]:
def update_meta(path: Path | str, *, key: str, value) -> tuple[bool, dict]:
"""Update a single YAML frontmatter key. value=None deletes the key."""
target = Path(path)
if not target.is_file():
@ -184,9 +386,10 @@ def write_property_update(path: Path | str, key: str, value) -> tuple[bool, dict
return True, {"path": str(target), "key": key, "value": value}
def write_rename(
def rename_file(
file_store,
vault_root: Path | str,
*,
old_path: Path | str,
new_path: Path | str,
) -> tuple[bool, dict]:
@ -275,10 +478,20 @@ def write_rename(
}
def write_archive(
def delete_file(path: Path | str) -> tuple[bool, dict]:
"""Delete a file. Watcher removes from store + graph."""
target = Path(path)
if not target.exists():
return False, {"path": str(target), "error": "not found"}
target.unlink()
return True, {"path": str(target), "deleted": True}
def archive_file(
vault_root: Path | str,
path: Path | str,
archive_dir_name: str = "Archive",
*,
archive_dir: str = "Archive",
) -> tuple[bool, dict]:
"""Archive a file: flip `status: archived`, then move under `<vault>/<archive_dir>/`.
@ -299,16 +512,16 @@ def write_archive(
"error": f"path is outside vault_root {vault}",
}
dst = vault / archive_dir_name / rel
dst = vault / archive_dir / rel
if dst.exists():
return False, {
"path": str(src),
"error": f"archive destination already exists: {dst}",
}
ok, prop_payload = write_property_update(src, "status", "archived")
ok, prop_payload = update_meta(src, key="status", value="archived")
if not ok:
return False, {**prop_payload, "stage": "property_update"}
return False, {**prop_payload, "stage": "update_meta"}
dst.parent.mkdir(parents=True, exist_ok=True)
try:
@ -327,213 +540,24 @@ def write_archive(
# ===========================================================================
# Section 2 — MFS reads (primary-key lookups against the file_store cache)
# ===========================================================================
#
# These don't touch the projection indexes — they hit the in-memory file
# meta + edge cache that the file_store maintains as a mirror of disk.
# For body / chunk content reads, `read_file` does fall through to the
# disk to get the latest text (file_store cache may lag a write).
async def read_file(
file_store,
path: str,
*,
include_chunks: bool = False,
) -> dict:
"""Read frontmatter + body for one path. Optionally include parsed chunks.
On-disk frontmatter is the source of truth — the file_store cache may
lag a write that hasn't been picked up by the watcher yet.
"""
meta = file_store.get_file_meta(path)
result: dict = {"path": path, "exists": False}
if meta is not None:
edges = file_store.get_edges(path)
result.update({
"exists": True,
"metadata": meta.metadata,
"link": [e.model_dump(exclude_none=True) for e in edges],
})
file_path = Path(path)
if file_path.is_file():
raw = file_path.read_text(encoding="utf-8")
post = frontmatter.loads(raw)
result["exists"] = True
result["content"] = post.content
result["metadata"] = dict(post.metadata)
if include_chunks:
chunks = await file_store.get_chunks(path)
result["chunks"] = [c.model_dump(exclude_none=True) for c in chunks]
return result
def list_files(
file_store,
*,
path_prefix: str | None = None,
tags: list[str] | None = None,
metadata: dict | None = None,
limit: int = 100,
) -> dict:
"""List indexed files filtered by frontmatter exact-match, tags, and prefix.
Returns {items: [{path, metadata}], count}.
"""
metadata_filter = metadata or {}
tag_filter = tags or []
items: list[dict] = []
for path, meta in file_store.nodes.items():
if path_prefix and not path.startswith(path_prefix):
continue
md = meta.metadata or {}
if metadata_filter and any(md.get(k) != v for k, v in metadata_filter.items()):
continue
if tag_filter:
file_tags = set(md.get("tags", []) or [])
if not all(t in file_tags for t in tag_filter):
continue
items.append({"path": path, "metadata": md})
if len(items) >= limit:
break
return {"items": items, "count": len(items)}
def _edge_to_dict(file_meta, edge) -> dict:
return {
"path": file_meta.path,
"metadata": file_meta.metadata,
"predicate": edge.predicate,
"anchor": edge.anchor,
"alias": edge.alias,
"embed": edge.embed,
"source": edge.source,
"confidence": edge.confidence,
}
def links_of(file_store, path: str) -> dict:
"""Files that `path` links TO (resolved). Each entry carries the typed-edge predicate."""
return {
"path": path,
"links": [_edge_to_dict(m, e) for m, e in file_store.get_links(path)],
}
def backlinks_of(file_store, path: str) -> dict:
"""Files that link TO `path`. Each entry carries the typed-edge predicate."""
return {
"path": path,
"backlinks": [_edge_to_dict(m, e) for m, e in file_store.get_backlinks(path)],
}
def wikilink_lookup(file_store, wikilink: str) -> dict:
"""Resolve a `[[target]]` wikilink with full ambiguity context.
Distinct from `file_store.resolve_wikilink(target)` (which returns
the single resolved path or None) — this surface returns the rich
payload callers need to disambiguate:
unique resolution → {wikilink, path, exists: True,
ambiguous: False, candidates: [path]}
ambiguous → {wikilink, path: None, exists: False,
ambiguous: True, candidates: [...]}
dangling → {wikilink, path: None, exists: False,
ambiguous: False, candidates: []}
"""
# Path-form (`a/b` or `a/b.md`): file_store already returns
# exactly the one path that exists, or None.
if "/" in wikilink or wikilink.endswith(".md"):
hit = file_store.resolve_wikilink(wikilink)
return {
"wikilink": wikilink,
"path": hit,
"exists": hit is not None,
"ambiguous": False,
"candidates": [hit] if hit else [],
}
# Stem-form: candidates list reveals 0/1/N resolution.
candidates = file_store.wikilink_candidates(wikilink)
if len(candidates) == 1:
return {
"wikilink": wikilink,
"path": candidates[0],
"exists": True,
"ambiguous": False,
"candidates": candidates,
}
return {
"wikilink": wikilink,
"path": None,
"exists": False,
"ambiguous": len(candidates) > 1,
"candidates": candidates,
}
async def count_tokens(
token_counter,
*,
path: str | None = None,
text: str | None = None,
) -> dict:
"""Estimate tokens for a file body (frontmatter excluded) or raw text.
Powers the Maintainer's split-trigger. Exactly one of `path` / `text`
must be provided.
"""
if path:
target = Path(path)
if not target.is_file():
return {"path": str(target), "error": "file not found"}
raw = target.read_text(encoding="utf-8")
post = frontmatter.loads(raw)
body = post.content
tokens = await token_counter.count(messages=[], text=body)
return {
"source": "file",
"path": str(target.resolve()),
"tokens": tokens,
"body_chars": len(body),
}
if text:
tokens = await token_counter.count(messages=[], text=text)
return {
"source": "text",
"tokens": tokens,
"body_chars": len(text),
}
return {"error": "one of `path` or `text` is required"}
def iter_files(file_store) -> Iterator[tuple[str, FileMetadata]]:
"""Walk every indexed (path, FileMetadata). Used by Maintainer scans.
Equivalent to `file_store.nodes.items()`, exposed here so consumers
don't have to know about the underlying cache attribute name.
"""
return iter(file_store.nodes.items())
# ===========================================================================
# Section 3 — Projection queries (Vector / FTS / File Graph)
# Section 3 — Projection Queries
# ===========================================================================
#
# Per `structure.md` §"核心引擎", the Vector index, FTS5 index, and File
# Graph are downstream **projections** of the MFS — they can be wholly
# rebuilt from disk. These functions are the read entry to those
# projections; `Retriever` composes them with policy (V+K weighting,
# projections; the Retriever composes them with policy (V+K weighting,
# graph BFS, intent routing) for ranked retrieval.
#
# Projections are write-only-via-Watcher: the API surface deliberately
# exposes no setters. To "update" a projection, write to the MFS (Section
# 2) and let the Watcher rebuild.
async def vector_search(
async def search_vector(
file_store,
query: str,
*,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
@ -541,9 +565,10 @@ async def vector_search(
return await file_store.vector_search(query, limit, chunk_filter)
async def keyword_search(
async def search_keyword(
file_store,
query: str,
*,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
@ -567,12 +592,12 @@ def extract_anchors(file_store, text: str) -> list[str]:
return file_store.extract_anchor_paths(text)
async def chunks_by_paths(file_store, paths: Iterable[str]) -> list[FileChunk]:
async def get_chunks(file_store, paths: Iterable[str]) -> list[FileChunk]:
"""Batch fetch chunks across many paths (used by graph-walk retrieval)."""
return await file_store.get_chunks_by_paths(paths)
def make_chunk_filter(
def make_filter(
file_store,
*,
paths: list[str] | None = None,
@ -583,267 +608,7 @@ def make_chunk_filter(
return file_store.filter(paths=paths, tags=tags, exclude_paths=exclude_paths)
def all_ambiguous_wikilinks(file_store) -> dict[str, list[str]]:
def find_collisions(file_store) -> dict[str, list[str]]:
"""Every stem that resolves to >1 path. Used by Maintainer.lint."""
return file_store.all_ambiguous_wikilinks()
# ===========================================================================
# Section 4 — Agent toolkit
# ===========================================================================
#
# `MemoryIO` adapts the read/write helpers above as `agentscope.tool`
# callables, with a vault-root containment check on writes and an audit
# trail every consumer can inspect after the agent's run. Used by the
# Ingestor's ReActAgent.
def _text_response(payload: Any) -> ToolResponse:
text = json.dumps(_to_jsonable(payload), ensure_ascii=False, indent=2)
return ToolResponse(content=[TextBlock(type="text", text=text)])
class MemoryIO:
"""Agent-facing tool surface over the Memory File System.
Args:
file_store: The vault's FileStore (provides graph + index).
vault_root: Containment boundary — write tools refuse paths
that escape it.
Attributes:
audit: Every successful or failed write is appended here. Read
this after the agent's run to reconstruct the mutation trail.
"""
_TOOL_NAMES = (
"memory_get",
"memory_list",
"memory_resolve_wikilink",
"memory_backlinks",
"memory_links",
"memory_create",
"memory_update",
"memory_property_update",
"memory_rename",
"memory_delete",
"memory_archive",
)
def __init__(self, file_store, vault_root: str | Path):
self.file_store = file_store
self.vault_root = Path(vault_root).resolve()
self.audit: list[dict] = []
def register_all(self, toolkit: Toolkit | None = None) -> Toolkit:
"""Register every memory_* method on `toolkit` (or a fresh one)."""
toolkit = toolkit or Toolkit()
for name in self._TOOL_NAMES:
toolkit.register_tool_function(
getattr(self, name), namesake_strategy="override",
)
return toolkit
# -- Internals --------------------------------------------------------
def _resolve(self, p: str) -> Path:
pp = Path(p)
if not pp.is_absolute():
pp = self.vault_root / pp
return pp.resolve()
def _under_vault(self, p: Path) -> bool:
try:
p.relative_to(self.vault_root)
return True
except (ValueError, OSError):
return False
def _record(self, op: str, ok: bool, **fields) -> dict:
entry = {"op": op, "ok": ok, **fields}
self.audit.append(entry)
return entry
# -- Read tools (delegate to module-level helpers) --------------------
async def memory_get(self, path: str, include_chunks: bool = False) -> ToolResponse:
"""Read a memory file (frontmatter + body, optional chunks).
Args:
path (str): Absolute path to the file.
include_chunks (bool): Include parsed chunk metadata.
"""
target = self._resolve(path)
result = await read_file(self.file_store, str(target), include_chunks=include_chunks)
return _text_response(result)
async def memory_list(
self,
path_prefix: str | None = None,
tags: list[str] | None = None,
metadata: dict | None = None,
limit: int = 100,
) -> ToolResponse:
"""List indexed vault files filtered by prefix, tags, and frontmatter.
Args:
path_prefix (str | None): Restrict to paths starting with this prefix.
tags (list[str] | None): All tags must be present on a file.
metadata (dict | None): Exact-match filter on frontmatter keys.
limit (int): Cap on returned items.
"""
return _text_response(list_files(
self.file_store,
path_prefix=path_prefix, tags=tags,
metadata=metadata, limit=limit,
))
async def memory_resolve_wikilink(self, wikilink: str) -> ToolResponse:
"""Resolve a `[[wikilink]]` to an absolute path.
Args:
wikilink (str): The wikilink target, e.g. `Topic` or `topics/Topic`.
"""
return _text_response(wikilink_lookup(self.file_store, wikilink))
async def memory_backlinks(self, path: str) -> ToolResponse:
"""List files linking TO a given path.
Args:
path (str): Absolute path to inspect.
"""
target = self._resolve(path)
return _text_response(backlinks_of(self.file_store, str(target)))
async def memory_links(self, path: str) -> ToolResponse:
"""List files a given path links to.
Args:
path (str): Absolute path to inspect.
"""
target = self._resolve(path)
return _text_response(links_of(self.file_store, str(target)))
# -- Write tools (delegate to write_*; record audit) ------------------
async def memory_create(
self,
path: str,
metadata: dict | None = None,
content: str = "",
overwrite: bool = False,
force: bool = False,
) -> ToolResponse:
"""Create a new markdown file in the vault.
Args:
path (str): Target path (resolved against vault_root if relative).
metadata (dict | None): YAML frontmatter for the new file.
content (str): Body markdown.
overwrite (bool): Allow overwriting an existing file.
force (bool): Allow creates that introduce stem-form wikilink ambiguity.
"""
target = self._resolve(path)
if not self._under_vault(target):
entry = self._record("create", False, path=str(target),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_create(
self.file_store, target,
metadata=dict(metadata or {}), content=content,
overwrite=overwrite, force=force,
)
entry = self._record("create", ok, path=str(target), result=payload)
return _text_response(entry)
async def memory_update(
self,
path: str,
old_string: str,
new_string: str,
replace_all: bool = False,
) -> ToolResponse:
"""Edit a file body by exact-string substitution.
Use a unique snippet for `old_string`. To append, pass the file's
tail as `old_string` and `tail + new_content` as `new_string`.
Args:
path (str): Absolute path to the file.
old_string (str): Exact text to replace (must be unique unless replace_all).
new_string (str): Replacement text.
replace_all (bool): Replace every occurrence instead of just one.
"""
target = self._resolve(path)
if not self._under_vault(target):
entry = self._record("update", False, path=str(target),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_update(target, old_string, new_string, replace_all=replace_all)
entry = self._record("update", ok, path=str(target), result=payload)
return _text_response(entry)
async def memory_property_update(self, path: str, key: str, value: Any = None) -> ToolResponse:
"""Update one YAML frontmatter key on a file (value=null deletes it).
Args:
path (str): Absolute path to the file.
key (str): Frontmatter key.
value: New value, or null to delete the key.
"""
target = self._resolve(path)
if not self._under_vault(target):
entry = self._record("property_update", False, path=str(target),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_property_update(target, key, value)
entry = self._record("property_update", ok, path=str(target), result=payload)
return _text_response(entry)
async def memory_rename(self, old_path: str, new_path: str) -> ToolResponse:
"""Rename a file and rewrite cross-vault wikilinks.
Args:
old_path (str): Current absolute path.
new_path (str): Target absolute path.
"""
old_p = self._resolve(old_path)
new_p = self._resolve(new_path)
if not self._under_vault(old_p) or not self._under_vault(new_p):
entry = self._record("rename", False, old_path=str(old_p), new_path=str(new_p),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_rename(self.file_store, self.vault_root, old_p, new_p)
entry = self._record("rename", ok, old_path=str(old_p), new_path=str(new_p), result=payload)
return _text_response(entry)
async def memory_delete(self, path: str) -> ToolResponse:
"""Delete a file from the vault.
Args:
path (str): Absolute path to the file.
"""
target = self._resolve(path)
if not self._under_vault(target):
entry = self._record("delete", False, path=str(target),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_delete(target)
entry = self._record("delete", ok, path=str(target), result=payload)
return _text_response(entry)
async def memory_archive(self, path: str, archive_dir: str = "Archive") -> ToolResponse:
"""Flip `status: archived` and move file under `<vault>/<archive_dir>/`.
Args:
path (str): Absolute path to the file.
archive_dir (str): Subdirectory name under vault_root for archives.
"""
target = self._resolve(path)
if not self._under_vault(target):
entry = self._record("archive", False, path=str(target),
error=f"path is outside vault_root {self.vault_root}")
return _text_response(entry)
ok, payload = write_archive(self.vault_root, target, archive_dir)
entry = self._record("archive", ok, path=str(target), result=payload)
return _text_response(entry)

598
reme2/memory/memory_toolkit.py Normal file
View file

@ -0,0 +1,598 @@
"""Memory toolkit — schema-bound projection of the Memory File System.
Layered on top of `reme2.memory.memory_io` (the pure core engine). One
`BaseStep` subclass per memory_* tool, each exposing TWO class methods:
* `execute()` — the MCP surface. Reads parameters from
`RuntimeContext` and writes the result through `_set_answer`.
* a method named after the tool (e.g. `memory_get`) — the agent
toolkit surface. Takes explicit parameters, returns a
`ToolResponse`. agentscope's `Toolkit.register_tool_function`
introspects the signature directly — no hand-authored JSON schema.
`build_memory_toolkit(app_context, audit, toolkit)` instantiates every
registered memory_* step and binds its tool method to a `Toolkit`.
Each instance carries an `audit` list so the host can surface what the
agent actually called.
Schema policy (status state machine, path templates) lives at the top
of this file as pure helpers; the relevant write tools (`memory_create`
/ `memory_property_update`) call them. `force=True` is the single
escape hatch — bypasses BOTH the policy gates here and the
wikilink-uniqueness gate downstream in `memory_io.create_file`.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
import frontmatter
from agentscope.message import TextBlock
from agentscope.tool import Toolkit, ToolResponse
from . import memory_io
from ..component import R
from ..component.base_step import BaseStep
from ..component.runtime_response import _set_answer, _to_jsonable
from ..enumeration import ComponentEnum
# ===========================================================================
# Section 1 — Schema policy
# ===========================================================================
_STATUS_STATES = ("active", "distilled", "archived")
_STATUS_TRANSITIONS: dict[str, set[str]] = {
"active": {"active", "distilled"},
"distilled": {"distilled", "archived"},
"archived": {"archived"},
}
def validate_status_transition(prior, requested) -> str | None:
"""Return error string if the requested status transition is invalid,
else None. Files without a prior status accept any initial value
(so first-write doesn't get blocked)."""
if requested is None:
return None # delete operation
if requested not in _STATUS_STATES:
return (
f"invalid status {requested!r}; must be one of "
f"{list(_STATUS_STATES)}"
)
if prior in _STATUS_STATES and requested not in _STATUS_TRANSITIONS[prior]:
return (
f"status transition {prior!r} → {requested!r} not allowed; "
f"state machine is single-direction "
f"active → distilled → archived"
)
return None
def validate_path_template(path: Path, vault_root: Path | None) -> str | None:
"""Return error string if `path` doesn't match a known agent-facing
template, else None. When `vault_root` is unknown, skip the check.
Allowed templates (relative to vault_root):
topics/{folder}/{name}.md — topic file
events/{date}/{name}/{filename} — event index OR sibling material
Archive/... — archive moves can land anywhere
"""
if vault_root is None:
return None
try:
rel = path.resolve().relative_to(vault_root)
except ValueError:
return f"path {path} is outside vault_root {vault_root}"
parts = rel.parts
if not parts:
return "path has no components relative to vault_root"
head = parts[0]
if head == "Archive":
return None
if head == "topics" and len(parts) >= 3:
return None
if head == "events" and len(parts) >= 4:
return None
return (
f"path {rel} doesn't match a known template — expected one of: "
f"topics/{{folder}}/{{name}}.md, "
f"events/{{date}}/{{name}}/{{filename}}, or Archive/..."
)
def update_status(
path: Path | str,
*,
value,
force: bool = False,
) -> tuple[bool, dict]:
"""Schema-aware status flip. Reads current status from disk, validates
the transition, then delegates to `memory_io.update_meta`."""
target = Path(path)
if not force:
prior = None
if target.is_file():
try:
prior = frontmatter.loads(
target.read_text(encoding="utf-8"),
).metadata.get("status")
except Exception:
prior = None
err = validate_status_transition(prior, value)
if err is not None:
return False, {
"path": str(target),
"key": "status",
"error": err,
"prior": prior,
"requested": value,
}
return memory_io.update_meta(target, key="status", value=value)
def create_file_with_schema(
file_store,
path: Path,
*,
metadata: dict,
content: str,
overwrite: bool = False,
force: bool = False,
) -> tuple[bool, dict]:
"""Schema-aware file create. Validates the path template (unless
`force=True`), then delegates to `memory_io.create_file` (which
still enforces the wikilink-uniqueness graph invariant — that gate
lives in the engine because it's structural, not business policy).
`force=True` bypasses BOTH the template gate here and the wikilink
gate downstream — it's the single escape hatch for any caller that
intentionally needs to step outside conventions.
"""
if not force:
vault_root = getattr(file_store, "vault_root", None)
template_err = validate_path_template(path, vault_root)
if template_err is not None:
return False, {
"path": str(path),
"error": template_err,
"hint": (
"place topics under topics/{folder}/{name}.md and "
"events under events/{date}/{name}/...; pass "
"force=true only if you intentionally need a "
"non-template path"
),
}
return memory_io.create_file(
file_store, path,
metadata=metadata, content=content,
overwrite=overwrite, force=force,
)
# ===========================================================================
# Section 2 — Tool response helper
# ===========================================================================
def _tool_response(
op: str,
ok: bool,
payload: Any,
audit: list[dict] | None = None,
) -> ToolResponse:
"""Wrap a tool-method result as a `ToolResponse` and optionally
append an audit row."""
if audit is not None:
entry = {"op": op, "ok": ok}
if isinstance(payload, dict):
entry.update(payload)
else:
entry["result"] = payload
audit.append(entry)
text = json.dumps(_to_jsonable(payload), ensure_ascii=False, indent=2)
return ToolResponse(content=[TextBlock(type="text", text=text)])
# ===========================================================================
# Section 3 — Memory steps (one BaseStep per tool, two surfaces each)
# ===========================================================================
@R.register("memory_get")
class MemoryGet(BaseStep):
"""Read a single memory file (frontmatter + body, optional chunks)."""
audit: list[dict] | None = None # set by build_memory_toolkit
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
include_chunks: bool = bool(self.context.get("include_chunks", False))
assert path, "path is required"
result = await memory_io.get_file(self.file_store, path, include_chunks=include_chunks)
_set_answer(self.context, result)
async def memory_get(self, path: str, include_chunks: bool = False) -> ToolResponse:
"""Read a single memory file (frontmatter + body, optional chunks)."""
result = await memory_io.get_file(self.file_store, path, include_chunks=include_chunks)
return _tool_response("memory_get", True, result, audit=self.audit)
@R.register("memory_list")
class MemoryList(BaseStep):
"""List indexed files filtered by frontmatter fields, tags, or path prefix."""
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
result = memory_io.list_files(
self.file_store,
path_prefix=self.context.get("path_prefix"),
tags=self.context.get("tags") or [],
metadata=self.context.get("metadata") or {},
limit=int(self.context.get("limit") or 100),
)
_set_answer(self.context, result)
async def memory_list(
self,
path_prefix: str | None = None,
tags: list[str] | None = None,
metadata: dict | None = None,
limit: int = 100,
) -> ToolResponse:
"""List indexed files filtered by frontmatter fields, tags, or path prefix."""
result = memory_io.list_files(
self.file_store,
path_prefix=path_prefix,
tags=tags or [],
metadata=metadata or {},
limit=limit,
)
return _tool_response("memory_list", True, result, audit=self.audit)
@R.register("memory_backlinks")
class MemoryBacklinks(BaseStep):
"""Files linking TO a given path. Each entry carries the typed-edge predicate."""
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
assert path, "path is required"
_set_answer(self.context, memory_io.get_backlinks(self.file_store, path))
async def memory_backlinks(self, path: str) -> ToolResponse:
"""Files linking TO a given path. Each entry carries the typed-edge predicate."""
result = memory_io.get_backlinks(self.file_store, path)
return _tool_response("memory_backlinks", True, result, audit=self.audit)
@R.register("memory_links")
class MemoryLinks(BaseStep):
"""Files a given path links to (resolved). Each entry carries the typed-edge predicate."""
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
assert path, "path is required"
_set_answer(self.context, memory_io.get_links(self.file_store, path))
async def memory_links(self, path: str) -> ToolResponse:
"""Files a given path links to (resolved). Each entry carries the typed-edge predicate."""
result = memory_io.get_links(self.file_store, path)
return _tool_response("memory_links", True, result, audit=self.audit)
@R.register("memory_resolve_wikilink")
class MemoryResolveWikilink(BaseStep):
"""Resolve a `[[target]]` wikilink with full ambiguity context."""
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
wikilink: str = self.context.get("wikilink", "") or ""
assert wikilink, "wikilink is required"
payload = memory_io.resolve_wikilink(self.file_store, wikilink)
self.context.response.success = bool(payload.get("exists"))
_set_answer(self.context, payload)
async def memory_resolve_wikilink(self, wikilink: str) -> ToolResponse:
"""Resolve a `[[target]]` wikilink with full ambiguity context."""
payload = memory_io.resolve_wikilink(self.file_store, wikilink)
return _tool_response(
"memory_resolve_wikilink",
bool(payload.get("exists")),
payload,
audit=self.audit,
)
@R.register("memory_create")
class MemoryCreate(BaseStep):
"""Create a markdown file. Path-template gate + wikilink-uniqueness
gate both fire unless `force=True`."""
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
metadata: dict = dict(self.context.get("metadata") or {})
content: str = self.context.get("content", "") or ""
overwrite: bool = bool(self.context.get("overwrite", False))
force: bool = bool(self.context.get("force", False))
assert path, "path is required"
target = Path(path)
ok, payload = create_file_with_schema(
self.file_store, target,
metadata=metadata, content=content,
overwrite=overwrite, force=force,
)
self.context.response.success = ok
if ok:
payload = {**payload, "path": str(target.resolve())}
_set_answer(self.context, payload)
async def memory_create(
self,
path: str,
metadata: dict | None = None,
content: str = "",
overwrite: bool = False,
force: bool = False,
) -> ToolResponse:
"""Create a markdown file. Path-template gate + wikilink-uniqueness
gate both fire unless `force=True`."""
target = Path(path)
ok, payload = create_file_with_schema(
self.file_store, target,
metadata=dict(metadata or {}),
content=content,
overwrite=overwrite,
force=force,
)
if ok:
payload = {**payload, "path": str(target.resolve())}
return _tool_response("memory_create", ok, payload, audit=self.audit)
@R.register("memory_delete")
class MemoryDelete(BaseStep):
"""Delete a file. Watcher removes from store + graph."""
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
assert path, "path is required"
ok, payload = memory_io.delete_file(path)
self.context.response.success = ok
_set_answer(self.context, payload)
async def memory_delete(self, path: str) -> ToolResponse:
"""Delete a file. Watcher removes from store + graph."""
ok, payload = memory_io.delete_file(path)
return _tool_response("memory_delete", ok, payload, audit=self.audit)
@R.register("memory_rename")
class MemoryRename(BaseStep):
"""Rename a file and rewrite incoming wikilinks across the vault."""
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
old_path: str = self.context.get("old_path", "") or ""
new_path: str = self.context.get("new_path", "") or ""
assert old_path and new_path, "old_path and new_path are required"
watcher = self.app_context.components["file_watcher"]["default"] # type: ignore[index,union-attr]
vault_root = Path(watcher.watch_path).resolve() # type: ignore[union-attr]
ok, payload = memory_io.rename_file(
self.file_store, vault_root,
old_path=old_path, new_path=new_path,
)
self.context.response.success = ok
_set_answer(self.context, payload)
async def memory_rename(self, old_path: str, new_path: str) -> ToolResponse:
"""Rename a file and rewrite incoming wikilinks across the vault."""
watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default")
vault_root = (
Path(getattr(watcher, "watch_path", ".")).resolve() if watcher else Path.cwd()
)
ok, payload = memory_io.rename_file(
self.file_store, vault_root,
old_path=old_path, new_path=new_path,
)
return _tool_response("memory_rename", ok, payload, audit=self.audit)
@R.register("memory_property_update")
class MemoryPropertyUpdate(BaseStep):
"""Update a single YAML frontmatter key. value=null deletes the key.
When key=='status', enforces the active → distilled → archived
state machine via `update_status`. Pass `force=True` to bypass.
Other keys go through the bare engine."""
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
key: str = self.context.get("key", "") or ""
value = self.context.get("value")
force: bool = bool(self.context.get("force", False))
assert path and key, "path and key are required"
if key == "status":
ok, payload = update_status(path, value=value, force=force)
else:
ok, payload = memory_io.update_meta(path, key=key, value=value)
self.context.response.success = ok
_set_answer(self.context, payload)
async def memory_property_update(
self,
path: str,
key: str,
value: Any = None,
force: bool = False,
) -> ToolResponse:
"""Update a single YAML frontmatter key. value=null deletes the key.
When key=='status', enforces the active → distilled → archived
state machine. Pass `force=True` to bypass."""
if key == "status":
ok, payload = update_status(path, value=value, force=force)
else:
ok, payload = memory_io.update_meta(path, key=key, value=value)
return _tool_response("memory_property_update", ok, payload, audit=self.audit)
@R.register("memory_update")
class MemoryUpdate(BaseStep):
"""Edit-style content update: replace `old_string` with `new_string`."""
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
old_string: str = self.context.get("old_string", "") or ""
new_string: str = self.context.get("new_string", "") or ""
replace_all: bool = bool(self.context.get("replace_all", False))
assert path, "path is required"
ok, payload = memory_io.update_body(
path, old_string=old_string, new_string=new_string, replace_all=replace_all,
)
self.context.response.success = ok
_set_answer(self.context, payload)
async def memory_update(
self,
path: str,
old_string: str,
new_string: str,
replace_all: bool = False,
) -> ToolResponse:
"""Edit-style content update: replace `old_string` with `new_string`."""
ok, payload = memory_io.update_body(
path, old_string=old_string, new_string=new_string, replace_all=replace_all,
)
return _tool_response("memory_update", ok, payload, audit=self.audit)
@R.register("memory_archive")
class MemoryArchive(BaseStep):
"""Archive a file: flip `status: archived` then move to `<vault>/Archive/`."""
audit: list[dict] | None = None
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
archive_dir_name: str = self.context.get("archive_dir", "Archive") or "Archive"
assert path, "path is required"
watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default")
vault_root = Path(getattr(watcher, "watch_path", ".")).resolve() if watcher else Path.cwd()
ok, payload = memory_io.archive_file(vault_root, path, archive_dir=archive_dir_name)
self.context.response.success = ok
_set_answer(self.context, payload)
async def memory_archive(self, path: str, archive_dir: str = "Archive") -> ToolResponse:
"""Archive a file: flip `status: archived` then move to `<vault>/<archive_dir>/`."""
watcher = self._get_component_optional(ComponentEnum.FILE_WATCHER, "default")
vault_root = (
Path(getattr(watcher, "watch_path", ".")).resolve() if watcher else Path.cwd()
)
ok, payload = memory_io.archive_file(vault_root, path, archive_dir=archive_dir)
return _tool_response("memory_archive", ok, payload, audit=self.audit)
@R.register("memory_count_tokens")
class MemoryCountTokens(BaseStep):
"""Estimate tokens for a file body or raw text. One of `path`/`text` required.
MCP-only — token counting is an editor concern, not part of the
R-M-W loop. No agent toolkit projection."""
async def execute(self):
assert self.context is not None
path: str = self.context.get("path", "") or ""
text: str = self.context.get("text", "") or ""
result = await memory_io.count_tokens(
self.as_token_counter,
path=path or None,
text=text or None,
)
self.context.response.success = "error" not in result
_set_answer(self.context, result)
# ===========================================================================
# Section 4 — Toolkit factory
# ===========================================================================
# The 11 memory_* tools the Ingestor's ReActAgent gets bound to.
# Each name doubles as the BaseStep registration key AND the tool method
# name on that step.
MEMORY_TOOL_NAMES: tuple[str, ...] = (
"memory_get",
"memory_list",
"memory_resolve_wikilink",
"memory_backlinks",
"memory_links",
"memory_create",
"memory_update",
"memory_property_update",
"memory_rename",
"memory_delete",
"memory_archive",
)
def build_memory_toolkit(
app_context,
audit: list[dict] | None = None,
toolkit: Toolkit | None = None,
) -> Toolkit:
"""Bind every memory_* step's tool method to an agentscope `Toolkit`.
For each name in `MEMORY_TOOL_NAMES`, instantiates the registered
BaseStep against `app_context`, attaches the shared `audit` list,
and registers the same-named class method as a tool function.
agentscope introspects the method signature directly — there is no
separate JSON schema layer.
"""
toolkit = toolkit or Toolkit()
for name in MEMORY_TOOL_NAMES:
step_cls = R.get(ComponentEnum.STEP, name)
if step_cls is None:
continue
instance = step_cls(app_context=app_context)
instance.audit = audit # type: ignore[attr-defined]
toolkit.register_tool_function(
getattr(instance, name),
namesake_strategy="override",
)
return toolkit

147
reme2/memory/protocol.md Normal file
View file

@ -0,0 +1,147 @@
# Memory Protocol
Single source of truth for vault schema, conventions, and the R-M-W
write loop. Consumed by:
- **Ingestor's embedded ReAct prompt** (`reme2/memory/ingestor.yaml` —
injected as `{protocol}` at load time).
- **Strong-agent SKILL** (`reme-plugin/skills/reme/SKILL.md` —
transcluded so the host agent sees the same rules).
Anything that defines schema invariants, path templates, write tool
semantics, or the R-M-W decision tree belongs here. Anything role-
specific (caller framing, audit trail expectations, summary
requirements) stays in the consumer.
## Vault layout
- **Topics** — long-lived cognitive memory at `topics/{folder}/{name}.md`.
A **folder topic** has `folder == name`; it's the cluster's index head.
Short wikilink `[[X]]` resolves to the folder topic if one exists,
else falls back to a unique same-stem file.
- **Events** — fact log of one session at
`events/{YYYY-MM-DD}/{name}/{name}.md`. The `.md` is the **index**
inside a folder; sibling files are **materials** (raw conversation,
tool outputs, data dumps). The index lists them under `## Materials`.
## Frontmatter — 4 schema axes
Every memory declares 4 orthogonal axes. The legacy `category` field is
auto-translated to these axes for back-compat reads, but new writes
should set the axes directly.
| Axis | Values | Meaning |
|---|---|---|
| `lifecycle` | `streaming` / `evolving` / `frozen` | streaming = events (decay/archive); evolving = topics (long-lived, edited); frozen = materials (immutable references) |
| `scope` | `instance` / `class` | instance = a specific moment / object; class = abstract concept / role / pattern |
| `source` | `auto` / `curated` / `derived` | auto = system-captured; curated = human/LLM intent; derived = computed from other memories |
| `role` | `observation` / `claim` / `question` / `profile` / `concept` / `method` / `reference` / `fundamentals` | cognitive role — drives ranking + role-specific validation |
### Conditional fields
- `confidence` ∈ {⏳, ✅, ❌} — **REQUIRED** when `role: claim` (legacy
categories `thesis` / `model`). Same gate applies to `role: question`
(legacy `questions`).
- `status` ∈ {`active`, `distilled`, `archived`} — meaningful only for
`lifecycle: streaming`. Topic-style memories ignore it.
- `originSessionId` — should be set when `source: auto`.
### Standard identity fields
`title`, `description`, `tags`, `created`, `updated`, `topics`,
`parent`. Use today's date for `created` / `updated` on new writes.
## Cross-file references
`[[wikilink]]` syntax. Two forms:
- **Stem form** `[[X]]` — resolved against the file_store's stem index;
prefers the folder topic if one exists.
- **Path form** `[[topics/X/X]]` or `[[topics/X/X.md]]` — anchored at
the vault root.
## Status state machine
`active → distilled → archived` (single direction, no skip). A reverse
or skip transition will be flagged by the Maintainer.
## Wikilink uniqueness
Every create path routes through `MemoryCreate.write`, which refuses to
introduce ambiguity (existing `[[X]]` would resolve to ≥2 paths). When
rejected, the response includes a `suggested_name`. Retry with that, or
pick a domain-specific qualifier (`Apple-Inc` beats `Apple-2`). Never
bypass with `force=true` unless you fully understand the ambiguity.
## Available tools
### Read tools (gather context BEFORE writing)
- `memory_get(path, include_chunks=False)` — full file content +
frontmatter. On an event index, follow `## Materials` and read each
artifact whose content you need.
- `memory_list(path_prefix=None, tags=None, metadata=None, limit=100)`
— list indexed files filtered by prefix / tags / frontmatter.
- `memory_resolve_wikilink(wikilink)` — resolve `[[X]]` to a path;
flags ambiguity / dangling.
- `memory_backlinks(path)` — files linking TO the given path.
- `memory_links(path)` — files the given path links to.
- `memory_search(query, …)` — hybrid (vector + keyword) chunk search.
- `memory_graph_search(query, seeds, graph_depth, …)` — vector +
keyword + graph BFS fusion.
### Write tools (mutations are SSOT-routed; each returns success +
payload + records to audit)
- `memory_update(path, old_string, new_string, replace_all=False)` —
body edit by exact-string substitution. Use a tail snippet to append.
- `memory_property_update(path, key, value)` — change one frontmatter
key (`value=null` deletes). Use this to flip status.
- `memory_create(path, metadata, content, overwrite=False, force=False)`
— new file. Reserve for genuinely NEW topics. Do NOT use for events
(`sync` owns events). All paths must be ABSOLUTE under vault_root.
- `memory_rename(old_path, new_path)` — move file + rewrite cross-vault
wikilinks. Refuses on destination conflict or stem ambiguity.
- `memory_delete(path)` — remove a file.
- `memory_archive(path)` — flip `status: archived` and move under
`<vault>/Archive/`.
### Hot-write helper (deterministic, no LLM)
- `sync(name, description?, content?, topics?, tags?, materials?,
on_date?)` — idempotent upsert of an event FOLDER per `(date, name)`.
Reuse the same `name` across calls in one thread to keep extending
the same folder. Refuses on `status: distilled` / `archived` and
returns a `suggested_name`.
## R-M-W decision rules
Apply in order. Stop at the first match.
1. **SKIP** — if material is ALREADY covered by existing topics, reply
with a single line `SKIP: <one-line reason>` and call no tools.
2. **CONTRADICT** — if material CONTRADICTS an existing block, use
`memory_update` with a unique snippet of the outdated text and the
corrected replacement.
3. **EXTEND** — if material EXTENDS an existing topic, use
`memory_update` with a unique TAIL snippet of the existing body, and
`new_string = tail + blank line + new content`.
4. **CREATE** — if material warrants a GENUINELY NEW topic, use
`memory_create` at `topics/{folder}/{name}.md`. Do NOT
`memory_create` under `events/` — `sync` owns that path.
5. **STATUS FLIP** — after integrating an event's content into a
topic, flip that event's status to `distilled` with
`memory_property_update`.
## Operating principles
- **Read before write.** Always inspect related topics before deciding
CONTRADICT vs EXTEND vs CREATE. The wikilink-uniqueness gate refuses
blind creates; reading first prevents wasted attempts.
- **Minimal edits.** Edit only what must change. Don't restructure
while updating content.
- **Frontmatter on create.** Always include reasonable frontmatter:
the 4 axes, `title`, `created`, `updated`, plus `confidence` when
`role: claim` or `role: question`.
- **Never delete unless asked.** Distillation flips status; it does
not remove events.

22
reme2/memory/retriever.py
View file

@ -95,7 +95,7 @@ class BaseRetriever(BaseStep):
"""
assert self.context is not None
ctx = self.context
chunk_filter = memory_io.make_chunk_filter(
chunk_filter = memory_io.make_filter(
self.file_store,
paths=ctx.get("paths") or None,
tags=ctx.get("tags") or None,
@ -116,9 +116,9 @@ class BaseRetriever(BaseStep):
class HybridRetriever(BaseRetriever):
"""V + K (+ optional graph BFS) fusion retriever.
Composes the file_store's single-channel primitives (`vector_search`,
`keyword_search`, `expand_neighbors`, `extract_anchor_paths`,
`get_chunks_by_paths`). Knobs (`vector_weight`, `graph_weight`,
Composes the engine API's projection primitives (`search_vector`,
`search_keyword`, `expand_neighbors`, `extract_anchors`,
`get_chunks`). Knobs (`vector_weight`, `graph_weight`,
`graph_depth`, `graph_decay`, `graph_direction`, `graph_mode`,
`graph_per_path_cap`, `anchor_expand`, `candidate_multiplier`) are
constructor defaults; `graph_search` accepts per-call overrides for
@ -173,8 +173,8 @@ class HybridRetriever(BaseRetriever):
text_weight = 1.0 - self.vector_weight
if fs.vector_enabled and fs.fts_enabled:
v_task = memory_io.vector_search(fs, query, candidates, chunk_filter)
k_task = memory_io.keyword_search(fs, query, candidates, chunk_filter)
v_task = memory_io.search_vector(fs, query, limit=candidates, chunk_filter=chunk_filter)
k_task = memory_io.search_keyword(fs, query, limit=candidates, chunk_filter=chunk_filter)
v_results, k_results = await asyncio.gather(v_task, k_task)
if not k_results:
@ -186,9 +186,9 @@ class HybridRetriever(BaseRetriever):
v_results, k_results, self.vector_weight, text_weight,
)[:max_results]
elif fs.vector_enabled:
results = await memory_io.vector_search(fs, query, max_results, chunk_filter)
results = await memory_io.search_vector(fs, query, limit=max_results, chunk_filter=chunk_filter)
elif fs.fts_enabled:
results = await memory_io.keyword_search(fs, query, max_results, chunk_filter)
results = await memory_io.search_keyword(fs, query, limit=max_results, chunk_filter=chunk_filter)
else:
results = []
@ -264,8 +264,8 @@ class HybridRetriever(BaseRetriever):
# 1. V + K in parallel (each is a no-op when its backend is disabled).
if query:
v_task = memory_io.vector_search(fs, query, candidate_count, chunk_filter)
k_task = memory_io.keyword_search(fs, query, candidate_count, chunk_filter)
v_task = memory_io.search_vector(fs, query, limit=candidate_count, chunk_filter=chunk_filter)
k_task = memory_io.search_keyword(fs, query, limit=candidate_count, chunk_filter=chunk_filter)
v_results, k_results = await asyncio.gather(v_task, k_task)
else:
v_results, k_results = [], []
@ -296,7 +296,7 @@ class HybridRetriever(BaseRetriever):
extra_paths = set(hops) - vk_paths
if chunk_filter is not None and chunk_filter.resolved_paths is not None:
extra_paths &= chunk_filter.resolved_paths
extra_chunks = await memory_io.chunks_by_paths(fs, extra_paths)
extra_chunks = await memory_io.get_chunks(fs, extra_paths)
# Per-path cap so a hub topic with N chunks doesn't flood results.
by_path: dict[str, list] = defaultdict(list)
for c in extra_chunks:

79
reme2/memory/schema/__init__.py Normal file
View file

@ -0,0 +1,79 @@
"""Memory Schema — vault domain "constitution".
Per `structure.md` §"全局架构蓝图", the Schema layer sits between the
domain-agnostic Core Engine and the three Memory Services. Engines store
arbitrary markdown + open frontmatter dicts; Services read this Schema
to know what a "well-formed memory" is and shape the vault accordingly.
Public API:
Memory — single typed class for every vault frontmatter
Lifecycle / Scope / Source / Role — the four behavioral axes (StrEnum)
Status — streaming-only state (active / distilled / archived)
Confidence — claim-only confidence (PENDING / VERIFIED / REJECTED)
parse_frontmatter(raw) → (Memory | None, errors) tolerant parser
*_PRESET dicts — common axis combos for hot-write paths
preset_for_category(name) — legacy category → preset lookup
LEGACY_AXES_FROM_CATEGORY — back-fill table; auto-applied by Memory's
pre-validator so legacy vaults parse free.
Lives in `reme2/memory/` (not `reme2/schema/` and not `reme2/component/`)
because the Schema is owned by the memory services that consume it. The
engine never imports from here.
"""
from .memory import (
LEGACY_AXES_FROM_CATEGORY,
Confidence,
Lifecycle,
Memory,
Role,
Scope,
Source,
Status,
)
from .parser import parse_frontmatter
from .presets import (
CONCEPT_PRESET,
EVENT_PRESET,
FUNDAMENTALS_PRESET,
MATERIAL_PRESET,
METHOD_PRESET,
MODEL_PRESET,
PRESETS_BY_CATEGORY,
PROFILE_PRESET,
QUESTIONS_PRESET,
THESIS_PRESET,
TOOL_PRESET,
preset_for_category,
)
__all__ = [
# core
"Memory",
"Lifecycle",
"Scope",
"Source",
"Role",
"Status",
"Confidence",
"LEGACY_AXES_FROM_CATEGORY",
# parser
"parse_frontmatter",
# presets
"EVENT_PRESET",
"PROFILE_PRESET",
"CONCEPT_PRESET",
"THESIS_PRESET",
"MODEL_PRESET",
"QUESTIONS_PRESET",
"METHOD_PRESET",
"TOOL_PRESET",
"FUNDAMENTALS_PRESET",
"MATERIAL_PRESET",
"PRESETS_BY_CATEGORY",
"preset_for_category",
]

317
reme2/memory/schema/memory.py Normal file
View file

@ -0,0 +1,317 @@
"""Memory — the typed shape of a vault frontmatter.
Single class covers every memory in the vault. The four orthogonal axes
(`lifecycle / scope / source / role`) define the *behavioral shape* —
services read off them to decide decay rules, retrieval ranking, merge
eligibility, etc., instead of switching on a `category` string.
Role-conditional fields (`confidence`, `status`, `origin_session_id`)
are validated declaratively via `model_validator(mode='after')`.
Legacy migration: a `model_validator(mode='before')` recognizes the old
`category` field and back-fills the 4 axes via `LEGACY_AXES_FROM_CATEGORY`,
so vault files written before this schema continue to parse without a
migration script.
Lives in `reme2/memory/schema/` (with the services that consume it),
NOT in `reme2/component/` — the engine is domain-agnostic and never
imports this module.
"""
from __future__ import annotations
from datetime import date, datetime
from enum import StrEnum
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
# ---------------------------------------------------------------------------
# The four behavioral axes
# ---------------------------------------------------------------------------
class Lifecycle(StrEnum):
"""Mutability + decay disposition of a memory.
streaming — write-once events that decay/archive after a freshness window.
evolving — long-lived memories that get continuously edited; never decay.
frozen — immutable references / artifacts; never edited, never decay.
"""
STREAMING = "streaming"
EVOLVING = "evolving"
FROZEN = "frozen"
class Scope(StrEnum):
"""Referential range of a memory.
instance — refers to a specific moment / object / occurrence.
class — abstracts over many instances; a concept / role / pattern.
"""
INSTANCE = "instance"
CLASS = "class"
class Source(StrEnum):
"""Provenance of the memory.
auto — captured by the system from a session / tool output.
curated — written or shaped by a human (or LLM curator) on purpose.
derived — computed from other memories (Maintainer products, summaries).
"""
AUTO = "auto"
CURATED = "curated"
DERIVED = "derived"
class Role(StrEnum):
"""Cognitive role the memory plays.
Drives schema-aware ranking + role-specific field validation.
Adding a new role is a one-line change here plus (optionally) a preset.
"""
OBSERVATION = "observation" # what happened (events live here)
CLAIM = "claim" # an assertion needing confidence (thesis, model)
QUESTION = "question" # an open inquiry needing an answer
PROFILE = "profile" # entity description (person, org, system)
CONCEPT = "concept" # abstract idea / definition (company, sector, concept)
METHOD = "method" # procedure / how-to
REFERENCE = "reference" # pointer to external thing (tool, paper, code)
FUNDAMENTALS = "fundamentals" # foundational data / baseline facts
class Status(StrEnum):
"""Lifecycle state — meaningful only for `lifecycle == streaming`.
Topic-style memories (evolving / frozen) ignore this field.
"""
ACTIVE = "active"
DISTILLED = "distilled"
ARCHIVED = "archived"
class Confidence(StrEnum):
"""Required for `role == claim`. Wire format stays emoji.
PENDING — claim under investigation; outcome not yet decided.
VERIFIED — claim supported by evidence and currently held.
REJECTED — claim was investigated and disconfirmed; kept for history.
"""
PENDING = "⏳"
VERIFIED = "✅"
REJECTED = "❌"
# ---------------------------------------------------------------------------
# Legacy `category` → 4-axis mapping (auto-migration)
# ---------------------------------------------------------------------------
LEGACY_AXES_FROM_CATEGORY: dict[str, dict] = {
# Events: streaming-instance-auto-observation
"event": {
"lifecycle": Lifecycle.STREAMING,
"scope": Scope.INSTANCE,
"source": Source.AUTO,
"role": Role.OBSERVATION,
},
# Topic categories — all evolving / class / curated, role varies
"company": {
"lifecycle": Lifecycle.EVOLVING,
"scope": Scope.CLASS,
"source": Source.CURATED,
"role": Role.CONCEPT,
},
"sector": {
"lifecycle": Lifecycle.EVOLVING,
"scope": Scope.CLASS,
"source": Source.CURATED,
"role": Role.CONCEPT,
},
"concept": {
"lifecycle": Lifecycle.EVOLVING,
"scope": Scope.CLASS,
"source": Source.CURATED,
"role": Role.CONCEPT,
},
"method": {
"lifecycle": Lifecycle.EVOLVING,
"scope": Scope.CLASS,
"source": Source.CURATED,
"role": Role.METHOD,
},
"tool": {
"lifecycle": Lifecycle.EVOLVING,
"scope": Scope.CLASS,
"source": Source.CURATED,
"role": Role.REFERENCE,
},
"profile": {
"lifecycle": Lifecycle.EVOLVING,
"scope": Scope.CLASS,
"source": Source.CURATED,
"role": Role.PROFILE,
},
"thesis": {
"lifecycle": Lifecycle.EVOLVING,
"scope": Scope.CLASS,
"source": Source.CURATED,
"role": Role.CLAIM,
},
"model": {
"lifecycle": Lifecycle.EVOLVING,
"scope": Scope.CLASS,
"source": Source.CURATED,
"role": Role.CLAIM,
},
"questions": {
"lifecycle": Lifecycle.EVOLVING,
"scope": Scope.CLASS,
"source": Source.CURATED,
"role": Role.QUESTION,
},
"fundamentals": {
"lifecycle": Lifecycle.EVOLVING,
"scope": Scope.CLASS,
"source": Source.CURATED,
"role": Role.FUNDAMENTALS,
},
# Materials: frozen-instance-auto-reference (siblings of an event index)
"material": {
"lifecycle": Lifecycle.FROZEN,
"scope": Scope.INSTANCE,
"source": Source.AUTO,
"role": Role.REFERENCE,
},
}
# ---------------------------------------------------------------------------
# Memory — the single typed shape
# ---------------------------------------------------------------------------
class Memory(BaseModel):
"""The typed view of a vault frontmatter.
`extra="allow"` keeps domain-specific fields (market, ticker, etc.) in
the parsed object without polluting this base schema. `populate_by_name`
lets `originSessionId` (legacy camelCase) populate `origin_session_id`.
"""
model_config = ConfigDict(extra="allow", populate_by_name=True)
# -- Identity / common metadata ----------------------------------------
title: str = Field(default="")
description: str = Field(default="")
tags: list[str] = Field(default_factory=list)
created: date | None = None
updated: date | None = None
# -- The four behavioral axes ------------------------------------------
lifecycle: Lifecycle
scope: Scope
source: Source
role: Role
# -- Cross-cutting graph fields ----------------------------------------
topics: list[str] = Field(default_factory=list, description="Outbound wikilinks to class memories.")
parent: str | None = Field(default=None, description="Wikilink to owning memory (e.g. material → event).")
# -- Role / lifecycle / source-conditional fields ----------------------
confidence: Confidence | None = Field(
default=None,
description="Required when role == claim. Use ⏳ / ✅ / ❌ as wire form.",
)
status: Status | None = Field(
default=None,
description="Lifecycle state. Meaningful only when lifecycle == streaming.",
)
origin_session_id: str | None = Field(
default=None,
alias="originSessionId",
description="Capture session id. Set when source == auto.",
)
# -- Migration: pre-validate hook --------------------------------------
@model_validator(mode="before")
@classmethod
def _migrate_legacy_category(cls, data):
"""Back-fill 4 axes from old `category` field if present.
Idempotent: if axes are already populated, do nothing. The legacy
`category` field is preserved (via `extra="allow"`) so downstream
code that still reads it keeps working until we delete it.
"""
if not isinstance(data, dict):
return data
if "role" in data and "lifecycle" in data:
return data
cat = data.get("category")
if cat in LEGACY_AXES_FROM_CATEGORY:
data = {**LEGACY_AXES_FROM_CATEGORY[cat], **data}
return data
# -- Field coercion / cleanup ------------------------------------------
@field_validator("created", "updated", mode="before")
@classmethod
def _coerce_date(cls, v):
if v is None or isinstance(v, date):
return v
if isinstance(v, datetime):
return v.date()
if isinstance(v, str):
return datetime.fromisoformat(v).date()
return v
@field_validator("tags", "topics", mode="before")
@classmethod
def _strip_dedup(cls, v):
"""Strip whitespace + dedup string lists. Preserves order."""
if not isinstance(v, list):
return v
seen: dict[str, None] = {}
for item in v:
if isinstance(item, str):
stripped = item.strip()
if stripped:
seen.setdefault(stripped, None)
return list(seen.keys())
# -- Role-conditional checks -------------------------------------------
@model_validator(mode="after")
def _enforce_role_conditionals(self):
if self.role is Role.CLAIM and self.confidence is None:
raise ValueError(
"role='claim' requires explicit confidence "
"(one of ⏳ / ✅ / ❌)"
)
return self
# -- Convenience properties --------------------------------------------
@property
def is_active(self) -> bool:
"""True for streaming memories that are still mutable."""
return self.lifecycle is Lifecycle.STREAMING and self.status is Status.ACTIVE
@property
def is_terminal(self) -> bool:
"""True for memories no longer accepting writes (distilled / archived / frozen)."""
if self.lifecycle is Lifecycle.FROZEN:
return True
return self.status in (Status.DISTILLED, Status.ARCHIVED)

39
reme2/memory/schema/parser.py Normal file
View file

@ -0,0 +1,39 @@
"""Tolerant parser — never raises, always returns (memory_or_None, errors).
Used by:
- Maintainer.lint : surfaces schema violations on existing files
- any read path : turn raw frontmatter into a typed view when possible
For *write* paths (`sync`, Ingestor) call `Memory.model_validate`
directly so validation errors propagate as exceptions and stop the write.
"""
from __future__ import annotations
from pydantic import ValidationError
from .memory import Memory
def parse_frontmatter(raw: dict) -> tuple[Memory | None, list[str]]:
"""Tolerant parse. Returns (parsed, errors).
Success → (Memory(...), [])
Failure → (None, ["loc: msg", ...])
Migration is automatic via Memory's `model_validator(mode='before')`,
so frontmatter that only carries the legacy `category` field still
parses successfully.
"""
if not isinstance(raw, dict):
return None, [f"frontmatter must be a dict, got {type(raw).__name__}"]
try:
return Memory.model_validate(raw), []
except ValidationError as e:
msgs: list[str] = []
for err in e.errors(include_context=False, include_url=False):
loc = ".".join(str(x) for x in err.get("loc", ()))
msgs.append(f"{loc}: {err.get('msg', '')}".strip(": "))
return None, msgs
except Exception as e: # noqa: BLE001 — defensive: never raise from a tolerant parser
return None, [f"{type(e).__name__}: {e}"]

134
reme2/memory/schema/presets.py Normal file
View file

@ -0,0 +1,134 @@
"""Memory presets — common axis combinations for hot-write paths.
Hot-write tools (`sync`) and cold-write services (Ingestor R-M-W) start
with a preset, layer the agent's input on top, then validate the merged
dict against `Memory`. Adding a new memory shape = adding one preset
entry here + (optionally) a `LEGACY_AXES_FROM_CATEGORY` row in memory.py
for migration.
Presets carry only the 4 axes + optional default `status` for streaming
memories. Identity fields (title, description, tags, created, updated)
come from the caller.
Values are stored as **plain strings** (not StrEnum members) so they
flow cleanly through `frontmatter.dumps` → `yaml.dump`, which doesn't
know how to represent enum subclasses. Pydantic still coerces them
back into StrEnum members when `Memory.model_validate` runs.
"""
from __future__ import annotations
from .memory import Lifecycle, Role, Scope, Source, Status
EVENT_PRESET: dict = {
"lifecycle": Lifecycle.STREAMING.value,
"scope": Scope.INSTANCE.value,
"source": Source.AUTO.value,
"role": Role.OBSERVATION.value,
"status": Status.ACTIVE.value,
# Legacy compat: the `category` field is preserved by extra="allow",
# but emit it explicitly so old tooling that still reads `category`
# (hooks, scripts, downstream consumers) keeps working.
"category": "event",
}
PROFILE_PRESET: dict = {
"lifecycle": Lifecycle.EVOLVING.value,
"scope": Scope.CLASS.value,
"source": Source.CURATED.value,
"role": Role.PROFILE.value,
"category": "profile",
}
CONCEPT_PRESET: dict = {
"lifecycle": Lifecycle.EVOLVING.value,
"scope": Scope.CLASS.value,
"source": Source.CURATED.value,
"role": Role.CONCEPT.value,
"category": "concept",
}
THESIS_PRESET: dict = {
"lifecycle": Lifecycle.EVOLVING.value,
"scope": Scope.CLASS.value,
"source": Source.CURATED.value,
"role": Role.CLAIM.value,
"category": "thesis",
}
MODEL_PRESET: dict = {
"lifecycle": Lifecycle.EVOLVING.value,
"scope": Scope.CLASS.value,
"source": Source.CURATED.value,
"role": Role.CLAIM.value,
"category": "model",
}
QUESTIONS_PRESET: dict = {
"lifecycle": Lifecycle.EVOLVING.value,
"scope": Scope.CLASS.value,
"source": Source.CURATED.value,
"role": Role.QUESTION.value,
"category": "questions",
}
METHOD_PRESET: dict = {
"lifecycle": Lifecycle.EVOLVING.value,
"scope": Scope.CLASS.value,
"source": Source.CURATED.value,
"role": Role.METHOD.value,
"category": "method",
}
TOOL_PRESET: dict = {
"lifecycle": Lifecycle.EVOLVING.value,
"scope": Scope.CLASS.value,
"source": Source.CURATED.value,
"role": Role.REFERENCE.value,
"category": "tool",
}
FUNDAMENTALS_PRESET: dict = {
"lifecycle": Lifecycle.EVOLVING.value,
"scope": Scope.CLASS.value,
"source": Source.CURATED.value,
"role": Role.FUNDAMENTALS.value,
"category": "fundamentals",
}
MATERIAL_PRESET: dict = {
"lifecycle": Lifecycle.FROZEN.value,
"scope": Scope.INSTANCE.value,
"source": Source.AUTO.value,
"role": Role.REFERENCE.value,
"category": "material",
}
# Old `category` → preset, for migration / lookup use.
PRESETS_BY_CATEGORY: dict[str, dict] = {
"event": EVENT_PRESET,
"profile": PROFILE_PRESET,
"company": CONCEPT_PRESET,
"sector": CONCEPT_PRESET,
"concept": CONCEPT_PRESET,
"thesis": THESIS_PRESET,
"model": MODEL_PRESET,
"questions": QUESTIONS_PRESET,
"method": METHOD_PRESET,
"tool": TOOL_PRESET,
"fundamentals": FUNDAMENTALS_PRESET,
"material": MATERIAL_PRESET,
}
def preset_for_category(category: str) -> dict | None:
"""Look up the preset bound to a legacy category name.
Returns a fresh dict each call (callers may mutate it). Returns
None for unknown categories — caller decides whether to refuse or
fall through to a generic shape.
"""
p = PRESETS_BY_CATEGORY.get(category)
return dict(p) if p is not None else None

41
reme2/schema/vault/__init__.py
View file

@ -1,41 +0,0 @@
"""Vault business schemas — domain models for the markdown vault.
These are the *business objects* the user stores in the vault (Topic,
Event), distinct from the engine schemas in `reme2/schema/` (FileMetadata,
FileChunk, ChunkFilter — internal data types).
Single Topic class covers all categories under topics/ (no satellite /
tentacle subclassing); folder-topic is identified by path convention.
Lives under `reme2/schema/vault/` (not in `reme2/mcp/`) so memory
services (Maintainer.lint, Ingestor) can validate frontmatter without
importing the MCP transport layer — that's what was creating the
mcp ↔ memory dependency cycle.
"""
from .event import Event, EventStatus
from .frontmatter import VaultBaseFrontmatter, parse_frontmatter
from .registry import ALL_KNOWN_CATEGORIES, schema_for
from .topic import (
INDEX_CATEGORIES,
JUDGMENT_CATEGORIES,
CONTENT_CATEGORIES,
Confidence,
Topic,
TopicCategory,
)
__all__ = [
"ALL_KNOWN_CATEGORIES",
"CONTENT_CATEGORIES",
"Confidence",
"Event",
"EventStatus",
"INDEX_CATEGORIES",
"JUDGMENT_CATEGORIES",
"Topic",
"TopicCategory",
"VaultBaseFrontmatter",
"parse_frontmatter",
"schema_for",
]

16
reme2/schema/vault/event.py
View file

@ -1,16 +0,0 @@
"""Event schema — task process records under events/{date}/{name}/."""
from typing import Literal
from .frontmatter import VaultBaseFrontmatter
EventStatus = Literal["active", "distilled", "archived"]
class Event(VaultBaseFrontmatter):
"""events/{YYYY-MM-DD}/{name}/{name}.md."""
category: Literal["event"] = "event" # type: ignore[assignment]
status: EventStatus = "active"
topics: list[str] = []
originSessionId: str | None = None

38
reme2/schema/vault/frontmatter.py
View file

@ -1,38 +0,0 @@
"""Common frontmatter schema shared by all vault files."""
from datetime import date, datetime
from typing import Any
from pydantic import BaseModel, ConfigDict, Field, field_validator
class VaultBaseFrontmatter(BaseModel):
"""Fields present on every vault file."""
model_config = ConfigDict(extra="allow")
title: str = Field(...)
description: str = Field(default="")
tags: list[str] = Field(default_factory=list)
category: str = Field(...)
created: date | None = None
updated: date | None = None
@field_validator("created", "updated", mode="before")
@classmethod
def _coerce_date(cls, v):
if v is None or isinstance(v, date):
return v
if isinstance(v, datetime):
return v.date()
if isinstance(v, str):
return datetime.fromisoformat(v).date()
return v
def parse_frontmatter(raw: dict[str, Any]) -> VaultBaseFrontmatter:
"""Tolerant parse — never raises; missing required fields fall back to defaults."""
safe = dict(raw or {})
safe.setdefault("title", safe.get("name") or "")
safe.setdefault("category", "unknown")
return VaultBaseFrontmatter.model_validate(safe)

27
reme2/schema/vault/registry.py
View file

@ -1,27 +0,0 @@
"""Map vault category → strict pydantic model for validation.
Used by the Maintainer to derive lint rules from schema instead of
hardcoding. Two business object schemas:
- event → Event
- everything else → Topic (judgment subset enforced via model_validator)
"""
from typing import get_args
from pydantic import BaseModel
from .event import Event
from .topic import Topic, TopicCategory
_REGISTRY: dict[str, type[BaseModel]] = {"event": Event}
for _cat in get_args(TopicCategory):
_REGISTRY[_cat] = Topic
ALL_KNOWN_CATEGORIES: frozenset[str] = frozenset(_REGISTRY.keys())
def schema_for(category: str | None) -> type[BaseModel] | None:
"""Return the pydantic schema bound to a category, or None for unknown."""
if not category:
return None
return _REGISTRY.get(category)

47
reme2/schema/vault/topic.py
View file

@ -1,47 +0,0 @@
"""Topic schema — every .md under topics/ is a Topic.
No satellite/tentacle/folder note subclassing. Folder topic = Topic whose
filename equals its parent directory name (path convention; not a schema field).
"""
from typing import Literal
from pydantic import model_validator
from .frontmatter import VaultBaseFrontmatter
# Topic categories grouped by usage:
# - Index categories: usually appear as folder topics; may have ticker/market
# - Content categories: cluster siblings; thesis/model/questions need confidence
INDEX_CATEGORIES = {"company", "sector", "concept", "method", "tool", "profile"}
JUDGMENT_CATEGORIES = {"thesis", "model", "questions"}
CONTENT_CATEGORIES = JUDGMENT_CATEGORIES | {"fundamentals"}
TopicCategory = Literal[
"company", "sector", "concept", "method", "tool", "profile",
"thesis", "model", "questions", "fundamentals",
]
Confidence = Literal["⏳", "✅", "❌"]
class Topic(VaultBaseFrontmatter):
"""topics/{folder}/{name}.md — long-lived cognitive memory node.
Folder topic is identified by path convention: filename stem == parent
folder name. Not represented in this schema directly; checked at runtime
via `Path(p).stem == Path(p).parent.name`.
"""
category: TopicCategory # type: ignore[assignment]
market: str | None = None
ticker: str | None = None
confidence: Confidence | None = None
@model_validator(mode="after")
def _confidence_required_for_judgments(self):
if self.category in JUDGMENT_CATEGORIES and self.confidence is None:
raise ValueError(
f"category={self.category} requires explicit confidence "
f"(one of ⏳ / ✅ / ❌)"
)
return self

25
reme2/utils/vault_paths.py
View file

@ -1,8 +1,8 @@
"""Vault path generation + naming disambiguation.
Pure-Python helpers consumed by the MCP step shells (`sync`,
`topic_create`) and any future flow that needs to materialize a path
under the vault layout. No async, no MCP awareness — easy to unit-test.
Pure-Python helpers consumed by the `sync` MCP step shell and any
future flow that needs to materialize a path under the vault layout.
No async, no MCP awareness — easy to unit-test.
Wikilink uniqueness is a *graph* property, not a path-builder concern —
see `reme2.component.file_store.BaseFileStore.collisions_after_create`
@ -31,16 +31,6 @@ def event_path(
return Path(vault_root) / events_dir / on_date / name / f"{name}.md"
def topic_path(
vault_root: str | Path,
folder: str,
name: str,
topics_dir: str = "topics",
) -> Path:
"""topics/{folder}/{name}.md under the given vault root."""
return Path(vault_root) / topics_dir / folder / f"{name}.md"
def is_folder_topic(path: str | Path) -> bool:
"""True if filename stem == parent directory name (folder note convention)."""
p = Path(path)
@ -50,11 +40,10 @@ def is_folder_topic(path: str | Path) -> bool:
def next_suffixed_stem(taken: Iterable[str], base: str) -> str:
"""Lowest unused `<base>-N` (N≥2). Returns `base` itself if not taken.
Used by topic_create / sync when a same-stem (topic) or
same-name-on-same-day (event) collision is detected — the suggested
suffix is *advisory*; the create still rejects so the agent can pick
a domain-specific qualifier (e.g., `Apple-Inc` vs `Apple-Fruit`)
that carries more meaning than a numeric suffix.
Used by `sync` when a same-name-on-same-day collision is detected —
the suggested suffix is *advisory*; the create still rejects so the
agent can pick a domain-specific qualifier (e.g. `Apple-Inc` vs
`Apple-Fruit`) that carries more meaning than a numeric suffix.
Examples (with taken={"BABA", "BABA-2"}):
next_suffixed_stem(taken, "BABA") == "BABA-3"