feat: add file_graph component and integrate with LinkedFileParser

- Add file_graph import to component registry
- Register FILE_GRAPH enum in ComponentEnum
- Implement BaseFileGraph integration in LinkedFileParser
- Replace FileEdge with FileLink for better semantic clarity
- Add lazy resolution of file_graph from app_context
- Update file watcher logging to reflect links instead of edges

refactor: streamline MCP transport layer architecture

- Remove redundant step shells from reme2/mcp/steps/
- Consolidate all @R.register components to reme2.memory package
- Update server.py to import reme2.memory directly
- Revise README.md to document new architecture
- Simplify module dependencies and import structure
This commit is contained in:
huangsen 2026-05-14 11:35:02 +08:00
parent 56b94c5952
commit 6a5561f86a
52 changed files with 2583 additions and 1470 deletions

View file

@ -4,6 +4,7 @@ from . import as_llm
from . import as_llm_formatter
from . import as_token_counter
from . import file_store
from . import file_graph
from . import client
from . import embedding
from . import file_parser
@ -31,6 +32,7 @@ __all__ = [
"as_llm_formatter",
"as_token_counter",
"file_store",
"file_graph",
"client",
"embedding",
"file_parser",

View file

@ -21,11 +21,11 @@ class BaseComponent(ABC):
component_type = ComponentEnum.BASE
def __init__(
self,
name: str | None = None,
backend: str = "",
app_context: "ApplicationContext | None" = None,
**kwargs,
self,
name: str | None = None,
backend: str = "",
app_context: "ApplicationContext | None" = None,
**kwargs,
) -> None:
self.name: str = name or self.__class__.__name__
self.backend: str = backend

View file

@ -77,7 +77,7 @@ class BaseEmbeddingModel(BaseComponent):
if to_compute:
for i in range(0, len(to_compute), self.max_batch_size):
batch = to_compute[i: i + self.max_batch_size]
batch = to_compute[i : i + self.max_batch_size]
indices = [idx for idx, _ in batch]
texts = [text for _, text in batch]
@ -89,7 +89,7 @@ class BaseEmbeddingModel(BaseComponent):
break
except (TimeoutError, ConnectionError, OSError):
if attempt < self.max_retries - 1:
await asyncio.sleep(2 ** attempt)
await asyncio.sleep(2**attempt)
except Exception:
break
@ -184,4 +184,4 @@ class BaseEmbeddingModel(BaseComponent):
try:
np.savez(self.cache_path, keys=np.array(keys, dtype=str), embeddings=embeddings)
except Exception:
pass
pass

View file

@ -0,0 +1,22 @@
"""File graph module.
Backend-agnostic graph engine that owns the wikilink graph (nodes +
typed edges + traversal queries). Sister to ``file_store``, which owns
chunks and projections (vector / FTS).
Two backends ship today:
- ``LocalFileGraph`` (``@R.register("local")``) networkx
``MultiDiGraph`` + JSONL persistence. Default.
- ``Neo4jFileGraph`` (``@R.register("neo4j")``) property-graph
backed by Neo4j. Requires the ``neo4j`` driver.
"""
from .base_file_graph import BaseFileGraph
from .local_file_graph import LocalFileGraph
from .neo4j_file_graph import Neo4jFileGraph
__all__ = [
"BaseFileGraph",
"LocalFileGraph",
"Neo4jFileGraph",
]

View file

@ -0,0 +1,76 @@
"""Abstract base for the file-graph engine.
The file-graph owns ``FileNode`` records keyed by vault-relative path
and serves graph traversal over wikilink links. file_graph trusts
``FileLink.path`` directly there is no internal wikilink resolution.
Pre-resolution forms (raw stem-form wikilinks like ``[[Foo]]``) are a
vault convention resolved by the external ``utils.wikilink_resolver``
before nodes are upserted.
Contract six abstract methods, two blocks:
Node CRUD upsert_node, delete_node, get_node, iter_nodes
Link access get_outlinks, get_inlinks
"""
from __future__ import annotations
import re
from abc import abstractmethod
from collections.abc import AsyncIterator
from pathlib import Path
from ..base_component import BaseComponent
from ...enumeration import ComponentEnum
from ...schema import FileLink, FileNode
class BaseFileGraph(BaseComponent):
"""Pluggable file-graph backend — node CRUD + link adjacency."""
component_type = ComponentEnum.FILE_GRAPH
def __init__(self, store_name: str = "default", **kwargs):
super().__init__(**kwargs)
if not re.match(r"^[a-zA-Z0-9_]+$", store_name):
raise ValueError(
f"Invalid store name '{store_name}'. " f"Only alphanumeric and underscores allowed.",
)
self.store_name: str = store_name
self.working_dir: str = self.app_context.app_config.working_dir if self.app_context else ""
self.store_path: Path = Path(self.working_dir) / "file_graph" / store_name
self.store_path.mkdir(parents=True, exist_ok=True)
# -- Node CRUD ---------------------------------------------------------
@abstractmethod
async def upsert_node(self, node: FileNode) -> None:
"""Add or replace a node and its outgoing links."""
@abstractmethod
async def delete_node(self, path: str) -> FileNode | None:
"""Remove a node + its incident links. Returns the removed node, or None."""
@abstractmethod
async def get_node(self, path: str) -> FileNode | None:
"""Single-node lookup."""
@abstractmethod
def iter_nodes(self) -> AsyncIterator[tuple[str, FileNode]]:
"""Walk every (path, FileNode) in the graph."""
# -- Link access -------------------------------------------------------
@abstractmethod
async def get_outlinks(
self,
path: str,
) -> list[tuple[FileNode, FileLink]]:
"""Resolved outgoing links from ``path``."""
@abstractmethod
async def get_inlinks(
self,
path: str,
) -> list[tuple[FileNode, FileLink]]:
"""Resolved incoming links to ``path``."""

View file

@ -0,0 +1,152 @@
"""Local file-graph backend — networkx ``MultiDiGraph``.
Each ``FileNode`` lives as ``data['node']`` on the graph; each
``FileLink`` whose ``link.path`` exists in the graph lives as
``data['link']`` on a directed graph edge between the two file nodes.
file_graph trusts ``link.path`` directly there is no internal
wikilink resolution. The parser pipeline (with the external resolver)
is responsible for producing safe ``FileLink`` records where
``link.path`` is a real vault-relative target path. Stem ambiguity is
already handled there by emitting one link per candidate.
Late-arriving target: when ``upsert_node(B)`` runs, other nodes whose
links have ``link.path == B.path`` get those in-edges restored
(one O(N×L) sweep per upsert; cheap for vault sizes).
Persistence: pickle to ``graph.pkl`` on close, restored on _start.
"""
from __future__ import annotations
import pickle
from collections.abc import AsyncIterator
from pathlib import Path
from .base_file_graph import BaseFileGraph
from ..component_registry import R
from ...schema import FileLink, FileNode
@R.register("local")
class LocalFileGraph(BaseFileGraph):
"""Networkx-backed file graph. Trusts ``FileLink.path`` for adjacency."""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._graph = None # networkx.MultiDiGraph; set in _start
self._graph_file: Path = self.store_path / "graph.pkl"
# -- Lifecycle ---------------------------------------------------------
async def _start(self) -> None:
await super()._start()
try:
import networkx as nx
except ImportError as e:
raise ImportError(
"LocalFileGraph requires networkx. Install with `pip install networkx`.",
) from e
self._graph = self._load_graph(nx) or nx.MultiDiGraph()
self.logger.info(
f"LocalFileGraph '{self.store_name}' ready: "
f"{self._graph.number_of_nodes()} nodes, "
f"{self._graph.number_of_edges()} edges",
)
async def _close(self) -> None:
self._save_graph()
await super()._close()
def _load_graph(self, nx):
if not self._graph_file.exists():
return None
try:
with open(self._graph_file, "rb") as f:
graph = pickle.load(f)
if not isinstance(graph, nx.MultiDiGraph):
self.logger.warning(
f"{self._graph_file} is not a MultiDiGraph; ignoring",
)
return None
return graph
except Exception as e:
self.logger.exception(f"Failed to load {self._graph_file}: {e}")
return None
def _save_graph(self) -> None:
try:
tmp = self._graph_file.with_suffix(".tmp")
with open(tmp, "wb") as f:
pickle.dump(self._graph, f, protocol=pickle.HIGHEST_PROTOCOL)
tmp.replace(self._graph_file)
except Exception as e:
self.logger.exception(f"Failed to write {self._graph_file}: {e}")
# -- Node CRUD ---------------------------------------------------------
async def upsert_node(self, node: FileNode) -> None:
path = node.path
if self._graph.has_node(path):
self._graph.remove_node(path)
self._graph.add_node(path, node=node)
# Out-links from this node — trust link.path.
for link in node.links:
if link.path and self._graph.has_node(link.path):
self._graph.add_edge(path, link.path, link=link)
# Late-arriving target: restore in-links from other nodes whose
# links resolve here. Scan all other nodes' links; cheap for
# vault sizes (O(N×L_avg) per upsert).
for src_path, src_data in self._graph.nodes(data=True):
if src_path == path:
continue
src_node = src_data.get("node")
if src_node is None:
continue
for link in src_node.links:
if link.path == path:
self._graph.add_edge(src_path, path, link=link)
async def delete_node(self, path: str) -> FileNode | None:
if not self._graph.has_node(path):
return None
node = self._graph.nodes[path].get("node")
self._graph.remove_node(path)
return node
async def get_node(self, path: str) -> FileNode | None:
if not self._graph.has_node(path):
return None
return self._graph.nodes[path].get("node")
async def iter_nodes(self) -> AsyncIterator[tuple[str, FileNode]]:
for path, data in self._graph.nodes(data=True):
node = data.get("node")
if node is not None:
yield path, node
# -- Link access -------------------------------------------------------
async def get_outlinks(self, path: str) -> list[tuple[FileNode, FileLink]]:
if not self._graph.has_node(path):
return []
out: list[tuple[FileNode, FileLink]] = []
for _src, dst, data in self._graph.out_edges(path, data=True):
target = self._graph.nodes[dst].get("node")
link = data.get("link")
if target is not None and isinstance(link, FileLink):
out.append((target, link))
return out
async def get_inlinks(self, path: str) -> list[tuple[FileNode, FileLink]]:
if not self._graph.has_node(path):
return []
out: list[tuple[FileNode, FileLink]] = []
for src, _dst, data in self._graph.in_edges(path, data=True):
source = self._graph.nodes[src].get("node")
link = data.get("link")
if source is not None and isinstance(link, FileLink):
out.append((source, link))
return out

View file

@ -0,0 +1,323 @@
"""Neo4j-backed file graph.
Property-graph mapping:
(:File {path, st_mtime, title, description, tags, links_json,
extra_json})
-[:LINKS {idx, anchor, predicate}]->(:File)
``path`` is the unique key (constraint enforced on ``_start``).
Frontmatter goes into flat properties; arbitrary extras land in
``extra_json``. The full ``FileLink[]`` payload is also stored as
``links_json`` so we can recover the node losslessly even for links
whose target wasn't indexed at upsert time (will be linked later by
``_restore_inlinks``).
Adjacency policy: file_graph trusts ``FileLink.path`` directly no
internal wikilink resolution. The parser pipeline (with the external
resolver) produces safe links where ``link.path`` is already a
vault-relative target path.
Conditional dependency: the ``neo4j`` driver is loaded lazily; the
import error fires at ``_start`` (boot), not at first call.
"""
from __future__ import annotations
import json
from collections.abc import AsyncIterator
from typing import Any
from .base_file_graph import BaseFileGraph
from ..component_registry import R
from ...schema import FileLink, FileNode
from ...schema.file_node import FileFrontMatter
_TYPED_FRONTMATTER_FIELDS = {"title", "description", "tags"}
_LINK_FIELDS = {"path", "anchor", "predicate"}
@R.register("neo4j")
class Neo4jFileGraph(BaseFileGraph):
"""Neo4j-backed file graph; trusts ``FileLink.path`` for adjacency.
Connection params (constructor kwargs):
uri: bolt URL, e.g. ``bolt://localhost:7687``
user: auth user (default ``neo4j``)
password: auth password
database: target db name (default ``neo4j``)
"""
def __init__(
self,
uri: str = "bolt://localhost:7687",
user: str = "neo4j",
password: str = "neo4j",
database: str = "neo4j",
**kwargs,
):
super().__init__(**kwargs)
self._uri: str = uri
self._user: str = user
self._password: str = password
self._database: str = database
self._driver = None
# -- Lifecycle ---------------------------------------------------------
async def _start(self) -> None:
await super()._start()
try:
from neo4j import AsyncGraphDatabase
except ImportError as e:
raise ImportError(
"Neo4jFileGraph requires the neo4j driver. " "Install with `pip install neo4j`.",
) from e
self._driver = AsyncGraphDatabase.driver(
self._uri,
auth=(self._user, self._password),
)
async with self._session() as session:
await session.run(
"CREATE CONSTRAINT file_path_unique IF NOT EXISTS " "FOR (f:File) REQUIRE f.path IS UNIQUE",
)
self.logger.info(
f"Neo4jFileGraph '{self.store_name}' connected at " f"{self._uri}/{self._database}",
)
async def _close(self) -> None:
if self._driver is not None:
await self._driver.close()
self._driver = None
await super()._close()
def _session(self):
assert self._driver is not None, "Neo4jFileGraph not started"
return self._driver.session(database=self._database)
# -- Node CRUD ---------------------------------------------------------
async def upsert_node(self, node: FileNode) -> None:
props = self._node_props(node)
# Trust link.path; only emit LINKS for links with non-empty path.
link_payload = [
{
"idx": i,
"anchor": link.anchor,
"predicate": link.predicate,
"target": link.path,
}
for i, link in enumerate(node.links)
if link.path
]
async with self._session() as session:
await session.execute_write(
self._upsert_node_tx,
node.path,
props,
link_payload,
)
# Late-arriving target: restore in-links from other nodes whose
# links resolve here. Separate session call (not in tx) so it
# doesn't block the upsert ack.
await self._restore_inlinks(node.path)
@staticmethod
async def _upsert_node_tx(tx, path, props, links):
await tx.run(
"MERGE (f:File {path: $path}) SET f += $props",
path=path,
props=props,
)
await tx.run(
"MATCH (f:File {path: $path})-[r:LINKS]->() DELETE r",
path=path,
)
# MERGE only when the target node exists. ``OPTIONAL MATCH`` +
# ``WHERE t IS NOT NULL`` lets us batch this in one pass per link.
for link in links:
await tx.run(
"""
MATCH (s:File {path: $src})
OPTIONAL MATCH (t:File {path: $dst})
WITH s, t WHERE t IS NOT NULL
MERGE (s)-[r:LINKS {idx: $idx}]->(t)
SET r.anchor = $anchor, r.predicate = $predicate
""",
src=path,
dst=link["target"],
idx=link["idx"],
anchor=link["anchor"],
predicate=link["predicate"],
)
async def _restore_inlinks(self, path: str) -> None:
"""Walk other nodes' links, add LINKS to ``path`` where ``link.path``
matches. Uses ``links_json`` on each node no resolution logic."""
async with self._session() as session:
rec = await session.run(
"""
MATCH (f:File)
WHERE f.path <> $path
RETURN f.path AS p, f.links_json AS l
""",
path=path,
)
rows = [dict(r) async for r in rec]
for row in rows:
try:
links = json.loads(row.get("l") or "[]")
except json.JSONDecodeError:
continue
for i, link in enumerate(links):
if not isinstance(link, dict) or link.get("path") != path:
continue
async with self._session() as session:
await session.run(
"""
MATCH (s:File {path: $src})
MATCH (t:File {path: $dst})
MERGE (s)-[r:LINKS {idx: $idx}]->(t)
SET r.anchor = $anchor, r.predicate = $predicate
""",
src=row["p"],
dst=path,
idx=i,
anchor=link.get("anchor"),
predicate=link.get("predicate"),
)
async def delete_node(self, path: str) -> FileNode | None:
node = await self.get_node(path)
if node is None:
return None
async with self._session() as session:
await session.run(
"MATCH (f:File {path: $path}) DETACH DELETE f",
path=path,
)
return node
async def get_node(self, path: str) -> FileNode | None:
async with self._session() as session:
rec = await session.run(
"MATCH (f:File {path: $path}) RETURN f LIMIT 1",
path=path,
)
row = await rec.single()
return self._row_to_node(row["f"]) if row else None
async def iter_nodes(self) -> AsyncIterator[tuple[str, FileNode]]:
async with self._session() as session:
rec = await session.run("MATCH (f:File) RETURN f")
async for row in rec:
node = self._row_to_node(row["f"])
yield node.path, node
# -- Link access -------------------------------------------------------
async def get_outlinks(self, path: str) -> list[tuple[FileNode, FileLink]]:
async with self._session() as session:
rec = await session.run(
"""
MATCH (s:File {path: $path})-[r:LINKS]->(t:File)
RETURN t, r ORDER BY r.idx ASC
""",
path=path,
)
rows = [dict(row) async for row in rec]
return [(self._row_to_node(r["t"]), self._rel_to_link(r["r"], r["t"])) for r in rows]
async def get_inlinks(self, path: str) -> list[tuple[FileNode, FileLink]]:
async with self._session() as session:
rec = await session.run(
"""
MATCH (s:File)-[r:LINKS]->(t:File {path: $path})
RETURN s, r ORDER BY s.path ASC, r.idx ASC
""",
path=path,
)
rows = [dict(row) async for row in rec]
# ``rel_to_link`` needs the target's path to populate FileLink.path.
return [(self._row_to_node(r["s"]), self._rel_to_link(r["r"], target_path=path)) for r in rows]
# -- Internal: row ↔ schema marshaling ---------------------------------
@staticmethod
def _node_props(node: FileNode) -> dict[str, Any]:
fm = node.front_matter
extras = dict(fm.__pydantic_extra__ or {})
return {
"path": node.path,
"st_mtime": float(node.st_mtime),
"title": fm.title or "",
"description": fm.description or "",
"tags": list(fm.tags or []),
"links_json": json.dumps(
[link.model_dump(exclude_none=True) for link in node.links],
ensure_ascii=False,
),
"extra_json": json.dumps(extras, ensure_ascii=False, sort_keys=True),
}
@staticmethod
def _row_to_node(row) -> FileNode:
d = dict(row)
try:
extras = json.loads(d.get("extra_json") or "{}")
except json.JSONDecodeError:
extras = {}
try:
links_raw = json.loads(d.get("links_json") or "[]")
except json.JSONDecodeError:
links_raw = []
links: list[FileLink] = []
for link in links_raw:
if not isinstance(link, dict):
continue
# Defensive: strip any keys the schema doesn't recognise
# (e.g. legacy fields from prior schema versions).
clean = {k: v for k, v in link.items() if k in _LINK_FIELDS}
try:
links.append(FileLink(**clean))
except Exception:
continue
fm_kwargs: dict[str, Any] = {
"title": d.get("title", "") or "",
"description": d.get("description", "") or "",
"tags": d.get("tags") or None,
}
fm_kwargs.update(
{k: v for k, v in extras.items() if k not in _TYPED_FRONTMATTER_FIELDS},
)
return FileNode(
path=d["path"],
st_mtime=float(d.get("st_mtime", 0.0)),
links=links,
chunk_ids=[],
front_matter=FileFrontMatter(**fm_kwargs),
)
@staticmethod
def _rel_to_link(rel, target_path: Any = None) -> FileLink:
"""Reconstitute a ``FileLink`` from a Neo4j relationship.
The relationship row carries ``anchor`` and ``predicate``; the
target's ``path`` comes from the matched ``File`` node (passed
in by callers that have it on hand). For ``get_outlinks`` it's
the row's target node; for ``get_inlinks`` it's the path the
caller queried for. Either way, FileLink.path is set so the
link stays "safe by construction".
"""
d = dict(rel)
if hasattr(target_path, "get"):
# neo4j Node passed in
path = target_path.get("path", "")
else:
path = target_path or ""
return FileLink(
path=path,
anchor=d.get("anchor"),
predicate=d.get("predicate"),
)

View file

@ -17,7 +17,6 @@ class BaseFileParser(BaseComponent):
super().__init__(**kwargs)
self.working_dir = self.app_context.app_config.working_dir if self.app_context is not None else ""
def _get_relative_path(self, path: str | Path) -> str:
"""Get path relative to working_dir."""
file_path = Path(path).absolute()

View file

@ -40,7 +40,7 @@ class DefaultFileParser(BaseFileParser):
data = {}
front_matter = FileFrontMatter(**data)
remaining = text[end_idx + 4:].lstrip("\n")
remaining = text[end_idx + 4 :].lstrip("\n")
return front_matter, remaining
async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
@ -72,18 +72,25 @@ class DefaultFileParser(BaseFileParser):
if content_bytes[end - 1] == ord(b"\n"):
end_line -= 1
chunks.append(FileChunk(
path=rel_path,
start_line=start_line,
end_line=end_line,
text=chunk_text,
).set_hash_id())
chunks.append(
FileChunk(
path=rel_path,
start_line=start_line,
end_line=end_line,
text=chunk_text,
).set_hash_id(),
)
if end >= len(content_bytes):
break
start += step
return FileNode(path=rel_path,
st_mtime=stat.st_mtime,
front_matter=front_matter,
chunk_ids=[c.id for c in chunks]), chunks
return (
FileNode(
path=rel_path,
st_mtime=stat.st_mtime,
front_matter=front_matter,
chunk_ids=[c.id for c in chunks],
),
chunks,
)

View file

@ -16,25 +16,20 @@ from pathlib import Path
from typing import Any
import frontmatter
from mistletoe.block_token import (
CodeFence,
Document,
Heading,
List,
ListItem,
SetextHeading,
Table,
TableRow,
)
from mistletoe.markdown_renderer import BlankLine, MarkdownRenderer
from .base_file_parser import BaseFileParser
from ..component_registry import R
from ...schema import FileChunk, FileEdge, FileFrontMatter, FileNode
from ..file_graph import BaseFileGraph
from ...enumeration import ComponentEnum
from ...schema import (
FileChunk,
FileLink,
FileFrontMatter,
FileNode,
)
from ...utils import hash_text
_PART_RESERVE = 18 # worst-case "[Part NNN/NNN]\n\n" prefix
from ...utils.wikilink_resolver import text_to_links
# -- AST node + helpers ---------------------------------------------------
@ -63,7 +58,7 @@ class MdNode:
desc_toc: str = ""
def _heading_text(node: Any, renderer: MarkdownRenderer) -> str:
def _heading_text(node: Any, renderer) -> str:
"""Heading text without `#` markers (for outline)."""
rendered = renderer.render(node).rstrip("\n")
if rendered.startswith("#"):
@ -71,14 +66,15 @@ def _heading_text(node: Any, renderer: MarkdownRenderer) -> str:
return rendered.split("\n", 1)[0].strip()
def _dedup_edges(edges: list[FileEdge]) -> list[FileEdge]:
def _dedup_links(links: list[FileLink]) -> list[FileLink]:
"""Drop links with identical (path, predicate, anchor); preserve order."""
seen: set[tuple] = set()
out: list[FileEdge] = []
for e in edges:
key = (e.link, e.predicate)
out: list[FileLink] = []
for link in links:
key = (link.path, link.predicate, link.anchor)
if key not in seen:
seen.add(key)
out.append(e)
out.append(link)
return out
@ -132,41 +128,75 @@ class LinkedFileParser(BaseFileParser):
"""Markdown parser: frontmatter + wikilink edges + full-skeleton chunks."""
def __init__(
self,
encoding: str = "utf-8",
chunk_chars: int = 2000,
embed_toc: bool = True,
**kwargs,
self,
encoding: str = "utf-8",
chunk_chars: int = 2000,
embed_toc: bool = True,
file_graph: str = "default",
**kwargs,
):
super().__init__(**kwargs)
self.encoding = encoding
self.chunk_chars = max(100, chunk_chars)
self.embed_toc = embed_toc
self._file_graph_name: str = file_graph
def _resolve_file_graph(self) -> BaseFileGraph | None:
"""Lazily fetch the configured file_graph from app_context.
Lazy (rather than ``_start``) so the parser doesn't impose a
component start-order constraint, and so tests can construct
the parser without a graph wired up.
"""
if self.app_context is None:
return None
graphs = self.app_context.components.get(ComponentEnum.FILE_GRAPH, {})
graph = graphs.get(self._file_graph_name)
if graph is None:
return None
if not isinstance(graph, BaseFileGraph):
raise TypeError(
f"Expected BaseFileGraph, got {type(graph).__name__}",
)
return graph
async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
from mistletoe.markdown_renderer import MarkdownRenderer
from mistletoe.block_token import Document
file_path = Path(path)
post = frontmatter.loads(file_path.read_text(encoding=self.encoding))
absolute = str(file_path.absolute())
chunks: list[FileChunk] = []
if post.content and post.content.strip():
with MarkdownRenderer() as renderer:
tree = self._build_tree(Document(post.content), renderer)
chunks = self._chunk_node(tree, "", "", absolute, renderer)
chunks = self._chunk_node(tree, "", "", str(file_path), renderer)
links: list[FileLink] = []
graph = self._resolve_file_graph()
if graph is not None:
links = _dedup_links(await text_to_links(graph, post.content))
node = FileNode(
path=absolute,
path=str(file_path),
st_mtime=file_path.stat().st_mtime,
chunk_ids=[chunk.id for chunk in chunks],
edges=_dedup_edges(FileEdge.from_text(post.content)),
links=links,
front_matter=FileFrontMatter(**dict(post.metadata)),
)
return node, chunks
def _build_tree(self, doc: Any, renderer: MarkdownRenderer) -> MdNode:
def _build_tree(self, doc: Any, renderer) -> MdNode:
"""Heading-level stack folds mistletoe's flat children into nested
sections; non-headings attach as ``body`` to the current section
(or root before the first heading)."""
from mistletoe.markdown_renderer import BlankLine
from mistletoe.block_token import (
Heading,
SetextHeading,
)
root = MdNode(kind="root", start_line=1, end_line=1)
stack: list[MdNode] = [root]
for child in doc.children or []:
@ -189,18 +219,27 @@ class LinkedFileParser(BaseFileParser):
rendered = renderer.render(child).rstrip("\n")
if not rendered:
continue
stack[-1].children.append(MdNode(
kind="body", block=child, text=rendered,
start_line=line, end_line=line + rendered.count("\n"),
))
stack[-1].children.append(
MdNode(
kind="body",
block=child,
text=rendered,
start_line=line,
end_line=line + rendered.count("\n"),
),
)
_finalize(root)
return root
# -- Recursive chunker ------------------------------------------------
def _chunk_node(
self, node: MdNode, before: str, after: str,
path: str, renderer: MarkdownRenderer,
self,
node: MdNode,
before: str,
after: str,
path: str,
renderer,
) -> list[FileChunk]:
"""Try the whole subtree; on overflow split (leaf) or descend.
``before``/``after`` are TOC fragments that bracket each emitted
@ -216,8 +255,16 @@ class LinkedFileParser(BaseFileParser):
else:
before_self = before
if len(node.text) <= self.chunk_chars:
return [self._make_chunk(before_self, node.text, after,
node.start_line, node.end_line, path)]
return [
self._make_chunk(
before_self,
node.text,
after,
node.start_line,
node.end_line,
path,
),
]
if node.kind == "body":
return self._split_leaf(node, before, after, path, renderer)
after_inside = _toc_join(node.desc_toc, after)
@ -229,32 +276,65 @@ class LinkedFileParser(BaseFileParser):
for c in node.children:
if c.kind == "section":
if run:
chunks.extend(self._chunk_body_run(
run, before_self, after_inside, path, renderer))
chunks.extend(
self._chunk_body_run(
run,
before_self,
after_inside,
path,
renderer,
),
)
run = []
remaining = "\n\n".join(sub_tocs[sec_idx + 1:])
chunks.extend(self._chunk_node(
c, accumulated, _toc_join(remaining, after), path, renderer))
remaining = "\n\n".join(sub_tocs[sec_idx + 1 :])
chunks.extend(
self._chunk_node(
c,
accumulated,
_toc_join(remaining, after),
path,
renderer,
),
)
accumulated = _toc_join(accumulated, sub_tocs[sec_idx])
sec_idx += 1
else:
run.append(c)
if run:
chunks.extend(self._chunk_body_run(
run, before_self, after_inside, path, renderer))
chunks.extend(
self._chunk_body_run(
run,
before_self,
after_inside,
path,
renderer,
),
)
return chunks
def _chunk_body_run(
self, run: list[MdNode], before: str, after: str,
path: str, renderer: MarkdownRenderer,
self,
run: list[MdNode],
before: str,
after: str,
path: str,
renderer,
) -> list[FileChunk]:
"""Greedy-pack consecutive body siblings under the same TOC slot.
No ``[Part X/N]`` markers distinct blocks, not a leaf split.
Oversized single body recurses to ``_split_leaf``."""
composite_size = sum(len(b.text) for b in run) + 2 * max(0, len(run) - 1)
if composite_size <= self.chunk_chars:
return [self._make_chunk(before, "\n\n".join(b.text for b in run), after,
run[0].start_line, run[-1].end_line, path)]
return [
self._make_chunk(
before,
"\n\n".join(b.text for b in run),
after,
run[0].start_line,
run[-1].end_line,
path,
),
]
chunks: list[FileChunk] = []
bucket: list[MdNode] = []
@ -264,9 +344,16 @@ class LinkedFileParser(BaseFileParser):
nonlocal bucket, bucket_chars
if not bucket:
return
chunks.append(self._make_chunk(
before, "\n\n".join(b.text for b in bucket), after,
bucket[0].start_line, bucket[-1].end_line, path))
chunks.append(
self._make_chunk(
before,
"\n\n".join(b.text for b in bucket),
after,
bucket[0].start_line,
bucket[-1].end_line,
path,
),
)
bucket = []
bucket_chars = 0
@ -287,9 +374,19 @@ class LinkedFileParser(BaseFileParser):
# -- Leaf splitters: build (text, start, end) units, hand off to packer
def _split_leaf(
self, body: MdNode, before: str, after: str,
path: str, renderer: MarkdownRenderer,
self,
body: MdNode,
before: str,
after: str,
path: str,
renderer,
) -> list[FileChunk]:
from mistletoe.block_token import (
CodeFence,
List,
Table,
)
block = body.block
if isinstance(block, Table):
return self._split_table(body, before, after, path)
@ -300,9 +397,15 @@ class LinkedFileParser(BaseFileParser):
return self._split_lines(body, before, after, path)
def _split_table(
self, body: MdNode, before: str, after: str, path: str,
self,
body: MdNode,
before: str,
after: str,
path: str,
) -> list[FileChunk]:
"""Repeat header + separator on every chunk."""
from mistletoe.block_token import TableRow
lines = body.text.split("\n")
header, data = "\n".join(lines[:2]), lines[2:]
rows = [r for r in (body.block.children or []) if isinstance(r, TableRow)]
@ -312,11 +415,21 @@ class LinkedFileParser(BaseFileParser):
return rows[i].line_number if i < len(rows) and rows[i].line_number else base + i
units = [(text, line_of(i), line_of(i)) for i, text in enumerate(data)]
return self._emit_packed(units, before, after, path,
joiner="\n", wrap=f"{header}\n{{inner}}")
return self._emit_packed(
units,
before,
after,
path,
joiner="\n",
wrap=f"{header}\n{{inner}}",
)
def _split_code(
self, body: MdNode, before: str, after: str, path: str,
self,
body: MdNode,
before: str,
after: str,
path: str,
) -> list[FileChunk]:
"""Repeat fence opener + closer on every chunk."""
code = body.block
@ -327,17 +440,28 @@ class LinkedFileParser(BaseFileParser):
if not raw:
return []
start = body.start_line + 1
units = [(indent + ln, start + i, start + i)
for i, ln in enumerate(raw.split("\n"))]
return self._emit_packed(units, before, after, path, joiner="\n",
wrap=f"{opener}\n{{inner}}\n{fence}",
allow_empty=True)
units = [(indent + ln, start + i, start + i) for i, ln in enumerate(raw.split("\n"))]
return self._emit_packed(
units,
before,
after,
path,
joiner="\n",
wrap=f"{opener}\n{{inner}}\n{fence}",
allow_empty=True,
)
def _split_list(
self, body: MdNode, before: str, after: str,
path: str, renderer: MarkdownRenderer,
self,
body: MdNode,
before: str,
after: str,
path: str,
renderer,
) -> list[FileChunk]:
"""Pack list items; oversized items emit alone (overflow accepted)."""
from mistletoe.block_token import ListItem
items = [c for c in (body.block.children or []) if isinstance(c, ListItem)]
if not items:
return self._split_lines(body, before, after, path)
@ -348,28 +472,43 @@ class LinkedFileParser(BaseFileParser):
continue
line = it.line_number or body.start_line
units.append((text, line, line + text.count("\n")))
return self._emit_packed(units, before, after, path,
joiner="\n", wrap="{inner}")
return self._emit_packed(
units,
before,
after,
path,
joiner="\n",
wrap="{inner}",
)
def _split_lines(
self, body: MdNode, before: str, after: str, path: str,
self,
body: MdNode,
before: str,
after: str,
path: str,
) -> list[FileChunk]:
"""Last-resort line-greedy split for paragraphs / quotes / html."""
start = body.start_line
units = [(line, start + i, start + i)
for i, line in enumerate(body.text.split("\n"))]
return self._emit_packed(units, before, after, path,
joiner="\n", wrap="{inner}")
units = [(line, start + i, start + i) for i, line in enumerate(body.text.split("\n"))]
return self._emit_packed(
units,
before,
after,
path,
joiner="\n",
wrap="{inner}",
)
def _emit_packed(
self,
units: list[tuple[str, int, int]],
before: str,
after: str,
path: str,
joiner: str,
wrap: str,
allow_empty: bool = False,
self,
units: list[tuple[str, int, int]],
before: str,
after: str,
path: str,
joiner: str,
wrap: str,
allow_empty: bool = False,
) -> list[FileChunk]:
"""Greedy-pack units into ``wrap`` envelopes; emit each piece.
@ -379,7 +518,7 @@ class LinkedFileParser(BaseFileParser):
``[Part X/N]`` markers; single pieces don't.
"""
envelope = len(wrap.replace("{inner}", ""))
budget = max(64, self.chunk_chars - envelope - _PART_RESERVE)
budget = max(64, self.chunk_chars - envelope)
sep_len = len(joiner)
parts: list[tuple[str, int, int]] = []
@ -410,9 +549,15 @@ class LinkedFileParser(BaseFileParser):
return [
self._make_chunk(
before,
f"[Part {idx}/{total}]\n\n{wrap.replace('{inner}', inner)}"
if total > 1 else wrap.replace("{inner}", inner),
after, s, e, path,
(
f"[Part {idx}/{total}]\n\n{wrap.replace('{inner}', inner)}"
if total > 1
else wrap.replace("{inner}", inner)
),
after,
s,
e,
path,
)
for idx, (inner, s, e) in enumerate(parts, 1)
]
@ -420,13 +565,13 @@ class LinkedFileParser(BaseFileParser):
# -- Emit -------------------------------------------------------------
def _make_chunk(
self,
before: str,
content: str,
after: str,
start_line: int,
end_line: int,
path: str,
self,
before: str,
content: str,
after: str,
start_line: int,
end_line: int,
path: str,
) -> FileChunk:
"""Build one ``FileChunk`` — text is ``before + content + after``
when ``embed_toc``, otherwise just ``content``."""

View file

@ -12,11 +12,11 @@ class BaseFileStore(BaseComponent):
component_type = ComponentEnum.FILE_STORE
def __init__(
self,
store_name: str,
embedding_model: str = "default",
keyword_index: str = "default",
**kwargs,
self,
store_name: str,
embedding_model: str = "default",
keyword_index: str = "default",
**kwargs,
):
super().__init__(**kwargs)
if not re.match(r"^[a-zA-Z0-9_]+$", store_name):
@ -46,15 +46,13 @@ class BaseFileStore(BaseComponent):
self.keyword_index = None
await self.dump_file_nodes()
async def load_file_nodes(self):
...
async def load_file_nodes(self): ...
async def dump_file_nodes(self):
...
async def dump_file_nodes(self): ...
async def upsert_file(
self,
file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
self,
file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
) -> None:
"""Upsert a file and its chunks into the store."""

View file

@ -30,8 +30,10 @@ class LocalFileStore(BaseFileStore):
async def _start(self) -> None:
await super()._start()
await self._load_jsonl(self.chunks_path, self.file_chunks, FileChunk, "id")
self.logger.info(f"LocalFileStore '{self.store_name}' ready: "
f"{len(self.file_nodes)} nodes, {len(self.file_chunks)} chunks")
self.logger.info(
f"LocalFileStore '{self.store_name}' ready: "
f"{len(self.file_nodes)} nodes, {len(self.file_chunks)} chunks"
)
async def _close(self) -> None:
await self._dump_jsonl(self.chunks_path, list(self.file_chunks.values()))
@ -71,8 +73,8 @@ class LocalFileStore(BaseFileStore):
await self._dump_jsonl(self.nodes_path, list(self.file_nodes.values()))
async def upsert_file(
self,
file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
self,
file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
) -> None:
if isinstance(file, tuple):
file = [file]

View file

@ -17,16 +17,16 @@ class BaseFileWatcher(BaseComponent):
component_type = ComponentEnum.FILE_WATCHER
def __init__(
self,
watch_paths: list[str] | str,
suffix_filters: list[str] | None = None,
recursive: bool = True,
force_polling: bool = True,
debounce: int = 2000,
poll_delay_ms: int = 2000,
file_store: str = "default",
file_parser: str = "default",
**kwargs,
self,
watch_paths: list[str] | str,
suffix_filters: list[str] | None = None,
recursive: bool = True,
force_polling: bool = True,
debounce: int = 2000,
poll_delay_ms: int = 2000,
file_store: str = "default",
file_parser: str = "default",
**kwargs,
):
super().__init__(**kwargs)
watch_paths = [watch_paths] if isinstance(watch_paths, str) else watch_paths

View file

@ -39,13 +39,13 @@ class LiteFileWatcher(BaseFileWatcher):
try:
logger.info(f"Watching: {valid_paths}")
async for changes in awatch(
*valid_paths,
watch_filter=self.watch_filter,
recursive=self.recursive,
force_polling=self.force_polling,
debounce=self.debounce,
poll_delay_ms=self.poll_delay_ms,
stop_event=self._stop_event,
*valid_paths,
watch_filter=self.watch_filter,
recursive=self.recursive,
force_polling=self.force_polling,
debounce=self.debounce,
poll_delay_ms=self.poll_delay_ms,
stop_event=self._stop_event,
):
if self._stop_event.is_set():
break

View file

@ -36,7 +36,7 @@ class BaseJob(BaseComponent):
for raw in self.step_configs:
config = raw if isinstance(raw, ComponentConfig) else ComponentConfig(**raw)
if not config.backend:
raise ValueError(f"Step is missing the required 'backend' field")
raise ValueError("Step is missing the required 'backend' field")
step_cls = R.get(ComponentEnum.STEP, config.backend)
if not step_cls:
raise ValueError(

View file

@ -24,6 +24,7 @@ class BaseKeywordIndex(BaseComponent):
"""Initialize tokenizer and load existing index if available."""
if self.app_context is None:
from ..tokenizer import RegexTokenizer
self.tokenizer = RegexTokenizer(filter_stopwords=False)
else:
self.tokenizer = self.get_component(ComponentEnum.TOKENIZER, self.tokenizer_name)

View file

@ -20,6 +20,7 @@ class DocMeta(TypedDict):
len: Number of tokens in the document.
token_ids: Set of unique token IDs present in the document.
"""
len: int
token_ids: set[int]

View file

@ -4,10 +4,6 @@ Lives next to `runtime_context.py` because both are about the BaseStep
interface the response side, specifically. Used by every Step that
returns a JSON-shaped payload (memory_*, sync, the three memory
services).
Was previously at `reme2/mcp/steps/_common.py`, which leaked an MCP
dependency into `reme2/memory/` services that legitimately need to
serialize their results the moved location breaks that cycle.
"""
from __future__ import annotations

View file

@ -52,4 +52,4 @@ class RegexTokenizer(BaseTokenizer):
tokens = [t for t in tokens if t not in self._stopwords]
result.append(tokens)
return result
return result

View file

@ -22,6 +22,7 @@ def _expand_env_vars(value: Any) -> Any:
typos don't silently produce empty connection strings.
"""
if isinstance(value, str):
def repl(m: re.Match) -> str:
name = m.group(1)
default = m.group(2)
@ -31,6 +32,7 @@ def _expand_env_vars(value: Any) -> Any:
return default
raise ValueError(f"Config references undefined env var: {name}")
return v
return _ENV_VAR_RE.sub(repl, value)
if isinstance(value, dict):
return {k: _expand_env_vars(v) for k, v in value.items()}

View file

@ -3,9 +3,9 @@ ReMe重构
a. ✅ vault_root → working_dir @sen
2. file_parser
a. 抽象基类 parse: @jinli
. 输入是path相对路径
. 输入是path相对路径
ⅱ. 输出是FileMetadata & list[FileChunks] & list[FileEdge]
b. default parser 兼容老方案 @jinli
b. default parser 兼容老方案 @jinli
. 带overlap的chunking策略 不输出FileEdge
c. markdown parser @sen
. 根据markdown ast做chunk不需要overlap
@ -27,11 +27,11 @@ ReMe重构
ⅳ. 【核心】检索机制 vector bm25 graph 如何进行融合
4. file_watcher @jinli
a. 抽象基类
. on_start:
. on_start:
1. file_store 的start 在前加载graphfile_watcher在后递归扫描目录
a. 通过ms_time对比graphon_change 进行改动
ⅱ. on_change:
1. 更新/增加:
1. 更新/增加:
a. delete_chunks_by_path 更新数据库
b. upate_chunks_by_path 更新数据库
c. 更新graph
@ -44,14 +44,14 @@ MemorySchema
. title
ⅱ. desc
ⅲ. tags
ⅳ.
ⅳ.
2. memory文件结构目录
a. MEMORY.md
b. msg/files -> daily/YYYYMMDD/YYYYMMDD.md + xxxx.md
. YYYYMMDD.md
1. xxx -> xxxx.md
2. xxx -> xxxd.md
ⅱ.
ⅱ.
c. daily -> topic/topic_l1/topic_l1.md + xxx.md + topic_l2
d. proactive

View file

@ -28,6 +28,8 @@ class ComponentEnum(str, Enum):
FILE_STORE = "file_store"
FILE_GRAPH = "file_graph"
FILE_WATCHER = "file_watcher"
KEYWORD_INDEX = "keyword_index"

View file

@ -1,39 +1,31 @@
# reme2.mcp
Agent-facing MCP interface layer. Exposes the markdown vault under
`reme2/` (file_store + watcher + memory services) as MCP tools for
`claude-code` and other MCP clients.
MCP transport layer. Exposes the agent-facing tools registered in
`reme2.memory` over MCP for `claude-code` and other MCP clients.
## Layout
```
reme2/mcp/
├── __init__.py
├── server.py MCP server bootstrap; defaults to ../config/service.yaml
└── steps/ @R.register MCP step shells
├── memory_retriever.py memory_search + memory_graph_search
├── memory_lint.py read-only Maintainer projection (lint findings)
└── sync.py hot-path event-folder upsert
├── server.py MCP server bootstrap; defaults to ../config/service.yaml
└── test/ end-to-end profile smoke tests
```
The 12 `memory_*` primitives (create/update/property_update/rename/
delete/archive/get/list/links/backlinks/resolve_wikilink/count_tokens)
live one layer down in `reme2/memory/memory_toolkit.py` — each is a
single `BaseStep` subclass with two class methods: `execute()` for the
MCP path (this layer) and a same-named method for the agent toolkit
path that the Ingestor's ReActAgent consumes. Importing
`reme2.mcp.steps` triggers all `@R.register` registrations.
The MCP layer's job is to **project** existing primitives as MCP tools
— it owns no business logic. Everything else lives outside the
transport boundary so memory services don't form an import cycle:
That's it. Every `@R.register` step the agent invokes lives one layer
down in `reme2/memory/` — there are no MCP-specific shells, because
none of the steps depend on MCP transport details. Importing
`reme2.memory` triggers all `@R.register` registrations.
| Concern | Location |
|---|---|
| Memory File System primitives | `reme2/component/file_store/`, `file_watcher/`, `file_parser/` |
| Three memory services | `reme2/memory/` — Retriever / Ingestor / Maintainer |
| Three memory services | `reme2/memory/{retriever,ingestor,maintainer}.py` |
| Engine API (pure, schema-free) | `reme2/memory/memory_io.py` |
| Schema-bound tools (BaseStep + agent toolkit) | `reme2/memory/memory_toolkit.py` |
| Schema-bound write/read tools | `reme2/memory/memory_toolkit.py` |
| Search step shells | `reme2/memory/memory_search.py` |
| Lint step shell | `reme2/memory/memory_lint.py` |
| Hot-path event upsert | `reme2/memory/sync.py` |
| Memory schema (4 axes + presets + parser) | `reme2/memory/schema/` |
| Path templates + name disambiguation | `reme2/utils/vault_paths.py` |
| Step response serialization | `reme2/component/runtime_response.py` |
@ -44,9 +36,8 @@ reme2.mcp → reme2.memory (incl. memory.schema) → reme2.component / reme2
```
The Ingestor (`reme2/memory/ingestor.py`) self-registers its `ingest`
MCP face — there's no shell for it under `steps/`. Topic creation lives
inside the Ingestor's R-M-W loop; there is no separate `topic_create`
tool.
MCP face. Topic creation lives inside the Ingestor's R-M-W loop; there
is no separate `topic_create` tool.
## Run
@ -91,12 +82,12 @@ Configs live in `reme2/config/`:
### Tools exposed (expert profile)
| Tool | Path | Purpose |
| Tool | Source | Purpose |
|---|---|---|
| `sync` | steps/sync.py | Hot-path event-folder upsert (idempotent per `(date, name)`). |
| `sync` | reme2/memory/sync.py | Hot-path event-folder upsert (idempotent per `(date, name)`). |
| `ingest` | reme2/memory/ingestor.py | Cold-path LLM-driven distillation; owns topic creation. |
| `memory_search` / `memory_graph_search` | steps/memory_retriever.py | V+K hybrid + optional graph BFS. |
| `memory_lint` | steps/memory_lint.py | Read-only projection of Maintainer's lint findings. |
| `memory_search` / `memory_graph_search` | reme2/memory/memory_search.py | V+K hybrid + optional graph BFS. |
| `memory_lint` | reme2/memory/memory_lint.py | Read-only projection of Maintainer's lint findings. |
| `memory_get` / `memory_list` / `memory_links` / `memory_backlinks` / `memory_resolve_wikilink` | reme2/memory/memory_toolkit.py | Read primitives. |
| `memory_create` / `memory_update` / `memory_property_update` / `memory_rename` / `memory_delete` / `memory_archive` | reme2/memory/memory_toolkit.py | Raw write primitives (prefer `ingest` / `sync`). |
| `memory_count_tokens` | reme2/memory/memory_toolkit.py | Token estimation. |

View file

@ -1,20 +1,19 @@
"""reme2.mcp — MCP interface layer.
"""reme2.mcp — MCP transport layer.
The Agent-facing surface: server entrypoint + step shells that wrap
the three services in `reme2.memory` (Retriever, Ingestor, Maintainer)
plus the hot-write primitive (`sync`) and the raw `memory_*` write/read
tools that bypass services and land directly on the Memory File System.
Just the MCP server bootstrap (config loading, env-var overrides,
sidecar HTTP). All agent-facing tools live in ``reme2.memory`` and
register themselves there; this package only wires them into the MCP
transport via the ``mcp`` service component.
This package depends on `reme2.memory`, `reme2.utils`, `reme2.component`
never the reverse. The Memory schema (the typed shape of every
frontmatter) lives under `reme2.memory.schema/` so the services that
own validation can use it without an import cycle through this
transport layer.
Dependency direction is strict: ``reme2.mcp reme2.memory`` (and
through it ``reme2.component`` / ``reme2.utils``). Importing
``reme2.memory`` triggers all ``@R.register`` decorators for
``memory_*`` / ``sync`` / ``memory_search`` / ``memory_lint`` /
``ingest`` / ``maintainer`` / ``hybrid``.
Sub-packages:
steps/ - all @R.register MCP step shells (memory_toolkit,
memory_retriever, memory_lint, sync).
server.py - MCP server bootstrap (defaults to ../config/service.yaml).
Files:
server.py MCP server bootstrap (defaults to ../config/service.yaml).
test/ end-to-end profile smoke tests.
"""
__version__ = "0.1.0"

View file

@ -17,8 +17,8 @@ from pathlib import Path
# Make this work whether invoked as `python -m reme2.mcp.server` (repo
# root already on sys.path) or `python reme2/mcp/server.py` (only this
# file's dir is on sys.path — without this, `import reme2.mcp.steps`
# would fail with ModuleNotFoundError).
# file's dir is on sys.path — without this, `import reme2.memory` would
# fail with ModuleNotFoundError).
_REPO_ROOT = Path(__file__).resolve().parents[2]
if str(_REPO_ROOT) not in sys.path:
sys.path.insert(0, str(_REPO_ROOT))
@ -32,10 +32,7 @@ load_dotenv(_REPO_ROOT / ".env")
# Eagerly import side-effect modules so all @R.register() decorators run
# before Application introspects the registry.
import reme2 # noqa: E402,F401
import reme2.mcp.steps # noqa: E402,F401
import reme2.memory # noqa: E402,F401 -- registers retriever + maintainer
import reme2.memory.ingestor # noqa: E402,F401
import reme2.memory.summarizer # noqa: E402,F401
import reme2.memory # noqa: E402,F401 -- registers every agent-facing tool
import reme2.component.service.mcp_service # noqa: E402,F401
from reme2.application import Application # noqa: E402

View file

@ -1,40 +0,0 @@
"""MCP step shells — the @R.register classes the model invokes as MCP tools.
Three groups:
Hot-write primitives that bypass services and land on MFS directly:
sync idempotent event-folder upsert (event log)
memory_toolkit memory_create / update / property_update /
rename / delete / archive / get / list /
links / backlinks / resolve_wikilink /
count_tokens. Each step lives in
`reme2.memory.memory_toolkit` and exposes
BOTH an `execute()` (this MCP path) and a
same-named class method (the agent toolkit
path used by the Ingestor's ReActAgent).
Service delegates (thin shells over the three memory services):
memory_retriever memory_search + memory_graph_search; delegate
to the Retriever component
(`reme2.memory.retriever`).
memory_lint read-only projection of the Maintainer's lint
findings (`reme2.memory.maintainer`).
Topic creation is owned by the Ingestor (`reme2.memory.ingestor`) its
LLM-driven R-M-W loop decides when a new topic is warranted, applies the
schema preset, and routes the write through `memory_create`. There is no
separate `topic_create` MCP tool.
The Ingestor's MCP face (`ingest`) is registered from
`reme2.memory.ingestor`. Importing this package triggers all the step
registrations hosted here and in `reme2.memory.memory_toolkit`.
"""
from ...memory import memory_toolkit # noqa: F401 -- triggers @R.register for memory_*
from . import memory_lint # noqa: F401 -- triggers @R.register for memory_lint
from . import memory_retriever # noqa: F401 -- memory_search / memory_graph_search
from .sync import Sync
__all__ = [
"Sync",
]

View file

@ -22,7 +22,7 @@ from ._helpers import AppContext, make_context
SUITES = {
"expert": test_expert.CHECKS,
"expert": test_expert.CHECKS,
"service": test_service.CHECKS,
}
@ -35,8 +35,7 @@ async def _run_suite(profile: str, checks: list[tuple[str, callable]]) -> tuple[
passed = 0
try:
ctx, tmp = await make_context(profile)
print(f" bootstrapped: vault={ctx.vault}, jobs={len(ctx.jobs)}, "
f"file_store={len(ctx.file_store)}")
print(f" bootstrapped: vault={ctx.vault}, jobs={len(ctx.jobs)}, " f"file_store={len(ctx.file_store)}")
for label, fn in checks:
try:
summary = await fn(ctx)
@ -69,7 +68,7 @@ async def _main() -> int:
for profile, (passed, total) in summary.items():
marker = "" if passed == total else ""
print(f" {marker} {profile}: {passed}/{total}")
failed += (total - passed)
failed += total - passed
return 0 if failed == 0 else failed

View file

@ -24,10 +24,7 @@ if str(_REPO_ROOT) not in sys.path:
# Eager-import side-effect modules so all @R.register() decorators run
# before Application introspects the registry.
import reme2 # noqa: E402,F401
import reme2.mcp.steps # noqa: E402,F401
import reme2.memory # noqa: E402,F401 -- registers retriever + maintainer
import reme2.memory.ingestor # noqa: E402,F401
import reme2.memory.summarizer # noqa: E402,F401
import reme2.memory # noqa: E402,F401 -- registers every agent-facing tool
from reme2.application import Application # noqa: E402
from reme2.config import parse_args # noqa: E402
@ -35,7 +32,7 @@ from reme2.config import parse_args # noqa: E402
CONFIG_DIR = _REPO_ROOT / "reme2" / "config"
PROFILES = {
"expert": CONFIG_DIR / "expert.yaml",
"expert": CONFIG_DIR / "expert.yaml",
"service": CONFIG_DIR / "service.yaml",
}
@ -106,8 +103,7 @@ async def wait_for_index(watcher, expected_min: int, timeout_s: float = 15.0) ->
last = now
await asyncio.sleep(0.25)
raise RuntimeError(
f"watcher did not reach >= {expected_min} files within {timeout_s}s "
f"(last seen: {last})"
f"watcher did not reach >= {expected_min} files within {timeout_s}s " f"(last seen: {last})",
)
@ -156,7 +152,9 @@ async def make_context(profile: str) -> tuple[AppContext, Path]:
vault.mkdir()
seed_vault(vault)
app = await build_app(PROFILES[profile], vault)
await wait_for_index(app.context.components["file_watcher"]["default"],
expected_min=SEED_FILE_COUNT)
await wait_for_index(
app.context.components["file_watcher"]["default"],
expected_min=SEED_FILE_COUNT,
)
jobs = sorted(app.context.jobs.keys())
return AppContext(app=app, vault=vault, jobs=jobs), tmp

View file

@ -75,19 +75,29 @@ async def check_memory_list(ctx: AppContext) -> str:
async def check_memory_search(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"memory_search", query="collaborates", max_results=3, min_score=0.0,
))
r = decode(
await ctx.app.run_job(
"memory_search",
query="collaborates",
max_results=3,
min_score=0.0,
),
)
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
assert len(hits) > 0, r
return f"{len(hits)} hits"
async def check_memory_graph_search(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"memory_graph_search", query="Alice", max_results=5, min_score=0.0,
graph_depth=1,
))
r = decode(
await ctx.app.run_job(
"memory_graph_search",
query="Alice",
max_results=5,
min_score=0.0,
graph_depth=1,
),
)
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
assert len(hits) > 0, r
# At least one result should carry a graph_hop annotation
@ -118,9 +128,12 @@ async def check_memory_resolve_wikilink(ctx: AppContext) -> str:
async def check_memory_count_tokens(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"memory_count_tokens", text="hello world from the smoke test",
))
r = decode(
await ctx.app.run_job(
"memory_count_tokens",
text="hello world from the smoke test",
),
)
assert isinstance(r, dict) and isinstance(r.get("tokens"), int), r
assert r["tokens"] > 0, r
return f"text → {r['tokens']} tokens"
@ -142,18 +155,20 @@ async def check_memory_lint(ctx: AppContext) -> str:
async def check_sync_create(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"sync",
name="suite-event",
description="full-profile suite",
content="## ops\n- ran the suite\n",
topics=["[[Alice]]"],
tags=["suite"],
materials=[
{"filename": "raw-prompt.md", "content": "# user prompt\n\nrun suite\n"},
{"filename": "tool-output.txt", "content": "exit=0\n"},
],
))
r = decode(
await ctx.app.run_job(
"sync",
name="suite-event",
description="full-profile suite",
content="## ops\n- ran the suite\n",
topics=["[[Alice]]"],
tags=["suite"],
materials=[
{"filename": "raw-prompt.md", "content": "# user prompt\n\nrun suite\n"},
{"filename": "tool-output.txt", "content": "exit=0\n"},
],
),
)
assert isinstance(r, dict), r
assert r.get("created") is True and r.get("action") == "created", r
materials = r.get("materials", [])
@ -176,17 +191,19 @@ async def check_sync_create(ctx: AppContext) -> str:
async def check_sync_append(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"sync",
name="suite-event",
content="## follow-up\n- second pass\n",
topics=["[[Bob]]"], # union with [[Alice]]
tags=["follow-up"],
materials=[
{"filename": "tool-output.txt", "content": "second run\n"}, # collision
{"filename": "summary.md", "content": "# summary\n"},
],
))
r = decode(
await ctx.app.run_job(
"sync",
name="suite-event",
content="## follow-up\n- second pass\n",
topics=["[[Bob]]"], # union with [[Alice]]
tags=["follow-up"],
materials=[
{"filename": "tool-output.txt", "content": "second run\n"}, # collision
{"filename": "summary.md", "content": "# summary\n"},
],
),
)
assert isinstance(r, dict), r
assert r.get("created") is False and r.get("action") == "appended", r
appended = r.get("materials", [])
@ -204,11 +221,18 @@ async def check_sync_append(ctx: AppContext) -> str:
async def check_sync_refuse_distilled(ctx: AppContext) -> str:
index_path = str(getattr(ctx, "_suite_event_index"))
await ctx.app.run_job(
"memory_property_update", path=index_path, key="status", value="distilled",
"memory_property_update",
path=index_path,
key="status",
value="distilled",
)
r = decode(
await ctx.app.run_job(
"sync",
name="suite-event",
content="should be refused",
),
)
r = decode(await ctx.app.run_job(
"sync", name="suite-event", content="should be refused",
))
assert isinstance(r, dict) and "error" in r, r
assert r.get("status") == "distilled", r
assert r.get("suggested_name"), r
@ -220,20 +244,22 @@ async def check_sync_refuse_distilled(ctx: AppContext) -> str:
async def check_memory_create(ctx: AppContext) -> str:
target = ctx.abs_path("topics", "Carol", "Carol.md")
r = decode(await ctx.app.run_job(
"memory_create",
path=target,
metadata={
"title": "Carol",
"lifecycle": "evolving",
"scope": "class",
"source": "curated",
"role": "profile",
"category": "profile",
"tags": ["person"],
},
content="# Carol\n\nKnows [[Alice]].\n",
))
r = decode(
await ctx.app.run_job(
"memory_create",
path=target,
metadata={
"title": "Carol",
"lifecycle": "evolving",
"scope": "class",
"source": "curated",
"role": "profile",
"category": "profile",
"tags": ["person"],
},
content="# Carol\n\nKnows [[Alice]].\n",
),
)
assert isinstance(r, dict) and r.get("created") is True, r
assert "error" not in r, r
assert Path(target).is_file(), target
@ -243,12 +269,14 @@ async def check_memory_create(ctx: AppContext) -> str:
async def check_memory_update(ctx: AppContext) -> str:
carol = ctx.abs_path("topics", "Carol", "Carol.md")
r = decode(await ctx.app.run_job(
"memory_update",
path=carol,
old_string="Knows [[Alice]].",
new_string="Knows [[Alice]] and [[Bob]].",
))
r = decode(
await ctx.app.run_job(
"memory_update",
path=carol,
old_string="Knows [[Alice]].",
new_string="Knows [[Alice]] and [[Bob]].",
),
)
assert isinstance(r, dict) and r.get("replaced", 0) >= 1, r
assert "Bob" in Path(carol).read_text(encoding="utf-8"), "edit not on disk"
return f"body edit applied (replaced={r['replaced']})"
@ -256,22 +284,30 @@ async def check_memory_update(ctx: AppContext) -> str:
async def check_memory_property_update(ctx: AppContext) -> str:
carol = ctx.abs_path("topics", "Carol", "Carol.md")
r = decode(await ctx.app.run_job(
"memory_property_update", path=carol, key="confidence", value="",
))
r = decode(
await ctx.app.run_job(
"memory_property_update",
path=carol,
key="confidence",
value="",
),
)
assert isinstance(r, dict) and "error" not in r, r
assert r.get("key") == "confidence" and r.get("value") == "", r
assert "confidence: ✅" in Path(carol).read_text(encoding="utf-8"), \
"property write not on disk"
assert "confidence: ✅" in Path(carol).read_text(encoding="utf-8"), "property write not on disk"
return "set confidence=✅"
async def check_memory_rename(ctx: AppContext) -> str:
src = ctx.abs_path("topics", "Carol", "Carol.md")
dst = ctx.abs_path("topics", "Carol", "Carol-renamed.md")
r = decode(await ctx.app.run_job(
"memory_rename", old_path=src, new_path=dst,
))
r = decode(
await ctx.app.run_job(
"memory_rename",
old_path=src,
new_path=dst,
),
)
assert isinstance(r, dict) and "error" not in r, r
assert r.get("new_path") and Path(r["new_path"]).is_file(), r
assert not Path(src).exists(), f"old path still on disk: {src}"
@ -291,8 +327,7 @@ async def check_memory_archive(ctx: AppContext) -> str:
async def check_memory_delete(ctx: AppContext) -> str:
target = getattr(ctx, "_carol_archived", None) \
or ctx.abs_path("topics", "Carol", "Carol-renamed.md")
target = getattr(ctx, "_carol_archived", None) or ctx.abs_path("topics", "Carol", "Carol-renamed.md")
r = decode(await ctx.app.run_job("memory_delete", path=target))
assert isinstance(r, dict) and r.get("deleted") is True, r
assert not Path(target).exists(), target
@ -306,14 +341,20 @@ async def check_schema_path_template_refuses(ctx: AppContext) -> str:
"""memory_create rejects paths outside topics/{X}/{Y}.md,
events/{date}/{name}/..., or Archive/..."""
target = ctx.abs_path("notes", "freeform.md") # outside any template
r = decode(await ctx.app.run_job(
"memory_create",
path=target,
metadata={"title": "freeform",
"lifecycle": "evolving", "scope": "class",
"source": "curated", "role": "concept"},
content="should be refused",
))
r = decode(
await ctx.app.run_job(
"memory_create",
path=target,
metadata={
"title": "freeform",
"lifecycle": "evolving",
"scope": "class",
"source": "curated",
"role": "concept",
},
content="should be refused",
),
)
assert isinstance(r, dict), r
assert "error" in r and "template" in r["error"].lower(), r
assert not Path(target).exists(), "file shouldn't have been written"
@ -323,15 +364,21 @@ async def check_schema_path_template_refuses(ctx: AppContext) -> str:
async def check_schema_path_template_force_bypass(ctx: AppContext) -> str:
"""force=True bypasses the template gate."""
target = ctx.abs_path("notes", "forced.md")
r = decode(await ctx.app.run_job(
"memory_create",
path=target,
metadata={"title": "forced",
"lifecycle": "evolving", "scope": "class",
"source": "curated", "role": "concept"},
content="forced through",
force=True,
))
r = decode(
await ctx.app.run_job(
"memory_create",
path=target,
metadata={
"title": "forced",
"lifecycle": "evolving",
"scope": "class",
"source": "curated",
"role": "concept",
},
content="forced through",
force=True,
),
)
assert isinstance(r, dict) and r.get("created") is True, r
assert Path(target).is_file(), target
return "force=True bypassed"
@ -342,12 +389,14 @@ async def check_schema_status_skip_refused(ctx: AppContext) -> str:
Trying distilled active is reverse must refuse."""
event_index = getattr(ctx, "_suite_event_index", None)
assert event_index is not None, "suite-event index not staged"
r = decode(await ctx.app.run_job(
"memory_property_update",
path=str(event_index),
key="status",
value="active",
))
r = decode(
await ctx.app.run_job(
"memory_property_update",
path=str(event_index),
key="status",
value="active",
),
)
assert isinstance(r, dict), r
assert "error" in r and "transition" in r["error"].lower(), r
assert r.get("prior") == "distilled", r
@ -358,12 +407,14 @@ async def check_schema_status_invalid_value(ctx: AppContext) -> str:
"""Random string for status is refused before any state-machine check."""
event_index = getattr(ctx, "_suite_event_index", None)
assert event_index is not None, "suite-event index not staged"
r = decode(await ctx.app.run_job(
"memory_property_update",
path=str(event_index),
key="status",
value="bogus",
))
r = decode(
await ctx.app.run_job(
"memory_property_update",
path=str(event_index),
key="status",
value="bogus",
),
)
assert isinstance(r, dict), r
assert "error" in r and "invalid" in r["error"].lower(), r
return "refused status='bogus'"
@ -374,20 +425,27 @@ async def check_schema_status_force_bypass(ctx: AppContext) -> str:
intentionally needs to step outside conventions."""
event_index = getattr(ctx, "_suite_event_index", None)
assert event_index is not None, "suite-event index not staged"
r = decode(await ctx.app.run_job(
"memory_property_update",
path=str(event_index),
key="status",
value="active",
force=True,
))
r = decode(
await ctx.app.run_job(
"memory_property_update",
path=str(event_index),
key="status",
value="active",
force=True,
),
)
assert isinstance(r, dict), r
assert "error" not in r, r
# restore for any downstream checks
decode(await ctx.app.run_job(
"memory_property_update",
path=str(event_index), key="status", value="distilled", force=True,
))
decode(
await ctx.app.run_job(
"memory_property_update",
path=str(event_index),
key="status",
value="distilled",
force=True,
),
)
return "force=True bypassed"
@ -398,28 +456,28 @@ async def check_schema_status_force_bypass(ctx: AppContext) -> str:
# rely on side effects from earlier ones (e.g. sync.append needs
# sync.create to have run).
CHECKS: list[tuple[str, callable]] = [
("registry", check_registry),
("memory_get", check_memory_get),
("memory_list", check_memory_list),
("memory_search", check_memory_search),
("memory_graph_search", check_memory_graph_search),
("memory_links", check_memory_links),
("memory_backlinks", check_memory_backlinks),
("memory_resolve_wikilink", check_memory_resolve_wikilink),
("memory_count_tokens", check_memory_count_tokens),
("memory_lint", check_memory_lint),
("sync.create", check_sync_create),
("sync.append", check_sync_append),
("sync.refuse_distilled", check_sync_refuse_distilled),
("memory_create", check_memory_create),
("memory_update", check_memory_update),
("memory_property_update", check_memory_property_update),
("memory_rename", check_memory_rename),
("memory_archive", check_memory_archive),
("memory_delete", check_memory_delete),
("schema.path_template_refuses", check_schema_path_template_refuses),
("schema.path_template_force", check_schema_path_template_force_bypass),
("schema.status_skip_refused", check_schema_status_skip_refused),
("schema.status_invalid_value", check_schema_status_invalid_value),
("schema.status_force", check_schema_status_force_bypass),
("registry", check_registry),
("memory_get", check_memory_get),
("memory_list", check_memory_list),
("memory_search", check_memory_search),
("memory_graph_search", check_memory_graph_search),
("memory_links", check_memory_links),
("memory_backlinks", check_memory_backlinks),
("memory_resolve_wikilink", check_memory_resolve_wikilink),
("memory_count_tokens", check_memory_count_tokens),
("memory_lint", check_memory_lint),
("sync.create", check_sync_create),
("sync.append", check_sync_append),
("sync.refuse_distilled", check_sync_refuse_distilled),
("memory_create", check_memory_create),
("memory_update", check_memory_update),
("memory_property_update", check_memory_property_update),
("memory_rename", check_memory_rename),
("memory_archive", check_memory_archive),
("memory_delete", check_memory_delete),
("schema.path_template_refuses", check_schema_path_template_refuses),
("schema.path_template_force", check_schema_path_template_force_bypass),
("schema.status_skip_refused", check_schema_status_skip_refused),
("schema.status_invalid_value", check_schema_status_invalid_value),
("schema.status_force", check_schema_status_force_bypass),
]

View file

@ -31,9 +31,7 @@ async def check_registry(ctx: AppContext) -> str:
missing = [j for j in EXPECTED_JOBS if j not in ctx.jobs]
assert not missing, f"missing jobs: {missing}"
extras = [j for j in ctx.jobs if j not in EXPECTED_JOBS]
assert not extras, (
f"curated profile leaked extra jobs: {extras} — keep the surface tight"
)
assert not extras, f"curated profile leaked extra jobs: {extras} — keep the surface tight"
return f"{len(ctx.jobs)} jobs registered"
@ -41,9 +39,14 @@ async def check_registry(ctx: AppContext) -> str:
async def check_retrieve_basic(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"retrieve", query="Alice Bob", max_results=5, min_score=0.0,
))
r = decode(
await ctx.app.run_job(
"retrieve",
query="Alice Bob",
max_results=5,
min_score=0.0,
),
)
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
assert len(hits) > 0, r
return f"{len(hits)} hits"
@ -51,13 +54,15 @@ async def check_retrieve_basic(ctx: AppContext) -> str:
async def check_retrieve_anchored(ctx: AppContext) -> str:
"""Wikilink-anchored mode — `[[Project X]]` in the query seeds BFS."""
r = decode(await ctx.app.run_job(
"retrieve",
query="What touches [[Project X]]?",
max_results=5,
min_score=0.0,
graph_depth=1,
))
r = decode(
await ctx.app.run_job(
"retrieve",
query="What touches [[Project X]]?",
max_results=5,
min_score=0.0,
graph_depth=1,
),
)
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
assert len(hits) > 0, r
paths = {h.get("path") for h in hits if isinstance(h, dict)}
@ -70,14 +75,16 @@ async def check_retrieve_anchored(ctx: AppContext) -> str:
async def check_retrieve_topic_seeded(ctx: AppContext) -> str:
"""Topic-rooted mode — explicit `seeds=[...]` instead of inline wikilink."""
project_x = ctx.abs_path("topics", "Project X", "Project X.md")
r = decode(await ctx.app.run_job(
"retrieve",
query="collaborates",
max_results=5,
min_score=0.0,
seeds=[project_x],
graph_depth=1,
))
r = decode(
await ctx.app.run_job(
"retrieve",
query="collaborates",
max_results=5,
min_score=0.0,
seeds=[project_x],
graph_depth=1,
),
)
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
assert len(hits) > 0, r
return f"{len(hits)} hits (seeded at Project X)"
@ -87,19 +94,21 @@ async def check_retrieve_topic_seeded(ctx: AppContext) -> str:
async def check_remember_log_create(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"remember",
mode="log",
name="curated-event",
description="curated-profile suite",
content="## ops\n- ran the curated suite\n",
topics=["[[Alice]]"],
tags=["curated"],
materials=[
{"filename": "raw-prompt.md", "content": "# user prompt\n\nrun curated suite\n"},
{"filename": "tool-output.txt", "content": "exit=0\n"},
],
))
r = decode(
await ctx.app.run_job(
"remember",
mode="log",
name="curated-event",
description="curated-profile suite",
content="## ops\n- ran the curated suite\n",
topics=["[[Alice]]"],
tags=["curated"],
materials=[
{"filename": "raw-prompt.md", "content": "# user prompt\n\nrun curated suite\n"},
{"filename": "tool-output.txt", "content": "exit=0\n"},
],
),
)
assert isinstance(r, dict), r
assert r.get("created") is True and r.get("action") == "created", r
materials = r.get("materials", [])
@ -118,17 +127,19 @@ async def check_remember_log_create(ctx: AppContext) -> str:
async def check_remember_log_append(ctx: AppContext) -> str:
r = decode(await ctx.app.run_job(
"remember",
mode="log",
name="curated-event",
content="## follow-up\n- second pass\n",
topics=["[[Bob]]"],
tags=["follow-up"],
materials=[
{"filename": "tool-output.txt", "content": "second run\n"}, # collision
],
))
r = decode(
await ctx.app.run_job(
"remember",
mode="log",
name="curated-event",
content="## follow-up\n- second pass\n",
topics=["[[Bob]]"],
tags=["follow-up"],
materials=[
{"filename": "tool-output.txt", "content": "second run\n"}, # collision
],
),
)
assert isinstance(r, dict), r
assert r.get("created") is False and r.get("action") == "appended", r
appended = r.get("materials", [])
@ -150,9 +161,14 @@ async def check_remember_log_refuse_distilled(ctx: AppContext) -> str:
# Give the watcher a moment to re-parse before sync re-reads frontmatter.
await wait_for_index(ctx.watcher, expected_min=len(ctx.file_store))
r = decode(await ctx.app.run_job(
"remember", mode="log", name="curated-event", content="should be refused",
))
r = decode(
await ctx.app.run_job(
"remember",
mode="log",
name="curated-event",
content="should be refused",
),
)
assert isinstance(r, dict) and "error" in r, r
assert r.get("status") == "distilled", r
assert r.get("suggested_name"), r
@ -164,26 +180,28 @@ async def check_remember_log_refuse_distilled(ctx: AppContext) -> str:
async def check_remember_distill_degraded(ctx: AppContext) -> str:
target = ctx.abs_path("topics", "curated-ingested", "curated-ingested.md")
r = decode(await ctx.app.run_job(
"remember",
# mode defaults to "distill"
content="# curated-ingested\n\nproduced by the curated test suite.\n",
target_path=target,
metadata={
"title": "curated-ingested",
"lifecycle": "evolving",
"scope": "class",
"source": "curated",
"role": "concept",
"category": "concept",
},
))
r = decode(
await ctx.app.run_job(
"remember",
# mode defaults to "distill"
content="# curated-ingested\n\nproduced by the curated test suite.\n",
target_path=target,
metadata={
"title": "curated-ingested",
"lifecycle": "evolving",
"scope": "class",
"source": "curated",
"role": "concept",
"category": "concept",
},
),
)
assert isinstance(r, dict), r
applied = r.get("applied") or []
assert len(applied) == 1 and applied[0].get("ok") is True, r
assert r.get("used_llm") is False, "expected degraded path (no LLM)"
assert Path(target).is_file(), target
return f"applied=1, used_llm=False"
return "applied=1, used_llm=False"
# ---------- maintain (lint + decay sweep) -----------------------------
@ -206,9 +224,13 @@ async def check_maintain_dry_run(ctx: AppContext) -> str:
async def check_maintain_targeted(ctx: AppContext) -> str:
"""target_prefix narrows scan; events/ subtree should yield the
one event folder created earlier in this suite."""
r = decode(await ctx.app.run_job(
"maintain", target_prefix="events/", dry_run=True,
))
r = decode(
await ctx.app.run_job(
"maintain",
target_prefix="events/",
dry_run=True,
),
)
assert isinstance(r, dict), r
assert r["scanned"] >= 1, r # at least the suite's own event folder
return f"scanned={r['scanned']} under events/"
@ -218,14 +240,14 @@ async def check_maintain_targeted(ctx: AppContext) -> str:
CHECKS: list[tuple[str, callable]] = [
("registry", check_registry),
("retrieve.basic", check_retrieve_basic),
("retrieve.anchored", check_retrieve_anchored),
("retrieve.topic_seeded", check_retrieve_topic_seeded),
("remember.log_create", check_remember_log_create),
("remember.log_append", check_remember_log_append),
("remember.log_refuse_distilled", check_remember_log_refuse_distilled),
("remember.distill_degraded", check_remember_distill_degraded),
("maintain.dry_run", check_maintain_dry_run),
("maintain.targeted", check_maintain_targeted),
("registry", check_registry),
("retrieve.basic", check_retrieve_basic),
("retrieve.anchored", check_retrieve_anchored),
("retrieve.topic_seeded", check_retrieve_topic_seeded),
("remember.log_create", check_remember_log_create),
("remember.log_append", check_remember_log_append),
("remember.log_refuse_distilled", check_remember_log_refuse_distilled),
("remember.distill_degraded", check_remember_distill_degraded),
("maintain.dry_run", check_maintain_dry_run),
("maintain.targeted", check_maintain_targeted),
]

View file

@ -1,22 +1,41 @@
"""Memory subsystem — the three services on top of the core engine.
"""Memory subsystem — agent-facing services + tools on top of the core engine.
Per the architecture blueprint:
Three services (read / cold-write / treatment) plus the agent-callable
write/read tools that operate on the Memory File System. Every
``@R.register`` step a host agent might invoke lives here, regardless
of whether it's reached via MCP, HTTP, or direct Python.
- retriever.py Read service. V + K + Graph BFS fusion + intent
routing. Registered as a Step (`backend: hybrid`);
MCP step shells in `reme2.mcp.steps.memory_retriever`
instantiate it on demand.
- ingestor.py Cold-write service. LLM-driven R-M-W curator.
Triggered on explicit handoff (task end / SessionEnd).
- maintainer.py Treatment service. Background Merge / Split / Decay /
Lint, woken by cron or thresholds.
- summarizer.py Auxiliary used by the services.
Services
retriever.py Read service V + K + Graph BFS fusion + intent
routing. Registered as ``hybrid``; consumed by
the search shells below.
ingestor.py Cold-write service LLM-driven R-M-W curator
(also registers an ``ingest`` MCP face).
maintainer.py Treatment service Merge / Split / Decay / Lint,
woken by cron or thresholds.
summarizer.py Auxiliary used by the services.
Hot-write MCP step shells (sync, memory_*) live in
`reme2.mcp.steps`, NOT here they bypass services and write MFS
directly. Importing this package triggers @R.register on the three
services so configs that name them resolve at boot.
Tools (each ``@R.register`` exposes a step the agent invokes by name)
memory_io.py Pure engine API (no schema, no Step boilerplate).
memory_toolkit.py 12 ``memory_*`` primitives get / list / links /
backlinks / resolve_wikilink / count_tokens +
create / update / property_update / rename /
delete / archive.
memory_search.py ``memory_search`` + ``memory_graph_search``
shells over the Retriever service.
memory_lint.py ``memory_lint`` narrowed projection of the
Maintainer's lint pass.
sync.py ``sync`` hot-path event-folder upsert
(deterministic, no LLM).
Importing this package triggers every ``@R.register`` so configs that
name these tools resolve at boot.
"""
from . import retriever # noqa: F401 -- runs @R.register("hybrid")
from . import maintainer # noqa: F401 -- runs @R.register("maintainer")
from . import retriever # noqa: F401 -- @R.register("hybrid")
from . import ingestor # noqa: F401 -- @R.register("ingestor")
from . import maintainer # noqa: F401 -- @R.register("maintainer")
from . import memory_toolkit # noqa: F401 -- @R.register("memory_*") x12
from . import memory_search # noqa: F401 -- @R.register("memory_search", "memory_graph_search")
from . import memory_lint # noqa: F401 -- @R.register("memory_lint")
from . import sync # noqa: F401 -- @R.register("sync")

View file

@ -107,9 +107,12 @@ class Ingestor(BaseStep):
return
if mode != "distill":
self.context.response.success = False
_set_answer(self.context, {
"error": f"unknown mode {mode!r}; expected 'log' or 'distill'",
})
_set_answer(
self.context,
{
"error": f"unknown mode {mode!r}; expected 'log' or 'distill'",
},
)
return
content: str = self.context.get("content", "") or ""
@ -182,18 +185,22 @@ class Ingestor(BaseStep):
supported. Useful for tests and bootstrap scripts."""
result = IngestResult(used_llm=False)
if not target_path:
result.failed.append({
"op": "create",
"ok": False,
"error": "degraded path: target_path is required when no LLM is configured",
})
result.failed.append(
{
"op": "create",
"ok": False,
"error": "degraded path: target_path is required when no LLM is configured",
}
)
return result
path = Path(target_path)
if not path.is_absolute():
path = self._working_dir() / path
ok, payload = create_file(
self.file_store, path,
metadata=metadata, content=content,
self.file_store,
path,
metadata=metadata,
content=content,
)
bucket = result.applied if ok else result.failed
bucket.append({"op": "create", "ok": ok, "path": str(path), "result": payload})
@ -203,9 +210,8 @@ class Ingestor(BaseStep):
"""Hot-path event-folder upsert — same code path as the standalone
`sync` step. Lazy-instantiated so we don't pay the construction
cost on every distill call."""
from ..mcp.steps.sync import Sync
from .sync import Sync
if getattr(self, "_sync_step", None) is None:
self._sync_step = Sync(app_context=self.app_context)
await self._sync_step(self.context)

View file

@ -70,7 +70,6 @@ from pydantic import BaseModel, Field
from ..component import R
from ..component.base_step import BaseStep
from ..component.runtime_response import _set_answer
from ..enumeration import ComponentEnum
from . import memory_io
from .schema import parse_frontmatter
@ -171,12 +170,12 @@ class FileSignal(BaseModel):
"""
path: str
relpath: str = "" # path relative to working_dir, "" if outside
lifecycle: str = "" # streaming / evolving / frozen
scope: str = "" # instance / class
source: str = "" # auto / curated / derived
role: str = "" # observation / claim / question / ...
category: str = "" # legacy field, kept for migration windows
relpath: str = "" # path relative to working_dir, "" if outside
lifecycle: str = "" # streaming / evolving / frozen
scope: str = "" # instance / class
source: str = "" # auto / curated / derived
role: str = "" # observation / claim / question / ...
category: str = "" # legacy field, kept for migration windows
status: str = ""
age_days: int = 0
metadata: dict = Field(default_factory=dict)
@ -335,22 +334,24 @@ class Maintainer(BaseStep):
pass
if target_prefix and not relpath.startswith(target_prefix):
continue
fm = meta.metadata or {}
fm = meta.front_matter.model_dump()
age_seconds = max(0.0, now - (meta.st_mtime or now))
parsed, _ = parse_frontmatter(fm)
signals.append(FileSignal(
path=path,
relpath=relpath,
lifecycle=str(parsed.lifecycle.value) if parsed else "",
scope=str(parsed.scope.value) if parsed else "",
source=str(parsed.source.value) if parsed else "",
role=str(parsed.role.value) if parsed else "",
category=str(fm.get("category") or ""),
status=str(fm.get("status") or ""),
age_days=int(age_seconds // 86400),
metadata=fm,
declared_topics=list(fm.get("topics") or []),
))
signals.append(
FileSignal(
path=path,
relpath=relpath,
lifecycle=str(parsed.lifecycle.value) if parsed else "",
scope=str(parsed.scope.value) if parsed else "",
source=str(parsed.source.value) if parsed else "",
role=str(parsed.role.value) if parsed else "",
category=str(fm.get("category") or ""),
status=str(fm.get("status") or ""),
age_days=int(age_seconds // 86400),
metadata=fm,
declared_topics=list(fm.get("topics") or []),
),
)
return signals
# -- proposers ----------------------------------------------------------
@ -361,30 +362,41 @@ class Maintainer(BaseStep):
for sig in signals:
for link in sig.declared_topics:
if not memory_io.resolve_wikilink(self.file_store, link)["exists"]:
out.append(LintFinding(
path=sig.path, kind="broken_wikilink",
detail=f"unresolved wikilink {link!r}",
))
out.append(
LintFinding(
path=sig.path,
kind="broken_wikilink",
detail=f"unresolved wikilink {link!r}",
),
)
# Memory schema check: tolerant parse, surface every error
# the parser collected. Empty `errors` ↔ valid frontmatter.
_, errors = parse_frontmatter(sig.metadata)
for err in errors:
out.append(LintFinding(
path=sig.path, kind="schema_violation",
detail=f"Memory: {err}"[:240],
))
out.append(
LintFinding(
path=sig.path,
kind="schema_violation",
detail=f"Memory: {err}"[:240],
),
)
# Stem collisions: the engine API exposes the ambiguous-stem map.
ambig = memory_io.find_collisions(self.file_store)
for stem, paths in ambig.items():
for p in paths:
out.append(LintFinding(
path=p, kind="stem_collision",
detail=f"stem {stem!r} also claimed by {[x for x in paths if x != p]}",
))
out.append(
LintFinding(
path=p,
kind="stem_collision",
detail=f"stem {stem!r} also claimed by {[x for x in paths if x != p]}",
),
)
return out
def _propose_decay(
self, signals: list[FileSignal], decay_days: int,
self,
signals: list[FileSignal],
decay_days: int,
) -> list[DecayOp]:
"""Distilled streaming memories past the freshness window.
@ -401,14 +413,18 @@ class Maintainer(BaseStep):
continue
if sig.age_days < decay_days:
continue
out.append(DecayOp(
path=sig.path, age_days=sig.age_days,
reason=f"streaming {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)",
))
out.append(
DecayOp(
path=sig.path,
age_days=sig.age_days,
reason=f"streaming {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)",
),
)
return out
async def _propose_enrich(
self, signals: list[FileSignal],
self,
signals: list[FileSignal],
) -> list[EnrichOp]:
"""Bare wikilinks → typed via inline-bracketed Dataview wrap.
@ -418,13 +434,13 @@ class Maintainer(BaseStep):
2. Read body, ask the LLM (one call per file) to assign one
of `ALLOWED_PREDICATES` to each bare target or 'skip'.
3. For each accepted (target, predicate), locate each bare
occurrence span via `FileEdge.from_text` and build a unique
occurrence span via `iter_links` and build a unique
context window snippet.
4. Emit EnrichOp(path, target, predicate, old_string, new_string,
confidence, reason). The apply step calls memory_update.
Idempotency: re-running won't re-enrich already-typed edges
(filter is `predicate is None`). The OOV-tolerant FileEdge
Idempotency: re-running won't re-enrich already-typed links
(filter is `predicate is None`). The OOV-tolerant FileLink
validator means a malformed LLM output collapses to None at the
next parse, surfacing it again next sweep bounded retries
avoid infinite re-enrichment loops.
@ -432,7 +448,8 @@ class Maintainer(BaseStep):
return []
async def _propose_discover(
self, signals: list[FileSignal],
self,
signals: list[FileSignal],
) -> list[DiscoverOp]:
"""Discover edges absent from body — append to `## Relations`.
@ -453,7 +470,9 @@ class Maintainer(BaseStep):
return []
async def _propose_merge(
self, signals: list[FileSignal], threshold: float,
self,
signals: list[FileSignal],
threshold: float,
) -> list[MergeOp]:
"""Cluster near-duplicate topics → MergeOps (LLM-assisted).
@ -468,7 +487,9 @@ class Maintainer(BaseStep):
return []
async def _propose_split(
self, signals: list[FileSignal], token_threshold: int,
self,
signals: list[FileSignal],
token_threshold: int,
) -> list[SplitOp]:
"""Topics over the token threshold → SplitOps (LLM-assisted).
@ -483,7 +504,8 @@ class Maintainer(BaseStep):
# -- conflict resolution ------------------------------------------------
def _resolve_conflicts(
self, proposed: list[BaseModel],
self,
proposed: list[BaseModel],
) -> tuple[list[BaseModel], list[BaseModel]]:
"""Apply the conflict matrix; return (kept_plan, dropped).
@ -588,7 +610,8 @@ class Maintainer(BaseStep):
# -- apply --------------------------------------------------------------
async def _apply(
self, plan: list[BaseModel],
self,
plan: list[BaseModel],
) -> tuple[list[dict], list[dict]]:
"""Execute the plan in fixed order. Each phase yields audit dicts.
@ -613,11 +636,13 @@ class Maintainer(BaseStep):
result = await self._apply_one(op)
applied.append({"op": op.op, **result}) # type: ignore[attr-defined]
except Exception as e:
failed.append({
"op": op.op, # type: ignore[attr-defined]
"payload": _dump(op),
"error": f"{type(e).__name__}: {e}",
})
failed.append(
{
"op": op.op, # type: ignore[attr-defined]
"payload": _dump(op),
"error": f"{type(e).__name__}: {e}",
}
)
return applied, failed
async def _apply_one(self, op: BaseModel) -> dict:
@ -626,28 +651,44 @@ class Maintainer(BaseStep):
return {"path": op.path, "kind": op.kind, "noop": True}
if isinstance(op, EnrichOp):
# TODO: wire to memory_io.update_body once apply-side is enabled.
return {"path": op.path, "target": op.target, "predicate": op.predicate,
"status": "pending_apply",
"would": "memory_update wraps bare [[X]] as [predicate:: [[X]]]"}
return {
"path": op.path,
"target": op.target,
"predicate": op.predicate,
"status": "pending_apply",
"would": "memory_update wraps bare [[X]] as [predicate:: [[X]]]",
}
if isinstance(op, DiscoverOp):
# TODO: append Dataview lines under ## Relations (create heading
# if absent), then call file_store invalidation.
return {"path": op.path, "edges_added": len(op.edges),
"status": "pending_apply",
"would": "append predicate:: [[Y]] under ## Relations heading"}
return {
"path": op.path,
"edges_added": len(op.edges),
"status": "pending_apply",
"would": "append predicate:: [[Y]] under ## Relations heading",
}
if isinstance(op, DecayOp):
# TODO: wire to MemoryArchive once we're ready to actually
# move files from a cron context. For now, surface intent.
return {"path": op.path, "status": "pending_apply",
"would": "flip status=archived + move to archive_dir"}
return {
"path": op.path,
"status": "pending_apply",
"would": "flip status=archived + move to archive_dir",
}
if isinstance(op, MergeOp):
return {"canonical": op.canonical, "sources": op.sources,
"status": "pending_apply",
"would": "merge bodies + rewrite incoming wikilinks + archive sources"}
return {
"canonical": op.canonical,
"sources": op.sources,
"status": "pending_apply",
"would": "merge bodies + rewrite incoming wikilinks + archive sources",
}
if isinstance(op, SplitOp):
return {"source": op.source, "sections": len(op.sections),
"status": "pending_apply",
"would": "extract sections + replace with [[…]] stubs"}
return {
"source": op.source,
"sections": len(op.sections),
"status": "pending_apply",
"would": "extract sections + replace with [[…]] stubs",
}
raise TypeError(f"unknown op type: {type(op).__name__}")

View file

@ -3,16 +3,21 @@
The .md files are the SSOT (per `structure.md` §"核心引擎"). The engine
is layered:
Memory File System Watcher & Parser Projections (vector / FTS / graph)
Memory File System Watcher & Parser Projections (vector / FTS)
(write entry) (incremental) (read entry, derived)
This module is the **single public API surface** over that engine. Every
consumer MCP step shells, the three memory services (Retriever,
Ingestor, Maintainer), and the agent toolkit (`memory_toolkit`) talks
to the engine through these functions, not by reaching into
`BaseFileStore` directly. That keeps `file_store` an implementation
detail (could be local sqlite, remote, etc.) and gives the layering one
place to evolve.
consumer agent-facing steps, the three memory services (Retriever,
Ingestor, Maintainer), and the toolkit (`memory_toolkit`) talks to
the engine through these functions, not by reaching into
``BaseFileStore`` directly.
Layering note. The slim ``BaseFileStore`` interface only owns the
search projections (vector / FTS) plus atomic file upserts/deletes.
Iteration, single-node lookup, and the wikilink graph live HERE we
walk ``LocalFileStore._nodes`` directly (engine-layer peer) and compute
links on-the-fly from each ``FileNode.links``. There is no precomputed
graph index; the graph is a derivation of the SSoT.
Naming convention:
- Verb-first: `get_file`, `create_file`, `search_vector`.
@ -21,11 +26,6 @@ Naming convention:
- Pure-disk writes (`update_body`, `update_meta`, `delete_file`,
`archive_file`) don't take `file_store` — they hit the filesystem
and the watcher picks them up. The asymmetry is honest.
Vault-convention helpers wikilink resolution, graph-walk ranking,
chunk filtering, stem collisions live here on top of the slim
`BaseFileStore` (which only manages graph + chunks). The engine itself
remains domain-agnostic.
"""
from __future__ import annotations
@ -38,21 +38,85 @@ from pathlib import Path
import frontmatter
from ..schema import ChunkFilter, FileChunk, FileNode, extract_wikilinks
from ..schema.file_edge import _WIKILINK_RE
from ..component.file_store.base_file_store import BaseFileStore
from ..schema import ChunkFilter, FileChunk, FileLink, FileNode, extract_wikilinks
from ..schema.file_link import _WIKILINK_RE
from ..utils.wikilink_resolver import (
resolve_wikilink as _resolve_wikilink,
wikilink_candidates,
)
# ===========================================================================
# Internal helpers — engine-layer access to the concrete node index
# ===========================================================================
#
# ``BaseFileStore`` only declares the search/upsert contract; iteration
# and single-node lookup live on the concrete impl (``LocalFileStore``).
# memory_io is a peer at the engine layer, so reaching into ``_nodes``
# is intentional — every iteration / graph walk funnels through here so
# the day the engine grows a public ``iter_nodes()``, this is the only
# place to swap.
def _nodes(file_store: BaseFileStore) -> dict[str, FileNode]:
"""The concrete in-memory ``{path: FileNode}`` index."""
return file_store._nodes # type: ignore[attr-defined]
def _meta(node: FileNode) -> dict:
"""Full frontmatter dict for a node — typed fields (title/description/
tags) merged with any ``extra=allow`` extras."""
return node.front_matter.model_dump()
def _get_outlinks(
file_store: BaseFileStore,
path: str,
) -> list[tuple[FileNode, FileLink]]:
"""Resolved outgoing links from ``path`` — ``[(target_node, link), ...]``."""
node = _nodes(file_store).get(path)
if node is None:
return []
out: list[tuple[FileNode, FileLink]] = []
for link in node.links:
target = _resolve_wikilink(file_store, link.path)
if target is None:
continue
target_node = _nodes(file_store).get(target)
if target_node is not None:
out.append((target_node, link))
return out
def _get_inlinks(
file_store: BaseFileStore,
path: str,
) -> list[tuple[FileNode, FileLink]]:
"""Resolved incoming links to ``path`` — linear scan over all nodes."""
out: list[tuple[FileNode, FileLink]] = []
for src_node in _nodes(file_store).values():
for link in src_node.links:
target = _resolve_wikilink(file_store, link.path)
if target == path:
out.append((src_node, link))
return out
def _filter_to_dict(chunk_filter: ChunkFilter | None) -> dict:
"""Serialize ``ChunkFilter`` for the search engine's ``search_filter`` arg."""
if chunk_filter is None:
return {}
return chunk_filter.model_dump(mode="json")
# ===========================================================================
# Section 1 — MFS Reads
# ===========================================================================
async def get_file(
file_store,
file_store: BaseFileStore,
path: str,
*,
include_chunks: bool = False,
@ -62,14 +126,16 @@ async def get_file(
On-disk frontmatter is the source of truth the file_store cache may
lag a write that hasn't been picked up by the watcher yet.
"""
node = file_store.get_node_by_path(path)
node = await file_store.get_node_by_path(path) # type: ignore[attr-defined]
result: dict = {"path": path, "exists": False}
if node is not None:
result.update({
"exists": True,
"metadata": node.metadata,
"link": [e.model_dump(exclude_none=True) for e in node.edges],
})
result.update(
{
"exists": True,
"metadata": _meta(node),
"link": [link.model_dump(exclude_none=True) for link in node.links],
}
)
file_path = Path(path)
if file_path.is_file():
@ -80,13 +146,13 @@ async def get_file(
result["metadata"] = dict(post.metadata)
if include_chunks:
chunks = await file_store.get_chunks_by_path(path)
chunks = await file_store.get_chunks_by_path(path) # type: ignore[attr-defined]
result["chunks"] = [c.model_dump(exclude_none=True) for c in chunks]
return result
def list_files(
file_store,
file_store: BaseFileStore,
*,
path_prefix: str | None = None,
tags: list[str] | None = None,
@ -97,14 +163,14 @@ def list_files(
metadata_filter = metadata or {}
tag_filter = tags or []
items: list[dict] = []
for path, node in file_store.nodes.items():
for path, node in _nodes(file_store).items():
if path_prefix and not path.startswith(path_prefix):
continue
md = node.metadata or {}
md = _meta(node)
if metadata_filter and any(md.get(k) != v for k, v in metadata_filter.items()):
continue
if tag_filter:
file_tags = set(md.get("tags", []) or [])
file_tags = set(md.get("tags") or [])
if not all(t in file_tags for t in tag_filter):
continue
items.append({"path": path, "metadata": md})
@ -113,32 +179,32 @@ def list_files(
return {"items": items, "count": len(items)}
def _edge_to_dict(node, edge) -> dict:
def _link_to_dict(node: FileNode, link: FileLink) -> dict:
return {
"path": node.path,
"metadata": node.metadata,
"predicate": edge.predicate,
"anchor": edge.anchor,
"metadata": _meta(node),
"predicate": link.predicate,
"anchor": link.anchor,
}
def get_links(file_store, path: str) -> dict:
"""Files that `path` links TO (resolved). Each entry carries the typed-edge predicate."""
def get_links(file_store: BaseFileStore, path: str) -> dict:
"""Files that `path` links TO (resolved). Each entry carries the typed-link predicate."""
return {
"path": path,
"links": [_edge_to_dict(m, e) for m, e in file_store.get_links(path)],
"links": [_link_to_dict(m, link) for m, link in _get_outlinks(file_store, path)],
}
def get_backlinks(file_store, path: str) -> dict:
"""Files that link TO `path`. Each entry carries the typed-edge predicate."""
def get_backlinks(file_store: BaseFileStore, path: str) -> dict:
"""Files that link TO `path`. Each entry carries the typed-link predicate."""
return {
"path": path,
"backlinks": [_edge_to_dict(m, e) for m, e in file_store.get_backlinks(path)],
"backlinks": [_link_to_dict(m, link) for m, link in _get_inlinks(file_store, path)],
}
def resolve_wikilink(file_store, wikilink: str) -> dict:
def resolve_wikilink(file_store: BaseFileStore, wikilink: str) -> dict:
"""Resolve a `[[target]]` wikilink with full ambiguity context.
Returns:
@ -177,9 +243,9 @@ def resolve_wikilink(file_store, wikilink: str) -> dict:
}
def iter_files(file_store) -> Iterator[tuple[str, FileNode]]:
def iter_files(file_store: BaseFileStore) -> Iterator[tuple[str, FileNode]]:
"""Walk every indexed (path, FileNode). Used by Maintainer scans."""
return iter(file_store.nodes.items())
return iter(_nodes(file_store).items())
async def count_tokens(
@ -234,7 +300,7 @@ def _replace_wikilink_targets(text: str, mapping: dict[str, str]) -> str:
def create_file(
file_store,
file_store: BaseFileStore,
path: Path,
*,
metadata: dict,
@ -328,7 +394,7 @@ def update_meta(path: Path | str, *, key: str, value) -> tuple[bool, dict]:
def rename_file(
file_store,
file_store: BaseFileStore,
working_dir: Path | str,
*,
old_path: Path | str,
@ -349,8 +415,7 @@ def rename_file(
if conflicts:
return False, {
"error": (
f"stem `[[{new_p.stem}]]` would resolve ambiguously "
f"to {len(conflicts) + 1} paths after this rename"
f"stem `[[{new_p.stem}]]` would resolve ambiguously " f"to {len(conflicts) + 1} paths after this rename"
),
"conflicts": conflicts,
"hint": (
@ -375,7 +440,7 @@ def rename_file(
except ValueError:
pass
referring_paths = [m.path for m, _ in file_store.get_backlinks(str(old_p))]
referring_paths = [m.path for m, _ in _get_inlinks(file_store, str(old_p))]
new_p.parent.mkdir(parents=True, exist_ok=True)
old_p.rename(new_p)
@ -469,43 +534,50 @@ def archive_file(
async def search_vector(
file_store,
file_store: BaseFileStore,
query: str,
*,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""Vector similarity over the chunk-level Vector projection."""
return await file_store.vector_search(query, limit, chunk_filter)
return await file_store.vector_search(query, limit, _filter_to_dict(chunk_filter))
async def search_keyword(
file_store,
file_store: BaseFileStore,
query: str,
*,
limit: int,
chunk_filter: ChunkFilter | None = None,
) -> list[FileChunk]:
"""FTS5 keyword search over the chunk-level keyword projection."""
return await file_store.keyword_search(query, limit, chunk_filter)
return await file_store.keyword_search(query, limit, _filter_to_dict(chunk_filter))
async def get_chunks(file_store, paths: Iterable[str]) -> list[FileChunk]:
async def get_chunks(
file_store: BaseFileStore,
paths: Iterable[str],
) -> list[FileChunk]:
"""Batch fetch chunks across many paths."""
return await file_store.get_chunks_by_paths(paths)
out: list[FileChunk] = []
for p in paths:
out.extend(await file_store.get_chunks_by_path(p)) # type: ignore[attr-defined]
return out
# ===========================================================================
# Section 4 — Graph helpers (BFS / scoring / collisions)
# ===========================================================================
#
# Layered on top of the slim `BaseFileStore` (which only owns the
# graph + chunks). These are vault conventions: folder-note preference,
# wikilink-uniqueness gates, BFS for memory_graph_search.
# All graph traversal walks ``FileNode.links`` directly via ``_get_outlinks``
# / ``_get_inlinks``. There's no precomputed adjacency index — every BFS
# resolves wikilinks per call. Cheap enough for typical vault sizes;
# revisit if the maintainer's lint pass becomes a hotspot.
def expand_neighbors(
file_store,
file_store: BaseFileStore,
seeds: Iterable[str],
*,
depth: int = 1,
@ -518,10 +590,11 @@ def expand_neighbors(
if depth < 0:
raise ValueError(f"depth must be >= 0, got {depth}")
nodes = _nodes(file_store)
seen: dict[str, int] = {}
frontier: deque[tuple[str, int]] = deque()
for path in seeds:
if path in file_store and path not in seen:
if path in nodes and path not in seen:
seen[path] = 0
frontier.append((path, 0))
@ -532,9 +605,9 @@ def expand_neighbors(
neighbors: list[str] = []
if direction in ("out", "both"):
neighbors.extend(m.path for m, _ in file_store.get_links(path))
neighbors.extend(m.path for m, _ in _get_outlinks(file_store, path))
if direction in ("in", "both"):
neighbors.extend(m.path for m, _ in file_store.get_backlinks(path))
neighbors.extend(m.path for m, _ in _get_inlinks(file_store, path))
if per_node_cap is not None and len(neighbors) > per_node_cap:
neighbors = neighbors[:per_node_cap]
@ -549,7 +622,7 @@ def expand_neighbors(
def subgraph_score(
file_store,
file_store: BaseFileStore,
seeds: Iterable[str],
*,
decay: float = 0.5,
@ -561,12 +634,16 @@ def subgraph_score(
if not (0.0 <= decay <= 1.0):
raise ValueError(f"decay must be in [0, 1], got {decay}")
hops = expand_neighbors(
file_store, seeds, depth=depth, direction=direction, per_node_cap=per_node_cap,
file_store,
seeds,
depth=depth,
direction=direction,
per_node_cap=per_node_cap,
)
return {path: decay ** hop for path, hop in hops.items()}
return {path: decay**hop for path, hop in hops.items()}
def extract_anchors(file_store, text: str) -> list[str]:
def extract_anchors(file_store: BaseFileStore, text: str) -> list[str]:
"""Pull `[[X]]` anchors from `text` and resolve each (deduped)."""
seen: set[str] = set()
out: list[str] = []
@ -578,7 +655,10 @@ def extract_anchors(file_store, text: str) -> list[str]:
return out
def collisions_after_create(file_store, proposed_path: str | Path) -> list[str]:
def collisions_after_create(
file_store: BaseFileStore,
proposed_path: str | Path,
) -> list[str]:
"""Existing paths that would conflict with adding `proposed_path`.
Folder-note rule: if `proposed_path`'s parent dir name == its stem,
@ -591,23 +671,26 @@ def collisions_after_create(file_store, proposed_path: str | Path) -> list[str]:
proposed_abs = str(p.resolve())
is_folder_note = p.parent.name == stem
folder_hits = [
path for path in file_store.nodes
if Path(path).stem == stem and Path(path).parent.name == stem
and path != proposed_abs
]
stem_hits = [
path for path in file_store.get_paths_by_stem(stem)
if path != proposed_abs
]
folder_hits: list[str] = []
stem_hits: list[str] = []
for path in _nodes(file_store):
if path == proposed_abs:
continue
path_obj = Path(path)
if path_obj.stem != stem:
continue
if path_obj.parent.name == stem:
folder_hits.append(path)
else:
stem_hits.append(path)
if is_folder_note:
return folder_hits
return folder_hits + [sp for sp in stem_hits if sp not in folder_hits]
return folder_hits + stem_hits
def make_filter(
file_store,
file_store: BaseFileStore,
*,
paths: list[str] | None = None,
tags: list[str] | None = None,
@ -617,17 +700,13 @@ def make_filter(
cf = ChunkFilter(paths=paths, tags=tags, exclude_paths=exclude_paths)
if cf.is_empty():
return cf
cf.resolved_paths = {
p for p, n in file_store.nodes.items() if cf.match_metadata(p, n.metadata)
}
cf.resolved_paths = {p for p, n in _nodes(file_store).items() if cf.match_metadata(p, _meta(n))}
return cf
def find_collisions(file_store) -> dict[str, list[str]]:
def find_collisions(file_store: BaseFileStore) -> dict[str, list[str]]:
"""Every stem that resolves to >1 path. Used by Maintainer.lint."""
out: dict[str, list[str]] = {}
for stem in file_store._stems:
cands = wikilink_candidates(file_store, stem)
if len(cands) > 1:
out[stem] = cands
return out
by_stem: dict[str, list[str]] = {}
for path in _nodes(file_store):
by_stem.setdefault(Path(path).stem, []).append(path)
return {s: ps for s, ps in by_stem.items() if len(ps) > 1}

View file

@ -12,11 +12,10 @@ agent doesn't need to know about decay/merge/split knobs.
from __future__ import annotations
from ...component import R
from ...component.base_step import BaseStep
from ...component.runtime_response import _set_answer
from ...enumeration import ComponentEnum
from ...memory.maintainer import Maintainer
from ..component import R
from ..component.base_step import BaseStep
from ..component.runtime_response import _set_answer
from ..enumeration import ComponentEnum
@R.register("memory_lint")
@ -57,14 +56,18 @@ class MemoryLint(BaseStep):
# action lives in `proposed` (LintFindings never get applied or
# dropped). Reshape into a focused response.
import json
raw = self.context.response.answer
audit = json.loads(raw) if isinstance(raw, str) else (raw or {})
findings = audit.get("proposed") or []
_set_answer(self.context, {
"scanned": audit.get("scanned", 0),
"findings": findings,
"target_prefix": target_prefix,
"ran_at": audit.get("ran_at", ""),
})
_set_answer(
self.context,
{
"scanned": audit.get("scanned", 0),
"findings": findings,
"target_prefix": target_prefix,
"ran_at": audit.get("ran_at", ""),
},
)
self.context.response.success = True

View file

@ -1,26 +1,26 @@
"""Memory retriever steps — thin MCP-facing wrappers over `BaseRetriever`.
"""Memory search step shells — thin wrappers over the Retriever service.
Per the architecture blueprint, retrieval policy (V+K hybrid + graph
BFS fusion + ranking + intent routing) is the **Retriever service**
(`reme2.memory.retriever`), registered as `ComponentEnum.RETRIEVER`.
These steps are the MCP projection: they translate `RuntimeContext`
(paths/tags/exclude_paths filter, per-call knob overrides) into the
retriever's call surface, then serialize results for the API response
(joining file metadata onto each chunk so callers don't have to fire a
`memory_get` per hit).
BFS fusion + ranking + intent routing) lives in the **Retriever
service** (`reme2.memory.retriever`), registered as
``ComponentEnum.RETRIEVER``. The two steps here are the agent-facing
projection: they translate ``RuntimeContext`` (paths/tags/exclude_paths
filter, per-call knob overrides) into the retriever's call surface,
then serialize results for the response (joining file metadata onto
each chunk so callers don't have to fire a ``memory_get`` per hit).
Direct primary-key file ops (read/list/links/backlinks) live in
`memory_io` those aren't retrieval.
``memory_io`` those aren't retrieval.
"""
from __future__ import annotations
from ...component import R
from ...component.base_step import BaseStep
from ...component.runtime_response import _set_answer
from ...enumeration import ComponentEnum
from ...memory import memory_io
from ...memory.retriever import BaseRetriever, HybridRetriever
from ..component import R
from ..component.base_step import BaseStep
from ..component.runtime_response import _set_answer
from ..enumeration import ComponentEnum
from . import memory_io
from .retriever import BaseRetriever, HybridRetriever
# Per-shell singleton cache: instantiating the retriever is cheap (it
@ -61,9 +61,16 @@ def _resolve_retriever(step: BaseStep) -> BaseRetriever:
cls = HybridRetriever
knob_keys = (
"vector_weight", "graph_weight", "graph_depth", "graph_decay",
"graph_direction", "graph_mode", "graph_per_path_cap",
"candidate_multiplier", "anchor_expand", "file_store",
"vector_weight",
"graph_weight",
"graph_depth",
"graph_decay",
"graph_direction",
"graph_mode",
"graph_per_path_cap",
"candidate_multiplier",
"anchor_expand",
"file_store",
)
init_kwargs = {k: step.kwargs[k] for k in knob_keys if k in step.kwargs}
init_kwargs["app_context"] = step.app_context
@ -84,9 +91,9 @@ def _serialize_chunk(chunk, file_store, extras: dict | None = None) -> dict:
`extras` lets the caller attach step-specific fields (e.g. `graph_hop`).
"""
item = chunk.model_dump(exclude_none=True, exclude={"embedding"})
node = file_store.get_node_by_path(chunk.path)
node = file_store._nodes.get(chunk.path) # engine-layer peer access
if node is not None:
item["file_metadata"] = node.metadata
item["file_metadata"] = node.front_matter.model_dump()
item["file_st_mtime"] = node.st_mtime
else:
item["file_metadata"] = None

View file

@ -46,9 +46,9 @@ from ..enumeration import ComponentEnum
_STATUS_STATES = ("active", "distilled", "archived")
_STATUS_TRANSITIONS: dict[str, set[str]] = {
"active": {"active", "distilled"},
"active": {"active", "distilled"},
"distilled": {"distilled", "archived"},
"archived": {"archived"},
"archived": {"archived"},
}
@ -59,10 +59,7 @@ def validate_status_transition(prior, requested) -> str | None:
if requested is None:
return None # delete operation
if requested not in _STATUS_STATES:
return (
f"invalid status {requested!r}; must be one of "
f"{list(_STATUS_STATES)}"
)
return f"invalid status {requested!r}; must be one of " f"{list(_STATUS_STATES)}"
if prior in _STATUS_STATES and requested not in _STATUS_TRANSITIONS[prior]:
return (
f"status transition {prior!r}{requested!r} not allowed; "
@ -167,9 +164,12 @@ def create_file_with_schema(
),
}
return memory_io.create_file(
file_store, path,
metadata=metadata, content=content,
overwrite=overwrite, force=force,
file_store,
path,
metadata=metadata,
content=content,
overwrite=overwrite,
force=force,
)
@ -336,9 +336,12 @@ class MemoryCreate(BaseStep):
target = Path(path)
ok, payload = create_file_with_schema(
self.file_store, target,
metadata=metadata, content=content,
overwrite=overwrite, force=force,
self.file_store,
target,
metadata=metadata,
content=content,
overwrite=overwrite,
force=force,
)
self.context.response.success = ok
if ok:
@ -357,7 +360,8 @@ class MemoryCreate(BaseStep):
gate both fire unless `force=True`."""
target = Path(path)
ok, payload = create_file_with_schema(
self.file_store, target,
self.file_store,
target,
metadata=dict(metadata or {}),
content=content,
overwrite=overwrite,
@ -403,8 +407,10 @@ class MemoryRename(BaseStep):
working_dir = Path(self.file_store.working_dir or Path.cwd()).resolve()
ok, payload = memory_io.rename_file(
self.file_store, working_dir,
old_path=old_path, new_path=new_path,
self.file_store,
working_dir,
old_path=old_path,
new_path=new_path,
)
self.context.response.success = ok
_set_answer(self.context, payload)
@ -414,8 +420,10 @@ class MemoryRename(BaseStep):
vr = getattr(self.file_store, "working_dir", None)
working_dir = Path(vr).resolve() if vr else Path.cwd()
ok, payload = memory_io.rename_file(
self.file_store, working_dir,
old_path=old_path, new_path=new_path,
self.file_store,
working_dir,
old_path=old_path,
new_path=new_path,
)
return _tool_response("memory_rename", ok, payload, audit=self.audit)
@ -477,7 +485,10 @@ class MemoryUpdate(BaseStep):
replace_all: bool = bool(self.context.get("replace_all", False))
assert path, "path is required"
ok, payload = memory_io.update_body(
path, old_string=old_string, new_string=new_string, replace_all=replace_all,
path,
old_string=old_string,
new_string=new_string,
replace_all=replace_all,
)
self.context.response.success = ok
_set_answer(self.context, payload)
@ -491,7 +502,10 @@ class MemoryUpdate(BaseStep):
) -> ToolResponse:
"""Edit-style content update: replace `old_string` with `new_string`."""
ok, payload = memory_io.update_body(
path, old_string=old_string, new_string=new_string, replace_all=replace_all,
path,
old_string=old_string,
new_string=new_string,
replace_all=replace_all,
)
return _tool_response("memory_update", ok, payload, audit=self.audit)

View file

@ -16,8 +16,8 @@ Two surfaces:
standard Step lifecycle + `self.file_store` property + per-instance
configuration via constructor kwargs. The default `execute()` dispatch
just runs `graph_search` so the retriever can also be used directly as
a step inside a job pipeline; the MCP step shells in
`reme2.mcp.steps.memory_retriever` instead bypass `execute` and call
a step inside a job pipeline; the search step shells in
`reme2.memory.memory_search` instead bypass `execute` and call
`search` / `graph_search` directly so they own the result-serialization
shape.
"""
@ -183,7 +183,10 @@ class HybridRetriever(BaseRetriever):
results = k_results[:max_results]
else:
results = self._merge_vk(
v_results, k_results, self.vector_weight, text_weight,
v_results,
k_results,
self.vector_weight,
text_weight,
)[:max_results]
elif fs.embedding_model:
results = await memory_io.search_vector(fs, query, limit=max_results, chunk_filter=chunk_filter)
@ -287,7 +290,7 @@ class HybridRetriever(BaseRetriever):
hops = memory_io.expand_neighbors(fs, seed_paths, depth=gd, direction=gdr)
else:
hops = {}
graph_scores: dict[str, float] = {p: gdc ** h for p, h in hops.items()}
graph_scores: dict[str, float] = {p: gdc**h for p, h in hops.items()}
# 4. Pull graph-only chunks (paths in expansion but not in VK).
# Skip in 'boost' mode — boost only re-ranks VK, never adds candidates.

View file

@ -76,14 +76,14 @@ class Role(StrEnum):
Adding a new role is a one-line change here plus (optionally) a preset.
"""
OBSERVATION = "observation" # what happened (events live here)
CLAIM = "claim" # an assertion needing confidence (thesis, model)
QUESTION = "question" # an open inquiry needing an answer
PROFILE = "profile" # entity description (person, org, system)
CONCEPT = "concept" # abstract idea / definition (company, sector, concept)
METHOD = "method" # procedure / how-to
REFERENCE = "reference" # pointer to external thing (tool, paper, code)
FUNDAMENTALS = "fundamentals" # foundational data / baseline facts
OBSERVATION = "observation" # what happened (events live here)
CLAIM = "claim" # an assertion needing confidence (thesis, model)
QUESTION = "question" # an open inquiry needing an answer
PROFILE = "profile" # entity description (person, org, system)
CONCEPT = "concept" # abstract idea / definition (company, sector, concept)
METHOD = "method" # procedure / how-to
REFERENCE = "reference" # pointer to external thing (tool, paper, code)
FUNDAMENTALS = "fundamentals" # foundational data / baseline facts
class Status(StrEnum):
@ -206,6 +206,7 @@ class MemoryFileNode(FileNode):
the parsed object without polluting this base schema. `populate_by_name`
lets `originSessionId` (legacy camelCase) populate `origin_session_id`.
"""
# -- Identity / common metadata ----------------------------------------
title: str = Field(default="")
@ -274,8 +275,7 @@ class MemoryFileNode(FileNode):
def _enforce_role_conditionals(self):
if self.role is Role.CLAIM and self.confidence is None:
raise ValueError(
"role='claim' requires explicit confidence "
"(one of ⏳ / ✅ / ❌)"
"role='claim' requires explicit confidence " "(one of ⏳ / ✅ / ❌)",
)
return self

View file

@ -31,7 +31,6 @@ Zero LLM cost.
"""
import json
import os
import re
from collections.abc import Iterable
from datetime import date as date_type, datetime, timezone
@ -42,8 +41,8 @@ from pydantic import ValidationError
from reme2.component import R
from reme2.component.base_step import BaseStep
from reme2.memory.memory_io import collisions_after_create, create_file
from reme2.memory.schema import EVENT_PRESET, MemoryFileNode
from .memory_io import collisions_after_create, create_file
from .schema import EVENT_PRESET, MemoryFileNode
_SAFE_FILENAME_RE = re.compile(r"^[a-zA-Z0-9._-]+$")
@ -213,10 +212,7 @@ class Sync(BaseStep):
"""Filenames in `folder` excluding the index, sorted for stability."""
if not folder.is_dir():
return []
return sorted(
entry.name for entry in folder.iterdir()
if entry.is_file() and entry.name != index_filename
)
return sorted(entry.name for entry in folder.iterdir() if entry.is_file() and entry.name != index_filename)
@staticmethod
def _union(prior: list, incoming: list) -> list:
@ -262,20 +258,31 @@ class Sync(BaseStep):
await self._append(target, content, materials, topics, tags)
else:
await self._create(
target, name, description, content, materials, topics, tags,
on_date, origin_session_id,
target,
name,
description,
content,
materials,
topics,
tags,
on_date,
origin_session_id,
)
async def _create(
self, target: Path, name: str, description: str, content: str,
materials: list[dict], topics: list[str], tags: list[str],
on_date, origin_session_id,
self,
target: Path,
name: str,
description: str,
content: str,
materials: list[dict],
topics: list[str],
tags: list[str],
on_date,
origin_session_id,
) -> None:
today = date_type.today().isoformat()
on_date_str = (
on_date.isoformat() if isinstance(on_date, date_type)
else (on_date or today)
)
on_date_str = on_date.isoformat() if isinstance(on_date, date_type) else (on_date or today)
# Start from EVENT_PRESET (4 axes + status + legacy `category`),
# layer caller-supplied identity fields on top.
metadata: dict = {
@ -291,16 +298,20 @@ class Sync(BaseStep):
metadata["originSessionId"] = origin_session_id
try:
MemoryFileNode.model_validate({
"path": str(target.resolve()),
"st_mtime": 0.0,
**metadata,
})
MemoryFileNode.model_validate(
{
"path": str(target.resolve()),
"st_mtime": 0.0,
**metadata,
}
)
except ValidationError as e:
self._set_error({
"error": "MemoryFileNode schema validation failed",
"details": e.errors(include_context=False, include_url=False),
})
self._set_error(
{
"error": "MemoryFileNode schema validation failed",
"details": e.errors(include_context=False, include_url=False),
}
)
return
graph = self.file_store
@ -308,32 +319,38 @@ class Sync(BaseStep):
if conflicts:
taken = {Path(p).stem for p in graph.nodes}
suggested_name = _next_suffixed_stem(taken, name)
self._set_error({
"error": (
f"stem `[[{name}]]` would resolve ambiguously "
f"to {len(conflicts) + 1} paths after this create"
),
"conflicts": conflicts,
"suggested_name": suggested_name,
"hint": (
f"retry with name='{suggested_name}', or pick a "
f"semantic qualifier (e.g. '{name}-followup')."
),
})
self._set_error(
{
"error": (
f"stem `[[{name}]]` would resolve ambiguously "
f"to {len(conflicts) + 1} paths after this create"
),
"conflicts": conflicts,
"suggested_name": suggested_name,
"hint": (
f"retry with name='{suggested_name}', or pick a "
f"semantic qualifier (e.g. '{name}-followup')."
),
}
)
return
material_filenames = [m["filename"] for m in materials]
index_body = self._emit_body(content, material_filenames)
ok, payload = create_file(
self.file_store, target,
metadata=metadata, content=index_body,
self.file_store,
target,
metadata=metadata,
content=index_body,
)
if not ok:
self._set_error({
"path": str(target.resolve()),
"error": payload.get("error", "create failed"),
"details": payload,
})
self._set_error(
{
"path": str(target.resolve()),
"error": payload.get("error", "create failed"),
"details": payload,
}
)
return
material_paths: list[str] = []
@ -348,19 +365,25 @@ class Sync(BaseStep):
continue
material_paths.append(str(material_path.resolve()))
self._set_ok({
"path": str(target.resolve()),
"category": "event",
"status": "active",
"topics": topics,
"materials": material_paths,
"created": True,
"action": "created",
})
self._set_ok(
{
"path": str(target.resolve()),
"category": "event",
"status": "active",
"topics": topics,
"materials": material_paths,
"created": True,
"action": "created",
}
)
async def _append(
self, target: Path, content: str, materials: list[dict],
topics: list[str], tags: list[str],
self,
target: Path,
content: str,
materials: list[dict],
topics: list[str],
tags: list[str],
) -> None:
# Read current frontmatter + body.
try:
@ -379,15 +402,17 @@ class Sync(BaseStep):
taken = {Path(p).stem for p in graph.nodes}
base = target.stem
suggested = _next_suffixed_stem(taken, base)
self._set_error({
"path": str(target.resolve()),
"error": (
f"event `{base}` already exists with status={status!r}; "
f"pick a new name to start a fresh thread"
),
"status": status,
"suggested_name": suggested,
})
self._set_error(
{
"path": str(target.resolve()),
"error": (
f"event `{base}` already exists with status={status!r}; "
f"pick a new name to start a fresh thread"
),
"status": status,
"suggested_name": suggested,
}
)
return
folder = target.parent
@ -420,10 +445,7 @@ class Sync(BaseStep):
if content.strip():
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
update_section = f"## Update — {ts}\n\n{content.rstrip()}\n"
narrative = (
f"{narrative.rstrip()}\n\n{update_section}"
if narrative else update_section
)
narrative = f"{narrative.rstrip()}\n\n{update_section}" if narrative else update_section
all_filenames_sorted = sorted(set(existing_filenames))
new_body = self._emit_body(narrative, all_filenames_sorted)
@ -434,17 +456,21 @@ class Sync(BaseStep):
meta["updated"] = date_type.today().isoformat()
try:
MemoryFileNode.model_validate({
"path": str(target.resolve()),
"st_mtime": 0.0,
**meta,
})
MemoryFileNode.model_validate(
{
"path": str(target.resolve()),
"st_mtime": 0.0,
**meta,
}
)
except ValidationError as e:
self._set_error({
"path": str(target.resolve()),
"error": "MemoryFileNode schema validation failed on append",
"details": e.errors(include_context=False, include_url=False),
})
self._set_error(
{
"path": str(target.resolve()),
"error": "MemoryFileNode schema validation failed on append",
"details": e.errors(include_context=False, include_url=False),
}
)
return
new_post = frontmatter.Post(new_body, **meta)
@ -454,12 +480,14 @@ class Sync(BaseStep):
self._set_error({"path": str(target.resolve()), "error": f"write failed: {e}"})
return
self._set_ok({
"path": str(target.resolve()),
"category": "event",
"status": "active",
"topics": meta["topics"],
"materials": new_material_paths,
"created": False,
"action": "appended",
})
self._set_ok(
{
"path": str(target.resolve()),
"category": "event",
"status": "active",
"topics": meta["topics"],
"materials": new_material_paths,
"created": False,
"action": "appended",
}
)

View file

@ -5,7 +5,7 @@ from .as_msg_stat import AsBlockStat, AsMsgStat
from .emb_node import EmbNode
from .chunk_filter import ChunkFilter
from .file_chunk import FileChunk
from .file_edge import FileEdge, extract_wikilinks
from .file_link import FileLink, extract_wikilinks, iter_links
from .file_node import FileFrontMatter, FileNode
from .request import Request
from .response import Response
@ -20,11 +20,12 @@ __all__ = [
"EmbNode",
"ChunkFilter",
"FileChunk",
"FileEdge",
"FileLink",
"FileFrontMatter",
"FileNode",
"Request",
"Response",
"StreamChunk",
"extract_wikilinks",
"iter_links",
]

View file

@ -12,14 +12,14 @@ class EmbNode(BaseModel):
embedding: np.ndarray | None = Field(default=None)
metadata: dict = Field(default_factory=dict)
@field_validator('embedding', mode='before')
@field_validator("embedding", mode="before")
@classmethod
def validate_embedding(cls, v):
if v is None:
return v
return np.array(v, dtype=np.float16)
@field_serializer('embedding')
@field_serializer("embedding")
def serialize_embedding(self, v: np.ndarray | None, _info):
if v is None:
return None

View file

@ -15,5 +15,6 @@ class FileChunk(EmbNode):
def set_hash_id(self):
from ..utils import hash_text
self.id = hash_text(" ".join([self.path, str(self.start_line), str(self.end_line), self.text]))
return self
return self

View file

@ -1,224 +0,0 @@
"""FileEdge — typed wikilink edge between vault files.
Single source of truth for the edge **schema** and the **inline parser**
(`FileEdge.from_text`) that recovers edges from body text. Edges live
exclusively in body text (frontmatter is not walked); the predicate
vocabulary is **open** any identifier-shaped token
(`[A-Za-z][A-Za-z0-9_]*`) the parser sees is preserved verbatim on
`FileEdge.predicate`. Vocabulary curation, if any, is the maintainer's
job, not the schema's.
## Inline forms recognised by `FileEdge.from_text`
[[X]] bare wikilink predicate=None
extends:: [[X]] line-level Dataview predicate="extends"
[extends:: [[X]]] inline-bracketed predicate="extends"
Multi-target every wikilink under one typed context inherits its
predicate (any separator works, not just commas):
extends:: [[A]], [[B]] line-level multi 2 edges, both "extends"
extends:: [[A]] and [[B]] prose-style multi 2 edges, both "extends"
[concerns:: [[A]], [[B]]] inline multi 2 edges, both "concerns"
extends:: [[A#s1]], [[B#s2]] multi w/ anchors → anchors preserved per link
Context precedence is **inline-bracketed > line-level > bare**
a wikilink inside a `[predicate:: ]` envelope is typed by that
envelope even if the line happens to start `predicate:: `.
"""
from __future__ import annotations
import re
from pydantic import BaseModel, ConfigDict, Field
# -- Regexes (module-private) ---------------------------------------------
# Wikilink. The optional `!` embed marker and `|alias` are matched but
# not captured — both are presentational and dropped from the edge.
# `target` is the file part; `anchor` is the optional `#…` suffix; we
# rejoin them into a single `link` string at edge-construction time.
_WIKILINK_RE = re.compile(
r"""
(?:!)?
\[\[
(?P<target>[^\]\|\#\n]+?)
(?:\#(?P<anchor>[^\]\|\n]+))?
(?:\|[^\]\n]+)?
\]\]
""",
re.VERBOSE,
)
# Line-level Dataview field. Anchored MULTILINE; allows leading bullet
# (`-`/`*`/`+`) so `- extends:: [[X]]` works inside Markdown lists.
_DATAVIEW_LINE_RE = re.compile(
r"^[ \t]*(?:[-*+][ \t]+)?(?P<predicate>[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P<value>.+?)\s*$",
re.MULTILINE,
)
# Inline-bracketed field opener: `[predicate::`. The matching `]` is
# located by depth-counted scan because the value may contain
# `[[wikilink]]` whose inner brackets are part of the value.
_INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P<predicate>[A-Za-z][A-Za-z0-9_]*)\s*::\s*")
def _iter_inline_fields(text: str) -> list[tuple[int, int, str]]:
"""Find inline-bracketed `[predicate:: …]` field spans by depth scan.
Returns a list of ``(start, end, predicate)`` triples. Newlines
terminate the scan: an inline field that spans a line break is
treated as malformed (matches Dataview semantics) and skipped.
"""
out: list[tuple[int, int, str]] = []
for m in _INLINE_FIELD_OPEN_RE.finditer(text):
depth = 1 # the outer '[' was the regex's first character
i = m.end()
n = len(text)
while i < n:
c = text[i]
if c == "\n":
break
if c == "[":
depth += 1
elif c == "]":
depth -= 1
if depth == 0:
out.append((m.start(), i + 1, m.group("predicate")))
break
i += 1
return out
# -- Schema ---------------------------------------------------------------
class FileEdge(BaseModel):
"""2-field minimal edge model: ``link`` + ``predicate``.
``link`` preserves the wikilink as written, including any ``#anchor``
suffix (e.g. ``"X"`` or ``"X#sec"``). The presentational ``|alias``
and ``!`` embed prefix are discarded by the parser. ``path`` and
``anchor`` are derived ``@property``s that split ``link`` on the
first ``#`` for callers that want either part without re-splitting
the string themselves.
"""
model_config = ConfigDict(extra="forbid")
link: str = Field(
...,
description=(
"Wikilink as written, preserving any '#anchor' suffix "
"(e.g. 'X' or 'X#sec'). Display alias and embed prefix discarded."
),
)
predicate: str | None = Field(
default=None,
description="Typed-edge predicate (Dataview-style). None for bare [[X]].",
)
@property
def path(self) -> str:
"""File-part of ``link`` (before any ``#anchor``).
For ``"X"`` returns ``"X"``; for ``"X#sec"`` returns ``"X"``;
for ``"topics/Foo#bar"`` returns ``"topics/Foo"``. Always
returns a non-empty string because ``link`` is required and the
regex never matches an empty target.
"""
return self.link.split("#", 1)[0].strip()
@property
def anchor(self) -> str | None:
"""Heading or block anchor parsed from ``link`` (text after first ``#``).
Returns ``None`` if the link has no anchor or the anchor is empty.
"""
if "#" not in self.link:
return None
tail = self.link.split("#", 1)[1].strip()
return tail or None
@classmethod
def _from_match(cls, wm: re.Match, *, predicate: str | None) -> FileEdge:
target = wm.group("target").strip()
anchor = wm.group("anchor")
link = f"{target}#{anchor.strip()}" if anchor else target
return cls(link=link, predicate=predicate)
@classmethod
def from_text(cls, text: str) -> list[FileEdge]:
"""Extract all edges from body text in source order.
Single pass: every wikilink in the text becomes one edge, and
its ``predicate`` is decided by the surrounding context with
precedence **inline-bracketed > line-level > bare**:
* ``[predicate:: [[X]]]`` ``predicate="predicate"``
* ``predicate:: [[X]]`` ``predicate="predicate"`` (line-level Dataview)
* ``[[X]]`` ``predicate=None`` (bare)
No consumed-span bookkeeping, no second sort: ``finditer``
already yields wikilinks in source order, and per-position
classification is unambiguous.
"""
if not text:
return []
# Inline-bracketed `[predicate:: ...]` envelopes need a
# depth-counted scan (regex can't match balanced `[[…]]` inside).
inline_spans = _iter_inline_fields(text)
return [
cls._from_match(wm, predicate=_predicate_for(text, wm.start(), inline_spans))
for wm in _WIKILINK_RE.finditer(text)
]
def _predicate_for(
text: str,
pos: int,
inline_spans: list[tuple[int, int, str]],
) -> str | None:
"""Resolve the predicate governing a wikilink at offset ``pos``.
Checks the two typed-edge contexts in precedence order; falls
through to ``None`` (bare) when neither applies.
"""
# 1. Inline-bracketed envelope `[predicate:: …]` containing pos.
for field_start, field_end, predicate in inline_spans:
if field_start <= pos < field_end:
return predicate
# 2. Line-level `predicate:: value` whose value range covers pos.
line_start = text.rfind("\n", 0, pos) + 1
line_end = text.find("\n", pos)
if line_end == -1:
line_end = len(text)
m = _DATAVIEW_LINE_RE.match(text[line_start:line_end])
if m and line_start + m.start("value") <= pos:
return m.group("predicate")
# 3. Bare wikilink — no predicate.
return None
# -- Public utility (target-only fast path) -------------------------------
def extract_wikilinks(text: str) -> list[str]:
"""Flat list of wikilink **file targets** in body text (no dedup).
Returns just the file part of each wikilink (before any ``#anchor``)
because callers feed the result to `resolve_wikilink`, which matches
against vault file stems / paths and would not recognise an anchor
suffix. Single regex pass cheaper than `FileEdge.from_text` when
callers don't need predicates (e.g. ingestor's auto-discovery hint,
memory_io anchor resolution).
"""
if not text:
return []
return [m.group("target").strip() for m in _WIKILINK_RE.finditer(text)]

192
reme2/schema/file_link.py Normal file
View file

@ -0,0 +1,192 @@
"""FileLink — typed wikilink between vault files.
One type, two states (the value of ``path`` distinguishes them):
* **pre-resolution** ``path`` holds the raw wikilink target as
written, e.g. ``"Foo"`` or ``"topics/Bar"``. Produced by
``iter_links(text)``: a pure regex pass over body text, no graph
access.
* **resolved** ``path`` holds the vault-relative path
file_graph stores, e.g. ``"topics/Foo/Foo.md"``. Produced by
``utils.wikilink_resolver.resolve_links(graph, links)`` (or the
one-shot ``text_to_links(graph, text)``), which expands stem
ambiguity into one ``FileLink`` per candidate path.
file_graph trusts ``link.path`` directly for adjacency: it only ever
stores resolved links. The pre-resolution form is internal pipeline
plumbing.
## Inline forms recognised by ``iter_links``
[[X]] bare wikilink predicate=None
extends:: [[X]] line-level Dataview predicate="extends"
[extends:: [[X]]] inline-bracketed predicate="extends"
Multi-target every wikilink under one typed context inherits its
predicate (any separator works, not just commas):
extends:: [[A]], [[B]] line-level multi 2 links, both "extends"
extends:: [[A]] and [[B]] prose-style multi 2 links, both "extends"
[concerns:: [[A]], [[B]]] inline multi 2 links, both "concerns"
extends:: [[A#s1]], [[B#s2]] multi w/ anchors → anchors preserved per link
Context precedence is **inline-bracketed > line-level > bare**
a wikilink inside a ``[predicate:: ]`` envelope is typed by that
envelope even if the line happens to start ``predicate:: ``.
"""
from __future__ import annotations
import re
from pydantic import BaseModel, ConfigDict, Field
# -- Regexes (module-private) ---------------------------------------------
_WIKILINK_RE = re.compile(
r"""
(?:!)?
\[\[
(?P<target>[^\]\|\#\n]+?)
(?:\#(?P<anchor>[^\]\|\n]+))?
(?:\|[^\]\n]+)?
\]\]
""",
re.VERBOSE,
)
_DATAVIEW_LINE_RE = re.compile(
r"^[ \t]*(?:[-*+][ \t]+)?(?P<predicate>[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P<value>.+?)\s*$",
re.MULTILINE,
)
_INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P<predicate>[A-Za-z][A-Za-z0-9_]*)\s*::\s*")
def _iter_inline_fields(text: str) -> list[tuple[int, int, str]]:
"""Find inline-bracketed ``[predicate:: …]`` field spans by depth scan."""
out: list[tuple[int, int, str]] = []
for m in _INLINE_FIELD_OPEN_RE.finditer(text):
depth = 1
i = m.end()
n = len(text)
while i < n:
c = text[i]
if c == "\n":
break
if c == "[":
depth += 1
elif c == "]":
depth -= 1
if depth == 0:
out.append((m.start(), i + 1, m.group("predicate")))
break
i += 1
return out
def _predicate_for(
text: str,
pos: int,
inline_spans: list[tuple[int, int, str]],
) -> str | None:
"""Resolve the predicate governing a wikilink at offset ``pos``."""
for field_start, field_end, predicate in inline_spans:
if field_start <= pos < field_end:
return predicate
line_start = text.rfind("\n", 0, pos) + 1
line_end = text.find("\n", pos)
if line_end == -1:
line_end = len(text)
m = _DATAVIEW_LINE_RE.match(text[line_start:line_end])
if m and line_start + m.start("value") <= pos:
return m.group("predicate")
return None
# -- Schema ---------------------------------------------------------------
class FileLink(BaseModel):
"""Typed wikilink — ``(path, anchor, predicate)``.
Single type for both states (see module docstring): ``path`` is
a raw wikilink target before resolution, a vault-relative resolved
path after.
Fields:
path wikilink target. Raw target text in pre-resolution
form (e.g. ``"Foo"``, ``"topics/Bar"``);
vault-relative resolved path in stored form
(e.g. ``"topics/Foo/Foo.md"``).
anchor heading or block anchor (text after ``#`` in the
wikilink). Pass-through across resolution.
predicate Dataview-style typed-link predicate; ``None`` for bare.
"""
model_config = ConfigDict(extra="forbid")
path: str = Field(
...,
description=(
"Wikilink target. Pre-resolution: the raw target as written "
"(e.g. 'Foo'). Resolved: the vault-relative path file_graph "
"stores. Stem ambiguity is resolved BEFORE construction of "
"the resolved form by emitting one FileLink per candidate."
),
)
anchor: str | None = Field(
default=None,
description="Heading or block anchor (text after '#'). None if absent.",
)
predicate: str | None = Field(
default=None,
description="Typed-link predicate (Dataview-style). None for bare [[X]].",
)
# -- Public extraction utilities ------------------------------------------
def iter_links(text: str) -> list[FileLink]:
"""Extract every wikilink in ``text`` as a pre-resolution ``FileLink``.
Each emitted ``FileLink`` has ``path`` set to the raw wikilink
target as written (the file portion only the ``#anchor`` lives
in its own field). Predicate is decided by surrounding context
with precedence **inline-bracketed > line-level > bare**.
Pure function: no graph access. Pass the result through
``utils.wikilink_resolver.resolve_links`` (or one-shot
``text_to_links``) to get the resolved form file_graph stores.
"""
if not text:
return []
inline_spans = _iter_inline_fields(text)
links: list[FileLink] = []
for wm in _WIKILINK_RE.finditer(text):
target = wm.group("target").strip()
anchor_raw = wm.group("anchor")
anchor = anchor_raw.strip() if anchor_raw else ""
links.append(
FileLink(
path=target,
anchor=anchor or None,
predicate=_predicate_for(text, wm.start(), inline_spans),
),
)
return links
def extract_wikilinks(text: str) -> list[str]:
"""Flat list of wikilink **file targets** in body text (no dedup).
Returns just the file part of each wikilink (before any ``#anchor``).
Single regex pass cheaper than ``iter_links`` when callers don't
need predicates (e.g. ``extract_anchors`` for query seeding in the
retriever).
"""
if not text:
return []
return [m.group("target").strip() for m in _WIKILINK_RE.finditer(text)]

View file

@ -1,6 +1,6 @@
from pydantic import BaseModel, ConfigDict, Field
from .file_edge import FileEdge
from .file_link import FileLink
class FileFrontMatter(BaseModel):
@ -18,6 +18,6 @@ class FileFrontMatter(BaseModel):
class FileNode(BaseModel):
path: str = Field(default=...)
st_mtime: float = Field(default=...)
edges: list[FileEdge] = Field(default_factory=list)
links: list[FileLink] = Field(default_factory=list)
chunk_ids: list[str] = Field(default_factory=list)
front_matter: FileFrontMatter = Field(default_factory=FileFrontMatter)

View file

@ -34,7 +34,8 @@ def keyword_score(query: str, text: str) -> float:
def filter_chunks(
chunks: list[FileChunk], chunk_filter: ChunkFilter | None,
chunks: list[FileChunk],
chunk_filter: ChunkFilter | None,
) -> list[FileChunk]:
"""Restrict `chunks` to those whose path passes the (compiled) filter."""
if chunk_filter is None or chunk_filter.resolved_paths is None:

View file

@ -1,56 +1,285 @@
"""Wikilink resolution — pure functions over a `BaseFileStore`'s graph.
"""Wikilink resolver — vault convention over ``BaseFileGraph``.
Wikilink resolution is a vault convention (path/stem forms, folder-note
preference) layered on top of the engine's plain `dict[path, FileNode]`.
Keeping it out of `BaseFileStore` keeps the engine domain-agnostic.
Stem-form wikilinks like ``[[Foo]]`` are a vault convention. The
file_graph engine doesn't know about them — it only stores nodes and
trusts ``FileLink.path`` for adjacency. This module bridges the gap,
applying:
* **folder-note rule** ``[[X]]`` prefers ``topics/X/X.md`` over a
sibling ``topics/X.md`` when both exist
* **stem ambiguity** ``[[Foo]]`` matching multiple paths is handled
by **emitting multiple ``FileLink`` records, one per candidate**
(file_graph then has unambiguous adjacency)
* **path-form passthrough** ``[[foo/bar]]`` and ``[[foo/bar.md]]``
resolve directly against the graph (vault-relative paths; no
working_dir absolutization)
All paths are vault-relative ``graph.iter_nodes()`` returns the
key form file_graph stores, and emitted ``FileLink.path`` matches.
All functions are stateless they walk ``graph.iter_nodes()`` per
call. For batch operations (``resolve_links`` over many links) the
stem index is built once and reused.
Seven entry points mapped to call sites:
resolve single link path | None
used by: ``extract_anchors``, ``memory_resolve_wikilink``
candidates stem [path] (folder-note ordered first)
used by: ``memory_resolve_wikilink`` (ambiguity report)
collisions all stems with >1 path
used by: ``maintainer.lint``
collisions_for paths conflicting with a proposed new path
used by: ``memory_create``, ``sync`` (preflight)
extract_anchors parse [[X]] from text + resolve, dedup
used by: ``retriever`` (query anchor seeds)
resolve_links ``[FileLink]`` (raw path) ``[FileLink]`` (resolved)
core resolution; stem ambiguity expands
text_to_links one-shot: ``iter_links(text)`` + ``resolve_links``
used by: parser pipeline (before ``upsert_node``)
"""
from __future__ import annotations
from collections.abc import Iterable
from pathlib import Path
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from reme2.component.file_store.base_file_store import BaseFileStore
from ..component.file_graph.base_file_graph import BaseFileGraph
from ..schema import FileLink
from ..schema.file_link import extract_wikilinks, iter_links
from .logger_utils import get_logger
def wikilink_candidates(store: BaseFileStore, target: str) -> list[str]:
"""Paths `[[target]]` would resolve to (folder-note hit wins).
_logger = get_logger()
Folder-note convention: `topics/X/X.md` is the cluster head for
bare `[[X]]`, beating any sibling that just shares the stem.
# =========================================================================
# Internal helpers
# =========================================================================
async def _build_stem_index(graph: BaseFileGraph) -> dict[str, list[str]]:
"""Walk once, group paths by stem. Used by batch hot paths."""
out: dict[str, list[str]] = {}
async for path, _ in graph.iter_nodes():
out.setdefault(Path(path).stem, []).append(path)
return out
def _split_link(link: str) -> tuple[str, str]:
"""Return ``(target, anchor)``. Anchor empty if no ``#``.
For raw link strings supplied by external callers (which may still
arrive in ``[[A#section]]`` form). Pre-extracted ``FileLink``
records already carry ``path`` and ``anchor`` separately.
"""
folder_hits = sorted(
p for p in store._stems.get(target, ()) if Path(p).parent.name == target
)
if not link:
return "", ""
if "#" not in link:
return link.strip(), ""
target_raw, anchor_raw = link.split("#", 1)
return target_raw.strip(), anchor_raw.strip()
def _is_path_form(target: str) -> bool:
return "/" in target or target.endswith(".md")
def _filter_stem_candidates(stem: str, paths: list[str]) -> list[str]:
"""Apply folder-note rule: if any folder-note exists, ONLY folder-notes
are candidates; otherwise all sibling-stems are candidates."""
if not paths:
return []
folder_hits = sorted(p for p in paths if Path(p).parent.name == stem)
if folder_hits:
return folder_hits
return store.get_paths_by_stem(target)
return sorted(paths)
def resolve_wikilink(store: BaseFileStore, wikilink: str) -> str | None:
"""Resolve a `[[target]]` to one absolute path.
# =========================================================================
# Public API — single-shot lookups
# =========================================================================
- Path form (`topics/X/X` or `topics/X/X.md`) relative to `working_dir`.
- Stem form (`X`) folder-note preference, else unique stem hit.
- Ambiguous stems return None and log a warning.
async def resolve(graph: BaseFileGraph, link: str) -> str | None:
"""Resolve a single wikilink to **one** vault-relative path, or None.
Returns None if dangling or ambiguous (with warning on ambiguity).
Use ``resolve_links`` for the multi-link expansion semantics.
"""
target = wikilink.strip()
target, _ = _split_link(link)
if not target:
return None
if "/" in target or target.endswith(".md"):
if store.working_dir is None:
return None
if _is_path_form(target):
candidate = target if target.endswith(".md") else f"{target}.md"
abs_candidate = str((store.working_dir / candidate).resolve())
return abs_candidate if abs_candidate in store.nodes else None
return candidate if await graph.get_node(candidate) else None
candidates = wikilink_candidates(store, target)
if len(candidates) == 1:
return candidates[0]
if len(candidates) > 1:
store.logger.warning(
f"Wikilink [[{target}]] is ambiguous, candidates: {candidates}",
paths = [p async for p, _ in graph.iter_nodes() if Path(p).stem == target]
candidates_for_stem = _filter_stem_candidates(target, paths)
if len(candidates_for_stem) == 1:
return candidates_for_stem[0]
if len(candidates_for_stem) > 1:
_logger.warning(
f"Wikilink [[{target}]] is ambiguous, " f"candidates: {candidates_for_stem}",
)
return None
async def candidates(graph: BaseFileGraph, stem: str) -> list[str]:
"""All paths a ``[[stem]]`` could match. Folder-note hits ordered first."""
folder_hits: list[str] = []
stem_hits: list[str] = []
async for path, _ in graph.iter_nodes():
p = Path(path)
if p.stem != stem:
continue
if p.parent.name == stem:
folder_hits.append(path)
else:
stem_hits.append(path)
if folder_hits:
return sorted(folder_hits)
return sorted(stem_hits)
async def collisions(graph: BaseFileGraph) -> dict[str, list[str]]:
"""Every stem that resolves to >1 path. Used by maintainer.lint."""
stem_index = await _build_stem_index(graph)
return {stem: sorted(paths) for stem, paths in stem_index.items() if len(paths) > 1}
async def collisions_for(
graph: BaseFileGraph,
proposed_path: str | Path,
) -> list[str]:
"""Existing paths that would conflict with adding ``proposed_path``.
``proposed_path`` is treated as vault-relative (matching the form
``graph.iter_nodes()`` returns). Folder-note rule: when the
proposed path is itself a folder-note (parent dir name == stem),
only colliding folder-notes are returned. Otherwise all paths
sharing the stem are returned.
"""
p = Path(proposed_path)
stem = p.stem
proposed_str = str(p)
is_folder_note = p.parent.name == stem
folder_hits: list[str] = []
stem_hits: list[str] = []
async for path, _ in graph.iter_nodes():
if path == proposed_str:
continue
path_obj = Path(path)
if path_obj.stem != stem:
continue
if path_obj.parent.name == stem:
folder_hits.append(path)
else:
stem_hits.append(path)
if is_folder_note:
return sorted(folder_hits)
return sorted(folder_hits) + sorted(stem_hits)
async def extract_anchors(graph: BaseFileGraph, text: str) -> list[str]:
"""Pull ``[[X]]`` from ``text``, resolve each, dedup in source order.
Uses single-target ``resolve`` semantics ambiguous stems return
no anchor. (For multi-link expansion at *write* time, see
``resolve_links``; ``extract_anchors`` is for read-time seeding
where a single deterministic target is wanted.)
"""
if not text:
return []
seen: set[str] = set()
out: list[str] = []
for raw in extract_wikilinks(text):
hit = await resolve(graph, raw)
if hit is not None and hit not in seen:
seen.add(hit)
out.append(hit)
return out
# =========================================================================
# Public API — parser pipeline
# =========================================================================
async def resolve_links(
graph: BaseFileGraph,
links: Iterable[FileLink],
) -> list[FileLink]:
"""Resolve pre-resolution ``FileLink`` records against the graph.
Each input link has ``path`` holding a raw wikilink target (the
extractor form). Returns FileLinks with ``path`` rewritten to the
vault-relative resolved path file_graph stores. ``anchor`` and
``predicate`` pass through unchanged.
Stem expansion semantics one input link produces zero, one, or
many output links:
* path-form, target indexed 1 link
* path-form, dangling 0 links
* stem-form, 1 folder-note hit 1 link
* stem-form, N folder-note hits N links (one per)
* stem-form, 0 folder-notes, 1 sibling 1 link
* stem-form, 0 folder-notes, N siblings N links (one per)
* stem-form, dangling 0 links
Build the stem index lazily: only paid if at least one input is
stem-form.
"""
link_list = list(links)
if not link_list:
return []
stem_index: dict[str, list[str]] | None = None
out: list[FileLink] = []
for link in link_list:
target = link.path
if not target:
continue
if _is_path_form(target):
candidate = target if target.endswith(".md") else f"{target}.md"
if await graph.get_node(candidate) is None:
continue
out.append(
FileLink(
path=candidate,
anchor=link.anchor,
predicate=link.predicate,
),
)
continue
# Stem form — may expand into multiple links.
if stem_index is None:
stem_index = await _build_stem_index(graph)
for chosen in _filter_stem_candidates(target, stem_index.get(target, [])):
out.append(
FileLink(
path=chosen,
anchor=link.anchor,
predicate=link.predicate,
),
)
return out
async def text_to_links(
graph: BaseFileGraph,
text: str,
) -> list[FileLink]:
"""One-shot: extract wikilinks from ``text`` and resolve to safe links.
Equivalent to ``await resolve_links(graph, iter_links(text))``.
The parser pipeline's single-call entry point.
"""
return await resolve_links(graph, iter_links(text))

View file

@ -26,20 +26,25 @@ def main() -> None:
)
ap.add_argument("path", help="Path to a markdown file.")
ap.add_argument(
"--chunk-chars", type=int, default=2000,
help="Max characters per chunk content (default: 2000). "
"Excludes TOC skeleton when embed_toc is on.",
"--chunk-chars",
type=int,
default=2000,
help="Max characters per chunk content (default: 2000). " "Excludes TOC skeleton when embed_toc is on.",
)
ap.add_argument(
"--no-toc", action="store_true",
"--no-toc",
action="store_true",
help="Disable the full-doc TOC skeleton wrap; chunks become plain content.",
)
ap.add_argument(
"--show-edges", action="store_true",
help="Print extracted FileEdges before chunks.",
"--show-edges",
action="store_true",
help="Print extracted FileLinks before chunks.",
)
ap.add_argument(
"--preview", type=int, default=0,
"--preview",
type=int,
default=0,
help="Truncate each chunk to N chars in output (0 = full text).",
)
args = ap.parse_args()
@ -59,12 +64,12 @@ def main() -> None:
sizes = [len(c.text) for c in chunks]
print(f"chars min/avg/max: {min(sizes)} / {sum(sizes)//len(sizes)} / {max(sizes)}")
if args.show_edges:
print(f"\nedges ({len(node.edges)}):")
for e in node.edges:
print(f"\nlinks ({len(node.links)}):")
for link in node.links:
print(
f"{e.link}"
+ (f" predicate={e.predicate}" if e.predicate else "")
+ (f" anchor={e.anchor}" if e.anchor else "")
f"{link.path}"
+ (f" predicate={link.predicate}" if link.predicate else "")
+ (f" anchor={link.anchor}" if link.anchor else ""),
)
for i, c in enumerate(chunks):

View file

@ -1,348 +0,0 @@
"""FileEdge unit tests — body-only edge extraction.
Covers the three legal inline forms (bare / line-level Dataview /
inline-bracketed Dataview), multi-target expansion, dedup against
typed wrappers, open-vocabulary predicate pass-through, the explicit
decision that frontmatter is no longer walked for links, and the
2-field schema (`link` + `predicate`) with `anchor` as a derived
property and `alias` / `embed` discarded at parse time.
"""
from reme2.schema.file_edge import FileEdge, extract_wikilinks
# --------------------------------------------------------------------------
# Bare wikilinks
# --------------------------------------------------------------------------
def test_bare_wikilink():
edges = FileEdge.from_text("see [[X]]")
assert len(edges) == 1
assert edges[0].link == "X"
assert edges[0].predicate is None
assert edges[0].anchor is None
def test_anchor_alias_embed_collapse_into_link():
"""`![[X#sec|Alias]]` → link='X#sec' (alias and embed dropped)."""
edges = FileEdge.from_text("![[X#sec|Alias]]")
assert len(edges) == 1
assert edges[0].link == "X#sec"
# anchor is a derived property parsed from link.
assert edges[0].anchor == "sec"
def test_alias_only_drops_to_link():
edges = FileEdge.from_text("[[X|Alias]]")
assert len(edges) == 1
assert edges[0].link == "X"
assert edges[0].anchor is None
def test_embed_only_drops_to_link():
edges = FileEdge.from_text("![[X]]")
assert len(edges) == 1
assert edges[0].link == "X"
# --------------------------------------------------------------------------
# Line-level Dataview
# --------------------------------------------------------------------------
def test_line_level_field():
edges = FileEdge.from_text("extends:: [[Source Topic]]")
assert len(edges) == 1
assert edges[0].link == "Source Topic"
assert edges[0].predicate == "extends"
def test_line_level_multi_target():
edges = FileEdge.from_text("concerns:: [[A]], [[B]], [[C]]")
assert [(e.link, e.predicate) for e in edges] == [
("A", "concerns"),
("B", "concerns"),
("C", "concerns"),
]
def test_line_level_with_bullet():
edges = FileEdge.from_text("- extends:: [[X]]\n * concerns:: [[Y]]")
assert [(e.link, e.predicate) for e in edges] == [
("X", "extends"),
("Y", "concerns"),
]
# --------------------------------------------------------------------------
# Multi-edge cases — many wikilinks under one or several typed contexts
# --------------------------------------------------------------------------
def test_multi_target_with_anchors_preserves_each():
"""Each comma-separated target keeps its own anchor in `link`."""
edges = FileEdge.from_text("extends:: [[A#sec1]], [[B#sec2]], [[C]]")
assert [(e.link, e.anchor, e.predicate) for e in edges] == [
("A#sec1", "sec1", "extends"),
("B#sec2", "sec2", "extends"),
("C", None, "extends"),
]
def test_multi_target_non_comma_separator_still_typed():
"""Wikilinks anywhere in the value range (not just comma-separated)
inherit the line's predicate. Useful for prose-style fields."""
edges = FileEdge.from_text("extends:: [[A]] and also [[B]]")
assert [(e.link, e.predicate) for e in edges] == [
("A", "extends"),
("B", "extends"),
]
def test_multi_dataview_lines_each_multi_target():
"""Multiple Dataview lines each with multi-target → all edges typed
by their respective line's predicate."""
edges = FileEdge.from_text(
"extends:: [[A]], [[B]]\nrelates:: [[C]], [[D]]"
)
assert [(e.link, e.predicate) for e in edges] == [
("A", "extends"),
("B", "extends"),
("C", "relates"),
("D", "relates"),
]
def test_inline_bracketed_then_bare_on_same_line():
"""Inline-bracketed governs only the wikilinks inside its brackets;
a trailing bare wikilink on the same line stays bare."""
edges = FileEdge.from_text("[ext:: [[A]]] then [[B]]")
assert [(e.link, e.predicate) for e in edges] == [
("A", "ext"),
("B", None),
]
def test_mid_line_dataview_like_not_typed():
"""``predicate::`` only counts at line start (modulo bullet) —
a `predicate::` mid-line is just prose, so its wikilinks are bare."""
edges = FileEdge.from_text("[ext:: [[A]]] and concerns:: [[B]]")
assert [(e.link, e.predicate) for e in edges] == [
("A", "ext"),
("B", None), # `concerns::` mid-line is not Dataview
]
# --------------------------------------------------------------------------
# Inline-bracketed Dataview
# --------------------------------------------------------------------------
def test_inline_bracketed():
edges = FileEdge.from_text("This [extends:: [[Y]]] something else.")
assert len(edges) == 1
assert edges[0].link == "Y"
assert edges[0].predicate == "extends"
def test_inline_bracketed_multi_target():
edges = FileEdge.from_text("[concerns:: [[A]], [[B]]]")
assert [(e.link, e.predicate) for e in edges] == [
("A", "concerns"),
("B", "concerns"),
]
def test_inline_bracketed_skips_cross_line():
# A `[predicate:: ...]` that spans a newline is malformed → the inner
# wikilink falls back to bare; the unmatched `[` does not eat tail text.
edges = FileEdge.from_text("[extends:: [[X]]\nbad]")
assert len(edges) == 1
assert edges[0].link == "X"
assert edges[0].predicate is None
# --------------------------------------------------------------------------
# Dedup: a wikilink inside a typed wrapper should not double-emit
# --------------------------------------------------------------------------
def test_inline_bracketed_does_not_double_emit():
edges = FileEdge.from_text("see [extends:: [[X]]] again.")
assert len(edges) == 1
assert edges[0].predicate == "extends"
def test_line_level_value_does_not_double_emit():
edges = FileEdge.from_text("extends:: [[X]]")
assert len(edges) == 1
def test_typed_and_bare_coexist_for_same_target():
edges = FileEdge.from_text("extends:: [[X]]\nFree text mentioning [[X]] again.")
links_preds = sorted(((e.link, e.predicate or "") for e in edges))
assert links_preds == [("X", ""), ("X", "extends")]
# --------------------------------------------------------------------------
# Open-vocabulary predicates — any identifier-shaped token passes through
# --------------------------------------------------------------------------
def test_arbitrary_predicate_preserved():
edges = FileEdge.from_text("anything_goes:: [[X]]")
assert len(edges) == 1
assert edges[0].link == "X"
assert edges[0].predicate == "anything_goes"
def test_inline_arbitrary_predicate_preserved():
edges = FileEdge.from_text("[wat:: [[X]]]")
assert len(edges) == 1
assert edges[0].predicate == "wat"
def test_predicate_must_be_identifier_shaped():
# A leading digit fails the regex `[A-Za-z][A-Za-z0-9_]*` so the line is
# not recognised as a Dataview field — the wikilink falls back to bare.
edges = FileEdge.from_text("123bad:: [[X]]")
assert len(edges) == 1
assert edges[0].link == "X"
assert edges[0].predicate is None
def test_file_edge_accepts_any_predicate_string():
e = FileEdge(link="X", predicate="totally_made_up")
assert e.predicate == "totally_made_up"
# --------------------------------------------------------------------------
# Frontmatter is NOT walked for links — explicit regression
# --------------------------------------------------------------------------
def test_frontmatter_links_block_is_ignored():
# Even if a YAML-shaped string sits at the top of body, FileEdge.from_text
# only operates on body. We pass body text directly here, so this test
# asserts the API surface no longer accepts a metadata dict.
import inspect
sig = inspect.signature(FileEdge.from_text)
assert list(sig.parameters.keys()) == ["text"], (
"FileEdge.from_text should accept body text only — frontmatter walk removed"
)
def test_no_frontmatter_walker_exported():
from reme2.schema import file_edge as fe
for removed in (
"parse_wikilinks_from_metadata",
"extract_wikilinks_from_metadata",
"extract_inline_fields",
"extract_typed_edges",
"InlineField",
):
assert not hasattr(fe, removed), (
f"{removed} should have been removed when YAML edges were dropped"
)
# --------------------------------------------------------------------------
# FileEdge schema
# --------------------------------------------------------------------------
def test_file_edge_extra_forbid():
import pytest
from pydantic import ValidationError
with pytest.raises(ValidationError):
FileEdge(link="X", target="X") # type: ignore[call-arg]
def test_file_edge_minimal_field_set():
"""Stored fields are just `link` and `predicate`. `path` and `anchor`
are `@property`s (not stored fields) so they shouldn't appear in
`model_dump()`. `target` / `alias` / `embed` were dropped entirely."""
e = FileEdge(link="X")
dumped = e.model_dump()
assert set(dumped.keys()) == {"link", "predicate"}
for removed in ("target", "alias", "embed", "anchor", "path"):
assert removed not in dumped
def test_anchor_property_parses_from_link():
"""`anchor` is derived from `link`, not stored separately."""
assert FileEdge(link="X").anchor is None
assert FileEdge(link="X#sec").anchor == "sec"
assert FileEdge(link="X#sec#more").anchor == "sec#more"
# Empty anchor is treated as no anchor.
assert FileEdge(link="X#").anchor is None
assert FileEdge(link="X# ").anchor is None
def test_path_property_parses_from_link():
"""`path` is derived from `link` — file part before any `#anchor`."""
assert FileEdge(link="X").path == "X"
assert FileEdge(link="X#sec").path == "X"
assert FileEdge(link="topics/Foo").path == "topics/Foo"
assert FileEdge(link="topics/Foo#bar").path == "topics/Foo"
# Multiple '#' — only first splits; the rest live in anchor.
assert FileEdge(link="X#a#b").path == "X"
assert FileEdge(link="X#a#b").anchor == "a#b"
# Empty anchor → path is still the full prefix.
assert FileEdge(link="X#").path == "X"
def test_path_anchor_roundtrip_via_link():
"""Reconstructing `link` from `path` + `anchor` yields the original."""
for link in ("X", "X#sec", "topics/Foo", "topics/Foo#bar", "X#a#b"):
e = FileEdge(link=link)
rebuilt = e.path if not e.anchor else f"{e.path}#{e.anchor}"
assert rebuilt == link, f"{link!r}{rebuilt!r}"
def test_anchor_is_not_constructor_arg():
"""Since `anchor` and `path` are properties, passing them to the
constructor should fail (extra='forbid')."""
import pytest
from pydantic import ValidationError
with pytest.raises(ValidationError):
FileEdge(link="X", anchor="sec") # type: ignore[call-arg]
with pytest.raises(ValidationError):
FileEdge(link="X", path="X") # type: ignore[call-arg]
# --------------------------------------------------------------------------
# Back-compat: extract_wikilinks returns flat target list (used by ingestor)
# --------------------------------------------------------------------------
def test_extract_wikilinks_flat_targets():
targets = extract_wikilinks("see [[X]] and extends:: [[Y]] and [extends:: [[Z]]]")
assert targets == ["X", "Y", "Z"]
def test_extract_wikilinks_strips_anchor():
"""`extract_wikilinks` returns just the file part — anchor stripped
so callers can feed it to `resolve_wikilink`."""
targets = extract_wikilinks("see [[X#sec]] and ![[Y#a|alias]]")
assert targets == ["X", "Y"]
# --------------------------------------------------------------------------
# Source ordering stability
# --------------------------------------------------------------------------
def test_edges_sorted_by_source_position():
body = (
"intro [[First]] then\n"
"extends:: [[Second]]\n"
"tail [[Third]]\n"
)
edges = FileEdge.from_text(body)
assert [e.link for e in edges] == ["First", "Second", "Third"]

345
tests/test_file_link.py Normal file
View file

@ -0,0 +1,345 @@
"""FileLink unit tests — body-only wikilink extraction.
Covers:
* three legal inline forms (bare / line-level Dataview / inline-bracketed
Dataview) and multi-target expansion within each
* dedup against typed wrappers (no double-emit when a wikilink lives
inside a typed envelope)
* open-vocabulary predicate pass-through and the identifier-shape gate
* the explicit decision that frontmatter is no longer walked for links
(extraction takes body text only)
* the 3-field schema: ``(path, anchor, predicate)`` all real fields,
extra='forbid'
Pre-resolution form: ``iter_links(text)`` returns ``FileLink`` records with
``path`` set to the raw wikilink target as written. The resolver
(``utils.wikilink_resolver``) rewrites ``path`` to the vault-relative
resolved path; here we only test the extractor.
"""
import inspect
import pytest
from pydantic import ValidationError
from reme2.schema import FileLink
from reme2.schema.file_link import extract_wikilinks, iter_links
# --------------------------------------------------------------------------
# Bare wikilinks
# --------------------------------------------------------------------------
def test_bare_wikilink():
links = iter_links("see [[X]]")
assert len(links) == 1
assert links[0].path == "X"
assert links[0].predicate is None
assert links[0].anchor is None
def test_anchor_split_from_target():
"""``[[X#sec]]`` → path='X', anchor='sec' (separate field, not derived)."""
links = iter_links("![[X#sec|Alias]]")
assert len(links) == 1
assert links[0].path == "X"
assert links[0].anchor == "sec"
def test_alias_only_dropped():
links = iter_links("[[X|Alias]]")
assert len(links) == 1
assert links[0].path == "X"
assert links[0].anchor is None
def test_embed_only_dropped():
links = iter_links("![[X]]")
assert len(links) == 1
assert links[0].path == "X"
# --------------------------------------------------------------------------
# Line-level Dataview
# --------------------------------------------------------------------------
def test_line_level_field():
links = iter_links("extends:: [[Source Topic]]")
assert len(links) == 1
assert links[0].path == "Source Topic"
assert links[0].predicate == "extends"
def test_line_level_multi_target():
links = iter_links("concerns:: [[A]], [[B]], [[C]]")
assert [(link.path, link.predicate) for link in links] == [
("A", "concerns"),
("B", "concerns"),
("C", "concerns"),
]
def test_line_level_with_bullet():
links = iter_links("- extends:: [[X]]\n * concerns:: [[Y]]")
assert [(link.path, link.predicate) for link in links] == [
("X", "extends"),
("Y", "concerns"),
]
# --------------------------------------------------------------------------
# Multi-link cases — many wikilinks under one or several typed contexts
# --------------------------------------------------------------------------
def test_multi_target_with_anchors_preserves_each():
"""Each comma-separated target keeps its own anchor as a separate field."""
links = iter_links("extends:: [[A#sec1]], [[B#sec2]], [[C]]")
assert [(link.path, link.anchor, link.predicate) for link in links] == [
("A", "sec1", "extends"),
("B", "sec2", "extends"),
("C", None, "extends"),
]
def test_multi_target_non_comma_separator_still_typed():
"""Wikilinks anywhere in the value range (not just comma-separated)
inherit the line's predicate. Useful for prose-style fields."""
links = iter_links("extends:: [[A]] and also [[B]]")
assert [(link.path, link.predicate) for link in links] == [
("A", "extends"),
("B", "extends"),
]
def test_multi_dataview_lines_each_multi_target():
"""Multiple Dataview lines each with multi-target → all links typed
by their respective line's predicate."""
links = iter_links(
"extends:: [[A]], [[B]]\nrelates:: [[C]], [[D]]",
)
assert [(link.path, link.predicate) for link in links] == [
("A", "extends"),
("B", "extends"),
("C", "relates"),
("D", "relates"),
]
def test_inline_bracketed_then_bare_on_same_line():
"""Inline-bracketed governs only the wikilinks inside its brackets;
a trailing bare wikilink on the same line stays bare."""
links = iter_links("[ext:: [[A]]] then [[B]]")
assert [(link.path, link.predicate) for link in links] == [
("A", "ext"),
("B", None),
]
def test_mid_line_dataview_like_not_typed():
"""``predicate::`` only counts at line start (modulo bullet) —
a ``predicate::`` mid-line is just prose, so its wikilinks are bare."""
links = iter_links("[ext:: [[A]]] and concerns:: [[B]]")
assert [(link.path, link.predicate) for link in links] == [
("A", "ext"),
("B", None), # `concerns::` mid-line is not Dataview
]
# --------------------------------------------------------------------------
# Inline-bracketed Dataview
# --------------------------------------------------------------------------
def test_inline_bracketed():
links = iter_links("This [extends:: [[Y]]] something else.")
assert len(links) == 1
assert links[0].path == "Y"
assert links[0].predicate == "extends"
def test_inline_bracketed_multi_target():
links = iter_links("[concerns:: [[A]], [[B]]]")
assert [(link.path, link.predicate) for link in links] == [
("A", "concerns"),
("B", "concerns"),
]
def test_inline_bracketed_skips_cross_line():
# A `[predicate:: ...]` that spans a newline is malformed → the inner
# wikilink falls back to bare; the unmatched `[` does not eat tail text.
links = iter_links("[extends:: [[X]]\nbad]")
assert len(links) == 1
assert links[0].path == "X"
assert links[0].predicate is None
# --------------------------------------------------------------------------
# Dedup: a wikilink inside a typed wrapper should not double-emit
# --------------------------------------------------------------------------
def test_inline_bracketed_does_not_double_emit():
links = iter_links("see [extends:: [[X]]] again.")
assert len(links) == 1
assert links[0].predicate == "extends"
def test_line_level_value_does_not_double_emit():
links = iter_links("extends:: [[X]]")
assert len(links) == 1
def test_typed_and_bare_coexist_for_same_target():
links = iter_links("extends:: [[X]]\nFree text mentioning [[X]] again.")
paths_preds = sorted(((link.path, link.predicate or "") for link in links))
assert paths_preds == [("X", ""), ("X", "extends")]
# --------------------------------------------------------------------------
# Open-vocabulary predicates — any identifier-shaped token passes through
# --------------------------------------------------------------------------
def test_arbitrary_predicate_preserved():
links = iter_links("anything_goes:: [[X]]")
assert len(links) == 1
assert links[0].path == "X"
assert links[0].predicate == "anything_goes"
def test_inline_arbitrary_predicate_preserved():
links = iter_links("[wat:: [[X]]]")
assert len(links) == 1
assert links[0].predicate == "wat"
def test_predicate_must_be_identifier_shaped():
# A leading digit fails the regex `[A-Za-z][A-Za-z0-9_]*` so the line is
# not recognised as a Dataview field — the wikilink falls back to bare.
links = iter_links("123bad:: [[X]]")
assert len(links) == 1
assert links[0].path == "X"
assert links[0].predicate is None
def test_file_link_accepts_any_predicate_string():
link = FileLink(path="X", predicate="totally_made_up")
assert link.predicate == "totally_made_up"
# --------------------------------------------------------------------------
# Frontmatter is NOT walked for links — explicit regression
# --------------------------------------------------------------------------
def test_iter_links_takes_body_text_only():
"""``iter_links`` operates on body text — no frontmatter walking."""
sig = inspect.signature(iter_links)
assert list(sig.parameters.keys()) == ["text"], "iter_links should accept body text only — frontmatter walk removed"
def test_no_frontmatter_walker_exported():
from reme2.schema import file_link as fl
for removed in (
"parse_wikilinks_from_metadata",
"extract_wikilinks_from_metadata",
"extract_inline_fields",
"extract_typed_edges",
"InlineField",
):
assert not hasattr(fl, removed), f"{removed} should have been removed when YAML links were dropped"
# --------------------------------------------------------------------------
# FileLink schema
# --------------------------------------------------------------------------
def test_file_link_extra_forbid():
with pytest.raises(ValidationError):
FileLink(path="X", target="X") # type: ignore[call-arg]
def test_file_link_field_set():
"""Stored fields are ``(path, anchor, predicate)`` — no others."""
link = FileLink(path="X")
dumped = link.model_dump()
assert set(dumped.keys()) == {"path", "anchor", "predicate"}
assert dumped == {"path": "X", "anchor": None, "predicate": None}
def test_file_link_dump_excludes_none_when_asked():
link = FileLink(path="X")
assert link.model_dump(exclude_none=True) == {"path": "X"}
def test_file_link_full_construction():
link = FileLink(path="topics/Foo.md", anchor="sec", predicate="extends")
assert link.path == "topics/Foo.md"
assert link.anchor == "sec"
assert link.predicate == "extends"
def test_file_link_path_required():
with pytest.raises(ValidationError):
FileLink() # type: ignore[call-arg]
# --------------------------------------------------------------------------
# Anchor extraction edge cases — anchor is a real field, captured by the
# regex's named group (not derived from path string-splitting)
# --------------------------------------------------------------------------
def test_anchor_extraction_edge_cases():
"""``[[X]]`` → no anchor; ``[[X#sec]]`` → anchor='sec'.
The wikilink regex requires the anchor capture to be one or more
chars, so a literal ``[[X#]]`` doesn't match the regex at all (the
trailing ``#`` makes it invalid syntax). Whitespace-only anchors
are treated as no anchor (stripped to empty None)."""
assert iter_links("[[X]]")[0].anchor is None
assert iter_links("[[X#sec]]")[0].anchor == "sec"
# `[[X#]]` is not a valid wikilink — anchor group requires 1+ chars.
assert iter_links("[[X#]]") == []
# `[[X# ]]` matches but strips to empty → anchor=None.
assert iter_links("[[X# ]]")[0].anchor is None
def test_anchor_inside_pipe_alias_still_extracted():
"""Anchor is captured before the alias pipe."""
links = iter_links("[[topics/Foo#sec|Display]]")
assert links[0].path == "topics/Foo"
assert links[0].anchor == "sec"
# --------------------------------------------------------------------------
# Back-compat: extract_wikilinks returns flat target list (used by retriever)
# --------------------------------------------------------------------------
def test_extract_wikilinks_flat_targets():
targets = extract_wikilinks("see [[X]] and extends:: [[Y]] and [extends:: [[Z]]]")
assert targets == ["X", "Y", "Z"]
def test_extract_wikilinks_strips_anchor():
"""``extract_wikilinks`` returns just the file part — anchor stripped
so callers can feed it to ``resolve``."""
targets = extract_wikilinks("see [[X#sec]] and ![[Y#a|alias]]")
assert targets == ["X", "Y"]
# --------------------------------------------------------------------------
# Source ordering stability
# --------------------------------------------------------------------------
def test_links_sorted_by_source_position():
body = "intro [[First]] then\n" "extends:: [[Second]]\n" "tail [[Third]]\n"
links = iter_links(body)
assert [link.path for link in links] == ["First", "Second", "Third"]

View file

@ -45,19 +45,7 @@ def _all_headings(text: str) -> list[str]:
def test_tree_groups_under_headings():
txt = (
"# Top\n"
"para1\n"
"\n"
"## Sub A\n"
"para2\n"
"\n"
"### Deeper\n"
"para3\n"
"\n"
"## Sub B\n"
"para4\n"
)
txt = "# Top\n" "para1\n" "\n" "## Sub A\n" "para2\n" "\n" "### Deeper\n" "para3\n" "\n" "## Sub B\n" "para4\n"
from mistletoe.block_token import Document
from mistletoe.markdown_renderer import MarkdownRenderer
@ -142,23 +130,13 @@ def test_every_chunk_lists_every_doc_heading():
expected_headings = {"# Doc", "## Section A", "### Subsection", "## Section B"}
for c in chunks:
present = set(_all_headings(c.text))
assert expected_headings <= present, (
f"chunk missing headings {expected_headings - present}: {c.text!r}"
)
assert expected_headings <= present, f"chunk missing headings {expected_headings - present}: {c.text!r}"
def test_owner_section_holds_chunk_content():
"""The chunk's content sits directly under its owner heading — not
under any other section's heading."""
txt = (
"# Doc\n"
"\n"
"## A\n"
"alpha alpha alpha alpha here\n"
"\n"
"## B\n"
"bravo bravo bravo bravo here\n"
)
txt = "# Doc\n" "\n" "## A\n" "alpha alpha alpha alpha here\n" "\n" "## B\n" "bravo bravo bravo bravo here\n"
chunks = _parser(60)._chunk(txt, "/x.md")
a_chunk = next(c for c in chunks if "alpha" in c.text)
b_chunk = next(c for c in chunks if "bravo" in c.text)
@ -383,15 +361,7 @@ def test_paragraph_line_split_keeps_skeleton():
def test_embed_toc_off_strips_skeleton():
"""With embed_toc=False, chunks contain only their own content —
no full-doc heading skeleton wrapping them."""
txt = (
"# Doc\n"
"\n"
"## A\n"
"para A long content here\n"
"\n"
"## B\n"
"para B long content here\n"
)
txt = "# Doc\n" "\n" "## A\n" "para A long content here\n" "\n" "## B\n" "para B long content here\n"
chunks = _parser(60, embed_toc=False)._chunk(txt, "/x.md")
a = next(c for c in chunks if "para A" in c.text)
b = next(c for c in chunks if "para B" in c.text)
@ -456,9 +426,9 @@ def test_body_run_budget_excludes_toc():
txt = (
"# Top\n"
"## Sub long heading title here\n"
"abcdefghij abcdefghij abcdefghij\n" # 32 chars body
"abcdefghij abcdefghij abcdefghij\n" # 32 chars body
"\n"
"klmnopqrst klmnopqrst klmnopqrst\n" # 32 chars body
"klmnopqrst klmnopqrst klmnopqrst\n" # 32 chars body
)
# 80 chars budget covers the joined body (32+2+32=66) but is
# smaller than body+TOC under old (counting) semantics (~120).
@ -590,10 +560,7 @@ def test_paragraph_split_marks_parts():
"delta line four with extra padding text here\n"
)
chunks = _parser(100)._chunk(txt, "/x.md")
para_chunks = [
c for c in chunks
if any(t in c.text for t in ("alpha", "beta", "gamma", "delta"))
]
para_chunks = [c for c in chunks if any(t in c.text for t in ("alpha", "beta", "gamma", "delta"))]
assert len(para_chunks) >= 2
n = len(para_chunks)
for i, c in enumerate(para_chunks, 1):
@ -602,15 +569,7 @@ def test_paragraph_split_marks_parts():
def test_single_piece_leaf_has_no_part_marker():
"""When a leaf block fits in one piece, no [Part] prefix is added."""
txt = (
"# Doc\n"
"\n"
"## T\n"
"\n"
"| a | b |\n"
"|---|---|\n"
"| 1 | 2 |\n"
)
txt = "# Doc\n" "\n" "## T\n" "\n" "| a | b |\n" "|---|---|\n" "| 1 | 2 |\n"
chunks = _parser(500)._chunk(txt, "/x.md")
assert len(chunks) == 1
assert "[Part" not in chunks[0].text
@ -642,11 +601,11 @@ def test_part_marker_absent_for_body_run_packing():
def test_chunk_line_ranges_track_source():
txt = (
"# Top\n" # line 1
"intro\n" # line 2
"# Top\n" # line 1
"intro\n" # line 2
"\n"
"## Sub\n" # line 4
"body\n" # line 5
"## Sub\n" # line 4
"body\n" # line 5
)
chunks = _parser(500)._chunk(txt, "/x.md")
assert len(chunks) == 1