diff --git a/reme2/component/__init__.py b/reme2/component/__init__.py index cda27c37..a82cdd12 100644 --- a/reme2/component/__init__.py +++ b/reme2/component/__init__.py @@ -4,6 +4,7 @@ from . import as_llm from . import as_llm_formatter from . import as_token_counter from . import file_store +from . import file_graph from . import client from . import embedding from . import file_parser @@ -31,6 +32,7 @@ __all__ = [ "as_llm_formatter", "as_token_counter", "file_store", + "file_graph", "client", "embedding", "file_parser", diff --git a/reme2/component/base_component.py b/reme2/component/base_component.py index c7df66db..1ac9b29c 100644 --- a/reme2/component/base_component.py +++ b/reme2/component/base_component.py @@ -21,11 +21,11 @@ class BaseComponent(ABC): component_type = ComponentEnum.BASE def __init__( - self, - name: str | None = None, - backend: str = "", - app_context: "ApplicationContext | None" = None, - **kwargs, + self, + name: str | None = None, + backend: str = "", + app_context: "ApplicationContext | None" = None, + **kwargs, ) -> None: self.name: str = name or self.__class__.__name__ self.backend: str = backend diff --git a/reme2/component/embedding/base_embedding_model.py b/reme2/component/embedding/base_embedding_model.py index 6bf73b99..6172115b 100644 --- a/reme2/component/embedding/base_embedding_model.py +++ b/reme2/component/embedding/base_embedding_model.py @@ -77,7 +77,7 @@ class BaseEmbeddingModel(BaseComponent): if to_compute: for i in range(0, len(to_compute), self.max_batch_size): - batch = to_compute[i: i + self.max_batch_size] + batch = to_compute[i : i + self.max_batch_size] indices = [idx for idx, _ in batch] texts = [text for _, text in batch] @@ -89,7 +89,7 @@ class BaseEmbeddingModel(BaseComponent): break except (TimeoutError, ConnectionError, OSError): if attempt < self.max_retries - 1: - await asyncio.sleep(2 ** attempt) + await asyncio.sleep(2**attempt) except Exception: break @@ -184,4 +184,4 @@ class BaseEmbeddingModel(BaseComponent): try: np.savez(self.cache_path, keys=np.array(keys, dtype=str), embeddings=embeddings) except Exception: - pass \ No newline at end of file + pass diff --git a/reme2/component/file_graph/__init__.py b/reme2/component/file_graph/__init__.py new file mode 100644 index 00000000..63a4f59b --- /dev/null +++ b/reme2/component/file_graph/__init__.py @@ -0,0 +1,22 @@ +"""File graph module. + +Backend-agnostic graph engine that owns the wikilink graph (nodes + +typed edges + traversal queries). Sister to ``file_store``, which owns +chunks and projections (vector / FTS). + +Two backends ship today: + - ``LocalFileGraph`` (``@R.register("local")``) — networkx + ``MultiDiGraph`` + JSONL persistence. Default. + - ``Neo4jFileGraph`` (``@R.register("neo4j")``) — property-graph + backed by Neo4j. Requires the ``neo4j`` driver. +""" + +from .base_file_graph import BaseFileGraph +from .local_file_graph import LocalFileGraph +from .neo4j_file_graph import Neo4jFileGraph + +__all__ = [ + "BaseFileGraph", + "LocalFileGraph", + "Neo4jFileGraph", +] diff --git a/reme2/component/file_graph/base_file_graph.py b/reme2/component/file_graph/base_file_graph.py new file mode 100644 index 00000000..6bce3696 --- /dev/null +++ b/reme2/component/file_graph/base_file_graph.py @@ -0,0 +1,76 @@ +"""Abstract base for the file-graph engine. + +The file-graph owns ``FileNode`` records keyed by vault-relative path +and serves graph traversal over wikilink links. file_graph trusts +``FileLink.path`` directly — there is no internal wikilink resolution. +Pre-resolution forms (raw stem-form wikilinks like ``[[Foo]]``) are a +vault convention resolved by the external ``utils.wikilink_resolver`` +before nodes are upserted. + +Contract — six abstract methods, two blocks: + + Node CRUD upsert_node, delete_node, get_node, iter_nodes + Link access get_outlinks, get_inlinks +""" + +from __future__ import annotations + +import re +from abc import abstractmethod +from collections.abc import AsyncIterator +from pathlib import Path + +from ..base_component import BaseComponent +from ...enumeration import ComponentEnum +from ...schema import FileLink, FileNode + + +class BaseFileGraph(BaseComponent): + """Pluggable file-graph backend — node CRUD + link adjacency.""" + + component_type = ComponentEnum.FILE_GRAPH + + def __init__(self, store_name: str = "default", **kwargs): + super().__init__(**kwargs) + if not re.match(r"^[a-zA-Z0-9_]+$", store_name): + raise ValueError( + f"Invalid store name '{store_name}'. " f"Only alphanumeric and underscores allowed.", + ) + self.store_name: str = store_name + self.working_dir: str = self.app_context.app_config.working_dir if self.app_context else "" + self.store_path: Path = Path(self.working_dir) / "file_graph" / store_name + self.store_path.mkdir(parents=True, exist_ok=True) + + # -- Node CRUD --------------------------------------------------------- + + @abstractmethod + async def upsert_node(self, node: FileNode) -> None: + """Add or replace a node and its outgoing links.""" + + @abstractmethod + async def delete_node(self, path: str) -> FileNode | None: + """Remove a node + its incident links. Returns the removed node, or None.""" + + @abstractmethod + async def get_node(self, path: str) -> FileNode | None: + """Single-node lookup.""" + + @abstractmethod + def iter_nodes(self) -> AsyncIterator[tuple[str, FileNode]]: + """Walk every (path, FileNode) in the graph.""" + + # -- Link access ------------------------------------------------------- + + @abstractmethod + async def get_outlinks( + self, + path: str, + ) -> list[tuple[FileNode, FileLink]]: + """Resolved outgoing links from ``path``.""" + + @abstractmethod + async def get_inlinks( + self, + path: str, + ) -> list[tuple[FileNode, FileLink]]: + """Resolved incoming links to ``path``.""" diff --git a/reme2/component/file_graph/local_file_graph.py b/reme2/component/file_graph/local_file_graph.py new file mode 100644 index 00000000..f587c1ee --- /dev/null +++ b/reme2/component/file_graph/local_file_graph.py @@ -0,0 +1,152 @@ +"""Local file-graph backend — networkx ``MultiDiGraph``. + +Each ``FileNode`` lives as ``data['node']`` on the graph; each +``FileLink`` whose ``link.path`` exists in the graph lives as +``data['link']`` on a directed graph edge between the two file nodes. + +file_graph trusts ``link.path`` directly — there is no internal +wikilink resolution. The parser pipeline (with the external resolver) +is responsible for producing safe ``FileLink`` records where +``link.path`` is a real vault-relative target path. Stem ambiguity is +already handled there by emitting one link per candidate. + +Late-arriving target: when ``upsert_node(B)`` runs, other nodes whose +links have ``link.path == B.path`` get those in-edges restored +(one O(N×L) sweep per upsert; cheap for vault sizes). + +Persistence: pickle to ``graph.pkl`` on close, restored on _start. +""" + +from __future__ import annotations + +import pickle +from collections.abc import AsyncIterator +from pathlib import Path + +from .base_file_graph import BaseFileGraph +from ..component_registry import R +from ...schema import FileLink, FileNode + + +@R.register("local") +class LocalFileGraph(BaseFileGraph): + """Networkx-backed file graph. Trusts ``FileLink.path`` for adjacency.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._graph = None # networkx.MultiDiGraph; set in _start + self._graph_file: Path = self.store_path / "graph.pkl" + + # -- Lifecycle --------------------------------------------------------- + + async def _start(self) -> None: + await super()._start() + try: + import networkx as nx + except ImportError as e: + raise ImportError( + "LocalFileGraph requires networkx. Install with `pip install networkx`.", + ) from e + self._graph = self._load_graph(nx) or nx.MultiDiGraph() + self.logger.info( + f"LocalFileGraph '{self.store_name}' ready: " + f"{self._graph.number_of_nodes()} nodes, " + f"{self._graph.number_of_edges()} edges", + ) + + async def _close(self) -> None: + self._save_graph() + await super()._close() + + def _load_graph(self, nx): + if not self._graph_file.exists(): + return None + try: + with open(self._graph_file, "rb") as f: + graph = pickle.load(f) + if not isinstance(graph, nx.MultiDiGraph): + self.logger.warning( + f"{self._graph_file} is not a MultiDiGraph; ignoring", + ) + return None + return graph + except Exception as e: + self.logger.exception(f"Failed to load {self._graph_file}: {e}") + return None + + def _save_graph(self) -> None: + try: + tmp = self._graph_file.with_suffix(".tmp") + with open(tmp, "wb") as f: + pickle.dump(self._graph, f, protocol=pickle.HIGHEST_PROTOCOL) + tmp.replace(self._graph_file) + except Exception as e: + self.logger.exception(f"Failed to write {self._graph_file}: {e}") + + # -- Node CRUD --------------------------------------------------------- + + async def upsert_node(self, node: FileNode) -> None: + path = node.path + if self._graph.has_node(path): + self._graph.remove_node(path) + self._graph.add_node(path, node=node) + + # Out-links from this node — trust link.path. + for link in node.links: + if link.path and self._graph.has_node(link.path): + self._graph.add_edge(path, link.path, link=link) + + # Late-arriving target: restore in-links from other nodes whose + # links resolve here. Scan all other nodes' links; cheap for + # vault sizes (O(N×L_avg) per upsert). + for src_path, src_data in self._graph.nodes(data=True): + if src_path == path: + continue + src_node = src_data.get("node") + if src_node is None: + continue + for link in src_node.links: + if link.path == path: + self._graph.add_edge(src_path, path, link=link) + + async def delete_node(self, path: str) -> FileNode | None: + if not self._graph.has_node(path): + return None + node = self._graph.nodes[path].get("node") + self._graph.remove_node(path) + return node + + async def get_node(self, path: str) -> FileNode | None: + if not self._graph.has_node(path): + return None + return self._graph.nodes[path].get("node") + + async def iter_nodes(self) -> AsyncIterator[tuple[str, FileNode]]: + for path, data in self._graph.nodes(data=True): + node = data.get("node") + if node is not None: + yield path, node + + # -- Link access ------------------------------------------------------- + + async def get_outlinks(self, path: str) -> list[tuple[FileNode, FileLink]]: + if not self._graph.has_node(path): + return [] + out: list[tuple[FileNode, FileLink]] = [] + for _src, dst, data in self._graph.out_edges(path, data=True): + target = self._graph.nodes[dst].get("node") + link = data.get("link") + if target is not None and isinstance(link, FileLink): + out.append((target, link)) + return out + + async def get_inlinks(self, path: str) -> list[tuple[FileNode, FileLink]]: + if not self._graph.has_node(path): + return [] + out: list[tuple[FileNode, FileLink]] = [] + for src, _dst, data in self._graph.in_edges(path, data=True): + source = self._graph.nodes[src].get("node") + link = data.get("link") + if source is not None and isinstance(link, FileLink): + out.append((source, link)) + return out diff --git a/reme2/component/file_graph/neo4j_file_graph.py b/reme2/component/file_graph/neo4j_file_graph.py new file mode 100644 index 00000000..df8b96bf --- /dev/null +++ b/reme2/component/file_graph/neo4j_file_graph.py @@ -0,0 +1,323 @@ +"""Neo4j-backed file graph. + +Property-graph mapping: + + (:File {path, st_mtime, title, description, tags, links_json, + extra_json}) + -[:LINKS {idx, anchor, predicate}]->(:File) + +``path`` is the unique key (constraint enforced on ``_start``). +Frontmatter goes into flat properties; arbitrary extras land in +``extra_json``. The full ``FileLink[]`` payload is also stored as +``links_json`` so we can recover the node losslessly even for links +whose target wasn't indexed at upsert time (will be linked later by +``_restore_inlinks``). + +Adjacency policy: file_graph trusts ``FileLink.path`` directly — no +internal wikilink resolution. The parser pipeline (with the external +resolver) produces safe links where ``link.path`` is already a +vault-relative target path. + +Conditional dependency: the ``neo4j`` driver is loaded lazily; the +import error fires at ``_start`` (boot), not at first call. +""" + +from __future__ import annotations + +import json +from collections.abc import AsyncIterator +from typing import Any + +from .base_file_graph import BaseFileGraph +from ..component_registry import R +from ...schema import FileLink, FileNode +from ...schema.file_node import FileFrontMatter + + +_TYPED_FRONTMATTER_FIELDS = {"title", "description", "tags"} +_LINK_FIELDS = {"path", "anchor", "predicate"} + + +@R.register("neo4j") +class Neo4jFileGraph(BaseFileGraph): + """Neo4j-backed file graph; trusts ``FileLink.path`` for adjacency. + + Connection params (constructor kwargs): + uri: bolt URL, e.g. ``bolt://localhost:7687`` + user: auth user (default ``neo4j``) + password: auth password + database: target db name (default ``neo4j``) + """ + + def __init__( + self, + uri: str = "bolt://localhost:7687", + user: str = "neo4j", + password: str = "neo4j", + database: str = "neo4j", + **kwargs, + ): + super().__init__(**kwargs) + self._uri: str = uri + self._user: str = user + self._password: str = password + self._database: str = database + self._driver = None + + # -- Lifecycle --------------------------------------------------------- + + async def _start(self) -> None: + await super()._start() + try: + from neo4j import AsyncGraphDatabase + except ImportError as e: + raise ImportError( + "Neo4jFileGraph requires the neo4j driver. " "Install with `pip install neo4j`.", + ) from e + self._driver = AsyncGraphDatabase.driver( + self._uri, + auth=(self._user, self._password), + ) + async with self._session() as session: + await session.run( + "CREATE CONSTRAINT file_path_unique IF NOT EXISTS " "FOR (f:File) REQUIRE f.path IS UNIQUE", + ) + self.logger.info( + f"Neo4jFileGraph '{self.store_name}' connected at " f"{self._uri}/{self._database}", + ) + + async def _close(self) -> None: + if self._driver is not None: + await self._driver.close() + self._driver = None + await super()._close() + + def _session(self): + assert self._driver is not None, "Neo4jFileGraph not started" + return self._driver.session(database=self._database) + + # -- Node CRUD --------------------------------------------------------- + + async def upsert_node(self, node: FileNode) -> None: + props = self._node_props(node) + # Trust link.path; only emit LINKS for links with non-empty path. + link_payload = [ + { + "idx": i, + "anchor": link.anchor, + "predicate": link.predicate, + "target": link.path, + } + for i, link in enumerate(node.links) + if link.path + ] + async with self._session() as session: + await session.execute_write( + self._upsert_node_tx, + node.path, + props, + link_payload, + ) + # Late-arriving target: restore in-links from other nodes whose + # links resolve here. Separate session call (not in tx) so it + # doesn't block the upsert ack. + await self._restore_inlinks(node.path) + + @staticmethod + async def _upsert_node_tx(tx, path, props, links): + await tx.run( + "MERGE (f:File {path: $path}) SET f += $props", + path=path, + props=props, + ) + await tx.run( + "MATCH (f:File {path: $path})-[r:LINKS]->() DELETE r", + path=path, + ) + # MERGE only when the target node exists. ``OPTIONAL MATCH`` + + # ``WHERE t IS NOT NULL`` lets us batch this in one pass per link. + for link in links: + await tx.run( + """ + MATCH (s:File {path: $src}) + OPTIONAL MATCH (t:File {path: $dst}) + WITH s, t WHERE t IS NOT NULL + MERGE (s)-[r:LINKS {idx: $idx}]->(t) + SET r.anchor = $anchor, r.predicate = $predicate + """, + src=path, + dst=link["target"], + idx=link["idx"], + anchor=link["anchor"], + predicate=link["predicate"], + ) + + async def _restore_inlinks(self, path: str) -> None: + """Walk other nodes' links, add LINKS to ``path`` where ``link.path`` + matches. Uses ``links_json`` on each node — no resolution logic.""" + async with self._session() as session: + rec = await session.run( + """ + MATCH (f:File) + WHERE f.path <> $path + RETURN f.path AS p, f.links_json AS l + """, + path=path, + ) + rows = [dict(r) async for r in rec] + for row in rows: + try: + links = json.loads(row.get("l") or "[]") + except json.JSONDecodeError: + continue + for i, link in enumerate(links): + if not isinstance(link, dict) or link.get("path") != path: + continue + async with self._session() as session: + await session.run( + """ + MATCH (s:File {path: $src}) + MATCH (t:File {path: $dst}) + MERGE (s)-[r:LINKS {idx: $idx}]->(t) + SET r.anchor = $anchor, r.predicate = $predicate + """, + src=row["p"], + dst=path, + idx=i, + anchor=link.get("anchor"), + predicate=link.get("predicate"), + ) + + async def delete_node(self, path: str) -> FileNode | None: + node = await self.get_node(path) + if node is None: + return None + async with self._session() as session: + await session.run( + "MATCH (f:File {path: $path}) DETACH DELETE f", + path=path, + ) + return node + + async def get_node(self, path: str) -> FileNode | None: + async with self._session() as session: + rec = await session.run( + "MATCH (f:File {path: $path}) RETURN f LIMIT 1", + path=path, + ) + row = await rec.single() + return self._row_to_node(row["f"]) if row else None + + async def iter_nodes(self) -> AsyncIterator[tuple[str, FileNode]]: + async with self._session() as session: + rec = await session.run("MATCH (f:File) RETURN f") + async for row in rec: + node = self._row_to_node(row["f"]) + yield node.path, node + + # -- Link access ------------------------------------------------------- + + async def get_outlinks(self, path: str) -> list[tuple[FileNode, FileLink]]: + async with self._session() as session: + rec = await session.run( + """ + MATCH (s:File {path: $path})-[r:LINKS]->(t:File) + RETURN t, r ORDER BY r.idx ASC + """, + path=path, + ) + rows = [dict(row) async for row in rec] + return [(self._row_to_node(r["t"]), self._rel_to_link(r["r"], r["t"])) for r in rows] + + async def get_inlinks(self, path: str) -> list[tuple[FileNode, FileLink]]: + async with self._session() as session: + rec = await session.run( + """ + MATCH (s:File)-[r:LINKS]->(t:File {path: $path}) + RETURN s, r ORDER BY s.path ASC, r.idx ASC + """, + path=path, + ) + rows = [dict(row) async for row in rec] + # ``rel_to_link`` needs the target's path to populate FileLink.path. + return [(self._row_to_node(r["s"]), self._rel_to_link(r["r"], target_path=path)) for r in rows] + + # -- Internal: row ↔ schema marshaling --------------------------------- + + @staticmethod + def _node_props(node: FileNode) -> dict[str, Any]: + fm = node.front_matter + extras = dict(fm.__pydantic_extra__ or {}) + return { + "path": node.path, + "st_mtime": float(node.st_mtime), + "title": fm.title or "", + "description": fm.description or "", + "tags": list(fm.tags or []), + "links_json": json.dumps( + [link.model_dump(exclude_none=True) for link in node.links], + ensure_ascii=False, + ), + "extra_json": json.dumps(extras, ensure_ascii=False, sort_keys=True), + } + + @staticmethod + def _row_to_node(row) -> FileNode: + d = dict(row) + try: + extras = json.loads(d.get("extra_json") or "{}") + except json.JSONDecodeError: + extras = {} + try: + links_raw = json.loads(d.get("links_json") or "[]") + except json.JSONDecodeError: + links_raw = [] + links: list[FileLink] = [] + for link in links_raw: + if not isinstance(link, dict): + continue + # Defensive: strip any keys the schema doesn't recognise + # (e.g. legacy fields from prior schema versions). + clean = {k: v for k, v in link.items() if k in _LINK_FIELDS} + try: + links.append(FileLink(**clean)) + except Exception: + continue + fm_kwargs: dict[str, Any] = { + "title": d.get("title", "") or "", + "description": d.get("description", "") or "", + "tags": d.get("tags") or None, + } + fm_kwargs.update( + {k: v for k, v in extras.items() if k not in _TYPED_FRONTMATTER_FIELDS}, + ) + return FileNode( + path=d["path"], + st_mtime=float(d.get("st_mtime", 0.0)), + links=links, + chunk_ids=[], + front_matter=FileFrontMatter(**fm_kwargs), + ) + + @staticmethod + def _rel_to_link(rel, target_path: Any = None) -> FileLink: + """Reconstitute a ``FileLink`` from a Neo4j relationship. + + The relationship row carries ``anchor`` and ``predicate``; the + target's ``path`` comes from the matched ``File`` node (passed + in by callers that have it on hand). For ``get_outlinks`` it's + the row's target node; for ``get_inlinks`` it's the path the + caller queried for. Either way, FileLink.path is set so the + link stays "safe by construction". + """ + d = dict(rel) + if hasattr(target_path, "get"): + # neo4j Node passed in + path = target_path.get("path", "") + else: + path = target_path or "" + return FileLink( + path=path, + anchor=d.get("anchor"), + predicate=d.get("predicate"), + ) diff --git a/reme2/component/file_parser/base_file_parser.py b/reme2/component/file_parser/base_file_parser.py index a37d1795..6caf9ce4 100644 --- a/reme2/component/file_parser/base_file_parser.py +++ b/reme2/component/file_parser/base_file_parser.py @@ -17,7 +17,6 @@ class BaseFileParser(BaseComponent): super().__init__(**kwargs) self.working_dir = self.app_context.app_config.working_dir if self.app_context is not None else "" - def _get_relative_path(self, path: str | Path) -> str: """Get path relative to working_dir.""" file_path = Path(path).absolute() diff --git a/reme2/component/file_parser/default_file_parser.py b/reme2/component/file_parser/default_file_parser.py index a726ea66..4fc57420 100644 --- a/reme2/component/file_parser/default_file_parser.py +++ b/reme2/component/file_parser/default_file_parser.py @@ -40,7 +40,7 @@ class DefaultFileParser(BaseFileParser): data = {} front_matter = FileFrontMatter(**data) - remaining = text[end_idx + 4:].lstrip("\n") + remaining = text[end_idx + 4 :].lstrip("\n") return front_matter, remaining async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: @@ -72,18 +72,25 @@ class DefaultFileParser(BaseFileParser): if content_bytes[end - 1] == ord(b"\n"): end_line -= 1 - chunks.append(FileChunk( - path=rel_path, - start_line=start_line, - end_line=end_line, - text=chunk_text, - ).set_hash_id()) + chunks.append( + FileChunk( + path=rel_path, + start_line=start_line, + end_line=end_line, + text=chunk_text, + ).set_hash_id(), + ) if end >= len(content_bytes): break start += step - return FileNode(path=rel_path, - st_mtime=stat.st_mtime, - front_matter=front_matter, - chunk_ids=[c.id for c in chunks]), chunks + return ( + FileNode( + path=rel_path, + st_mtime=stat.st_mtime, + front_matter=front_matter, + chunk_ids=[c.id for c in chunks], + ), + chunks, + ) diff --git a/reme2/component/file_parser/linked_file_parser.py b/reme2/component/file_parser/linked_file_parser.py index a07e0e3f..265e2405 100644 --- a/reme2/component/file_parser/linked_file_parser.py +++ b/reme2/component/file_parser/linked_file_parser.py @@ -16,25 +16,20 @@ from pathlib import Path from typing import Any import frontmatter -from mistletoe.block_token import ( - CodeFence, - Document, - Heading, - List, - ListItem, - SetextHeading, - Table, - TableRow, -) -from mistletoe.markdown_renderer import BlankLine, MarkdownRenderer + from .base_file_parser import BaseFileParser from ..component_registry import R -from ...schema import FileChunk, FileEdge, FileFrontMatter, FileNode +from ..file_graph import BaseFileGraph +from ...enumeration import ComponentEnum +from ...schema import ( + FileChunk, + FileLink, + FileFrontMatter, + FileNode, +) from ...utils import hash_text - -_PART_RESERVE = 18 # worst-case "[Part NNN/NNN]\n\n" prefix - +from ...utils.wikilink_resolver import text_to_links # -- AST node + helpers --------------------------------------------------- @@ -63,7 +58,7 @@ class MdNode: desc_toc: str = "" -def _heading_text(node: Any, renderer: MarkdownRenderer) -> str: +def _heading_text(node: Any, renderer) -> str: """Heading text without `#` markers (for outline).""" rendered = renderer.render(node).rstrip("\n") if rendered.startswith("#"): @@ -71,14 +66,15 @@ def _heading_text(node: Any, renderer: MarkdownRenderer) -> str: return rendered.split("\n", 1)[0].strip() -def _dedup_edges(edges: list[FileEdge]) -> list[FileEdge]: +def _dedup_links(links: list[FileLink]) -> list[FileLink]: + """Drop links with identical (path, predicate, anchor); preserve order.""" seen: set[tuple] = set() - out: list[FileEdge] = [] - for e in edges: - key = (e.link, e.predicate) + out: list[FileLink] = [] + for link in links: + key = (link.path, link.predicate, link.anchor) if key not in seen: seen.add(key) - out.append(e) + out.append(link) return out @@ -132,41 +128,75 @@ class LinkedFileParser(BaseFileParser): """Markdown parser: frontmatter + wikilink edges + full-skeleton chunks.""" def __init__( - self, - encoding: str = "utf-8", - chunk_chars: int = 2000, - embed_toc: bool = True, - **kwargs, + self, + encoding: str = "utf-8", + chunk_chars: int = 2000, + embed_toc: bool = True, + file_graph: str = "default", + **kwargs, ): super().__init__(**kwargs) self.encoding = encoding self.chunk_chars = max(100, chunk_chars) self.embed_toc = embed_toc + self._file_graph_name: str = file_graph + + def _resolve_file_graph(self) -> BaseFileGraph | None: + """Lazily fetch the configured file_graph from app_context. + + Lazy (rather than ``_start``) so the parser doesn't impose a + component start-order constraint, and so tests can construct + the parser without a graph wired up. + """ + if self.app_context is None: + return None + graphs = self.app_context.components.get(ComponentEnum.FILE_GRAPH, {}) + graph = graphs.get(self._file_graph_name) + if graph is None: + return None + if not isinstance(graph, BaseFileGraph): + raise TypeError( + f"Expected BaseFileGraph, got {type(graph).__name__}", + ) + return graph async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: + from mistletoe.markdown_renderer import MarkdownRenderer + from mistletoe.block_token import Document + file_path = Path(path) post = frontmatter.loads(file_path.read_text(encoding=self.encoding)) - absolute = str(file_path.absolute()) chunks: list[FileChunk] = [] if post.content and post.content.strip(): with MarkdownRenderer() as renderer: tree = self._build_tree(Document(post.content), renderer) - chunks = self._chunk_node(tree, "", "", absolute, renderer) + chunks = self._chunk_node(tree, "", "", str(file_path), renderer) + + links: list[FileLink] = [] + graph = self._resolve_file_graph() + if graph is not None: + links = _dedup_links(await text_to_links(graph, post.content)) node = FileNode( - path=absolute, + path=str(file_path), st_mtime=file_path.stat().st_mtime, chunk_ids=[chunk.id for chunk in chunks], - edges=_dedup_edges(FileEdge.from_text(post.content)), + links=links, front_matter=FileFrontMatter(**dict(post.metadata)), ) return node, chunks - def _build_tree(self, doc: Any, renderer: MarkdownRenderer) -> MdNode: + def _build_tree(self, doc: Any, renderer) -> MdNode: """Heading-level stack folds mistletoe's flat children into nested sections; non-headings attach as ``body`` to the current section (or root before the first heading).""" + from mistletoe.markdown_renderer import BlankLine + from mistletoe.block_token import ( + Heading, + SetextHeading, + ) + root = MdNode(kind="root", start_line=1, end_line=1) stack: list[MdNode] = [root] for child in doc.children or []: @@ -189,18 +219,27 @@ class LinkedFileParser(BaseFileParser): rendered = renderer.render(child).rstrip("\n") if not rendered: continue - stack[-1].children.append(MdNode( - kind="body", block=child, text=rendered, - start_line=line, end_line=line + rendered.count("\n"), - )) + stack[-1].children.append( + MdNode( + kind="body", + block=child, + text=rendered, + start_line=line, + end_line=line + rendered.count("\n"), + ), + ) _finalize(root) return root # -- Recursive chunker ------------------------------------------------ def _chunk_node( - self, node: MdNode, before: str, after: str, - path: str, renderer: MarkdownRenderer, + self, + node: MdNode, + before: str, + after: str, + path: str, + renderer, ) -> list[FileChunk]: """Try the whole subtree; on overflow split (leaf) or descend. ``before``/``after`` are TOC fragments that bracket each emitted @@ -216,8 +255,16 @@ class LinkedFileParser(BaseFileParser): else: before_self = before if len(node.text) <= self.chunk_chars: - return [self._make_chunk(before_self, node.text, after, - node.start_line, node.end_line, path)] + return [ + self._make_chunk( + before_self, + node.text, + after, + node.start_line, + node.end_line, + path, + ), + ] if node.kind == "body": return self._split_leaf(node, before, after, path, renderer) after_inside = _toc_join(node.desc_toc, after) @@ -229,32 +276,65 @@ class LinkedFileParser(BaseFileParser): for c in node.children: if c.kind == "section": if run: - chunks.extend(self._chunk_body_run( - run, before_self, after_inside, path, renderer)) + chunks.extend( + self._chunk_body_run( + run, + before_self, + after_inside, + path, + renderer, + ), + ) run = [] - remaining = "\n\n".join(sub_tocs[sec_idx + 1:]) - chunks.extend(self._chunk_node( - c, accumulated, _toc_join(remaining, after), path, renderer)) + remaining = "\n\n".join(sub_tocs[sec_idx + 1 :]) + chunks.extend( + self._chunk_node( + c, + accumulated, + _toc_join(remaining, after), + path, + renderer, + ), + ) accumulated = _toc_join(accumulated, sub_tocs[sec_idx]) sec_idx += 1 else: run.append(c) if run: - chunks.extend(self._chunk_body_run( - run, before_self, after_inside, path, renderer)) + chunks.extend( + self._chunk_body_run( + run, + before_self, + after_inside, + path, + renderer, + ), + ) return chunks def _chunk_body_run( - self, run: list[MdNode], before: str, after: str, - path: str, renderer: MarkdownRenderer, + self, + run: list[MdNode], + before: str, + after: str, + path: str, + renderer, ) -> list[FileChunk]: """Greedy-pack consecutive body siblings under the same TOC slot. No ``[Part X/N]`` markers — distinct blocks, not a leaf split. Oversized single body recurses to ``_split_leaf``.""" composite_size = sum(len(b.text) for b in run) + 2 * max(0, len(run) - 1) if composite_size <= self.chunk_chars: - return [self._make_chunk(before, "\n\n".join(b.text for b in run), after, - run[0].start_line, run[-1].end_line, path)] + return [ + self._make_chunk( + before, + "\n\n".join(b.text for b in run), + after, + run[0].start_line, + run[-1].end_line, + path, + ), + ] chunks: list[FileChunk] = [] bucket: list[MdNode] = [] @@ -264,9 +344,16 @@ class LinkedFileParser(BaseFileParser): nonlocal bucket, bucket_chars if not bucket: return - chunks.append(self._make_chunk( - before, "\n\n".join(b.text for b in bucket), after, - bucket[0].start_line, bucket[-1].end_line, path)) + chunks.append( + self._make_chunk( + before, + "\n\n".join(b.text for b in bucket), + after, + bucket[0].start_line, + bucket[-1].end_line, + path, + ), + ) bucket = [] bucket_chars = 0 @@ -287,9 +374,19 @@ class LinkedFileParser(BaseFileParser): # -- Leaf splitters: build (text, start, end) units, hand off to packer def _split_leaf( - self, body: MdNode, before: str, after: str, - path: str, renderer: MarkdownRenderer, + self, + body: MdNode, + before: str, + after: str, + path: str, + renderer, ) -> list[FileChunk]: + from mistletoe.block_token import ( + CodeFence, + List, + Table, + ) + block = body.block if isinstance(block, Table): return self._split_table(body, before, after, path) @@ -300,9 +397,15 @@ class LinkedFileParser(BaseFileParser): return self._split_lines(body, before, after, path) def _split_table( - self, body: MdNode, before: str, after: str, path: str, + self, + body: MdNode, + before: str, + after: str, + path: str, ) -> list[FileChunk]: """Repeat header + separator on every chunk.""" + from mistletoe.block_token import TableRow + lines = body.text.split("\n") header, data = "\n".join(lines[:2]), lines[2:] rows = [r for r in (body.block.children or []) if isinstance(r, TableRow)] @@ -312,11 +415,21 @@ class LinkedFileParser(BaseFileParser): return rows[i].line_number if i < len(rows) and rows[i].line_number else base + i units = [(text, line_of(i), line_of(i)) for i, text in enumerate(data)] - return self._emit_packed(units, before, after, path, - joiner="\n", wrap=f"{header}\n{{inner}}") + return self._emit_packed( + units, + before, + after, + path, + joiner="\n", + wrap=f"{header}\n{{inner}}", + ) def _split_code( - self, body: MdNode, before: str, after: str, path: str, + self, + body: MdNode, + before: str, + after: str, + path: str, ) -> list[FileChunk]: """Repeat fence opener + closer on every chunk.""" code = body.block @@ -327,17 +440,28 @@ class LinkedFileParser(BaseFileParser): if not raw: return [] start = body.start_line + 1 - units = [(indent + ln, start + i, start + i) - for i, ln in enumerate(raw.split("\n"))] - return self._emit_packed(units, before, after, path, joiner="\n", - wrap=f"{opener}\n{{inner}}\n{fence}", - allow_empty=True) + units = [(indent + ln, start + i, start + i) for i, ln in enumerate(raw.split("\n"))] + return self._emit_packed( + units, + before, + after, + path, + joiner="\n", + wrap=f"{opener}\n{{inner}}\n{fence}", + allow_empty=True, + ) def _split_list( - self, body: MdNode, before: str, after: str, - path: str, renderer: MarkdownRenderer, + self, + body: MdNode, + before: str, + after: str, + path: str, + renderer, ) -> list[FileChunk]: """Pack list items; oversized items emit alone (overflow accepted).""" + from mistletoe.block_token import ListItem + items = [c for c in (body.block.children or []) if isinstance(c, ListItem)] if not items: return self._split_lines(body, before, after, path) @@ -348,28 +472,43 @@ class LinkedFileParser(BaseFileParser): continue line = it.line_number or body.start_line units.append((text, line, line + text.count("\n"))) - return self._emit_packed(units, before, after, path, - joiner="\n", wrap="{inner}") + return self._emit_packed( + units, + before, + after, + path, + joiner="\n", + wrap="{inner}", + ) def _split_lines( - self, body: MdNode, before: str, after: str, path: str, + self, + body: MdNode, + before: str, + after: str, + path: str, ) -> list[FileChunk]: """Last-resort line-greedy split for paragraphs / quotes / html.""" start = body.start_line - units = [(line, start + i, start + i) - for i, line in enumerate(body.text.split("\n"))] - return self._emit_packed(units, before, after, path, - joiner="\n", wrap="{inner}") + units = [(line, start + i, start + i) for i, line in enumerate(body.text.split("\n"))] + return self._emit_packed( + units, + before, + after, + path, + joiner="\n", + wrap="{inner}", + ) def _emit_packed( - self, - units: list[tuple[str, int, int]], - before: str, - after: str, - path: str, - joiner: str, - wrap: str, - allow_empty: bool = False, + self, + units: list[tuple[str, int, int]], + before: str, + after: str, + path: str, + joiner: str, + wrap: str, + allow_empty: bool = False, ) -> list[FileChunk]: """Greedy-pack units into ``wrap`` envelopes; emit each piece. @@ -379,7 +518,7 @@ class LinkedFileParser(BaseFileParser): ``[Part X/N]`` markers; single pieces don't. """ envelope = len(wrap.replace("{inner}", "")) - budget = max(64, self.chunk_chars - envelope - _PART_RESERVE) + budget = max(64, self.chunk_chars - envelope) sep_len = len(joiner) parts: list[tuple[str, int, int]] = [] @@ -410,9 +549,15 @@ class LinkedFileParser(BaseFileParser): return [ self._make_chunk( before, - f"[Part {idx}/{total}]\n\n{wrap.replace('{inner}', inner)}" - if total > 1 else wrap.replace("{inner}", inner), - after, s, e, path, + ( + f"[Part {idx}/{total}]\n\n{wrap.replace('{inner}', inner)}" + if total > 1 + else wrap.replace("{inner}", inner) + ), + after, + s, + e, + path, ) for idx, (inner, s, e) in enumerate(parts, 1) ] @@ -420,13 +565,13 @@ class LinkedFileParser(BaseFileParser): # -- Emit ------------------------------------------------------------- def _make_chunk( - self, - before: str, - content: str, - after: str, - start_line: int, - end_line: int, - path: str, + self, + before: str, + content: str, + after: str, + start_line: int, + end_line: int, + path: str, ) -> FileChunk: """Build one ``FileChunk`` — text is ``before + content + after`` when ``embed_toc``, otherwise just ``content``.""" diff --git a/reme2/component/file_store/base_file_store.py b/reme2/component/file_store/base_file_store.py index 16f9e3b5..84bde311 100644 --- a/reme2/component/file_store/base_file_store.py +++ b/reme2/component/file_store/base_file_store.py @@ -12,11 +12,11 @@ class BaseFileStore(BaseComponent): component_type = ComponentEnum.FILE_STORE def __init__( - self, - store_name: str, - embedding_model: str = "default", - keyword_index: str = "default", - **kwargs, + self, + store_name: str, + embedding_model: str = "default", + keyword_index: str = "default", + **kwargs, ): super().__init__(**kwargs) if not re.match(r"^[a-zA-Z0-9_]+$", store_name): @@ -46,15 +46,13 @@ class BaseFileStore(BaseComponent): self.keyword_index = None await self.dump_file_nodes() - async def load_file_nodes(self): - ... + async def load_file_nodes(self): ... - async def dump_file_nodes(self): - ... + async def dump_file_nodes(self): ... async def upsert_file( - self, - file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]], + self, + file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]], ) -> None: """Upsert a file and its chunks into the store.""" diff --git a/reme2/component/file_store/local_file_store.py b/reme2/component/file_store/local_file_store.py index 09fb5d24..155847c3 100644 --- a/reme2/component/file_store/local_file_store.py +++ b/reme2/component/file_store/local_file_store.py @@ -30,8 +30,10 @@ class LocalFileStore(BaseFileStore): async def _start(self) -> None: await super()._start() await self._load_jsonl(self.chunks_path, self.file_chunks, FileChunk, "id") - self.logger.info(f"LocalFileStore '{self.store_name}' ready: " - f"{len(self.file_nodes)} nodes, {len(self.file_chunks)} chunks") + self.logger.info( + f"LocalFileStore '{self.store_name}' ready: " + f"{len(self.file_nodes)} nodes, {len(self.file_chunks)} chunks" + ) async def _close(self) -> None: await self._dump_jsonl(self.chunks_path, list(self.file_chunks.values())) @@ -71,8 +73,8 @@ class LocalFileStore(BaseFileStore): await self._dump_jsonl(self.nodes_path, list(self.file_nodes.values())) async def upsert_file( - self, - file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]], + self, + file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]], ) -> None: if isinstance(file, tuple): file = [file] diff --git a/reme2/component/file_watcher/base_file_watcher.py b/reme2/component/file_watcher/base_file_watcher.py index c1204958..b97242ca 100644 --- a/reme2/component/file_watcher/base_file_watcher.py +++ b/reme2/component/file_watcher/base_file_watcher.py @@ -17,16 +17,16 @@ class BaseFileWatcher(BaseComponent): component_type = ComponentEnum.FILE_WATCHER def __init__( - self, - watch_paths: list[str] | str, - suffix_filters: list[str] | None = None, - recursive: bool = True, - force_polling: bool = True, - debounce: int = 2000, - poll_delay_ms: int = 2000, - file_store: str = "default", - file_parser: str = "default", - **kwargs, + self, + watch_paths: list[str] | str, + suffix_filters: list[str] | None = None, + recursive: bool = True, + force_polling: bool = True, + debounce: int = 2000, + poll_delay_ms: int = 2000, + file_store: str = "default", + file_parser: str = "default", + **kwargs, ): super().__init__(**kwargs) watch_paths = [watch_paths] if isinstance(watch_paths, str) else watch_paths diff --git a/reme2/component/file_watcher/lite_file_watcher.py b/reme2/component/file_watcher/lite_file_watcher.py index 2a0331a9..3e73252f 100644 --- a/reme2/component/file_watcher/lite_file_watcher.py +++ b/reme2/component/file_watcher/lite_file_watcher.py @@ -39,13 +39,13 @@ class LiteFileWatcher(BaseFileWatcher): try: logger.info(f"Watching: {valid_paths}") async for changes in awatch( - *valid_paths, - watch_filter=self.watch_filter, - recursive=self.recursive, - force_polling=self.force_polling, - debounce=self.debounce, - poll_delay_ms=self.poll_delay_ms, - stop_event=self._stop_event, + *valid_paths, + watch_filter=self.watch_filter, + recursive=self.recursive, + force_polling=self.force_polling, + debounce=self.debounce, + poll_delay_ms=self.poll_delay_ms, + stop_event=self._stop_event, ): if self._stop_event.is_set(): break diff --git a/reme2/component/job/base_job.py b/reme2/component/job/base_job.py index 177467cb..1198c67e 100644 --- a/reme2/component/job/base_job.py +++ b/reme2/component/job/base_job.py @@ -36,7 +36,7 @@ class BaseJob(BaseComponent): for raw in self.step_configs: config = raw if isinstance(raw, ComponentConfig) else ComponentConfig(**raw) if not config.backend: - raise ValueError(f"Step is missing the required 'backend' field") + raise ValueError("Step is missing the required 'backend' field") step_cls = R.get(ComponentEnum.STEP, config.backend) if not step_cls: raise ValueError( diff --git a/reme2/component/keyword_index/base_keyword_index.py b/reme2/component/keyword_index/base_keyword_index.py index 82b957be..0ee134cd 100644 --- a/reme2/component/keyword_index/base_keyword_index.py +++ b/reme2/component/keyword_index/base_keyword_index.py @@ -24,6 +24,7 @@ class BaseKeywordIndex(BaseComponent): """Initialize tokenizer and load existing index if available.""" if self.app_context is None: from ..tokenizer import RegexTokenizer + self.tokenizer = RegexTokenizer(filter_stopwords=False) else: self.tokenizer = self.get_component(ComponentEnum.TOKENIZER, self.tokenizer_name) diff --git a/reme2/component/keyword_index/bm25_index.py b/reme2/component/keyword_index/bm25_index.py index 17dfa72d..2f80b698 100644 --- a/reme2/component/keyword_index/bm25_index.py +++ b/reme2/component/keyword_index/bm25_index.py @@ -20,6 +20,7 @@ class DocMeta(TypedDict): len: Number of tokens in the document. token_ids: Set of unique token IDs present in the document. """ + len: int token_ids: set[int] diff --git a/reme2/component/runtime_response.py b/reme2/component/runtime_response.py index 8eb26d49..7c4e95c5 100644 --- a/reme2/component/runtime_response.py +++ b/reme2/component/runtime_response.py @@ -4,10 +4,6 @@ Lives next to `runtime_context.py` because both are about the BaseStep interface — the response side, specifically. Used by every Step that returns a JSON-shaped payload (memory_*, sync, the three memory services). - -Was previously at `reme2/mcp/steps/_common.py`, which leaked an MCP -dependency into `reme2/memory/` services that legitimately need to -serialize their results — the moved location breaks that cycle. """ from __future__ import annotations diff --git a/reme2/component/tokenizer/regex_tokenizer.py b/reme2/component/tokenizer/regex_tokenizer.py index ad7df5b2..d31b1d7e 100644 --- a/reme2/component/tokenizer/regex_tokenizer.py +++ b/reme2/component/tokenizer/regex_tokenizer.py @@ -52,4 +52,4 @@ class RegexTokenizer(BaseTokenizer): tokens = [t for t in tokens if t not in self._stopwords] result.append(tokens) - return result \ No newline at end of file + return result diff --git a/reme2/config/config_parser.py b/reme2/config/config_parser.py index 72f8cb07..728ca6a6 100644 --- a/reme2/config/config_parser.py +++ b/reme2/config/config_parser.py @@ -22,6 +22,7 @@ def _expand_env_vars(value: Any) -> Any: typos don't silently produce empty connection strings. """ if isinstance(value, str): + def repl(m: re.Match) -> str: name = m.group(1) default = m.group(2) @@ -31,6 +32,7 @@ def _expand_env_vars(value: Any) -> Any: return default raise ValueError(f"Config references undefined env var: {name}") return v + return _ENV_VAR_RE.sub(repl, value) if isinstance(value, dict): return {k: _expand_env_vars(v) for k, v in value.items()} diff --git a/reme2/config/reme_todo.md b/reme2/config/reme_todo.md index edd93293..a3fecfa6 100644 --- a/reme2/config/reme_todo.md +++ b/reme2/config/reme_todo.md @@ -3,9 +3,9 @@ ReMe重构 a. ✅ vault_root → working_dir @sen 2. file_parser a. 抽象基类 parse: @jinli - ⅰ. 输入是path:相对路径 + ⅰ. 输入是path:相对路径 ⅱ. 输出是FileMetadata & list[FileChunks] & list[FileEdge] - b. default parser 兼容老方案 @jinli + b. default parser 兼容老方案 @jinli ⅰ. 带overlap的chunking策略 ,不输出FileEdge c. markdown parser @sen ⅰ. 根据markdown ast做chunk,不需要overlap @@ -27,11 +27,11 @@ ReMe重构 ⅳ. 【核心】检索机制 vector bm25 graph 如何进行融合 4. file_watcher @jinli a. 抽象基类 - ⅰ. on_start: + ⅰ. on_start: 1. file_store 的start 在前,加载graph,file_watcher在后,递归扫描目录 a. 通过ms_time对比graph,on_change 进行改动 ⅱ. on_change: - 1. 更新/增加: + 1. 更新/增加: a. delete_chunks_by_path 更新数据库 b. upate_chunks_by_path 更新数据库 c. 更新graph @@ -44,14 +44,14 @@ MemorySchema ⅰ. title ⅱ. desc ⅲ. tags - ⅳ. + ⅳ. 2. memory文件结构目录 a. MEMORY.md b. msg/files -> daily/YYYYMMDD/YYYYMMDD.md + xxxx.md ⅰ. YYYYMMDD.md 1. xxx -> xxxx.md 2. xxx -> xxxd.md - ⅱ. + ⅱ. c. daily -> topic/topic_l1/topic_l1.md + xxx.md + topic_l2 d. proactive diff --git a/reme2/enumeration/component_enum.py b/reme2/enumeration/component_enum.py index d3878720..9b5446ce 100644 --- a/reme2/enumeration/component_enum.py +++ b/reme2/enumeration/component_enum.py @@ -28,6 +28,8 @@ class ComponentEnum(str, Enum): FILE_STORE = "file_store" + FILE_GRAPH = "file_graph" + FILE_WATCHER = "file_watcher" KEYWORD_INDEX = "keyword_index" diff --git a/reme2/mcp/README.md b/reme2/mcp/README.md index 032f29f3..736426bb 100644 --- a/reme2/mcp/README.md +++ b/reme2/mcp/README.md @@ -1,39 +1,31 @@ # reme2.mcp -Agent-facing MCP interface layer. Exposes the markdown vault under -`reme2/` (file_store + watcher + memory services) as MCP tools for -`claude-code` and other MCP clients. +MCP transport layer. Exposes the agent-facing tools registered in +`reme2.memory` over MCP for `claude-code` and other MCP clients. ## Layout ``` reme2/mcp/ ├── __init__.py -├── server.py MCP server bootstrap; defaults to ../config/service.yaml -└── steps/ @R.register MCP step shells - ├── memory_retriever.py memory_search + memory_graph_search - ├── memory_lint.py read-only Maintainer projection (lint findings) - └── sync.py hot-path event-folder upsert +├── server.py MCP server bootstrap; defaults to ../config/service.yaml +└── test/ end-to-end profile smoke tests ``` -The 12 `memory_*` primitives (create/update/property_update/rename/ -delete/archive/get/list/links/backlinks/resolve_wikilink/count_tokens) -live one layer down in `reme2/memory/memory_toolkit.py` — each is a -single `BaseStep` subclass with two class methods: `execute()` for the -MCP path (this layer) and a same-named method for the agent toolkit -path that the Ingestor's ReActAgent consumes. Importing -`reme2.mcp.steps` triggers all `@R.register` registrations. - -The MCP layer's job is to **project** existing primitives as MCP tools -— it owns no business logic. Everything else lives outside the -transport boundary so memory services don't form an import cycle: +That's it. Every `@R.register` step the agent invokes lives one layer +down in `reme2/memory/` — there are no MCP-specific shells, because +none of the steps depend on MCP transport details. Importing +`reme2.memory` triggers all `@R.register` registrations. | Concern | Location | |---|---| | Memory File System primitives | `reme2/component/file_store/`, `file_watcher/`, `file_parser/` | -| Three memory services | `reme2/memory/` — Retriever / Ingestor / Maintainer | +| Three memory services | `reme2/memory/{retriever,ingestor,maintainer}.py` | | Engine API (pure, schema-free) | `reme2/memory/memory_io.py` | -| Schema-bound tools (BaseStep + agent toolkit) | `reme2/memory/memory_toolkit.py` | +| Schema-bound write/read tools | `reme2/memory/memory_toolkit.py` | +| Search step shells | `reme2/memory/memory_search.py` | +| Lint step shell | `reme2/memory/memory_lint.py` | +| Hot-path event upsert | `reme2/memory/sync.py` | | Memory schema (4 axes + presets + parser) | `reme2/memory/schema/` | | Path templates + name disambiguation | `reme2/utils/vault_paths.py` | | Step response serialization | `reme2/component/runtime_response.py` | @@ -44,9 +36,8 @@ reme2.mcp → reme2.memory (incl. memory.schema) → reme2.component / reme2 ``` The Ingestor (`reme2/memory/ingestor.py`) self-registers its `ingest` -MCP face — there's no shell for it under `steps/`. Topic creation lives -inside the Ingestor's R-M-W loop; there is no separate `topic_create` -tool. +MCP face. Topic creation lives inside the Ingestor's R-M-W loop; there +is no separate `topic_create` tool. ## Run @@ -91,12 +82,12 @@ Configs live in `reme2/config/`: ### Tools exposed (expert profile) -| Tool | Path | Purpose | +| Tool | Source | Purpose | |---|---|---| -| `sync` | steps/sync.py | Hot-path event-folder upsert (idempotent per `(date, name)`). | +| `sync` | reme2/memory/sync.py | Hot-path event-folder upsert (idempotent per `(date, name)`). | | `ingest` | reme2/memory/ingestor.py | Cold-path LLM-driven distillation; owns topic creation. | -| `memory_search` / `memory_graph_search` | steps/memory_retriever.py | V+K hybrid + optional graph BFS. | -| `memory_lint` | steps/memory_lint.py | Read-only projection of Maintainer's lint findings. | +| `memory_search` / `memory_graph_search` | reme2/memory/memory_search.py | V+K hybrid + optional graph BFS. | +| `memory_lint` | reme2/memory/memory_lint.py | Read-only projection of Maintainer's lint findings. | | `memory_get` / `memory_list` / `memory_links` / `memory_backlinks` / `memory_resolve_wikilink` | reme2/memory/memory_toolkit.py | Read primitives. | | `memory_create` / `memory_update` / `memory_property_update` / `memory_rename` / `memory_delete` / `memory_archive` | reme2/memory/memory_toolkit.py | Raw write primitives (prefer `ingest` / `sync`). | | `memory_count_tokens` | reme2/memory/memory_toolkit.py | Token estimation. | diff --git a/reme2/mcp/__init__.py b/reme2/mcp/__init__.py index 8cfebfd7..c3422006 100644 --- a/reme2/mcp/__init__.py +++ b/reme2/mcp/__init__.py @@ -1,20 +1,19 @@ -"""reme2.mcp — MCP interface layer. +"""reme2.mcp — MCP transport layer. -The Agent-facing surface: server entrypoint + step shells that wrap -the three services in `reme2.memory` (Retriever, Ingestor, Maintainer) -plus the hot-write primitive (`sync`) and the raw `memory_*` write/read -tools that bypass services and land directly on the Memory File System. +Just the MCP server bootstrap (config loading, env-var overrides, +sidecar HTTP). All agent-facing tools live in ``reme2.memory`` and +register themselves there; this package only wires them into the MCP +transport via the ``mcp`` service component. -This package depends on `reme2.memory`, `reme2.utils`, `reme2.component` -— never the reverse. The Memory schema (the typed shape of every -frontmatter) lives under `reme2.memory.schema/` so the services that -own validation can use it without an import cycle through this -transport layer. +Dependency direction is strict: ``reme2.mcp → reme2.memory`` (and +through it ``reme2.component`` / ``reme2.utils``). Importing +``reme2.memory`` triggers all ``@R.register`` decorators for +``memory_*`` / ``sync`` / ``memory_search`` / ``memory_lint`` / +``ingest`` / ``maintainer`` / ``hybrid``. -Sub-packages: - steps/ - all @R.register MCP step shells (memory_toolkit, - memory_retriever, memory_lint, sync). - server.py - MCP server bootstrap (defaults to ../config/service.yaml). +Files: + server.py — MCP server bootstrap (defaults to ../config/service.yaml). + test/ — end-to-end profile smoke tests. """ __version__ = "0.1.0" diff --git a/reme2/mcp/server.py b/reme2/mcp/server.py index 69db2858..5aff67b0 100644 --- a/reme2/mcp/server.py +++ b/reme2/mcp/server.py @@ -17,8 +17,8 @@ from pathlib import Path # Make this work whether invoked as `python -m reme2.mcp.server` (repo # root already on sys.path) or `python reme2/mcp/server.py` (only this -# file's dir is on sys.path — without this, `import reme2.mcp.steps` -# would fail with ModuleNotFoundError). +# file's dir is on sys.path — without this, `import reme2.memory` would +# fail with ModuleNotFoundError). _REPO_ROOT = Path(__file__).resolve().parents[2] if str(_REPO_ROOT) not in sys.path: sys.path.insert(0, str(_REPO_ROOT)) @@ -32,10 +32,7 @@ load_dotenv(_REPO_ROOT / ".env") # Eagerly import side-effect modules so all @R.register() decorators run # before Application introspects the registry. import reme2 # noqa: E402,F401 -import reme2.mcp.steps # noqa: E402,F401 -import reme2.memory # noqa: E402,F401 -- registers retriever + maintainer -import reme2.memory.ingestor # noqa: E402,F401 -import reme2.memory.summarizer # noqa: E402,F401 +import reme2.memory # noqa: E402,F401 -- registers every agent-facing tool import reme2.component.service.mcp_service # noqa: E402,F401 from reme2.application import Application # noqa: E402 diff --git a/reme2/mcp/steps/__init__.py b/reme2/mcp/steps/__init__.py deleted file mode 100644 index 41d8a7c5..00000000 --- a/reme2/mcp/steps/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -"""MCP step shells — the @R.register classes the model invokes as MCP tools. - -Three groups: - - Hot-write primitives that bypass services and land on MFS directly: - sync — idempotent event-folder upsert (event log) - memory_toolkit — memory_create / update / property_update / - rename / delete / archive / get / list / - links / backlinks / resolve_wikilink / - count_tokens. Each step lives in - `reme2.memory.memory_toolkit` and exposes - BOTH an `execute()` (this MCP path) and a - same-named class method (the agent toolkit - path used by the Ingestor's ReActAgent). - - Service delegates (thin shells over the three memory services): - memory_retriever — memory_search + memory_graph_search; delegate - to the Retriever component - (`reme2.memory.retriever`). - memory_lint — read-only projection of the Maintainer's lint - findings (`reme2.memory.maintainer`). - -Topic creation is owned by the Ingestor (`reme2.memory.ingestor`) — its -LLM-driven R-M-W loop decides when a new topic is warranted, applies the -schema preset, and routes the write through `memory_create`. There is no -separate `topic_create` MCP tool. - -The Ingestor's MCP face (`ingest`) is registered from -`reme2.memory.ingestor`. Importing this package triggers all the step -registrations hosted here and in `reme2.memory.memory_toolkit`. -""" - -from ...memory import memory_toolkit # noqa: F401 -- triggers @R.register for memory_* -from . import memory_lint # noqa: F401 -- triggers @R.register for memory_lint -from . import memory_retriever # noqa: F401 -- memory_search / memory_graph_search -from .sync import Sync - -__all__ = [ - "Sync", -] diff --git a/reme2/mcp/test/__main__.py b/reme2/mcp/test/__main__.py index a5f56302..9fc4ab90 100644 --- a/reme2/mcp/test/__main__.py +++ b/reme2/mcp/test/__main__.py @@ -22,7 +22,7 @@ from ._helpers import AppContext, make_context SUITES = { - "expert": test_expert.CHECKS, + "expert": test_expert.CHECKS, "service": test_service.CHECKS, } @@ -35,8 +35,7 @@ async def _run_suite(profile: str, checks: list[tuple[str, callable]]) -> tuple[ passed = 0 try: ctx, tmp = await make_context(profile) - print(f" bootstrapped: vault={ctx.vault}, jobs={len(ctx.jobs)}, " - f"file_store={len(ctx.file_store)}") + print(f" bootstrapped: vault={ctx.vault}, jobs={len(ctx.jobs)}, " f"file_store={len(ctx.file_store)}") for label, fn in checks: try: summary = await fn(ctx) @@ -69,7 +68,7 @@ async def _main() -> int: for profile, (passed, total) in summary.items(): marker = "✓" if passed == total else "✗" print(f" {marker} {profile}: {passed}/{total}") - failed += (total - passed) + failed += total - passed return 0 if failed == 0 else failed diff --git a/reme2/mcp/test/_helpers.py b/reme2/mcp/test/_helpers.py index 9ac14360..e0db9149 100644 --- a/reme2/mcp/test/_helpers.py +++ b/reme2/mcp/test/_helpers.py @@ -24,10 +24,7 @@ if str(_REPO_ROOT) not in sys.path: # Eager-import side-effect modules so all @R.register() decorators run # before Application introspects the registry. import reme2 # noqa: E402,F401 -import reme2.mcp.steps # noqa: E402,F401 -import reme2.memory # noqa: E402,F401 -- registers retriever + maintainer -import reme2.memory.ingestor # noqa: E402,F401 -import reme2.memory.summarizer # noqa: E402,F401 +import reme2.memory # noqa: E402,F401 -- registers every agent-facing tool from reme2.application import Application # noqa: E402 from reme2.config import parse_args # noqa: E402 @@ -35,7 +32,7 @@ from reme2.config import parse_args # noqa: E402 CONFIG_DIR = _REPO_ROOT / "reme2" / "config" PROFILES = { - "expert": CONFIG_DIR / "expert.yaml", + "expert": CONFIG_DIR / "expert.yaml", "service": CONFIG_DIR / "service.yaml", } @@ -106,8 +103,7 @@ async def wait_for_index(watcher, expected_min: int, timeout_s: float = 15.0) -> last = now await asyncio.sleep(0.25) raise RuntimeError( - f"watcher did not reach >= {expected_min} files within {timeout_s}s " - f"(last seen: {last})" + f"watcher did not reach >= {expected_min} files within {timeout_s}s " f"(last seen: {last})", ) @@ -156,7 +152,9 @@ async def make_context(profile: str) -> tuple[AppContext, Path]: vault.mkdir() seed_vault(vault) app = await build_app(PROFILES[profile], vault) - await wait_for_index(app.context.components["file_watcher"]["default"], - expected_min=SEED_FILE_COUNT) + await wait_for_index( + app.context.components["file_watcher"]["default"], + expected_min=SEED_FILE_COUNT, + ) jobs = sorted(app.context.jobs.keys()) return AppContext(app=app, vault=vault, jobs=jobs), tmp diff --git a/reme2/mcp/test/test_expert.py b/reme2/mcp/test/test_expert.py index 6dcfdad0..d98f4688 100644 --- a/reme2/mcp/test/test_expert.py +++ b/reme2/mcp/test/test_expert.py @@ -75,19 +75,29 @@ async def check_memory_list(ctx: AppContext) -> str: async def check_memory_search(ctx: AppContext) -> str: - r = decode(await ctx.app.run_job( - "memory_search", query="collaborates", max_results=3, min_score=0.0, - )) + r = decode( + await ctx.app.run_job( + "memory_search", + query="collaborates", + max_results=3, + min_score=0.0, + ), + ) hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) assert len(hits) > 0, r return f"{len(hits)} hits" async def check_memory_graph_search(ctx: AppContext) -> str: - r = decode(await ctx.app.run_job( - "memory_graph_search", query="Alice", max_results=5, min_score=0.0, - graph_depth=1, - )) + r = decode( + await ctx.app.run_job( + "memory_graph_search", + query="Alice", + max_results=5, + min_score=0.0, + graph_depth=1, + ), + ) hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) assert len(hits) > 0, r # At least one result should carry a graph_hop annotation @@ -118,9 +128,12 @@ async def check_memory_resolve_wikilink(ctx: AppContext) -> str: async def check_memory_count_tokens(ctx: AppContext) -> str: - r = decode(await ctx.app.run_job( - "memory_count_tokens", text="hello world from the smoke test", - )) + r = decode( + await ctx.app.run_job( + "memory_count_tokens", + text="hello world from the smoke test", + ), + ) assert isinstance(r, dict) and isinstance(r.get("tokens"), int), r assert r["tokens"] > 0, r return f"text → {r['tokens']} tokens" @@ -142,18 +155,20 @@ async def check_memory_lint(ctx: AppContext) -> str: async def check_sync_create(ctx: AppContext) -> str: - r = decode(await ctx.app.run_job( - "sync", - name="suite-event", - description="full-profile suite", - content="## ops\n- ran the suite\n", - topics=["[[Alice]]"], - tags=["suite"], - materials=[ - {"filename": "raw-prompt.md", "content": "# user prompt\n\nrun suite\n"}, - {"filename": "tool-output.txt", "content": "exit=0\n"}, - ], - )) + r = decode( + await ctx.app.run_job( + "sync", + name="suite-event", + description="full-profile suite", + content="## ops\n- ran the suite\n", + topics=["[[Alice]]"], + tags=["suite"], + materials=[ + {"filename": "raw-prompt.md", "content": "# user prompt\n\nrun suite\n"}, + {"filename": "tool-output.txt", "content": "exit=0\n"}, + ], + ), + ) assert isinstance(r, dict), r assert r.get("created") is True and r.get("action") == "created", r materials = r.get("materials", []) @@ -176,17 +191,19 @@ async def check_sync_create(ctx: AppContext) -> str: async def check_sync_append(ctx: AppContext) -> str: - r = decode(await ctx.app.run_job( - "sync", - name="suite-event", - content="## follow-up\n- second pass\n", - topics=["[[Bob]]"], # union with [[Alice]] - tags=["follow-up"], - materials=[ - {"filename": "tool-output.txt", "content": "second run\n"}, # collision - {"filename": "summary.md", "content": "# summary\n"}, - ], - )) + r = decode( + await ctx.app.run_job( + "sync", + name="suite-event", + content="## follow-up\n- second pass\n", + topics=["[[Bob]]"], # union with [[Alice]] + tags=["follow-up"], + materials=[ + {"filename": "tool-output.txt", "content": "second run\n"}, # collision + {"filename": "summary.md", "content": "# summary\n"}, + ], + ), + ) assert isinstance(r, dict), r assert r.get("created") is False and r.get("action") == "appended", r appended = r.get("materials", []) @@ -204,11 +221,18 @@ async def check_sync_append(ctx: AppContext) -> str: async def check_sync_refuse_distilled(ctx: AppContext) -> str: index_path = str(getattr(ctx, "_suite_event_index")) await ctx.app.run_job( - "memory_property_update", path=index_path, key="status", value="distilled", + "memory_property_update", + path=index_path, + key="status", + value="distilled", + ) + r = decode( + await ctx.app.run_job( + "sync", + name="suite-event", + content="should be refused", + ), ) - r = decode(await ctx.app.run_job( - "sync", name="suite-event", content="should be refused", - )) assert isinstance(r, dict) and "error" in r, r assert r.get("status") == "distilled", r assert r.get("suggested_name"), r @@ -220,20 +244,22 @@ async def check_sync_refuse_distilled(ctx: AppContext) -> str: async def check_memory_create(ctx: AppContext) -> str: target = ctx.abs_path("topics", "Carol", "Carol.md") - r = decode(await ctx.app.run_job( - "memory_create", - path=target, - metadata={ - "title": "Carol", - "lifecycle": "evolving", - "scope": "class", - "source": "curated", - "role": "profile", - "category": "profile", - "tags": ["person"], - }, - content="# Carol\n\nKnows [[Alice]].\n", - )) + r = decode( + await ctx.app.run_job( + "memory_create", + path=target, + metadata={ + "title": "Carol", + "lifecycle": "evolving", + "scope": "class", + "source": "curated", + "role": "profile", + "category": "profile", + "tags": ["person"], + }, + content="# Carol\n\nKnows [[Alice]].\n", + ), + ) assert isinstance(r, dict) and r.get("created") is True, r assert "error" not in r, r assert Path(target).is_file(), target @@ -243,12 +269,14 @@ async def check_memory_create(ctx: AppContext) -> str: async def check_memory_update(ctx: AppContext) -> str: carol = ctx.abs_path("topics", "Carol", "Carol.md") - r = decode(await ctx.app.run_job( - "memory_update", - path=carol, - old_string="Knows [[Alice]].", - new_string="Knows [[Alice]] and [[Bob]].", - )) + r = decode( + await ctx.app.run_job( + "memory_update", + path=carol, + old_string="Knows [[Alice]].", + new_string="Knows [[Alice]] and [[Bob]].", + ), + ) assert isinstance(r, dict) and r.get("replaced", 0) >= 1, r assert "Bob" in Path(carol).read_text(encoding="utf-8"), "edit not on disk" return f"body edit applied (replaced={r['replaced']})" @@ -256,22 +284,30 @@ async def check_memory_update(ctx: AppContext) -> str: async def check_memory_property_update(ctx: AppContext) -> str: carol = ctx.abs_path("topics", "Carol", "Carol.md") - r = decode(await ctx.app.run_job( - "memory_property_update", path=carol, key="confidence", value="✅", - )) + r = decode( + await ctx.app.run_job( + "memory_property_update", + path=carol, + key="confidence", + value="✅", + ), + ) assert isinstance(r, dict) and "error" not in r, r assert r.get("key") == "confidence" and r.get("value") == "✅", r - assert "confidence: ✅" in Path(carol).read_text(encoding="utf-8"), \ - "property write not on disk" + assert "confidence: ✅" in Path(carol).read_text(encoding="utf-8"), "property write not on disk" return "set confidence=✅" async def check_memory_rename(ctx: AppContext) -> str: src = ctx.abs_path("topics", "Carol", "Carol.md") dst = ctx.abs_path("topics", "Carol", "Carol-renamed.md") - r = decode(await ctx.app.run_job( - "memory_rename", old_path=src, new_path=dst, - )) + r = decode( + await ctx.app.run_job( + "memory_rename", + old_path=src, + new_path=dst, + ), + ) assert isinstance(r, dict) and "error" not in r, r assert r.get("new_path") and Path(r["new_path"]).is_file(), r assert not Path(src).exists(), f"old path still on disk: {src}" @@ -291,8 +327,7 @@ async def check_memory_archive(ctx: AppContext) -> str: async def check_memory_delete(ctx: AppContext) -> str: - target = getattr(ctx, "_carol_archived", None) \ - or ctx.abs_path("topics", "Carol", "Carol-renamed.md") + target = getattr(ctx, "_carol_archived", None) or ctx.abs_path("topics", "Carol", "Carol-renamed.md") r = decode(await ctx.app.run_job("memory_delete", path=target)) assert isinstance(r, dict) and r.get("deleted") is True, r assert not Path(target).exists(), target @@ -306,14 +341,20 @@ async def check_schema_path_template_refuses(ctx: AppContext) -> str: """memory_create rejects paths outside topics/{X}/{Y}.md, events/{date}/{name}/..., or Archive/...""" target = ctx.abs_path("notes", "freeform.md") # outside any template - r = decode(await ctx.app.run_job( - "memory_create", - path=target, - metadata={"title": "freeform", - "lifecycle": "evolving", "scope": "class", - "source": "curated", "role": "concept"}, - content="should be refused", - )) + r = decode( + await ctx.app.run_job( + "memory_create", + path=target, + metadata={ + "title": "freeform", + "lifecycle": "evolving", + "scope": "class", + "source": "curated", + "role": "concept", + }, + content="should be refused", + ), + ) assert isinstance(r, dict), r assert "error" in r and "template" in r["error"].lower(), r assert not Path(target).exists(), "file shouldn't have been written" @@ -323,15 +364,21 @@ async def check_schema_path_template_refuses(ctx: AppContext) -> str: async def check_schema_path_template_force_bypass(ctx: AppContext) -> str: """force=True bypasses the template gate.""" target = ctx.abs_path("notes", "forced.md") - r = decode(await ctx.app.run_job( - "memory_create", - path=target, - metadata={"title": "forced", - "lifecycle": "evolving", "scope": "class", - "source": "curated", "role": "concept"}, - content="forced through", - force=True, - )) + r = decode( + await ctx.app.run_job( + "memory_create", + path=target, + metadata={ + "title": "forced", + "lifecycle": "evolving", + "scope": "class", + "source": "curated", + "role": "concept", + }, + content="forced through", + force=True, + ), + ) assert isinstance(r, dict) and r.get("created") is True, r assert Path(target).is_file(), target return "force=True bypassed" @@ -342,12 +389,14 @@ async def check_schema_status_skip_refused(ctx: AppContext) -> str: Trying distilled → active is reverse — must refuse.""" event_index = getattr(ctx, "_suite_event_index", None) assert event_index is not None, "suite-event index not staged" - r = decode(await ctx.app.run_job( - "memory_property_update", - path=str(event_index), - key="status", - value="active", - )) + r = decode( + await ctx.app.run_job( + "memory_property_update", + path=str(event_index), + key="status", + value="active", + ), + ) assert isinstance(r, dict), r assert "error" in r and "transition" in r["error"].lower(), r assert r.get("prior") == "distilled", r @@ -358,12 +407,14 @@ async def check_schema_status_invalid_value(ctx: AppContext) -> str: """Random string for status is refused before any state-machine check.""" event_index = getattr(ctx, "_suite_event_index", None) assert event_index is not None, "suite-event index not staged" - r = decode(await ctx.app.run_job( - "memory_property_update", - path=str(event_index), - key="status", - value="bogus", - )) + r = decode( + await ctx.app.run_job( + "memory_property_update", + path=str(event_index), + key="status", + value="bogus", + ), + ) assert isinstance(r, dict), r assert "error" in r and "invalid" in r["error"].lower(), r return "refused status='bogus'" @@ -374,20 +425,27 @@ async def check_schema_status_force_bypass(ctx: AppContext) -> str: intentionally needs to step outside conventions.""" event_index = getattr(ctx, "_suite_event_index", None) assert event_index is not None, "suite-event index not staged" - r = decode(await ctx.app.run_job( - "memory_property_update", - path=str(event_index), - key="status", - value="active", - force=True, - )) + r = decode( + await ctx.app.run_job( + "memory_property_update", + path=str(event_index), + key="status", + value="active", + force=True, + ), + ) assert isinstance(r, dict), r assert "error" not in r, r # restore for any downstream checks - decode(await ctx.app.run_job( - "memory_property_update", - path=str(event_index), key="status", value="distilled", force=True, - )) + decode( + await ctx.app.run_job( + "memory_property_update", + path=str(event_index), + key="status", + value="distilled", + force=True, + ), + ) return "force=True bypassed" @@ -398,28 +456,28 @@ async def check_schema_status_force_bypass(ctx: AppContext) -> str: # rely on side effects from earlier ones (e.g. sync.append needs # sync.create to have run). CHECKS: list[tuple[str, callable]] = [ - ("registry", check_registry), - ("memory_get", check_memory_get), - ("memory_list", check_memory_list), - ("memory_search", check_memory_search), - ("memory_graph_search", check_memory_graph_search), - ("memory_links", check_memory_links), - ("memory_backlinks", check_memory_backlinks), - ("memory_resolve_wikilink", check_memory_resolve_wikilink), - ("memory_count_tokens", check_memory_count_tokens), - ("memory_lint", check_memory_lint), - ("sync.create", check_sync_create), - ("sync.append", check_sync_append), - ("sync.refuse_distilled", check_sync_refuse_distilled), - ("memory_create", check_memory_create), - ("memory_update", check_memory_update), - ("memory_property_update", check_memory_property_update), - ("memory_rename", check_memory_rename), - ("memory_archive", check_memory_archive), - ("memory_delete", check_memory_delete), - ("schema.path_template_refuses", check_schema_path_template_refuses), - ("schema.path_template_force", check_schema_path_template_force_bypass), - ("schema.status_skip_refused", check_schema_status_skip_refused), - ("schema.status_invalid_value", check_schema_status_invalid_value), - ("schema.status_force", check_schema_status_force_bypass), + ("registry", check_registry), + ("memory_get", check_memory_get), + ("memory_list", check_memory_list), + ("memory_search", check_memory_search), + ("memory_graph_search", check_memory_graph_search), + ("memory_links", check_memory_links), + ("memory_backlinks", check_memory_backlinks), + ("memory_resolve_wikilink", check_memory_resolve_wikilink), + ("memory_count_tokens", check_memory_count_tokens), + ("memory_lint", check_memory_lint), + ("sync.create", check_sync_create), + ("sync.append", check_sync_append), + ("sync.refuse_distilled", check_sync_refuse_distilled), + ("memory_create", check_memory_create), + ("memory_update", check_memory_update), + ("memory_property_update", check_memory_property_update), + ("memory_rename", check_memory_rename), + ("memory_archive", check_memory_archive), + ("memory_delete", check_memory_delete), + ("schema.path_template_refuses", check_schema_path_template_refuses), + ("schema.path_template_force", check_schema_path_template_force_bypass), + ("schema.status_skip_refused", check_schema_status_skip_refused), + ("schema.status_invalid_value", check_schema_status_invalid_value), + ("schema.status_force", check_schema_status_force_bypass), ] diff --git a/reme2/mcp/test/test_service.py b/reme2/mcp/test/test_service.py index 66241814..e356fc78 100644 --- a/reme2/mcp/test/test_service.py +++ b/reme2/mcp/test/test_service.py @@ -31,9 +31,7 @@ async def check_registry(ctx: AppContext) -> str: missing = [j for j in EXPECTED_JOBS if j not in ctx.jobs] assert not missing, f"missing jobs: {missing}" extras = [j for j in ctx.jobs if j not in EXPECTED_JOBS] - assert not extras, ( - f"curated profile leaked extra jobs: {extras} — keep the surface tight" - ) + assert not extras, f"curated profile leaked extra jobs: {extras} — keep the surface tight" return f"{len(ctx.jobs)} jobs registered" @@ -41,9 +39,14 @@ async def check_registry(ctx: AppContext) -> str: async def check_retrieve_basic(ctx: AppContext) -> str: - r = decode(await ctx.app.run_job( - "retrieve", query="Alice Bob", max_results=5, min_score=0.0, - )) + r = decode( + await ctx.app.run_job( + "retrieve", + query="Alice Bob", + max_results=5, + min_score=0.0, + ), + ) hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) assert len(hits) > 0, r return f"{len(hits)} hits" @@ -51,13 +54,15 @@ async def check_retrieve_basic(ctx: AppContext) -> str: async def check_retrieve_anchored(ctx: AppContext) -> str: """Wikilink-anchored mode — `[[Project X]]` in the query seeds BFS.""" - r = decode(await ctx.app.run_job( - "retrieve", - query="What touches [[Project X]]?", - max_results=5, - min_score=0.0, - graph_depth=1, - )) + r = decode( + await ctx.app.run_job( + "retrieve", + query="What touches [[Project X]]?", + max_results=5, + min_score=0.0, + graph_depth=1, + ), + ) hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) assert len(hits) > 0, r paths = {h.get("path") for h in hits if isinstance(h, dict)} @@ -70,14 +75,16 @@ async def check_retrieve_anchored(ctx: AppContext) -> str: async def check_retrieve_topic_seeded(ctx: AppContext) -> str: """Topic-rooted mode — explicit `seeds=[...]` instead of inline wikilink.""" project_x = ctx.abs_path("topics", "Project X", "Project X.md") - r = decode(await ctx.app.run_job( - "retrieve", - query="collaborates", - max_results=5, - min_score=0.0, - seeds=[project_x], - graph_depth=1, - )) + r = decode( + await ctx.app.run_job( + "retrieve", + query="collaborates", + max_results=5, + min_score=0.0, + seeds=[project_x], + graph_depth=1, + ), + ) hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else []) assert len(hits) > 0, r return f"{len(hits)} hits (seeded at Project X)" @@ -87,19 +94,21 @@ async def check_retrieve_topic_seeded(ctx: AppContext) -> str: async def check_remember_log_create(ctx: AppContext) -> str: - r = decode(await ctx.app.run_job( - "remember", - mode="log", - name="curated-event", - description="curated-profile suite", - content="## ops\n- ran the curated suite\n", - topics=["[[Alice]]"], - tags=["curated"], - materials=[ - {"filename": "raw-prompt.md", "content": "# user prompt\n\nrun curated suite\n"}, - {"filename": "tool-output.txt", "content": "exit=0\n"}, - ], - )) + r = decode( + await ctx.app.run_job( + "remember", + mode="log", + name="curated-event", + description="curated-profile suite", + content="## ops\n- ran the curated suite\n", + topics=["[[Alice]]"], + tags=["curated"], + materials=[ + {"filename": "raw-prompt.md", "content": "# user prompt\n\nrun curated suite\n"}, + {"filename": "tool-output.txt", "content": "exit=0\n"}, + ], + ), + ) assert isinstance(r, dict), r assert r.get("created") is True and r.get("action") == "created", r materials = r.get("materials", []) @@ -118,17 +127,19 @@ async def check_remember_log_create(ctx: AppContext) -> str: async def check_remember_log_append(ctx: AppContext) -> str: - r = decode(await ctx.app.run_job( - "remember", - mode="log", - name="curated-event", - content="## follow-up\n- second pass\n", - topics=["[[Bob]]"], - tags=["follow-up"], - materials=[ - {"filename": "tool-output.txt", "content": "second run\n"}, # collision - ], - )) + r = decode( + await ctx.app.run_job( + "remember", + mode="log", + name="curated-event", + content="## follow-up\n- second pass\n", + topics=["[[Bob]]"], + tags=["follow-up"], + materials=[ + {"filename": "tool-output.txt", "content": "second run\n"}, # collision + ], + ), + ) assert isinstance(r, dict), r assert r.get("created") is False and r.get("action") == "appended", r appended = r.get("materials", []) @@ -150,9 +161,14 @@ async def check_remember_log_refuse_distilled(ctx: AppContext) -> str: # Give the watcher a moment to re-parse before sync re-reads frontmatter. await wait_for_index(ctx.watcher, expected_min=len(ctx.file_store)) - r = decode(await ctx.app.run_job( - "remember", mode="log", name="curated-event", content="should be refused", - )) + r = decode( + await ctx.app.run_job( + "remember", + mode="log", + name="curated-event", + content="should be refused", + ), + ) assert isinstance(r, dict) and "error" in r, r assert r.get("status") == "distilled", r assert r.get("suggested_name"), r @@ -164,26 +180,28 @@ async def check_remember_log_refuse_distilled(ctx: AppContext) -> str: async def check_remember_distill_degraded(ctx: AppContext) -> str: target = ctx.abs_path("topics", "curated-ingested", "curated-ingested.md") - r = decode(await ctx.app.run_job( - "remember", - # mode defaults to "distill" - content="# curated-ingested\n\nproduced by the curated test suite.\n", - target_path=target, - metadata={ - "title": "curated-ingested", - "lifecycle": "evolving", - "scope": "class", - "source": "curated", - "role": "concept", - "category": "concept", - }, - )) + r = decode( + await ctx.app.run_job( + "remember", + # mode defaults to "distill" + content="# curated-ingested\n\nproduced by the curated test suite.\n", + target_path=target, + metadata={ + "title": "curated-ingested", + "lifecycle": "evolving", + "scope": "class", + "source": "curated", + "role": "concept", + "category": "concept", + }, + ), + ) assert isinstance(r, dict), r applied = r.get("applied") or [] assert len(applied) == 1 and applied[0].get("ok") is True, r assert r.get("used_llm") is False, "expected degraded path (no LLM)" assert Path(target).is_file(), target - return f"applied=1, used_llm=False" + return "applied=1, used_llm=False" # ---------- maintain (lint + decay sweep) ----------------------------- @@ -206,9 +224,13 @@ async def check_maintain_dry_run(ctx: AppContext) -> str: async def check_maintain_targeted(ctx: AppContext) -> str: """target_prefix narrows scan; events/ subtree should yield the one event folder created earlier in this suite.""" - r = decode(await ctx.app.run_job( - "maintain", target_prefix="events/", dry_run=True, - )) + r = decode( + await ctx.app.run_job( + "maintain", + target_prefix="events/", + dry_run=True, + ), + ) assert isinstance(r, dict), r assert r["scanned"] >= 1, r # at least the suite's own event folder return f"scanned={r['scanned']} under events/" @@ -218,14 +240,14 @@ async def check_maintain_targeted(ctx: AppContext) -> str: CHECKS: list[tuple[str, callable]] = [ - ("registry", check_registry), - ("retrieve.basic", check_retrieve_basic), - ("retrieve.anchored", check_retrieve_anchored), - ("retrieve.topic_seeded", check_retrieve_topic_seeded), - ("remember.log_create", check_remember_log_create), - ("remember.log_append", check_remember_log_append), - ("remember.log_refuse_distilled", check_remember_log_refuse_distilled), - ("remember.distill_degraded", check_remember_distill_degraded), - ("maintain.dry_run", check_maintain_dry_run), - ("maintain.targeted", check_maintain_targeted), + ("registry", check_registry), + ("retrieve.basic", check_retrieve_basic), + ("retrieve.anchored", check_retrieve_anchored), + ("retrieve.topic_seeded", check_retrieve_topic_seeded), + ("remember.log_create", check_remember_log_create), + ("remember.log_append", check_remember_log_append), + ("remember.log_refuse_distilled", check_remember_log_refuse_distilled), + ("remember.distill_degraded", check_remember_distill_degraded), + ("maintain.dry_run", check_maintain_dry_run), + ("maintain.targeted", check_maintain_targeted), ] diff --git a/reme2/memory/__init__.py b/reme2/memory/__init__.py index 86f14c23..0de3e04c 100644 --- a/reme2/memory/__init__.py +++ b/reme2/memory/__init__.py @@ -1,22 +1,41 @@ -"""Memory subsystem — the three services on top of the core engine. +"""Memory subsystem — agent-facing services + tools on top of the core engine. -Per the architecture blueprint: +Three services (read / cold-write / treatment) plus the agent-callable +write/read tools that operate on the Memory File System. Every +``@R.register`` step a host agent might invoke lives here, regardless +of whether it's reached via MCP, HTTP, or direct Python. - - retriever.py Read service. V + K + Graph BFS fusion + intent - routing. Registered as a Step (`backend: hybrid`); - MCP step shells in `reme2.mcp.steps.memory_retriever` - instantiate it on demand. - - ingestor.py Cold-write service. LLM-driven R-M-W curator. - Triggered on explicit handoff (task end / SessionEnd). - - maintainer.py Treatment service. Background Merge / Split / Decay / - Lint, woken by cron or thresholds. - - summarizer.py Auxiliary used by the services. + Services + retriever.py Read service — V + K + Graph BFS fusion + intent + routing. Registered as ``hybrid``; consumed by + the search shells below. + ingestor.py Cold-write service — LLM-driven R-M-W curator + (also registers an ``ingest`` MCP face). + maintainer.py Treatment service — Merge / Split / Decay / Lint, + woken by cron or thresholds. + summarizer.py Auxiliary used by the services. -Hot-write MCP step shells (sync, memory_*) live in -`reme2.mcp.steps`, NOT here — they bypass services and write MFS -directly. Importing this package triggers @R.register on the three -services so configs that name them resolve at boot. + Tools (each ``@R.register`` exposes a step the agent invokes by name) + memory_io.py Pure engine API (no schema, no Step boilerplate). + memory_toolkit.py 12 ``memory_*`` primitives — get / list / links / + backlinks / resolve_wikilink / count_tokens + + create / update / property_update / rename / + delete / archive. + memory_search.py ``memory_search`` + ``memory_graph_search`` — + shells over the Retriever service. + memory_lint.py ``memory_lint`` — narrowed projection of the + Maintainer's lint pass. + sync.py ``sync`` — hot-path event-folder upsert + (deterministic, no LLM). + +Importing this package triggers every ``@R.register`` so configs that +name these tools resolve at boot. """ -from . import retriever # noqa: F401 -- runs @R.register("hybrid") -from . import maintainer # noqa: F401 -- runs @R.register("maintainer") +from . import retriever # noqa: F401 -- @R.register("hybrid") +from . import ingestor # noqa: F401 -- @R.register("ingestor") +from . import maintainer # noqa: F401 -- @R.register("maintainer") +from . import memory_toolkit # noqa: F401 -- @R.register("memory_*") x12 +from . import memory_search # noqa: F401 -- @R.register("memory_search", "memory_graph_search") +from . import memory_lint # noqa: F401 -- @R.register("memory_lint") +from . import sync # noqa: F401 -- @R.register("sync") diff --git a/reme2/memory/ingestor.py b/reme2/memory/ingestor.py index f43bdccb..d44f4e7c 100644 --- a/reme2/memory/ingestor.py +++ b/reme2/memory/ingestor.py @@ -107,9 +107,12 @@ class Ingestor(BaseStep): return if mode != "distill": self.context.response.success = False - _set_answer(self.context, { - "error": f"unknown mode {mode!r}; expected 'log' or 'distill'", - }) + _set_answer( + self.context, + { + "error": f"unknown mode {mode!r}; expected 'log' or 'distill'", + }, + ) return content: str = self.context.get("content", "") or "" @@ -182,18 +185,22 @@ class Ingestor(BaseStep): supported. Useful for tests and bootstrap scripts.""" result = IngestResult(used_llm=False) if not target_path: - result.failed.append({ - "op": "create", - "ok": False, - "error": "degraded path: target_path is required when no LLM is configured", - }) + result.failed.append( + { + "op": "create", + "ok": False, + "error": "degraded path: target_path is required when no LLM is configured", + } + ) return result path = Path(target_path) if not path.is_absolute(): path = self._working_dir() / path ok, payload = create_file( - self.file_store, path, - metadata=metadata, content=content, + self.file_store, + path, + metadata=metadata, + content=content, ) bucket = result.applied if ok else result.failed bucket.append({"op": "create", "ok": ok, "path": str(path), "result": payload}) @@ -203,9 +210,8 @@ class Ingestor(BaseStep): """Hot-path event-folder upsert — same code path as the standalone `sync` step. Lazy-instantiated so we don't pay the construction cost on every distill call.""" - from ..mcp.steps.sync import Sync + from .sync import Sync if getattr(self, "_sync_step", None) is None: self._sync_step = Sync(app_context=self.app_context) await self._sync_step(self.context) - diff --git a/reme2/memory/maintainer.py b/reme2/memory/maintainer.py index 1a4eb721..538ad101 100644 --- a/reme2/memory/maintainer.py +++ b/reme2/memory/maintainer.py @@ -70,7 +70,6 @@ from pydantic import BaseModel, Field from ..component import R from ..component.base_step import BaseStep from ..component.runtime_response import _set_answer -from ..enumeration import ComponentEnum from . import memory_io from .schema import parse_frontmatter @@ -171,12 +170,12 @@ class FileSignal(BaseModel): """ path: str - relpath: str = "" # path relative to working_dir, "" if outside - lifecycle: str = "" # streaming / evolving / frozen - scope: str = "" # instance / class - source: str = "" # auto / curated / derived - role: str = "" # observation / claim / question / ... - category: str = "" # legacy field, kept for migration windows + relpath: str = "" # path relative to working_dir, "" if outside + lifecycle: str = "" # streaming / evolving / frozen + scope: str = "" # instance / class + source: str = "" # auto / curated / derived + role: str = "" # observation / claim / question / ... + category: str = "" # legacy field, kept for migration windows status: str = "" age_days: int = 0 metadata: dict = Field(default_factory=dict) @@ -335,22 +334,24 @@ class Maintainer(BaseStep): pass if target_prefix and not relpath.startswith(target_prefix): continue - fm = meta.metadata or {} + fm = meta.front_matter.model_dump() age_seconds = max(0.0, now - (meta.st_mtime or now)) parsed, _ = parse_frontmatter(fm) - signals.append(FileSignal( - path=path, - relpath=relpath, - lifecycle=str(parsed.lifecycle.value) if parsed else "", - scope=str(parsed.scope.value) if parsed else "", - source=str(parsed.source.value) if parsed else "", - role=str(parsed.role.value) if parsed else "", - category=str(fm.get("category") or ""), - status=str(fm.get("status") or ""), - age_days=int(age_seconds // 86400), - metadata=fm, - declared_topics=list(fm.get("topics") or []), - )) + signals.append( + FileSignal( + path=path, + relpath=relpath, + lifecycle=str(parsed.lifecycle.value) if parsed else "", + scope=str(parsed.scope.value) if parsed else "", + source=str(parsed.source.value) if parsed else "", + role=str(parsed.role.value) if parsed else "", + category=str(fm.get("category") or ""), + status=str(fm.get("status") or ""), + age_days=int(age_seconds // 86400), + metadata=fm, + declared_topics=list(fm.get("topics") or []), + ), + ) return signals # -- proposers ---------------------------------------------------------- @@ -361,30 +362,41 @@ class Maintainer(BaseStep): for sig in signals: for link in sig.declared_topics: if not memory_io.resolve_wikilink(self.file_store, link)["exists"]: - out.append(LintFinding( - path=sig.path, kind="broken_wikilink", - detail=f"unresolved wikilink {link!r}", - )) + out.append( + LintFinding( + path=sig.path, + kind="broken_wikilink", + detail=f"unresolved wikilink {link!r}", + ), + ) # Memory schema check: tolerant parse, surface every error # the parser collected. Empty `errors` ↔ valid frontmatter. _, errors = parse_frontmatter(sig.metadata) for err in errors: - out.append(LintFinding( - path=sig.path, kind="schema_violation", - detail=f"Memory: {err}"[:240], - )) + out.append( + LintFinding( + path=sig.path, + kind="schema_violation", + detail=f"Memory: {err}"[:240], + ), + ) # Stem collisions: the engine API exposes the ambiguous-stem map. ambig = memory_io.find_collisions(self.file_store) for stem, paths in ambig.items(): for p in paths: - out.append(LintFinding( - path=p, kind="stem_collision", - detail=f"stem {stem!r} also claimed by {[x for x in paths if x != p]}", - )) + out.append( + LintFinding( + path=p, + kind="stem_collision", + detail=f"stem {stem!r} also claimed by {[x for x in paths if x != p]}", + ), + ) return out def _propose_decay( - self, signals: list[FileSignal], decay_days: int, + self, + signals: list[FileSignal], + decay_days: int, ) -> list[DecayOp]: """Distilled streaming memories past the freshness window. @@ -401,14 +413,18 @@ class Maintainer(BaseStep): continue if sig.age_days < decay_days: continue - out.append(DecayOp( - path=sig.path, age_days=sig.age_days, - reason=f"streaming {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)", - )) + out.append( + DecayOp( + path=sig.path, + age_days=sig.age_days, + reason=f"streaming {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)", + ), + ) return out async def _propose_enrich( - self, signals: list[FileSignal], + self, + signals: list[FileSignal], ) -> list[EnrichOp]: """Bare wikilinks → typed via inline-bracketed Dataview wrap. @@ -418,13 +434,13 @@ class Maintainer(BaseStep): 2. Read body, ask the LLM (one call per file) to assign one of `ALLOWED_PREDICATES` to each bare target — or 'skip'. 3. For each accepted (target, predicate), locate each bare - occurrence span via `FileEdge.from_text` and build a unique + occurrence span via `iter_links` and build a unique context window snippet. 4. Emit EnrichOp(path, target, predicate, old_string, new_string, confidence, reason). The apply step calls memory_update. - Idempotency: re-running won't re-enrich already-typed edges - (filter is `predicate is None`). The OOV-tolerant FileEdge + Idempotency: re-running won't re-enrich already-typed links + (filter is `predicate is None`). The OOV-tolerant FileLink validator means a malformed LLM output collapses to None at the next parse, surfacing it again next sweep — bounded retries avoid infinite re-enrichment loops. @@ -432,7 +448,8 @@ class Maintainer(BaseStep): return [] async def _propose_discover( - self, signals: list[FileSignal], + self, + signals: list[FileSignal], ) -> list[DiscoverOp]: """Discover edges absent from body — append to `## Relations`. @@ -453,7 +470,9 @@ class Maintainer(BaseStep): return [] async def _propose_merge( - self, signals: list[FileSignal], threshold: float, + self, + signals: list[FileSignal], + threshold: float, ) -> list[MergeOp]: """Cluster near-duplicate topics → MergeOps (LLM-assisted). @@ -468,7 +487,9 @@ class Maintainer(BaseStep): return [] async def _propose_split( - self, signals: list[FileSignal], token_threshold: int, + self, + signals: list[FileSignal], + token_threshold: int, ) -> list[SplitOp]: """Topics over the token threshold → SplitOps (LLM-assisted). @@ -483,7 +504,8 @@ class Maintainer(BaseStep): # -- conflict resolution ------------------------------------------------ def _resolve_conflicts( - self, proposed: list[BaseModel], + self, + proposed: list[BaseModel], ) -> tuple[list[BaseModel], list[BaseModel]]: """Apply the conflict matrix; return (kept_plan, dropped). @@ -588,7 +610,8 @@ class Maintainer(BaseStep): # -- apply -------------------------------------------------------------- async def _apply( - self, plan: list[BaseModel], + self, + plan: list[BaseModel], ) -> tuple[list[dict], list[dict]]: """Execute the plan in fixed order. Each phase yields audit dicts. @@ -613,11 +636,13 @@ class Maintainer(BaseStep): result = await self._apply_one(op) applied.append({"op": op.op, **result}) # type: ignore[attr-defined] except Exception as e: - failed.append({ - "op": op.op, # type: ignore[attr-defined] - "payload": _dump(op), - "error": f"{type(e).__name__}: {e}", - }) + failed.append( + { + "op": op.op, # type: ignore[attr-defined] + "payload": _dump(op), + "error": f"{type(e).__name__}: {e}", + } + ) return applied, failed async def _apply_one(self, op: BaseModel) -> dict: @@ -626,28 +651,44 @@ class Maintainer(BaseStep): return {"path": op.path, "kind": op.kind, "noop": True} if isinstance(op, EnrichOp): # TODO: wire to memory_io.update_body once apply-side is enabled. - return {"path": op.path, "target": op.target, "predicate": op.predicate, - "status": "pending_apply", - "would": "memory_update wraps bare [[X]] as [predicate:: [[X]]]"} + return { + "path": op.path, + "target": op.target, + "predicate": op.predicate, + "status": "pending_apply", + "would": "memory_update wraps bare [[X]] as [predicate:: [[X]]]", + } if isinstance(op, DiscoverOp): # TODO: append Dataview lines under ## Relations (create heading # if absent), then call file_store invalidation. - return {"path": op.path, "edges_added": len(op.edges), - "status": "pending_apply", - "would": "append predicate:: [[Y]] under ## Relations heading"} + return { + "path": op.path, + "edges_added": len(op.edges), + "status": "pending_apply", + "would": "append predicate:: [[Y]] under ## Relations heading", + } if isinstance(op, DecayOp): # TODO: wire to MemoryArchive once we're ready to actually # move files from a cron context. For now, surface intent. - return {"path": op.path, "status": "pending_apply", - "would": "flip status=archived + move to archive_dir"} + return { + "path": op.path, + "status": "pending_apply", + "would": "flip status=archived + move to archive_dir", + } if isinstance(op, MergeOp): - return {"canonical": op.canonical, "sources": op.sources, - "status": "pending_apply", - "would": "merge bodies + rewrite incoming wikilinks + archive sources"} + return { + "canonical": op.canonical, + "sources": op.sources, + "status": "pending_apply", + "would": "merge bodies + rewrite incoming wikilinks + archive sources", + } if isinstance(op, SplitOp): - return {"source": op.source, "sections": len(op.sections), - "status": "pending_apply", - "would": "extract sections + replace with [[…]] stubs"} + return { + "source": op.source, + "sections": len(op.sections), + "status": "pending_apply", + "would": "extract sections + replace with [[…]] stubs", + } raise TypeError(f"unknown op type: {type(op).__name__}") diff --git a/reme2/memory/memory_io.py b/reme2/memory/memory_io.py index 6be0c6af..fca6fdb7 100644 --- a/reme2/memory/memory_io.py +++ b/reme2/memory/memory_io.py @@ -3,16 +3,21 @@ The .md files are the SSOT (per `structure.md` §"核心引擎"). The engine is layered: - Memory File System → Watcher & Parser → Projections (vector / FTS / graph) + Memory File System → Watcher & Parser → Projections (vector / FTS) (write entry) (incremental) (read entry, derived) This module is the **single public API surface** over that engine. Every -consumer — MCP step shells, the three memory services (Retriever, -Ingestor, Maintainer), and the agent toolkit (`memory_toolkit`) — talks -to the engine through these functions, not by reaching into -`BaseFileStore` directly. That keeps `file_store` an implementation -detail (could be local sqlite, remote, etc.) and gives the layering one -place to evolve. +consumer — agent-facing steps, the three memory services (Retriever, +Ingestor, Maintainer), and the toolkit (`memory_toolkit`) — talks to +the engine through these functions, not by reaching into +``BaseFileStore`` directly. + +Layering note. The slim ``BaseFileStore`` interface only owns the +search projections (vector / FTS) plus atomic file upserts/deletes. +Iteration, single-node lookup, and the wikilink graph live HERE — we +walk ``LocalFileStore._nodes`` directly (engine-layer peer) and compute +links on-the-fly from each ``FileNode.links``. There is no precomputed +graph index; the graph is a derivation of the SSoT. Naming convention: - Verb-first: `get_file`, `create_file`, `search_vector`. @@ -21,11 +26,6 @@ Naming convention: - Pure-disk writes (`update_body`, `update_meta`, `delete_file`, `archive_file`) don't take `file_store` — they hit the filesystem and the watcher picks them up. The asymmetry is honest. - -Vault-convention helpers — wikilink resolution, graph-walk ranking, -chunk filtering, stem collisions — live here on top of the slim -`BaseFileStore` (which only manages graph + chunks). The engine itself -remains domain-agnostic. """ from __future__ import annotations @@ -38,21 +38,85 @@ from pathlib import Path import frontmatter -from ..schema import ChunkFilter, FileChunk, FileNode, extract_wikilinks -from ..schema.file_edge import _WIKILINK_RE +from ..component.file_store.base_file_store import BaseFileStore +from ..schema import ChunkFilter, FileChunk, FileLink, FileNode, extract_wikilinks +from ..schema.file_link import _WIKILINK_RE from ..utils.wikilink_resolver import ( resolve_wikilink as _resolve_wikilink, wikilink_candidates, ) +# =========================================================================== +# Internal helpers — engine-layer access to the concrete node index +# =========================================================================== +# +# ``BaseFileStore`` only declares the search/upsert contract; iteration +# and single-node lookup live on the concrete impl (``LocalFileStore``). +# memory_io is a peer at the engine layer, so reaching into ``_nodes`` +# is intentional — every iteration / graph walk funnels through here so +# the day the engine grows a public ``iter_nodes()``, this is the only +# place to swap. + + +def _nodes(file_store: BaseFileStore) -> dict[str, FileNode]: + """The concrete in-memory ``{path: FileNode}`` index.""" + return file_store._nodes # type: ignore[attr-defined] + + +def _meta(node: FileNode) -> dict: + """Full frontmatter dict for a node — typed fields (title/description/ + tags) merged with any ``extra=allow`` extras.""" + return node.front_matter.model_dump() + + +def _get_outlinks( + file_store: BaseFileStore, + path: str, +) -> list[tuple[FileNode, FileLink]]: + """Resolved outgoing links from ``path`` — ``[(target_node, link), ...]``.""" + node = _nodes(file_store).get(path) + if node is None: + return [] + out: list[tuple[FileNode, FileLink]] = [] + for link in node.links: + target = _resolve_wikilink(file_store, link.path) + if target is None: + continue + target_node = _nodes(file_store).get(target) + if target_node is not None: + out.append((target_node, link)) + return out + + +def _get_inlinks( + file_store: BaseFileStore, + path: str, +) -> list[tuple[FileNode, FileLink]]: + """Resolved incoming links to ``path`` — linear scan over all nodes.""" + out: list[tuple[FileNode, FileLink]] = [] + for src_node in _nodes(file_store).values(): + for link in src_node.links: + target = _resolve_wikilink(file_store, link.path) + if target == path: + out.append((src_node, link)) + return out + + +def _filter_to_dict(chunk_filter: ChunkFilter | None) -> dict: + """Serialize ``ChunkFilter`` for the search engine's ``search_filter`` arg.""" + if chunk_filter is None: + return {} + return chunk_filter.model_dump(mode="json") + + # =========================================================================== # Section 1 — MFS Reads # =========================================================================== async def get_file( - file_store, + file_store: BaseFileStore, path: str, *, include_chunks: bool = False, @@ -62,14 +126,16 @@ async def get_file( On-disk frontmatter is the source of truth — the file_store cache may lag a write that hasn't been picked up by the watcher yet. """ - node = file_store.get_node_by_path(path) + node = await file_store.get_node_by_path(path) # type: ignore[attr-defined] result: dict = {"path": path, "exists": False} if node is not None: - result.update({ - "exists": True, - "metadata": node.metadata, - "link": [e.model_dump(exclude_none=True) for e in node.edges], - }) + result.update( + { + "exists": True, + "metadata": _meta(node), + "link": [link.model_dump(exclude_none=True) for link in node.links], + } + ) file_path = Path(path) if file_path.is_file(): @@ -80,13 +146,13 @@ async def get_file( result["metadata"] = dict(post.metadata) if include_chunks: - chunks = await file_store.get_chunks_by_path(path) + chunks = await file_store.get_chunks_by_path(path) # type: ignore[attr-defined] result["chunks"] = [c.model_dump(exclude_none=True) for c in chunks] return result def list_files( - file_store, + file_store: BaseFileStore, *, path_prefix: str | None = None, tags: list[str] | None = None, @@ -97,14 +163,14 @@ def list_files( metadata_filter = metadata or {} tag_filter = tags or [] items: list[dict] = [] - for path, node in file_store.nodes.items(): + for path, node in _nodes(file_store).items(): if path_prefix and not path.startswith(path_prefix): continue - md = node.metadata or {} + md = _meta(node) if metadata_filter and any(md.get(k) != v for k, v in metadata_filter.items()): continue if tag_filter: - file_tags = set(md.get("tags", []) or []) + file_tags = set(md.get("tags") or []) if not all(t in file_tags for t in tag_filter): continue items.append({"path": path, "metadata": md}) @@ -113,32 +179,32 @@ def list_files( return {"items": items, "count": len(items)} -def _edge_to_dict(node, edge) -> dict: +def _link_to_dict(node: FileNode, link: FileLink) -> dict: return { "path": node.path, - "metadata": node.metadata, - "predicate": edge.predicate, - "anchor": edge.anchor, + "metadata": _meta(node), + "predicate": link.predicate, + "anchor": link.anchor, } -def get_links(file_store, path: str) -> dict: - """Files that `path` links TO (resolved). Each entry carries the typed-edge predicate.""" +def get_links(file_store: BaseFileStore, path: str) -> dict: + """Files that `path` links TO (resolved). Each entry carries the typed-link predicate.""" return { "path": path, - "links": [_edge_to_dict(m, e) for m, e in file_store.get_links(path)], + "links": [_link_to_dict(m, link) for m, link in _get_outlinks(file_store, path)], } -def get_backlinks(file_store, path: str) -> dict: - """Files that link TO `path`. Each entry carries the typed-edge predicate.""" +def get_backlinks(file_store: BaseFileStore, path: str) -> dict: + """Files that link TO `path`. Each entry carries the typed-link predicate.""" return { "path": path, - "backlinks": [_edge_to_dict(m, e) for m, e in file_store.get_backlinks(path)], + "backlinks": [_link_to_dict(m, link) for m, link in _get_inlinks(file_store, path)], } -def resolve_wikilink(file_store, wikilink: str) -> dict: +def resolve_wikilink(file_store: BaseFileStore, wikilink: str) -> dict: """Resolve a `[[target]]` wikilink with full ambiguity context. Returns: @@ -177,9 +243,9 @@ def resolve_wikilink(file_store, wikilink: str) -> dict: } -def iter_files(file_store) -> Iterator[tuple[str, FileNode]]: +def iter_files(file_store: BaseFileStore) -> Iterator[tuple[str, FileNode]]: """Walk every indexed (path, FileNode). Used by Maintainer scans.""" - return iter(file_store.nodes.items()) + return iter(_nodes(file_store).items()) async def count_tokens( @@ -234,7 +300,7 @@ def _replace_wikilink_targets(text: str, mapping: dict[str, str]) -> str: def create_file( - file_store, + file_store: BaseFileStore, path: Path, *, metadata: dict, @@ -328,7 +394,7 @@ def update_meta(path: Path | str, *, key: str, value) -> tuple[bool, dict]: def rename_file( - file_store, + file_store: BaseFileStore, working_dir: Path | str, *, old_path: Path | str, @@ -349,8 +415,7 @@ def rename_file( if conflicts: return False, { "error": ( - f"stem `[[{new_p.stem}]]` would resolve ambiguously " - f"to {len(conflicts) + 1} paths after this rename" + f"stem `[[{new_p.stem}]]` would resolve ambiguously " f"to {len(conflicts) + 1} paths after this rename" ), "conflicts": conflicts, "hint": ( @@ -375,7 +440,7 @@ def rename_file( except ValueError: pass - referring_paths = [m.path for m, _ in file_store.get_backlinks(str(old_p))] + referring_paths = [m.path for m, _ in _get_inlinks(file_store, str(old_p))] new_p.parent.mkdir(parents=True, exist_ok=True) old_p.rename(new_p) @@ -469,43 +534,50 @@ def archive_file( async def search_vector( - file_store, + file_store: BaseFileStore, query: str, *, limit: int, chunk_filter: ChunkFilter | None = None, ) -> list[FileChunk]: """Vector similarity over the chunk-level Vector projection.""" - return await file_store.vector_search(query, limit, chunk_filter) + return await file_store.vector_search(query, limit, _filter_to_dict(chunk_filter)) async def search_keyword( - file_store, + file_store: BaseFileStore, query: str, *, limit: int, chunk_filter: ChunkFilter | None = None, ) -> list[FileChunk]: """FTS5 keyword search over the chunk-level keyword projection.""" - return await file_store.keyword_search(query, limit, chunk_filter) + return await file_store.keyword_search(query, limit, _filter_to_dict(chunk_filter)) -async def get_chunks(file_store, paths: Iterable[str]) -> list[FileChunk]: +async def get_chunks( + file_store: BaseFileStore, + paths: Iterable[str], +) -> list[FileChunk]: """Batch fetch chunks across many paths.""" - return await file_store.get_chunks_by_paths(paths) + out: list[FileChunk] = [] + for p in paths: + out.extend(await file_store.get_chunks_by_path(p)) # type: ignore[attr-defined] + return out # =========================================================================== # Section 4 — Graph helpers (BFS / scoring / collisions) # =========================================================================== # -# Layered on top of the slim `BaseFileStore` (which only owns the -# graph + chunks). These are vault conventions: folder-note preference, -# wikilink-uniqueness gates, BFS for memory_graph_search. +# All graph traversal walks ``FileNode.links`` directly via ``_get_outlinks`` +# / ``_get_inlinks``. There's no precomputed adjacency index — every BFS +# resolves wikilinks per call. Cheap enough for typical vault sizes; +# revisit if the maintainer's lint pass becomes a hotspot. def expand_neighbors( - file_store, + file_store: BaseFileStore, seeds: Iterable[str], *, depth: int = 1, @@ -518,10 +590,11 @@ def expand_neighbors( if depth < 0: raise ValueError(f"depth must be >= 0, got {depth}") + nodes = _nodes(file_store) seen: dict[str, int] = {} frontier: deque[tuple[str, int]] = deque() for path in seeds: - if path in file_store and path not in seen: + if path in nodes and path not in seen: seen[path] = 0 frontier.append((path, 0)) @@ -532,9 +605,9 @@ def expand_neighbors( neighbors: list[str] = [] if direction in ("out", "both"): - neighbors.extend(m.path for m, _ in file_store.get_links(path)) + neighbors.extend(m.path for m, _ in _get_outlinks(file_store, path)) if direction in ("in", "both"): - neighbors.extend(m.path for m, _ in file_store.get_backlinks(path)) + neighbors.extend(m.path for m, _ in _get_inlinks(file_store, path)) if per_node_cap is not None and len(neighbors) > per_node_cap: neighbors = neighbors[:per_node_cap] @@ -549,7 +622,7 @@ def expand_neighbors( def subgraph_score( - file_store, + file_store: BaseFileStore, seeds: Iterable[str], *, decay: float = 0.5, @@ -561,12 +634,16 @@ def subgraph_score( if not (0.0 <= decay <= 1.0): raise ValueError(f"decay must be in [0, 1], got {decay}") hops = expand_neighbors( - file_store, seeds, depth=depth, direction=direction, per_node_cap=per_node_cap, + file_store, + seeds, + depth=depth, + direction=direction, + per_node_cap=per_node_cap, ) - return {path: decay ** hop for path, hop in hops.items()} + return {path: decay**hop for path, hop in hops.items()} -def extract_anchors(file_store, text: str) -> list[str]: +def extract_anchors(file_store: BaseFileStore, text: str) -> list[str]: """Pull `[[X]]` anchors from `text` and resolve each (deduped).""" seen: set[str] = set() out: list[str] = [] @@ -578,7 +655,10 @@ def extract_anchors(file_store, text: str) -> list[str]: return out -def collisions_after_create(file_store, proposed_path: str | Path) -> list[str]: +def collisions_after_create( + file_store: BaseFileStore, + proposed_path: str | Path, +) -> list[str]: """Existing paths that would conflict with adding `proposed_path`. Folder-note rule: if `proposed_path`'s parent dir name == its stem, @@ -591,23 +671,26 @@ def collisions_after_create(file_store, proposed_path: str | Path) -> list[str]: proposed_abs = str(p.resolve()) is_folder_note = p.parent.name == stem - folder_hits = [ - path for path in file_store.nodes - if Path(path).stem == stem and Path(path).parent.name == stem - and path != proposed_abs - ] - stem_hits = [ - path for path in file_store.get_paths_by_stem(stem) - if path != proposed_abs - ] + folder_hits: list[str] = [] + stem_hits: list[str] = [] + for path in _nodes(file_store): + if path == proposed_abs: + continue + path_obj = Path(path) + if path_obj.stem != stem: + continue + if path_obj.parent.name == stem: + folder_hits.append(path) + else: + stem_hits.append(path) if is_folder_note: return folder_hits - return folder_hits + [sp for sp in stem_hits if sp not in folder_hits] + return folder_hits + stem_hits def make_filter( - file_store, + file_store: BaseFileStore, *, paths: list[str] | None = None, tags: list[str] | None = None, @@ -617,17 +700,13 @@ def make_filter( cf = ChunkFilter(paths=paths, tags=tags, exclude_paths=exclude_paths) if cf.is_empty(): return cf - cf.resolved_paths = { - p for p, n in file_store.nodes.items() if cf.match_metadata(p, n.metadata) - } + cf.resolved_paths = {p for p, n in _nodes(file_store).items() if cf.match_metadata(p, _meta(n))} return cf -def find_collisions(file_store) -> dict[str, list[str]]: +def find_collisions(file_store: BaseFileStore) -> dict[str, list[str]]: """Every stem that resolves to >1 path. Used by Maintainer.lint.""" - out: dict[str, list[str]] = {} - for stem in file_store._stems: - cands = wikilink_candidates(file_store, stem) - if len(cands) > 1: - out[stem] = cands - return out + by_stem: dict[str, list[str]] = {} + for path in _nodes(file_store): + by_stem.setdefault(Path(path).stem, []).append(path) + return {s: ps for s, ps in by_stem.items() if len(ps) > 1} diff --git a/reme2/mcp/steps/memory_lint.py b/reme2/memory/memory_lint.py similarity index 84% rename from reme2/mcp/steps/memory_lint.py rename to reme2/memory/memory_lint.py index 29b5007b..197205e2 100644 --- a/reme2/mcp/steps/memory_lint.py +++ b/reme2/memory/memory_lint.py @@ -12,11 +12,10 @@ agent doesn't need to know about decay/merge/split knobs. from __future__ import annotations -from ...component import R -from ...component.base_step import BaseStep -from ...component.runtime_response import _set_answer -from ...enumeration import ComponentEnum -from ...memory.maintainer import Maintainer +from ..component import R +from ..component.base_step import BaseStep +from ..component.runtime_response import _set_answer +from ..enumeration import ComponentEnum @R.register("memory_lint") @@ -57,14 +56,18 @@ class MemoryLint(BaseStep): # action lives in `proposed` (LintFindings never get applied or # dropped). Reshape into a focused response. import json + raw = self.context.response.answer audit = json.loads(raw) if isinstance(raw, str) else (raw or {}) findings = audit.get("proposed") or [] - _set_answer(self.context, { - "scanned": audit.get("scanned", 0), - "findings": findings, - "target_prefix": target_prefix, - "ran_at": audit.get("ran_at", ""), - }) + _set_answer( + self.context, + { + "scanned": audit.get("scanned", 0), + "findings": findings, + "target_prefix": target_prefix, + "ran_at": audit.get("ran_at", ""), + }, + ) self.context.response.success = True diff --git a/reme2/mcp/steps/memory_retriever.py b/reme2/memory/memory_search.py similarity index 84% rename from reme2/mcp/steps/memory_retriever.py rename to reme2/memory/memory_search.py index 961d73ce..8e34c9ad 100644 --- a/reme2/mcp/steps/memory_retriever.py +++ b/reme2/memory/memory_search.py @@ -1,26 +1,26 @@ -"""Memory retriever steps — thin MCP-facing wrappers over `BaseRetriever`. +"""Memory search step shells — thin wrappers over the Retriever service. Per the architecture blueprint, retrieval policy (V+K hybrid + graph -BFS fusion + ranking + intent routing) is the **Retriever service** -(`reme2.memory.retriever`), registered as `ComponentEnum.RETRIEVER`. -These steps are the MCP projection: they translate `RuntimeContext` -(paths/tags/exclude_paths filter, per-call knob overrides) into the -retriever's call surface, then serialize results for the API response -(joining file metadata onto each chunk so callers don't have to fire a -`memory_get` per hit). +BFS fusion + ranking + intent routing) lives in the **Retriever +service** (`reme2.memory.retriever`), registered as +``ComponentEnum.RETRIEVER``. The two steps here are the agent-facing +projection: they translate ``RuntimeContext`` (paths/tags/exclude_paths +filter, per-call knob overrides) into the retriever's call surface, +then serialize results for the response (joining file metadata onto +each chunk so callers don't have to fire a ``memory_get`` per hit). Direct primary-key file ops (read/list/links/backlinks) live in -`memory_io` — those aren't retrieval. +``memory_io`` — those aren't retrieval. """ from __future__ import annotations -from ...component import R -from ...component.base_step import BaseStep -from ...component.runtime_response import _set_answer -from ...enumeration import ComponentEnum -from ...memory import memory_io -from ...memory.retriever import BaseRetriever, HybridRetriever +from ..component import R +from ..component.base_step import BaseStep +from ..component.runtime_response import _set_answer +from ..enumeration import ComponentEnum +from . import memory_io +from .retriever import BaseRetriever, HybridRetriever # Per-shell singleton cache: instantiating the retriever is cheap (it @@ -61,9 +61,16 @@ def _resolve_retriever(step: BaseStep) -> BaseRetriever: cls = HybridRetriever knob_keys = ( - "vector_weight", "graph_weight", "graph_depth", "graph_decay", - "graph_direction", "graph_mode", "graph_per_path_cap", - "candidate_multiplier", "anchor_expand", "file_store", + "vector_weight", + "graph_weight", + "graph_depth", + "graph_decay", + "graph_direction", + "graph_mode", + "graph_per_path_cap", + "candidate_multiplier", + "anchor_expand", + "file_store", ) init_kwargs = {k: step.kwargs[k] for k in knob_keys if k in step.kwargs} init_kwargs["app_context"] = step.app_context @@ -84,9 +91,9 @@ def _serialize_chunk(chunk, file_store, extras: dict | None = None) -> dict: `extras` lets the caller attach step-specific fields (e.g. `graph_hop`). """ item = chunk.model_dump(exclude_none=True, exclude={"embedding"}) - node = file_store.get_node_by_path(chunk.path) + node = file_store._nodes.get(chunk.path) # engine-layer peer access if node is not None: - item["file_metadata"] = node.metadata + item["file_metadata"] = node.front_matter.model_dump() item["file_st_mtime"] = node.st_mtime else: item["file_metadata"] = None diff --git a/reme2/memory/memory_toolkit.py b/reme2/memory/memory_toolkit.py index ff3c5e71..0530d210 100644 --- a/reme2/memory/memory_toolkit.py +++ b/reme2/memory/memory_toolkit.py @@ -46,9 +46,9 @@ from ..enumeration import ComponentEnum _STATUS_STATES = ("active", "distilled", "archived") _STATUS_TRANSITIONS: dict[str, set[str]] = { - "active": {"active", "distilled"}, + "active": {"active", "distilled"}, "distilled": {"distilled", "archived"}, - "archived": {"archived"}, + "archived": {"archived"}, } @@ -59,10 +59,7 @@ def validate_status_transition(prior, requested) -> str | None: if requested is None: return None # delete operation if requested not in _STATUS_STATES: - return ( - f"invalid status {requested!r}; must be one of " - f"{list(_STATUS_STATES)}" - ) + return f"invalid status {requested!r}; must be one of " f"{list(_STATUS_STATES)}" if prior in _STATUS_STATES and requested not in _STATUS_TRANSITIONS[prior]: return ( f"status transition {prior!r} → {requested!r} not allowed; " @@ -167,9 +164,12 @@ def create_file_with_schema( ), } return memory_io.create_file( - file_store, path, - metadata=metadata, content=content, - overwrite=overwrite, force=force, + file_store, + path, + metadata=metadata, + content=content, + overwrite=overwrite, + force=force, ) @@ -336,9 +336,12 @@ class MemoryCreate(BaseStep): target = Path(path) ok, payload = create_file_with_schema( - self.file_store, target, - metadata=metadata, content=content, - overwrite=overwrite, force=force, + self.file_store, + target, + metadata=metadata, + content=content, + overwrite=overwrite, + force=force, ) self.context.response.success = ok if ok: @@ -357,7 +360,8 @@ class MemoryCreate(BaseStep): gate both fire unless `force=True`.""" target = Path(path) ok, payload = create_file_with_schema( - self.file_store, target, + self.file_store, + target, metadata=dict(metadata or {}), content=content, overwrite=overwrite, @@ -403,8 +407,10 @@ class MemoryRename(BaseStep): working_dir = Path(self.file_store.working_dir or Path.cwd()).resolve() ok, payload = memory_io.rename_file( - self.file_store, working_dir, - old_path=old_path, new_path=new_path, + self.file_store, + working_dir, + old_path=old_path, + new_path=new_path, ) self.context.response.success = ok _set_answer(self.context, payload) @@ -414,8 +420,10 @@ class MemoryRename(BaseStep): vr = getattr(self.file_store, "working_dir", None) working_dir = Path(vr).resolve() if vr else Path.cwd() ok, payload = memory_io.rename_file( - self.file_store, working_dir, - old_path=old_path, new_path=new_path, + self.file_store, + working_dir, + old_path=old_path, + new_path=new_path, ) return _tool_response("memory_rename", ok, payload, audit=self.audit) @@ -477,7 +485,10 @@ class MemoryUpdate(BaseStep): replace_all: bool = bool(self.context.get("replace_all", False)) assert path, "path is required" ok, payload = memory_io.update_body( - path, old_string=old_string, new_string=new_string, replace_all=replace_all, + path, + old_string=old_string, + new_string=new_string, + replace_all=replace_all, ) self.context.response.success = ok _set_answer(self.context, payload) @@ -491,7 +502,10 @@ class MemoryUpdate(BaseStep): ) -> ToolResponse: """Edit-style content update: replace `old_string` with `new_string`.""" ok, payload = memory_io.update_body( - path, old_string=old_string, new_string=new_string, replace_all=replace_all, + path, + old_string=old_string, + new_string=new_string, + replace_all=replace_all, ) return _tool_response("memory_update", ok, payload, audit=self.audit) diff --git a/reme2/memory/retriever.py b/reme2/memory/retriever.py index 8939d566..381a2754 100644 --- a/reme2/memory/retriever.py +++ b/reme2/memory/retriever.py @@ -16,8 +16,8 @@ Two surfaces: standard Step lifecycle + `self.file_store` property + per-instance configuration via constructor kwargs. The default `execute()` dispatch just runs `graph_search` so the retriever can also be used directly as -a step inside a job pipeline; the MCP step shells in -`reme2.mcp.steps.memory_retriever` instead bypass `execute` and call +a step inside a job pipeline; the search step shells in +`reme2.memory.memory_search` instead bypass `execute` and call `search` / `graph_search` directly so they own the result-serialization shape. """ @@ -183,7 +183,10 @@ class HybridRetriever(BaseRetriever): results = k_results[:max_results] else: results = self._merge_vk( - v_results, k_results, self.vector_weight, text_weight, + v_results, + k_results, + self.vector_weight, + text_weight, )[:max_results] elif fs.embedding_model: results = await memory_io.search_vector(fs, query, limit=max_results, chunk_filter=chunk_filter) @@ -287,7 +290,7 @@ class HybridRetriever(BaseRetriever): hops = memory_io.expand_neighbors(fs, seed_paths, depth=gd, direction=gdr) else: hops = {} - graph_scores: dict[str, float] = {p: gdc ** h for p, h in hops.items()} + graph_scores: dict[str, float] = {p: gdc**h for p, h in hops.items()} # 4. Pull graph-only chunks (paths in expansion but not in V∪K). # Skip in 'boost' mode — boost only re-ranks V∪K, never adds candidates. diff --git a/reme2/memory/schema/memory.py b/reme2/memory/schema/memory.py index 2e6417b6..7e71ff79 100644 --- a/reme2/memory/schema/memory.py +++ b/reme2/memory/schema/memory.py @@ -76,14 +76,14 @@ class Role(StrEnum): Adding a new role is a one-line change here plus (optionally) a preset. """ - OBSERVATION = "observation" # what happened (events live here) - CLAIM = "claim" # an assertion needing confidence (thesis, model) - QUESTION = "question" # an open inquiry needing an answer - PROFILE = "profile" # entity description (person, org, system) - CONCEPT = "concept" # abstract idea / definition (company, sector, concept) - METHOD = "method" # procedure / how-to - REFERENCE = "reference" # pointer to external thing (tool, paper, code) - FUNDAMENTALS = "fundamentals" # foundational data / baseline facts + OBSERVATION = "observation" # what happened (events live here) + CLAIM = "claim" # an assertion needing confidence (thesis, model) + QUESTION = "question" # an open inquiry needing an answer + PROFILE = "profile" # entity description (person, org, system) + CONCEPT = "concept" # abstract idea / definition (company, sector, concept) + METHOD = "method" # procedure / how-to + REFERENCE = "reference" # pointer to external thing (tool, paper, code) + FUNDAMENTALS = "fundamentals" # foundational data / baseline facts class Status(StrEnum): @@ -206,6 +206,7 @@ class MemoryFileNode(FileNode): the parsed object without polluting this base schema. `populate_by_name` lets `originSessionId` (legacy camelCase) populate `origin_session_id`. """ + # -- Identity / common metadata ---------------------------------------- title: str = Field(default="") @@ -274,8 +275,7 @@ class MemoryFileNode(FileNode): def _enforce_role_conditionals(self): if self.role is Role.CLAIM and self.confidence is None: raise ValueError( - "role='claim' requires explicit confidence " - "(one of ⏳ / ✅ / ❌)" + "role='claim' requires explicit confidence " "(one of ⏳ / ✅ / ❌)", ) return self diff --git a/reme2/mcp/steps/sync.py b/reme2/memory/sync.py similarity index 79% rename from reme2/mcp/steps/sync.py rename to reme2/memory/sync.py index 5a0ad551..5ffd6ab7 100644 --- a/reme2/mcp/steps/sync.py +++ b/reme2/memory/sync.py @@ -31,7 +31,6 @@ Zero LLM cost. """ import json -import os import re from collections.abc import Iterable from datetime import date as date_type, datetime, timezone @@ -42,8 +41,8 @@ from pydantic import ValidationError from reme2.component import R from reme2.component.base_step import BaseStep -from reme2.memory.memory_io import collisions_after_create, create_file -from reme2.memory.schema import EVENT_PRESET, MemoryFileNode +from .memory_io import collisions_after_create, create_file +from .schema import EVENT_PRESET, MemoryFileNode _SAFE_FILENAME_RE = re.compile(r"^[a-zA-Z0-9._-]+$") @@ -213,10 +212,7 @@ class Sync(BaseStep): """Filenames in `folder` excluding the index, sorted for stability.""" if not folder.is_dir(): return [] - return sorted( - entry.name for entry in folder.iterdir() - if entry.is_file() and entry.name != index_filename - ) + return sorted(entry.name for entry in folder.iterdir() if entry.is_file() and entry.name != index_filename) @staticmethod def _union(prior: list, incoming: list) -> list: @@ -262,20 +258,31 @@ class Sync(BaseStep): await self._append(target, content, materials, topics, tags) else: await self._create( - target, name, description, content, materials, topics, tags, - on_date, origin_session_id, + target, + name, + description, + content, + materials, + topics, + tags, + on_date, + origin_session_id, ) async def _create( - self, target: Path, name: str, description: str, content: str, - materials: list[dict], topics: list[str], tags: list[str], - on_date, origin_session_id, + self, + target: Path, + name: str, + description: str, + content: str, + materials: list[dict], + topics: list[str], + tags: list[str], + on_date, + origin_session_id, ) -> None: today = date_type.today().isoformat() - on_date_str = ( - on_date.isoformat() if isinstance(on_date, date_type) - else (on_date or today) - ) + on_date_str = on_date.isoformat() if isinstance(on_date, date_type) else (on_date or today) # Start from EVENT_PRESET (4 axes + status + legacy `category`), # layer caller-supplied identity fields on top. metadata: dict = { @@ -291,16 +298,20 @@ class Sync(BaseStep): metadata["originSessionId"] = origin_session_id try: - MemoryFileNode.model_validate({ - "path": str(target.resolve()), - "st_mtime": 0.0, - **metadata, - }) + MemoryFileNode.model_validate( + { + "path": str(target.resolve()), + "st_mtime": 0.0, + **metadata, + } + ) except ValidationError as e: - self._set_error({ - "error": "MemoryFileNode schema validation failed", - "details": e.errors(include_context=False, include_url=False), - }) + self._set_error( + { + "error": "MemoryFileNode schema validation failed", + "details": e.errors(include_context=False, include_url=False), + } + ) return graph = self.file_store @@ -308,32 +319,38 @@ class Sync(BaseStep): if conflicts: taken = {Path(p).stem for p in graph.nodes} suggested_name = _next_suffixed_stem(taken, name) - self._set_error({ - "error": ( - f"stem `[[{name}]]` would resolve ambiguously " - f"to {len(conflicts) + 1} paths after this create" - ), - "conflicts": conflicts, - "suggested_name": suggested_name, - "hint": ( - f"retry with name='{suggested_name}', or pick a " - f"semantic qualifier (e.g. '{name}-followup')." - ), - }) + self._set_error( + { + "error": ( + f"stem `[[{name}]]` would resolve ambiguously " + f"to {len(conflicts) + 1} paths after this create" + ), + "conflicts": conflicts, + "suggested_name": suggested_name, + "hint": ( + f"retry with name='{suggested_name}', or pick a " + f"semantic qualifier (e.g. '{name}-followup')." + ), + } + ) return material_filenames = [m["filename"] for m in materials] index_body = self._emit_body(content, material_filenames) ok, payload = create_file( - self.file_store, target, - metadata=metadata, content=index_body, + self.file_store, + target, + metadata=metadata, + content=index_body, ) if not ok: - self._set_error({ - "path": str(target.resolve()), - "error": payload.get("error", "create failed"), - "details": payload, - }) + self._set_error( + { + "path": str(target.resolve()), + "error": payload.get("error", "create failed"), + "details": payload, + } + ) return material_paths: list[str] = [] @@ -348,19 +365,25 @@ class Sync(BaseStep): continue material_paths.append(str(material_path.resolve())) - self._set_ok({ - "path": str(target.resolve()), - "category": "event", - "status": "active", - "topics": topics, - "materials": material_paths, - "created": True, - "action": "created", - }) + self._set_ok( + { + "path": str(target.resolve()), + "category": "event", + "status": "active", + "topics": topics, + "materials": material_paths, + "created": True, + "action": "created", + } + ) async def _append( - self, target: Path, content: str, materials: list[dict], - topics: list[str], tags: list[str], + self, + target: Path, + content: str, + materials: list[dict], + topics: list[str], + tags: list[str], ) -> None: # Read current frontmatter + body. try: @@ -379,15 +402,17 @@ class Sync(BaseStep): taken = {Path(p).stem for p in graph.nodes} base = target.stem suggested = _next_suffixed_stem(taken, base) - self._set_error({ - "path": str(target.resolve()), - "error": ( - f"event `{base}` already exists with status={status!r}; " - f"pick a new name to start a fresh thread" - ), - "status": status, - "suggested_name": suggested, - }) + self._set_error( + { + "path": str(target.resolve()), + "error": ( + f"event `{base}` already exists with status={status!r}; " + f"pick a new name to start a fresh thread" + ), + "status": status, + "suggested_name": suggested, + } + ) return folder = target.parent @@ -420,10 +445,7 @@ class Sync(BaseStep): if content.strip(): ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") update_section = f"## Update — {ts}\n\n{content.rstrip()}\n" - narrative = ( - f"{narrative.rstrip()}\n\n{update_section}" - if narrative else update_section - ) + narrative = f"{narrative.rstrip()}\n\n{update_section}" if narrative else update_section all_filenames_sorted = sorted(set(existing_filenames)) new_body = self._emit_body(narrative, all_filenames_sorted) @@ -434,17 +456,21 @@ class Sync(BaseStep): meta["updated"] = date_type.today().isoformat() try: - MemoryFileNode.model_validate({ - "path": str(target.resolve()), - "st_mtime": 0.0, - **meta, - }) + MemoryFileNode.model_validate( + { + "path": str(target.resolve()), + "st_mtime": 0.0, + **meta, + } + ) except ValidationError as e: - self._set_error({ - "path": str(target.resolve()), - "error": "MemoryFileNode schema validation failed on append", - "details": e.errors(include_context=False, include_url=False), - }) + self._set_error( + { + "path": str(target.resolve()), + "error": "MemoryFileNode schema validation failed on append", + "details": e.errors(include_context=False, include_url=False), + } + ) return new_post = frontmatter.Post(new_body, **meta) @@ -454,12 +480,14 @@ class Sync(BaseStep): self._set_error({"path": str(target.resolve()), "error": f"write failed: {e}"}) return - self._set_ok({ - "path": str(target.resolve()), - "category": "event", - "status": "active", - "topics": meta["topics"], - "materials": new_material_paths, - "created": False, - "action": "appended", - }) + self._set_ok( + { + "path": str(target.resolve()), + "category": "event", + "status": "active", + "topics": meta["topics"], + "materials": new_material_paths, + "created": False, + "action": "appended", + } + ) diff --git a/reme2/schema/__init__.py b/reme2/schema/__init__.py index 22d280c0..d691c1d0 100644 --- a/reme2/schema/__init__.py +++ b/reme2/schema/__init__.py @@ -5,7 +5,7 @@ from .as_msg_stat import AsBlockStat, AsMsgStat from .emb_node import EmbNode from .chunk_filter import ChunkFilter from .file_chunk import FileChunk -from .file_edge import FileEdge, extract_wikilinks +from .file_link import FileLink, extract_wikilinks, iter_links from .file_node import FileFrontMatter, FileNode from .request import Request from .response import Response @@ -20,11 +20,12 @@ __all__ = [ "EmbNode", "ChunkFilter", "FileChunk", - "FileEdge", + "FileLink", "FileFrontMatter", "FileNode", "Request", "Response", "StreamChunk", "extract_wikilinks", + "iter_links", ] diff --git a/reme2/schema/emb_node.py b/reme2/schema/emb_node.py index 7a261d3a..ec9257d4 100644 --- a/reme2/schema/emb_node.py +++ b/reme2/schema/emb_node.py @@ -12,14 +12,14 @@ class EmbNode(BaseModel): embedding: np.ndarray | None = Field(default=None) metadata: dict = Field(default_factory=dict) - @field_validator('embedding', mode='before') + @field_validator("embedding", mode="before") @classmethod def validate_embedding(cls, v): if v is None: return v return np.array(v, dtype=np.float16) - @field_serializer('embedding') + @field_serializer("embedding") def serialize_embedding(self, v: np.ndarray | None, _info): if v is None: return None diff --git a/reme2/schema/file_chunk.py b/reme2/schema/file_chunk.py index b30d803a..1414e2cc 100644 --- a/reme2/schema/file_chunk.py +++ b/reme2/schema/file_chunk.py @@ -15,5 +15,6 @@ class FileChunk(EmbNode): def set_hash_id(self): from ..utils import hash_text + self.id = hash_text(" ".join([self.path, str(self.start_line), str(self.end_line), self.text])) - return self \ No newline at end of file + return self diff --git a/reme2/schema/file_edge.py b/reme2/schema/file_edge.py deleted file mode 100644 index c2f61b80..00000000 --- a/reme2/schema/file_edge.py +++ /dev/null @@ -1,224 +0,0 @@ -"""FileEdge — typed wikilink edge between vault files. - -Single source of truth for the edge **schema** and the **inline parser** -(`FileEdge.from_text`) that recovers edges from body text. Edges live -exclusively in body text (frontmatter is not walked); the predicate -vocabulary is **open** — any identifier-shaped token -(`[A-Za-z][A-Za-z0-9_]*`) the parser sees is preserved verbatim on -`FileEdge.predicate`. Vocabulary curation, if any, is the maintainer's -job, not the schema's. - -## Inline forms recognised by `FileEdge.from_text` - - [[X]] bare wikilink → predicate=None - extends:: [[X]] line-level Dataview → predicate="extends" - [extends:: [[X]]] inline-bracketed → predicate="extends" - -Multi-target — every wikilink under one typed context inherits its -predicate (any separator works, not just commas): - - extends:: [[A]], [[B]] line-level multi → 2 edges, both "extends" - extends:: [[A]] and [[B]] prose-style multi → 2 edges, both "extends" - [concerns:: [[A]], [[B]]] inline multi → 2 edges, both "concerns" - extends:: [[A#s1]], [[B#s2]] multi w/ anchors → anchors preserved per link - -Context precedence is **inline-bracketed > line-level > bare** — -a wikilink inside a `[predicate:: …]` envelope is typed by that -envelope even if the line happens to start `predicate:: …`. -""" - -from __future__ import annotations - -import re - -from pydantic import BaseModel, ConfigDict, Field - - -# -- Regexes (module-private) --------------------------------------------- - -# Wikilink. The optional `!` embed marker and `|alias` are matched but -# not captured — both are presentational and dropped from the edge. -# `target` is the file part; `anchor` is the optional `#…` suffix; we -# rejoin them into a single `link` string at edge-construction time. -_WIKILINK_RE = re.compile( - r""" - (?:!)? - \[\[ - (?P[^\]\|\#\n]+?) - (?:\#(?P[^\]\|\n]+))? - (?:\|[^\]\n]+)? - \]\] - """, - re.VERBOSE, -) - -# Line-level Dataview field. Anchored MULTILINE; allows leading bullet -# (`-`/`*`/`+`) so `- extends:: [[X]]` works inside Markdown lists. -_DATAVIEW_LINE_RE = re.compile( - r"^[ \t]*(?:[-*+][ \t]+)?(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P.+?)\s*$", - re.MULTILINE, -) - -# Inline-bracketed field opener: `[predicate::`. The matching `]` is -# located by depth-counted scan because the value may contain -# `[[wikilink]]` whose inner brackets are part of the value. -_INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*") - - -def _iter_inline_fields(text: str) -> list[tuple[int, int, str]]: - """Find inline-bracketed `[predicate:: …]` field spans by depth scan. - - Returns a list of ``(start, end, predicate)`` triples. Newlines - terminate the scan: an inline field that spans a line break is - treated as malformed (matches Dataview semantics) and skipped. - """ - out: list[tuple[int, int, str]] = [] - for m in _INLINE_FIELD_OPEN_RE.finditer(text): - depth = 1 # the outer '[' was the regex's first character - i = m.end() - n = len(text) - while i < n: - c = text[i] - if c == "\n": - break - if c == "[": - depth += 1 - elif c == "]": - depth -= 1 - if depth == 0: - out.append((m.start(), i + 1, m.group("predicate"))) - break - i += 1 - return out - - -# -- Schema --------------------------------------------------------------- - - -class FileEdge(BaseModel): - """2-field minimal edge model: ``link`` + ``predicate``. - - ``link`` preserves the wikilink as written, including any ``#anchor`` - suffix (e.g. ``"X"`` or ``"X#sec"``). The presentational ``|alias`` - and ``!`` embed prefix are discarded by the parser. ``path`` and - ``anchor`` are derived ``@property``s that split ``link`` on the - first ``#`` for callers that want either part without re-splitting - the string themselves. - """ - - model_config = ConfigDict(extra="forbid") - - link: str = Field( - ..., - description=( - "Wikilink as written, preserving any '#anchor' suffix " - "(e.g. 'X' or 'X#sec'). Display alias and embed prefix discarded." - ), - ) - predicate: str | None = Field( - default=None, - description="Typed-edge predicate (Dataview-style). None for bare [[X]].", - ) - - @property - def path(self) -> str: - """File-part of ``link`` (before any ``#anchor``). - - For ``"X"`` returns ``"X"``; for ``"X#sec"`` returns ``"X"``; - for ``"topics/Foo#bar"`` returns ``"topics/Foo"``. Always - returns a non-empty string because ``link`` is required and the - regex never matches an empty target. - """ - return self.link.split("#", 1)[0].strip() - - @property - def anchor(self) -> str | None: - """Heading or block anchor parsed from ``link`` (text after first ``#``). - - Returns ``None`` if the link has no anchor or the anchor is empty. - """ - if "#" not in self.link: - return None - tail = self.link.split("#", 1)[1].strip() - return tail or None - - @classmethod - def _from_match(cls, wm: re.Match, *, predicate: str | None) -> FileEdge: - target = wm.group("target").strip() - anchor = wm.group("anchor") - link = f"{target}#{anchor.strip()}" if anchor else target - return cls(link=link, predicate=predicate) - - @classmethod - def from_text(cls, text: str) -> list[FileEdge]: - """Extract all edges from body text in source order. - - Single pass: every wikilink in the text becomes one edge, and - its ``predicate`` is decided by the surrounding context with - precedence **inline-bracketed > line-level > bare**: - - * ``[predicate:: [[X]]]`` → ``predicate="predicate"`` - * ``predicate:: [[X]]`` → ``predicate="predicate"`` (line-level Dataview) - * ``[[X]]`` → ``predicate=None`` (bare) - - No consumed-span bookkeeping, no second sort: ``finditer`` - already yields wikilinks in source order, and per-position - classification is unambiguous. - """ - if not text: - return [] - - # Inline-bracketed `[predicate:: ...]` envelopes need a - # depth-counted scan (regex can't match balanced `[[…]]` inside). - inline_spans = _iter_inline_fields(text) - - return [ - cls._from_match(wm, predicate=_predicate_for(text, wm.start(), inline_spans)) - for wm in _WIKILINK_RE.finditer(text) - ] - - -def _predicate_for( - text: str, - pos: int, - inline_spans: list[tuple[int, int, str]], -) -> str | None: - """Resolve the predicate governing a wikilink at offset ``pos``. - - Checks the two typed-edge contexts in precedence order; falls - through to ``None`` (bare) when neither applies. - """ - # 1. Inline-bracketed envelope `[predicate:: …]` containing pos. - for field_start, field_end, predicate in inline_spans: - if field_start <= pos < field_end: - return predicate - - # 2. Line-level `predicate:: value` whose value range covers pos. - line_start = text.rfind("\n", 0, pos) + 1 - line_end = text.find("\n", pos) - if line_end == -1: - line_end = len(text) - m = _DATAVIEW_LINE_RE.match(text[line_start:line_end]) - if m and line_start + m.start("value") <= pos: - return m.group("predicate") - - # 3. Bare wikilink — no predicate. - return None - - -# -- Public utility (target-only fast path) ------------------------------- - - -def extract_wikilinks(text: str) -> list[str]: - """Flat list of wikilink **file targets** in body text (no dedup). - - Returns just the file part of each wikilink (before any ``#anchor``) - because callers feed the result to `resolve_wikilink`, which matches - against vault file stems / paths and would not recognise an anchor - suffix. Single regex pass — cheaper than `FileEdge.from_text` when - callers don't need predicates (e.g. ingestor's auto-discovery hint, - memory_io anchor resolution). - """ - if not text: - return [] - return [m.group("target").strip() for m in _WIKILINK_RE.finditer(text)] diff --git a/reme2/schema/file_link.py b/reme2/schema/file_link.py new file mode 100644 index 00000000..e50041d6 --- /dev/null +++ b/reme2/schema/file_link.py @@ -0,0 +1,192 @@ +"""FileLink — typed wikilink between vault files. + +One type, two states (the value of ``path`` distinguishes them): + + * **pre-resolution** — ``path`` holds the raw wikilink target as + written, e.g. ``"Foo"`` or ``"topics/Bar"``. Produced by + ``iter_links(text)``: a pure regex pass over body text, no graph + access. + * **resolved** — ``path`` holds the vault-relative path + file_graph stores, e.g. ``"topics/Foo/Foo.md"``. Produced by + ``utils.wikilink_resolver.resolve_links(graph, links)`` (or the + one-shot ``text_to_links(graph, text)``), which expands stem + ambiguity into one ``FileLink`` per candidate path. + +file_graph trusts ``link.path`` directly for adjacency: it only ever +stores resolved links. The pre-resolution form is internal pipeline +plumbing. + +## Inline forms recognised by ``iter_links`` + + [[X]] bare wikilink → predicate=None + extends:: [[X]] line-level Dataview → predicate="extends" + [extends:: [[X]]] inline-bracketed → predicate="extends" + +Multi-target — every wikilink under one typed context inherits its +predicate (any separator works, not just commas): + + extends:: [[A]], [[B]] line-level multi → 2 links, both "extends" + extends:: [[A]] and [[B]] prose-style multi → 2 links, both "extends" + [concerns:: [[A]], [[B]]] inline multi → 2 links, both "concerns" + extends:: [[A#s1]], [[B#s2]] multi w/ anchors → anchors preserved per link + +Context precedence is **inline-bracketed > line-level > bare** — +a wikilink inside a ``[predicate:: …]`` envelope is typed by that +envelope even if the line happens to start ``predicate:: …``. +""" + +from __future__ import annotations + +import re + +from pydantic import BaseModel, ConfigDict, Field + + +# -- Regexes (module-private) --------------------------------------------- + +_WIKILINK_RE = re.compile( + r""" + (?:!)? + \[\[ + (?P[^\]\|\#\n]+?) + (?:\#(?P[^\]\|\n]+))? + (?:\|[^\]\n]+)? + \]\] + """, + re.VERBOSE, +) + +_DATAVIEW_LINE_RE = re.compile( + r"^[ \t]*(?:[-*+][ \t]+)?(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P.+?)\s*$", + re.MULTILINE, +) + +_INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*") + + +def _iter_inline_fields(text: str) -> list[tuple[int, int, str]]: + """Find inline-bracketed ``[predicate:: …]`` field spans by depth scan.""" + out: list[tuple[int, int, str]] = [] + for m in _INLINE_FIELD_OPEN_RE.finditer(text): + depth = 1 + i = m.end() + n = len(text) + while i < n: + c = text[i] + if c == "\n": + break + if c == "[": + depth += 1 + elif c == "]": + depth -= 1 + if depth == 0: + out.append((m.start(), i + 1, m.group("predicate"))) + break + i += 1 + return out + + +def _predicate_for( + text: str, + pos: int, + inline_spans: list[tuple[int, int, str]], +) -> str | None: + """Resolve the predicate governing a wikilink at offset ``pos``.""" + for field_start, field_end, predicate in inline_spans: + if field_start <= pos < field_end: + return predicate + line_start = text.rfind("\n", 0, pos) + 1 + line_end = text.find("\n", pos) + if line_end == -1: + line_end = len(text) + m = _DATAVIEW_LINE_RE.match(text[line_start:line_end]) + if m and line_start + m.start("value") <= pos: + return m.group("predicate") + return None + + +# -- Schema --------------------------------------------------------------- + + +class FileLink(BaseModel): + """Typed wikilink — ``(path, anchor, predicate)``. + + Single type for both states (see module docstring): ``path`` is + a raw wikilink target before resolution, a vault-relative resolved + path after. + + Fields: + path wikilink target. Raw target text in pre-resolution + form (e.g. ``"Foo"``, ``"topics/Bar"``); + vault-relative resolved path in stored form + (e.g. ``"topics/Foo/Foo.md"``). + anchor heading or block anchor (text after ``#`` in the + wikilink). Pass-through across resolution. + predicate Dataview-style typed-link predicate; ``None`` for bare. + """ + + model_config = ConfigDict(extra="forbid") + + path: str = Field( + ..., + description=( + "Wikilink target. Pre-resolution: the raw target as written " + "(e.g. 'Foo'). Resolved: the vault-relative path file_graph " + "stores. Stem ambiguity is resolved BEFORE construction of " + "the resolved form by emitting one FileLink per candidate." + ), + ) + anchor: str | None = Field( + default=None, + description="Heading or block anchor (text after '#'). None if absent.", + ) + predicate: str | None = Field( + default=None, + description="Typed-link predicate (Dataview-style). None for bare [[X]].", + ) + + +# -- Public extraction utilities ------------------------------------------ + + +def iter_links(text: str) -> list[FileLink]: + """Extract every wikilink in ``text`` as a pre-resolution ``FileLink``. + + Each emitted ``FileLink`` has ``path`` set to the raw wikilink + target as written (the file portion only — the ``#anchor`` lives + in its own field). Predicate is decided by surrounding context + with precedence **inline-bracketed > line-level > bare**. + + Pure function: no graph access. Pass the result through + ``utils.wikilink_resolver.resolve_links`` (or one-shot + ``text_to_links``) to get the resolved form file_graph stores. + """ + if not text: + return [] + inline_spans = _iter_inline_fields(text) + links: list[FileLink] = [] + for wm in _WIKILINK_RE.finditer(text): + target = wm.group("target").strip() + anchor_raw = wm.group("anchor") + anchor = anchor_raw.strip() if anchor_raw else "" + links.append( + FileLink( + path=target, + anchor=anchor or None, + predicate=_predicate_for(text, wm.start(), inline_spans), + ), + ) + return links + + +def extract_wikilinks(text: str) -> list[str]: + """Flat list of wikilink **file targets** in body text (no dedup). + + Returns just the file part of each wikilink (before any ``#anchor``). + Single regex pass — cheaper than ``iter_links`` when callers don't + need predicates (e.g. ``extract_anchors`` for query seeding in the + retriever). + """ + if not text: + return [] + return [m.group("target").strip() for m in _WIKILINK_RE.finditer(text)] diff --git a/reme2/schema/file_node.py b/reme2/schema/file_node.py index 5c28ff11..c179bc56 100644 --- a/reme2/schema/file_node.py +++ b/reme2/schema/file_node.py @@ -1,6 +1,6 @@ from pydantic import BaseModel, ConfigDict, Field -from .file_edge import FileEdge +from .file_link import FileLink class FileFrontMatter(BaseModel): @@ -18,6 +18,6 @@ class FileFrontMatter(BaseModel): class FileNode(BaseModel): path: str = Field(default=...) st_mtime: float = Field(default=...) - edges: list[FileEdge] = Field(default_factory=list) + links: list[FileLink] = Field(default_factory=list) chunk_ids: list[str] = Field(default_factory=list) front_matter: FileFrontMatter = Field(default_factory=FileFrontMatter) diff --git a/reme2/utils/chunk_search.py b/reme2/utils/chunk_search.py index 07c7b358..970c1c97 100644 --- a/reme2/utils/chunk_search.py +++ b/reme2/utils/chunk_search.py @@ -34,7 +34,8 @@ def keyword_score(query: str, text: str) -> float: def filter_chunks( - chunks: list[FileChunk], chunk_filter: ChunkFilter | None, + chunks: list[FileChunk], + chunk_filter: ChunkFilter | None, ) -> list[FileChunk]: """Restrict `chunks` to those whose path passes the (compiled) filter.""" if chunk_filter is None or chunk_filter.resolved_paths is None: diff --git a/reme2/utils/wikilink_resolver.py b/reme2/utils/wikilink_resolver.py index 463a9980..e77de9ff 100644 --- a/reme2/utils/wikilink_resolver.py +++ b/reme2/utils/wikilink_resolver.py @@ -1,56 +1,285 @@ -"""Wikilink resolution — pure functions over a `BaseFileStore`'s graph. +"""Wikilink resolver — vault convention over ``BaseFileGraph``. -Wikilink resolution is a vault convention (path/stem forms, folder-note -preference) layered on top of the engine's plain `dict[path, FileNode]`. -Keeping it out of `BaseFileStore` keeps the engine domain-agnostic. +Stem-form wikilinks like ``[[Foo]]`` are a vault convention. The +file_graph engine doesn't know about them — it only stores nodes and +trusts ``FileLink.path`` for adjacency. This module bridges the gap, +applying: + +* **folder-note rule** — ``[[X]]`` prefers ``topics/X/X.md`` over a + sibling ``topics/X.md`` when both exist +* **stem ambiguity** — ``[[Foo]]`` matching multiple paths is handled + by **emitting multiple ``FileLink`` records, one per candidate** + (file_graph then has unambiguous adjacency) +* **path-form passthrough** — ``[[foo/bar]]`` and ``[[foo/bar.md]]`` + resolve directly against the graph (vault-relative paths; no + working_dir absolutization) + +All paths are vault-relative — ``graph.iter_nodes()`` returns the +key form file_graph stores, and emitted ``FileLink.path`` matches. + +All functions are stateless — they walk ``graph.iter_nodes()`` per +call. For batch operations (``resolve_links`` over many links) the +stem index is built once and reused. + +Seven entry points mapped to call sites: + + resolve single link → path | None + used by: ``extract_anchors``, ``memory_resolve_wikilink`` + candidates stem → [path] (folder-note ordered first) + used by: ``memory_resolve_wikilink`` (ambiguity report) + collisions all stems with >1 path + used by: ``maintainer.lint`` + collisions_for paths conflicting with a proposed new path + used by: ``memory_create``, ``sync`` (preflight) + extract_anchors parse [[X]] from text + resolve, dedup + used by: ``retriever`` (query anchor seeds) + resolve_links ``[FileLink]`` (raw path) → ``[FileLink]`` (resolved) + core resolution; stem ambiguity expands + text_to_links one-shot: ``iter_links(text)`` + ``resolve_links`` + used by: parser pipeline (before ``upsert_node``) """ from __future__ import annotations +from collections.abc import Iterable from pathlib import Path -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from reme2.component.file_store.base_file_store import BaseFileStore +from ..component.file_graph.base_file_graph import BaseFileGraph +from ..schema import FileLink +from ..schema.file_link import extract_wikilinks, iter_links +from .logger_utils import get_logger -def wikilink_candidates(store: BaseFileStore, target: str) -> list[str]: - """Paths `[[target]]` would resolve to (folder-note hit wins). +_logger = get_logger() - Folder-note convention: `topics/X/X.md` is the cluster head for - bare `[[X]]`, beating any sibling that just shares the stem. + +# ========================================================================= +# Internal helpers +# ========================================================================= + + +async def _build_stem_index(graph: BaseFileGraph) -> dict[str, list[str]]: + """Walk once, group paths by stem. Used by batch hot paths.""" + out: dict[str, list[str]] = {} + async for path, _ in graph.iter_nodes(): + out.setdefault(Path(path).stem, []).append(path) + return out + + +def _split_link(link: str) -> tuple[str, str]: + """Return ``(target, anchor)``. Anchor empty if no ``#``. + + For raw link strings supplied by external callers (which may still + arrive in ``[[A#section]]`` form). Pre-extracted ``FileLink`` + records already carry ``path`` and ``anchor`` separately. """ - folder_hits = sorted( - p for p in store._stems.get(target, ()) if Path(p).parent.name == target - ) + if not link: + return "", "" + if "#" not in link: + return link.strip(), "" + target_raw, anchor_raw = link.split("#", 1) + return target_raw.strip(), anchor_raw.strip() + + +def _is_path_form(target: str) -> bool: + return "/" in target or target.endswith(".md") + + +def _filter_stem_candidates(stem: str, paths: list[str]) -> list[str]: + """Apply folder-note rule: if any folder-note exists, ONLY folder-notes + are candidates; otherwise all sibling-stems are candidates.""" + if not paths: + return [] + folder_hits = sorted(p for p in paths if Path(p).parent.name == stem) if folder_hits: return folder_hits - return store.get_paths_by_stem(target) + return sorted(paths) -def resolve_wikilink(store: BaseFileStore, wikilink: str) -> str | None: - """Resolve a `[[target]]` to one absolute path. +# ========================================================================= +# Public API — single-shot lookups +# ========================================================================= - - Path form (`topics/X/X` or `topics/X/X.md`) → relative to `working_dir`. - - Stem form (`X`) → folder-note preference, else unique stem hit. - - Ambiguous stems return None and log a warning. + +async def resolve(graph: BaseFileGraph, link: str) -> str | None: + """Resolve a single wikilink to **one** vault-relative path, or None. + + Returns None if dangling or ambiguous (with warning on ambiguity). + Use ``resolve_links`` for the multi-link expansion semantics. """ - target = wikilink.strip() + target, _ = _split_link(link) if not target: return None - if "/" in target or target.endswith(".md"): - if store.working_dir is None: - return None + if _is_path_form(target): candidate = target if target.endswith(".md") else f"{target}.md" - abs_candidate = str((store.working_dir / candidate).resolve()) - return abs_candidate if abs_candidate in store.nodes else None + return candidate if await graph.get_node(candidate) else None - candidates = wikilink_candidates(store, target) - if len(candidates) == 1: - return candidates[0] - if len(candidates) > 1: - store.logger.warning( - f"Wikilink [[{target}]] is ambiguous, candidates: {candidates}", + paths = [p async for p, _ in graph.iter_nodes() if Path(p).stem == target] + candidates_for_stem = _filter_stem_candidates(target, paths) + if len(candidates_for_stem) == 1: + return candidates_for_stem[0] + if len(candidates_for_stem) > 1: + _logger.warning( + f"Wikilink [[{target}]] is ambiguous, " f"candidates: {candidates_for_stem}", ) return None + + +async def candidates(graph: BaseFileGraph, stem: str) -> list[str]: + """All paths a ``[[stem]]`` could match. Folder-note hits ordered first.""" + folder_hits: list[str] = [] + stem_hits: list[str] = [] + async for path, _ in graph.iter_nodes(): + p = Path(path) + if p.stem != stem: + continue + if p.parent.name == stem: + folder_hits.append(path) + else: + stem_hits.append(path) + if folder_hits: + return sorted(folder_hits) + return sorted(stem_hits) + + +async def collisions(graph: BaseFileGraph) -> dict[str, list[str]]: + """Every stem that resolves to >1 path. Used by maintainer.lint.""" + stem_index = await _build_stem_index(graph) + return {stem: sorted(paths) for stem, paths in stem_index.items() if len(paths) > 1} + + +async def collisions_for( + graph: BaseFileGraph, + proposed_path: str | Path, +) -> list[str]: + """Existing paths that would conflict with adding ``proposed_path``. + + ``proposed_path`` is treated as vault-relative (matching the form + ``graph.iter_nodes()`` returns). Folder-note rule: when the + proposed path is itself a folder-note (parent dir name == stem), + only colliding folder-notes are returned. Otherwise all paths + sharing the stem are returned. + """ + p = Path(proposed_path) + stem = p.stem + proposed_str = str(p) + is_folder_note = p.parent.name == stem + + folder_hits: list[str] = [] + stem_hits: list[str] = [] + async for path, _ in graph.iter_nodes(): + if path == proposed_str: + continue + path_obj = Path(path) + if path_obj.stem != stem: + continue + if path_obj.parent.name == stem: + folder_hits.append(path) + else: + stem_hits.append(path) + + if is_folder_note: + return sorted(folder_hits) + return sorted(folder_hits) + sorted(stem_hits) + + +async def extract_anchors(graph: BaseFileGraph, text: str) -> list[str]: + """Pull ``[[X]]`` from ``text``, resolve each, dedup in source order. + + Uses single-target ``resolve`` semantics — ambiguous stems return + no anchor. (For multi-link expansion at *write* time, see + ``resolve_links``; ``extract_anchors`` is for read-time seeding + where a single deterministic target is wanted.) + """ + if not text: + return [] + seen: set[str] = set() + out: list[str] = [] + for raw in extract_wikilinks(text): + hit = await resolve(graph, raw) + if hit is not None and hit not in seen: + seen.add(hit) + out.append(hit) + return out + + +# ========================================================================= +# Public API — parser pipeline +# ========================================================================= + + +async def resolve_links( + graph: BaseFileGraph, + links: Iterable[FileLink], +) -> list[FileLink]: + """Resolve pre-resolution ``FileLink`` records against the graph. + + Each input link has ``path`` holding a raw wikilink target (the + extractor form). Returns FileLinks with ``path`` rewritten to the + vault-relative resolved path file_graph stores. ``anchor`` and + ``predicate`` pass through unchanged. + + Stem expansion semantics — one input link produces zero, one, or + many output links: + + * path-form, target indexed → 1 link + * path-form, dangling → 0 links + * stem-form, 1 folder-note hit → 1 link + * stem-form, N folder-note hits → N links (one per) + * stem-form, 0 folder-notes, 1 sibling → 1 link + * stem-form, 0 folder-notes, N siblings → N links (one per) + * stem-form, dangling → 0 links + + Build the stem index lazily: only paid if at least one input is + stem-form. + """ + link_list = list(links) + if not link_list: + return [] + + stem_index: dict[str, list[str]] | None = None + out: list[FileLink] = [] + for link in link_list: + target = link.path + if not target: + continue + + if _is_path_form(target): + candidate = target if target.endswith(".md") else f"{target}.md" + if await graph.get_node(candidate) is None: + continue + out.append( + FileLink( + path=candidate, + anchor=link.anchor, + predicate=link.predicate, + ), + ) + continue + + # Stem form — may expand into multiple links. + if stem_index is None: + stem_index = await _build_stem_index(graph) + for chosen in _filter_stem_candidates(target, stem_index.get(target, [])): + out.append( + FileLink( + path=chosen, + anchor=link.anchor, + predicate=link.predicate, + ), + ) + + return out + + +async def text_to_links( + graph: BaseFileGraph, + text: str, +) -> list[FileLink]: + """One-shot: extract wikilinks from ``text`` and resolve to safe links. + + Equivalent to ``await resolve_links(graph, iter_links(text))``. + The parser pipeline's single-call entry point. + """ + return await resolve_links(graph, iter_links(text)) diff --git a/tests/inspect_md_parser.py b/tests/inspect_md_parser.py index 2e00d6e1..23ab0c32 100644 --- a/tests/inspect_md_parser.py +++ b/tests/inspect_md_parser.py @@ -26,20 +26,25 @@ def main() -> None: ) ap.add_argument("path", help="Path to a markdown file.") ap.add_argument( - "--chunk-chars", type=int, default=2000, - help="Max characters per chunk content (default: 2000). " - "Excludes TOC skeleton when embed_toc is on.", + "--chunk-chars", + type=int, + default=2000, + help="Max characters per chunk content (default: 2000). " "Excludes TOC skeleton when embed_toc is on.", ) ap.add_argument( - "--no-toc", action="store_true", + "--no-toc", + action="store_true", help="Disable the full-doc TOC skeleton wrap; chunks become plain content.", ) ap.add_argument( - "--show-edges", action="store_true", - help="Print extracted FileEdges before chunks.", + "--show-edges", + action="store_true", + help="Print extracted FileLinks before chunks.", ) ap.add_argument( - "--preview", type=int, default=0, + "--preview", + type=int, + default=0, help="Truncate each chunk to N chars in output (0 = full text).", ) args = ap.parse_args() @@ -59,12 +64,12 @@ def main() -> None: sizes = [len(c.text) for c in chunks] print(f"chars min/avg/max: {min(sizes)} / {sum(sizes)//len(sizes)} / {max(sizes)}") if args.show_edges: - print(f"\nedges ({len(node.edges)}):") - for e in node.edges: + print(f"\nlinks ({len(node.links)}):") + for link in node.links: print( - f" → {e.link}" - + (f" predicate={e.predicate}" if e.predicate else "") - + (f" anchor={e.anchor}" if e.anchor else "") + f" → {link.path}" + + (f" predicate={link.predicate}" if link.predicate else "") + + (f" anchor={link.anchor}" if link.anchor else ""), ) for i, c in enumerate(chunks): diff --git a/tests/test_file_edge.py b/tests/test_file_edge.py deleted file mode 100644 index 6c347351..00000000 --- a/tests/test_file_edge.py +++ /dev/null @@ -1,348 +0,0 @@ -"""FileEdge unit tests — body-only edge extraction. - -Covers the three legal inline forms (bare / line-level Dataview / -inline-bracketed Dataview), multi-target expansion, dedup against -typed wrappers, open-vocabulary predicate pass-through, the explicit -decision that frontmatter is no longer walked for links, and the -2-field schema (`link` + `predicate`) with `anchor` as a derived -property and `alias` / `embed` discarded at parse time. -""" - -from reme2.schema.file_edge import FileEdge, extract_wikilinks - - -# -------------------------------------------------------------------------- -# Bare wikilinks -# -------------------------------------------------------------------------- - - -def test_bare_wikilink(): - edges = FileEdge.from_text("see [[X]]") - assert len(edges) == 1 - assert edges[0].link == "X" - assert edges[0].predicate is None - assert edges[0].anchor is None - - -def test_anchor_alias_embed_collapse_into_link(): - """`![[X#sec|Alias]]` → link='X#sec' (alias and embed dropped).""" - edges = FileEdge.from_text("![[X#sec|Alias]]") - assert len(edges) == 1 - assert edges[0].link == "X#sec" - # anchor is a derived property parsed from link. - assert edges[0].anchor == "sec" - - -def test_alias_only_drops_to_link(): - edges = FileEdge.from_text("[[X|Alias]]") - assert len(edges) == 1 - assert edges[0].link == "X" - assert edges[0].anchor is None - - -def test_embed_only_drops_to_link(): - edges = FileEdge.from_text("![[X]]") - assert len(edges) == 1 - assert edges[0].link == "X" - - -# -------------------------------------------------------------------------- -# Line-level Dataview -# -------------------------------------------------------------------------- - - -def test_line_level_field(): - edges = FileEdge.from_text("extends:: [[Source Topic]]") - assert len(edges) == 1 - assert edges[0].link == "Source Topic" - assert edges[0].predicate == "extends" - - -def test_line_level_multi_target(): - edges = FileEdge.from_text("concerns:: [[A]], [[B]], [[C]]") - assert [(e.link, e.predicate) for e in edges] == [ - ("A", "concerns"), - ("B", "concerns"), - ("C", "concerns"), - ] - - -def test_line_level_with_bullet(): - edges = FileEdge.from_text("- extends:: [[X]]\n * concerns:: [[Y]]") - assert [(e.link, e.predicate) for e in edges] == [ - ("X", "extends"), - ("Y", "concerns"), - ] - - -# -------------------------------------------------------------------------- -# Multi-edge cases — many wikilinks under one or several typed contexts -# -------------------------------------------------------------------------- - - -def test_multi_target_with_anchors_preserves_each(): - """Each comma-separated target keeps its own anchor in `link`.""" - edges = FileEdge.from_text("extends:: [[A#sec1]], [[B#sec2]], [[C]]") - assert [(e.link, e.anchor, e.predicate) for e in edges] == [ - ("A#sec1", "sec1", "extends"), - ("B#sec2", "sec2", "extends"), - ("C", None, "extends"), - ] - - -def test_multi_target_non_comma_separator_still_typed(): - """Wikilinks anywhere in the value range (not just comma-separated) - inherit the line's predicate. Useful for prose-style fields.""" - edges = FileEdge.from_text("extends:: [[A]] and also [[B]]") - assert [(e.link, e.predicate) for e in edges] == [ - ("A", "extends"), - ("B", "extends"), - ] - - -def test_multi_dataview_lines_each_multi_target(): - """Multiple Dataview lines each with multi-target → all edges typed - by their respective line's predicate.""" - edges = FileEdge.from_text( - "extends:: [[A]], [[B]]\nrelates:: [[C]], [[D]]" - ) - assert [(e.link, e.predicate) for e in edges] == [ - ("A", "extends"), - ("B", "extends"), - ("C", "relates"), - ("D", "relates"), - ] - - -def test_inline_bracketed_then_bare_on_same_line(): - """Inline-bracketed governs only the wikilinks inside its brackets; - a trailing bare wikilink on the same line stays bare.""" - edges = FileEdge.from_text("[ext:: [[A]]] then [[B]]") - assert [(e.link, e.predicate) for e in edges] == [ - ("A", "ext"), - ("B", None), - ] - - -def test_mid_line_dataview_like_not_typed(): - """``predicate::`` only counts at line start (modulo bullet) — - a `predicate::` mid-line is just prose, so its wikilinks are bare.""" - edges = FileEdge.from_text("[ext:: [[A]]] and concerns:: [[B]]") - assert [(e.link, e.predicate) for e in edges] == [ - ("A", "ext"), - ("B", None), # `concerns::` mid-line is not Dataview - ] - - -# -------------------------------------------------------------------------- -# Inline-bracketed Dataview -# -------------------------------------------------------------------------- - - -def test_inline_bracketed(): - edges = FileEdge.from_text("This [extends:: [[Y]]] something else.") - assert len(edges) == 1 - assert edges[0].link == "Y" - assert edges[0].predicate == "extends" - - -def test_inline_bracketed_multi_target(): - edges = FileEdge.from_text("[concerns:: [[A]], [[B]]]") - assert [(e.link, e.predicate) for e in edges] == [ - ("A", "concerns"), - ("B", "concerns"), - ] - - -def test_inline_bracketed_skips_cross_line(): - # A `[predicate:: ...]` that spans a newline is malformed → the inner - # wikilink falls back to bare; the unmatched `[` does not eat tail text. - edges = FileEdge.from_text("[extends:: [[X]]\nbad]") - assert len(edges) == 1 - assert edges[0].link == "X" - assert edges[0].predicate is None - - -# -------------------------------------------------------------------------- -# Dedup: a wikilink inside a typed wrapper should not double-emit -# -------------------------------------------------------------------------- - - -def test_inline_bracketed_does_not_double_emit(): - edges = FileEdge.from_text("see [extends:: [[X]]] again.") - assert len(edges) == 1 - assert edges[0].predicate == "extends" - - -def test_line_level_value_does_not_double_emit(): - edges = FileEdge.from_text("extends:: [[X]]") - assert len(edges) == 1 - - -def test_typed_and_bare_coexist_for_same_target(): - edges = FileEdge.from_text("extends:: [[X]]\nFree text mentioning [[X]] again.") - links_preds = sorted(((e.link, e.predicate or "") for e in edges)) - assert links_preds == [("X", ""), ("X", "extends")] - - -# -------------------------------------------------------------------------- -# Open-vocabulary predicates — any identifier-shaped token passes through -# -------------------------------------------------------------------------- - - -def test_arbitrary_predicate_preserved(): - edges = FileEdge.from_text("anything_goes:: [[X]]") - assert len(edges) == 1 - assert edges[0].link == "X" - assert edges[0].predicate == "anything_goes" - - -def test_inline_arbitrary_predicate_preserved(): - edges = FileEdge.from_text("[wat:: [[X]]]") - assert len(edges) == 1 - assert edges[0].predicate == "wat" - - -def test_predicate_must_be_identifier_shaped(): - # A leading digit fails the regex `[A-Za-z][A-Za-z0-9_]*` so the line is - # not recognised as a Dataview field — the wikilink falls back to bare. - edges = FileEdge.from_text("123bad:: [[X]]") - assert len(edges) == 1 - assert edges[0].link == "X" - assert edges[0].predicate is None - - -def test_file_edge_accepts_any_predicate_string(): - e = FileEdge(link="X", predicate="totally_made_up") - assert e.predicate == "totally_made_up" - - -# -------------------------------------------------------------------------- -# Frontmatter is NOT walked for links — explicit regression -# -------------------------------------------------------------------------- - - -def test_frontmatter_links_block_is_ignored(): - # Even if a YAML-shaped string sits at the top of body, FileEdge.from_text - # only operates on body. We pass body text directly here, so this test - # asserts the API surface no longer accepts a metadata dict. - import inspect - - sig = inspect.signature(FileEdge.from_text) - assert list(sig.parameters.keys()) == ["text"], ( - "FileEdge.from_text should accept body text only — frontmatter walk removed" - ) - - -def test_no_frontmatter_walker_exported(): - from reme2.schema import file_edge as fe - - for removed in ( - "parse_wikilinks_from_metadata", - "extract_wikilinks_from_metadata", - "extract_inline_fields", - "extract_typed_edges", - "InlineField", - ): - assert not hasattr(fe, removed), ( - f"{removed} should have been removed when YAML edges were dropped" - ) - - -# -------------------------------------------------------------------------- -# FileEdge schema -# -------------------------------------------------------------------------- - - -def test_file_edge_extra_forbid(): - import pytest - from pydantic import ValidationError - - with pytest.raises(ValidationError): - FileEdge(link="X", target="X") # type: ignore[call-arg] - - -def test_file_edge_minimal_field_set(): - """Stored fields are just `link` and `predicate`. `path` and `anchor` - are `@property`s (not stored fields) so they shouldn't appear in - `model_dump()`. `target` / `alias` / `embed` were dropped entirely.""" - e = FileEdge(link="X") - dumped = e.model_dump() - assert set(dumped.keys()) == {"link", "predicate"} - for removed in ("target", "alias", "embed", "anchor", "path"): - assert removed not in dumped - - -def test_anchor_property_parses_from_link(): - """`anchor` is derived from `link`, not stored separately.""" - assert FileEdge(link="X").anchor is None - assert FileEdge(link="X#sec").anchor == "sec" - assert FileEdge(link="X#sec#more").anchor == "sec#more" - # Empty anchor is treated as no anchor. - assert FileEdge(link="X#").anchor is None - assert FileEdge(link="X# ").anchor is None - - -def test_path_property_parses_from_link(): - """`path` is derived from `link` — file part before any `#anchor`.""" - assert FileEdge(link="X").path == "X" - assert FileEdge(link="X#sec").path == "X" - assert FileEdge(link="topics/Foo").path == "topics/Foo" - assert FileEdge(link="topics/Foo#bar").path == "topics/Foo" - # Multiple '#' — only first splits; the rest live in anchor. - assert FileEdge(link="X#a#b").path == "X" - assert FileEdge(link="X#a#b").anchor == "a#b" - # Empty anchor → path is still the full prefix. - assert FileEdge(link="X#").path == "X" - - -def test_path_anchor_roundtrip_via_link(): - """Reconstructing `link` from `path` + `anchor` yields the original.""" - for link in ("X", "X#sec", "topics/Foo", "topics/Foo#bar", "X#a#b"): - e = FileEdge(link=link) - rebuilt = e.path if not e.anchor else f"{e.path}#{e.anchor}" - assert rebuilt == link, f"{link!r} → {rebuilt!r}" - - -def test_anchor_is_not_constructor_arg(): - """Since `anchor` and `path` are properties, passing them to the - constructor should fail (extra='forbid').""" - import pytest - from pydantic import ValidationError - - with pytest.raises(ValidationError): - FileEdge(link="X", anchor="sec") # type: ignore[call-arg] - with pytest.raises(ValidationError): - FileEdge(link="X", path="X") # type: ignore[call-arg] - - -# -------------------------------------------------------------------------- -# Back-compat: extract_wikilinks returns flat target list (used by ingestor) -# -------------------------------------------------------------------------- - - -def test_extract_wikilinks_flat_targets(): - targets = extract_wikilinks("see [[X]] and extends:: [[Y]] and [extends:: [[Z]]]") - assert targets == ["X", "Y", "Z"] - - -def test_extract_wikilinks_strips_anchor(): - """`extract_wikilinks` returns just the file part — anchor stripped - so callers can feed it to `resolve_wikilink`.""" - targets = extract_wikilinks("see [[X#sec]] and ![[Y#a|alias]]") - assert targets == ["X", "Y"] - - -# -------------------------------------------------------------------------- -# Source ordering stability -# -------------------------------------------------------------------------- - - -def test_edges_sorted_by_source_position(): - body = ( - "intro [[First]] then\n" - "extends:: [[Second]]\n" - "tail [[Third]]\n" - ) - edges = FileEdge.from_text(body) - assert [e.link for e in edges] == ["First", "Second", "Third"] diff --git a/tests/test_file_link.py b/tests/test_file_link.py new file mode 100644 index 00000000..2f5602dd --- /dev/null +++ b/tests/test_file_link.py @@ -0,0 +1,345 @@ +"""FileLink unit tests — body-only wikilink extraction. + +Covers: + * three legal inline forms (bare / line-level Dataview / inline-bracketed + Dataview) and multi-target expansion within each + * dedup against typed wrappers (no double-emit when a wikilink lives + inside a typed envelope) + * open-vocabulary predicate pass-through and the identifier-shape gate + * the explicit decision that frontmatter is no longer walked for links + (extraction takes body text only) + * the 3-field schema: ``(path, anchor, predicate)`` — all real fields, + extra='forbid' + +Pre-resolution form: ``iter_links(text)`` returns ``FileLink`` records with +``path`` set to the raw wikilink target as written. The resolver +(``utils.wikilink_resolver``) rewrites ``path`` to the vault-relative +resolved path; here we only test the extractor. +""" + +import inspect + +import pytest +from pydantic import ValidationError + +from reme2.schema import FileLink +from reme2.schema.file_link import extract_wikilinks, iter_links + + +# -------------------------------------------------------------------------- +# Bare wikilinks +# -------------------------------------------------------------------------- + + +def test_bare_wikilink(): + links = iter_links("see [[X]]") + assert len(links) == 1 + assert links[0].path == "X" + assert links[0].predicate is None + assert links[0].anchor is None + + +def test_anchor_split_from_target(): + """``[[X#sec]]`` → path='X', anchor='sec' (separate field, not derived).""" + links = iter_links("![[X#sec|Alias]]") + assert len(links) == 1 + assert links[0].path == "X" + assert links[0].anchor == "sec" + + +def test_alias_only_dropped(): + links = iter_links("[[X|Alias]]") + assert len(links) == 1 + assert links[0].path == "X" + assert links[0].anchor is None + + +def test_embed_only_dropped(): + links = iter_links("![[X]]") + assert len(links) == 1 + assert links[0].path == "X" + + +# -------------------------------------------------------------------------- +# Line-level Dataview +# -------------------------------------------------------------------------- + + +def test_line_level_field(): + links = iter_links("extends:: [[Source Topic]]") + assert len(links) == 1 + assert links[0].path == "Source Topic" + assert links[0].predicate == "extends" + + +def test_line_level_multi_target(): + links = iter_links("concerns:: [[A]], [[B]], [[C]]") + assert [(link.path, link.predicate) for link in links] == [ + ("A", "concerns"), + ("B", "concerns"), + ("C", "concerns"), + ] + + +def test_line_level_with_bullet(): + links = iter_links("- extends:: [[X]]\n * concerns:: [[Y]]") + assert [(link.path, link.predicate) for link in links] == [ + ("X", "extends"), + ("Y", "concerns"), + ] + + +# -------------------------------------------------------------------------- +# Multi-link cases — many wikilinks under one or several typed contexts +# -------------------------------------------------------------------------- + + +def test_multi_target_with_anchors_preserves_each(): + """Each comma-separated target keeps its own anchor as a separate field.""" + links = iter_links("extends:: [[A#sec1]], [[B#sec2]], [[C]]") + assert [(link.path, link.anchor, link.predicate) for link in links] == [ + ("A", "sec1", "extends"), + ("B", "sec2", "extends"), + ("C", None, "extends"), + ] + + +def test_multi_target_non_comma_separator_still_typed(): + """Wikilinks anywhere in the value range (not just comma-separated) + inherit the line's predicate. Useful for prose-style fields.""" + links = iter_links("extends:: [[A]] and also [[B]]") + assert [(link.path, link.predicate) for link in links] == [ + ("A", "extends"), + ("B", "extends"), + ] + + +def test_multi_dataview_lines_each_multi_target(): + """Multiple Dataview lines each with multi-target → all links typed + by their respective line's predicate.""" + links = iter_links( + "extends:: [[A]], [[B]]\nrelates:: [[C]], [[D]]", + ) + assert [(link.path, link.predicate) for link in links] == [ + ("A", "extends"), + ("B", "extends"), + ("C", "relates"), + ("D", "relates"), + ] + + +def test_inline_bracketed_then_bare_on_same_line(): + """Inline-bracketed governs only the wikilinks inside its brackets; + a trailing bare wikilink on the same line stays bare.""" + links = iter_links("[ext:: [[A]]] then [[B]]") + assert [(link.path, link.predicate) for link in links] == [ + ("A", "ext"), + ("B", None), + ] + + +def test_mid_line_dataview_like_not_typed(): + """``predicate::`` only counts at line start (modulo bullet) — + a ``predicate::`` mid-line is just prose, so its wikilinks are bare.""" + links = iter_links("[ext:: [[A]]] and concerns:: [[B]]") + assert [(link.path, link.predicate) for link in links] == [ + ("A", "ext"), + ("B", None), # `concerns::` mid-line is not Dataview + ] + + +# -------------------------------------------------------------------------- +# Inline-bracketed Dataview +# -------------------------------------------------------------------------- + + +def test_inline_bracketed(): + links = iter_links("This [extends:: [[Y]]] something else.") + assert len(links) == 1 + assert links[0].path == "Y" + assert links[0].predicate == "extends" + + +def test_inline_bracketed_multi_target(): + links = iter_links("[concerns:: [[A]], [[B]]]") + assert [(link.path, link.predicate) for link in links] == [ + ("A", "concerns"), + ("B", "concerns"), + ] + + +def test_inline_bracketed_skips_cross_line(): + # A `[predicate:: ...]` that spans a newline is malformed → the inner + # wikilink falls back to bare; the unmatched `[` does not eat tail text. + links = iter_links("[extends:: [[X]]\nbad]") + assert len(links) == 1 + assert links[0].path == "X" + assert links[0].predicate is None + + +# -------------------------------------------------------------------------- +# Dedup: a wikilink inside a typed wrapper should not double-emit +# -------------------------------------------------------------------------- + + +def test_inline_bracketed_does_not_double_emit(): + links = iter_links("see [extends:: [[X]]] again.") + assert len(links) == 1 + assert links[0].predicate == "extends" + + +def test_line_level_value_does_not_double_emit(): + links = iter_links("extends:: [[X]]") + assert len(links) == 1 + + +def test_typed_and_bare_coexist_for_same_target(): + links = iter_links("extends:: [[X]]\nFree text mentioning [[X]] again.") + paths_preds = sorted(((link.path, link.predicate or "") for link in links)) + assert paths_preds == [("X", ""), ("X", "extends")] + + +# -------------------------------------------------------------------------- +# Open-vocabulary predicates — any identifier-shaped token passes through +# -------------------------------------------------------------------------- + + +def test_arbitrary_predicate_preserved(): + links = iter_links("anything_goes:: [[X]]") + assert len(links) == 1 + assert links[0].path == "X" + assert links[0].predicate == "anything_goes" + + +def test_inline_arbitrary_predicate_preserved(): + links = iter_links("[wat:: [[X]]]") + assert len(links) == 1 + assert links[0].predicate == "wat" + + +def test_predicate_must_be_identifier_shaped(): + # A leading digit fails the regex `[A-Za-z][A-Za-z0-9_]*` so the line is + # not recognised as a Dataview field — the wikilink falls back to bare. + links = iter_links("123bad:: [[X]]") + assert len(links) == 1 + assert links[0].path == "X" + assert links[0].predicate is None + + +def test_file_link_accepts_any_predicate_string(): + link = FileLink(path="X", predicate="totally_made_up") + assert link.predicate == "totally_made_up" + + +# -------------------------------------------------------------------------- +# Frontmatter is NOT walked for links — explicit regression +# -------------------------------------------------------------------------- + + +def test_iter_links_takes_body_text_only(): + """``iter_links`` operates on body text — no frontmatter walking.""" + sig = inspect.signature(iter_links) + assert list(sig.parameters.keys()) == ["text"], "iter_links should accept body text only — frontmatter walk removed" + + +def test_no_frontmatter_walker_exported(): + from reme2.schema import file_link as fl + + for removed in ( + "parse_wikilinks_from_metadata", + "extract_wikilinks_from_metadata", + "extract_inline_fields", + "extract_typed_edges", + "InlineField", + ): + assert not hasattr(fl, removed), f"{removed} should have been removed when YAML links were dropped" + + +# -------------------------------------------------------------------------- +# FileLink schema +# -------------------------------------------------------------------------- + + +def test_file_link_extra_forbid(): + with pytest.raises(ValidationError): + FileLink(path="X", target="X") # type: ignore[call-arg] + + +def test_file_link_field_set(): + """Stored fields are ``(path, anchor, predicate)`` — no others.""" + link = FileLink(path="X") + dumped = link.model_dump() + assert set(dumped.keys()) == {"path", "anchor", "predicate"} + assert dumped == {"path": "X", "anchor": None, "predicate": None} + + +def test_file_link_dump_excludes_none_when_asked(): + link = FileLink(path="X") + assert link.model_dump(exclude_none=True) == {"path": "X"} + + +def test_file_link_full_construction(): + link = FileLink(path="topics/Foo.md", anchor="sec", predicate="extends") + assert link.path == "topics/Foo.md" + assert link.anchor == "sec" + assert link.predicate == "extends" + + +def test_file_link_path_required(): + with pytest.raises(ValidationError): + FileLink() # type: ignore[call-arg] + + +# -------------------------------------------------------------------------- +# Anchor extraction edge cases — anchor is a real field, captured by the +# regex's named group (not derived from path string-splitting) +# -------------------------------------------------------------------------- + + +def test_anchor_extraction_edge_cases(): + """``[[X]]`` → no anchor; ``[[X#sec]]`` → anchor='sec'. + The wikilink regex requires the anchor capture to be one or more + chars, so a literal ``[[X#]]`` doesn't match the regex at all (the + trailing ``#`` makes it invalid syntax). Whitespace-only anchors + are treated as no anchor (stripped to empty → None).""" + assert iter_links("[[X]]")[0].anchor is None + assert iter_links("[[X#sec]]")[0].anchor == "sec" + # `[[X#]]` is not a valid wikilink — anchor group requires 1+ chars. + assert iter_links("[[X#]]") == [] + # `[[X# ]]` matches but strips to empty → anchor=None. + assert iter_links("[[X# ]]")[0].anchor is None + + +def test_anchor_inside_pipe_alias_still_extracted(): + """Anchor is captured before the alias pipe.""" + links = iter_links("[[topics/Foo#sec|Display]]") + assert links[0].path == "topics/Foo" + assert links[0].anchor == "sec" + + +# -------------------------------------------------------------------------- +# Back-compat: extract_wikilinks returns flat target list (used by retriever) +# -------------------------------------------------------------------------- + + +def test_extract_wikilinks_flat_targets(): + targets = extract_wikilinks("see [[X]] and extends:: [[Y]] and [extends:: [[Z]]]") + assert targets == ["X", "Y", "Z"] + + +def test_extract_wikilinks_strips_anchor(): + """``extract_wikilinks`` returns just the file part — anchor stripped + so callers can feed it to ``resolve``.""" + targets = extract_wikilinks("see [[X#sec]] and ![[Y#a|alias]]") + assert targets == ["X", "Y"] + + +# -------------------------------------------------------------------------- +# Source ordering stability +# -------------------------------------------------------------------------- + + +def test_links_sorted_by_source_position(): + body = "intro [[First]] then\n" "extends:: [[Second]]\n" "tail [[Third]]\n" + links = iter_links(body) + assert [link.path for link in links] == ["First", "Second", "Third"] diff --git a/tests/test_md_chunker.py b/tests/test_md_chunker.py index ec128610..64f54945 100644 --- a/tests/test_md_chunker.py +++ b/tests/test_md_chunker.py @@ -45,19 +45,7 @@ def _all_headings(text: str) -> list[str]: def test_tree_groups_under_headings(): - txt = ( - "# Top\n" - "para1\n" - "\n" - "## Sub A\n" - "para2\n" - "\n" - "### Deeper\n" - "para3\n" - "\n" - "## Sub B\n" - "para4\n" - ) + txt = "# Top\n" "para1\n" "\n" "## Sub A\n" "para2\n" "\n" "### Deeper\n" "para3\n" "\n" "## Sub B\n" "para4\n" from mistletoe.block_token import Document from mistletoe.markdown_renderer import MarkdownRenderer @@ -142,23 +130,13 @@ def test_every_chunk_lists_every_doc_heading(): expected_headings = {"# Doc", "## Section A", "### Subsection", "## Section B"} for c in chunks: present = set(_all_headings(c.text)) - assert expected_headings <= present, ( - f"chunk missing headings {expected_headings - present}: {c.text!r}" - ) + assert expected_headings <= present, f"chunk missing headings {expected_headings - present}: {c.text!r}" def test_owner_section_holds_chunk_content(): """The chunk's content sits directly under its owner heading — not under any other section's heading.""" - txt = ( - "# Doc\n" - "\n" - "## A\n" - "alpha alpha alpha alpha here\n" - "\n" - "## B\n" - "bravo bravo bravo bravo here\n" - ) + txt = "# Doc\n" "\n" "## A\n" "alpha alpha alpha alpha here\n" "\n" "## B\n" "bravo bravo bravo bravo here\n" chunks = _parser(60)._chunk(txt, "/x.md") a_chunk = next(c for c in chunks if "alpha" in c.text) b_chunk = next(c for c in chunks if "bravo" in c.text) @@ -383,15 +361,7 @@ def test_paragraph_line_split_keeps_skeleton(): def test_embed_toc_off_strips_skeleton(): """With embed_toc=False, chunks contain only their own content — no full-doc heading skeleton wrapping them.""" - txt = ( - "# Doc\n" - "\n" - "## A\n" - "para A long content here\n" - "\n" - "## B\n" - "para B long content here\n" - ) + txt = "# Doc\n" "\n" "## A\n" "para A long content here\n" "\n" "## B\n" "para B long content here\n" chunks = _parser(60, embed_toc=False)._chunk(txt, "/x.md") a = next(c for c in chunks if "para A" in c.text) b = next(c for c in chunks if "para B" in c.text) @@ -456,9 +426,9 @@ def test_body_run_budget_excludes_toc(): txt = ( "# Top\n" "## Sub long heading title here\n" - "abcdefghij abcdefghij abcdefghij\n" # 32 chars body + "abcdefghij abcdefghij abcdefghij\n" # 32 chars body "\n" - "klmnopqrst klmnopqrst klmnopqrst\n" # 32 chars body + "klmnopqrst klmnopqrst klmnopqrst\n" # 32 chars body ) # 80 chars budget covers the joined body (32+2+32=66) but is # smaller than body+TOC under old (counting) semantics (~120). @@ -590,10 +560,7 @@ def test_paragraph_split_marks_parts(): "delta line four with extra padding text here\n" ) chunks = _parser(100)._chunk(txt, "/x.md") - para_chunks = [ - c for c in chunks - if any(t in c.text for t in ("alpha", "beta", "gamma", "delta")) - ] + para_chunks = [c for c in chunks if any(t in c.text for t in ("alpha", "beta", "gamma", "delta"))] assert len(para_chunks) >= 2 n = len(para_chunks) for i, c in enumerate(para_chunks, 1): @@ -602,15 +569,7 @@ def test_paragraph_split_marks_parts(): def test_single_piece_leaf_has_no_part_marker(): """When a leaf block fits in one piece, no [Part] prefix is added.""" - txt = ( - "# Doc\n" - "\n" - "## T\n" - "\n" - "| a | b |\n" - "|---|---|\n" - "| 1 | 2 |\n" - ) + txt = "# Doc\n" "\n" "## T\n" "\n" "| a | b |\n" "|---|---|\n" "| 1 | 2 |\n" chunks = _parser(500)._chunk(txt, "/x.md") assert len(chunks) == 1 assert "[Part" not in chunks[0].text @@ -642,11 +601,11 @@ def test_part_marker_absent_for_body_run_packing(): def test_chunk_line_ranges_track_source(): txt = ( - "# Top\n" # line 1 - "intro\n" # line 2 + "# Top\n" # line 1 + "intro\n" # line 2 "\n" - "## Sub\n" # line 4 - "body\n" # line 5 + "## Sub\n" # line 4 + "body\n" # line 5 ) chunks = _parser(500)._chunk(txt, "/x.md") assert len(chunks) == 1