mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-21 00:22:45 +00:00
feat: add file_graph component and integrate with LinkedFileParser
- Add file_graph import to component registry - Register FILE_GRAPH enum in ComponentEnum - Implement BaseFileGraph integration in LinkedFileParser - Replace FileEdge with FileLink for better semantic clarity - Add lazy resolution of file_graph from app_context - Update file watcher logging to reflect links instead of edges refactor: streamline MCP transport layer architecture - Remove redundant step shells from reme2/mcp/steps/ - Consolidate all @R.register components to reme2.memory package - Update server.py to import reme2.memory directly - Revise README.md to document new architecture - Simplify module dependencies and import structure
This commit is contained in:
parent
56b94c5952
commit
6a5561f86a
52 changed files with 2583 additions and 1470 deletions
|
|
@ -4,6 +4,7 @@ from . import as_llm
|
|||
from . import as_llm_formatter
|
||||
from . import as_token_counter
|
||||
from . import file_store
|
||||
from . import file_graph
|
||||
from . import client
|
||||
from . import embedding
|
||||
from . import file_parser
|
||||
|
|
@ -31,6 +32,7 @@ __all__ = [
|
|||
"as_llm_formatter",
|
||||
"as_token_counter",
|
||||
"file_store",
|
||||
"file_graph",
|
||||
"client",
|
||||
"embedding",
|
||||
"file_parser",
|
||||
|
|
|
|||
|
|
@ -21,11 +21,11 @@ class BaseComponent(ABC):
|
|||
component_type = ComponentEnum.BASE
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str | None = None,
|
||||
backend: str = "",
|
||||
app_context: "ApplicationContext | None" = None,
|
||||
**kwargs,
|
||||
self,
|
||||
name: str | None = None,
|
||||
backend: str = "",
|
||||
app_context: "ApplicationContext | None" = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.name: str = name or self.__class__.__name__
|
||||
self.backend: str = backend
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ class BaseEmbeddingModel(BaseComponent):
|
|||
|
||||
if to_compute:
|
||||
for i in range(0, len(to_compute), self.max_batch_size):
|
||||
batch = to_compute[i: i + self.max_batch_size]
|
||||
batch = to_compute[i : i + self.max_batch_size]
|
||||
indices = [idx for idx, _ in batch]
|
||||
texts = [text for _, text in batch]
|
||||
|
||||
|
|
@ -89,7 +89,7 @@ class BaseEmbeddingModel(BaseComponent):
|
|||
break
|
||||
except (TimeoutError, ConnectionError, OSError):
|
||||
if attempt < self.max_retries - 1:
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
await asyncio.sleep(2**attempt)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
|
|
@ -184,4 +184,4 @@ class BaseEmbeddingModel(BaseComponent):
|
|||
try:
|
||||
np.savez(self.cache_path, keys=np.array(keys, dtype=str), embeddings=embeddings)
|
||||
except Exception:
|
||||
pass
|
||||
pass
|
||||
|
|
|
|||
22
reme2/component/file_graph/__init__.py
Normal file
22
reme2/component/file_graph/__init__.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
"""File graph module.
|
||||
|
||||
Backend-agnostic graph engine that owns the wikilink graph (nodes +
|
||||
typed edges + traversal queries). Sister to ``file_store``, which owns
|
||||
chunks and projections (vector / FTS).
|
||||
|
||||
Two backends ship today:
|
||||
- ``LocalFileGraph`` (``@R.register("local")``) — networkx
|
||||
``MultiDiGraph`` + JSONL persistence. Default.
|
||||
- ``Neo4jFileGraph`` (``@R.register("neo4j")``) — property-graph
|
||||
backed by Neo4j. Requires the ``neo4j`` driver.
|
||||
"""
|
||||
|
||||
from .base_file_graph import BaseFileGraph
|
||||
from .local_file_graph import LocalFileGraph
|
||||
from .neo4j_file_graph import Neo4jFileGraph
|
||||
|
||||
__all__ = [
|
||||
"BaseFileGraph",
|
||||
"LocalFileGraph",
|
||||
"Neo4jFileGraph",
|
||||
]
|
||||
76
reme2/component/file_graph/base_file_graph.py
Normal file
76
reme2/component/file_graph/base_file_graph.py
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
"""Abstract base for the file-graph engine.
|
||||
|
||||
The file-graph owns ``FileNode`` records keyed by vault-relative path
|
||||
and serves graph traversal over wikilink links. file_graph trusts
|
||||
``FileLink.path`` directly — there is no internal wikilink resolution.
|
||||
Pre-resolution forms (raw stem-form wikilinks like ``[[Foo]]``) are a
|
||||
vault convention resolved by the external ``utils.wikilink_resolver``
|
||||
before nodes are upserted.
|
||||
|
||||
Contract — six abstract methods, two blocks:
|
||||
|
||||
Node CRUD upsert_node, delete_node, get_node, iter_nodes
|
||||
Link access get_outlinks, get_inlinks
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from abc import abstractmethod
|
||||
from collections.abc import AsyncIterator
|
||||
from pathlib import Path
|
||||
|
||||
from ..base_component import BaseComponent
|
||||
from ...enumeration import ComponentEnum
|
||||
from ...schema import FileLink, FileNode
|
||||
|
||||
|
||||
class BaseFileGraph(BaseComponent):
|
||||
"""Pluggable file-graph backend — node CRUD + link adjacency."""
|
||||
|
||||
component_type = ComponentEnum.FILE_GRAPH
|
||||
|
||||
def __init__(self, store_name: str = "default", **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
if not re.match(r"^[a-zA-Z0-9_]+$", store_name):
|
||||
raise ValueError(
|
||||
f"Invalid store name '{store_name}'. " f"Only alphanumeric and underscores allowed.",
|
||||
)
|
||||
self.store_name: str = store_name
|
||||
self.working_dir: str = self.app_context.app_config.working_dir if self.app_context else ""
|
||||
self.store_path: Path = Path(self.working_dir) / "file_graph" / store_name
|
||||
self.store_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# -- Node CRUD ---------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
async def upsert_node(self, node: FileNode) -> None:
|
||||
"""Add or replace a node and its outgoing links."""
|
||||
|
||||
@abstractmethod
|
||||
async def delete_node(self, path: str) -> FileNode | None:
|
||||
"""Remove a node + its incident links. Returns the removed node, or None."""
|
||||
|
||||
@abstractmethod
|
||||
async def get_node(self, path: str) -> FileNode | None:
|
||||
"""Single-node lookup."""
|
||||
|
||||
@abstractmethod
|
||||
def iter_nodes(self) -> AsyncIterator[tuple[str, FileNode]]:
|
||||
"""Walk every (path, FileNode) in the graph."""
|
||||
|
||||
# -- Link access -------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
async def get_outlinks(
|
||||
self,
|
||||
path: str,
|
||||
) -> list[tuple[FileNode, FileLink]]:
|
||||
"""Resolved outgoing links from ``path``."""
|
||||
|
||||
@abstractmethod
|
||||
async def get_inlinks(
|
||||
self,
|
||||
path: str,
|
||||
) -> list[tuple[FileNode, FileLink]]:
|
||||
"""Resolved incoming links to ``path``."""
|
||||
152
reme2/component/file_graph/local_file_graph.py
Normal file
152
reme2/component/file_graph/local_file_graph.py
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
"""Local file-graph backend — networkx ``MultiDiGraph``.
|
||||
|
||||
Each ``FileNode`` lives as ``data['node']`` on the graph; each
|
||||
``FileLink`` whose ``link.path`` exists in the graph lives as
|
||||
``data['link']`` on a directed graph edge between the two file nodes.
|
||||
|
||||
file_graph trusts ``link.path`` directly — there is no internal
|
||||
wikilink resolution. The parser pipeline (with the external resolver)
|
||||
is responsible for producing safe ``FileLink`` records where
|
||||
``link.path`` is a real vault-relative target path. Stem ambiguity is
|
||||
already handled there by emitting one link per candidate.
|
||||
|
||||
Late-arriving target: when ``upsert_node(B)`` runs, other nodes whose
|
||||
links have ``link.path == B.path`` get those in-edges restored
|
||||
(one O(N×L) sweep per upsert; cheap for vault sizes).
|
||||
|
||||
Persistence: pickle to ``graph.pkl`` on close, restored on _start.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pickle
|
||||
from collections.abc import AsyncIterator
|
||||
from pathlib import Path
|
||||
|
||||
from .base_file_graph import BaseFileGraph
|
||||
from ..component_registry import R
|
||||
from ...schema import FileLink, FileNode
|
||||
|
||||
|
||||
@R.register("local")
|
||||
class LocalFileGraph(BaseFileGraph):
|
||||
"""Networkx-backed file graph. Trusts ``FileLink.path`` for adjacency."""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self._graph = None # networkx.MultiDiGraph; set in _start
|
||||
self._graph_file: Path = self.store_path / "graph.pkl"
|
||||
|
||||
# -- Lifecycle ---------------------------------------------------------
|
||||
|
||||
async def _start(self) -> None:
|
||||
await super()._start()
|
||||
try:
|
||||
import networkx as nx
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"LocalFileGraph requires networkx. Install with `pip install networkx`.",
|
||||
) from e
|
||||
self._graph = self._load_graph(nx) or nx.MultiDiGraph()
|
||||
self.logger.info(
|
||||
f"LocalFileGraph '{self.store_name}' ready: "
|
||||
f"{self._graph.number_of_nodes()} nodes, "
|
||||
f"{self._graph.number_of_edges()} edges",
|
||||
)
|
||||
|
||||
async def _close(self) -> None:
|
||||
self._save_graph()
|
||||
await super()._close()
|
||||
|
||||
def _load_graph(self, nx):
|
||||
if not self._graph_file.exists():
|
||||
return None
|
||||
try:
|
||||
with open(self._graph_file, "rb") as f:
|
||||
graph = pickle.load(f)
|
||||
if not isinstance(graph, nx.MultiDiGraph):
|
||||
self.logger.warning(
|
||||
f"{self._graph_file} is not a MultiDiGraph; ignoring",
|
||||
)
|
||||
return None
|
||||
return graph
|
||||
except Exception as e:
|
||||
self.logger.exception(f"Failed to load {self._graph_file}: {e}")
|
||||
return None
|
||||
|
||||
def _save_graph(self) -> None:
|
||||
try:
|
||||
tmp = self._graph_file.with_suffix(".tmp")
|
||||
with open(tmp, "wb") as f:
|
||||
pickle.dump(self._graph, f, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
tmp.replace(self._graph_file)
|
||||
except Exception as e:
|
||||
self.logger.exception(f"Failed to write {self._graph_file}: {e}")
|
||||
|
||||
# -- Node CRUD ---------------------------------------------------------
|
||||
|
||||
async def upsert_node(self, node: FileNode) -> None:
|
||||
path = node.path
|
||||
if self._graph.has_node(path):
|
||||
self._graph.remove_node(path)
|
||||
self._graph.add_node(path, node=node)
|
||||
|
||||
# Out-links from this node — trust link.path.
|
||||
for link in node.links:
|
||||
if link.path and self._graph.has_node(link.path):
|
||||
self._graph.add_edge(path, link.path, link=link)
|
||||
|
||||
# Late-arriving target: restore in-links from other nodes whose
|
||||
# links resolve here. Scan all other nodes' links; cheap for
|
||||
# vault sizes (O(N×L_avg) per upsert).
|
||||
for src_path, src_data in self._graph.nodes(data=True):
|
||||
if src_path == path:
|
||||
continue
|
||||
src_node = src_data.get("node")
|
||||
if src_node is None:
|
||||
continue
|
||||
for link in src_node.links:
|
||||
if link.path == path:
|
||||
self._graph.add_edge(src_path, path, link=link)
|
||||
|
||||
async def delete_node(self, path: str) -> FileNode | None:
|
||||
if not self._graph.has_node(path):
|
||||
return None
|
||||
node = self._graph.nodes[path].get("node")
|
||||
self._graph.remove_node(path)
|
||||
return node
|
||||
|
||||
async def get_node(self, path: str) -> FileNode | None:
|
||||
if not self._graph.has_node(path):
|
||||
return None
|
||||
return self._graph.nodes[path].get("node")
|
||||
|
||||
async def iter_nodes(self) -> AsyncIterator[tuple[str, FileNode]]:
|
||||
for path, data in self._graph.nodes(data=True):
|
||||
node = data.get("node")
|
||||
if node is not None:
|
||||
yield path, node
|
||||
|
||||
# -- Link access -------------------------------------------------------
|
||||
|
||||
async def get_outlinks(self, path: str) -> list[tuple[FileNode, FileLink]]:
|
||||
if not self._graph.has_node(path):
|
||||
return []
|
||||
out: list[tuple[FileNode, FileLink]] = []
|
||||
for _src, dst, data in self._graph.out_edges(path, data=True):
|
||||
target = self._graph.nodes[dst].get("node")
|
||||
link = data.get("link")
|
||||
if target is not None and isinstance(link, FileLink):
|
||||
out.append((target, link))
|
||||
return out
|
||||
|
||||
async def get_inlinks(self, path: str) -> list[tuple[FileNode, FileLink]]:
|
||||
if not self._graph.has_node(path):
|
||||
return []
|
||||
out: list[tuple[FileNode, FileLink]] = []
|
||||
for src, _dst, data in self._graph.in_edges(path, data=True):
|
||||
source = self._graph.nodes[src].get("node")
|
||||
link = data.get("link")
|
||||
if source is not None and isinstance(link, FileLink):
|
||||
out.append((source, link))
|
||||
return out
|
||||
323
reme2/component/file_graph/neo4j_file_graph.py
Normal file
323
reme2/component/file_graph/neo4j_file_graph.py
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
"""Neo4j-backed file graph.
|
||||
|
||||
Property-graph mapping:
|
||||
|
||||
(:File {path, st_mtime, title, description, tags, links_json,
|
||||
extra_json})
|
||||
-[:LINKS {idx, anchor, predicate}]->(:File)
|
||||
|
||||
``path`` is the unique key (constraint enforced on ``_start``).
|
||||
Frontmatter goes into flat properties; arbitrary extras land in
|
||||
``extra_json``. The full ``FileLink[]`` payload is also stored as
|
||||
``links_json`` so we can recover the node losslessly even for links
|
||||
whose target wasn't indexed at upsert time (will be linked later by
|
||||
``_restore_inlinks``).
|
||||
|
||||
Adjacency policy: file_graph trusts ``FileLink.path`` directly — no
|
||||
internal wikilink resolution. The parser pipeline (with the external
|
||||
resolver) produces safe links where ``link.path`` is already a
|
||||
vault-relative target path.
|
||||
|
||||
Conditional dependency: the ``neo4j`` driver is loaded lazily; the
|
||||
import error fires at ``_start`` (boot), not at first call.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
from .base_file_graph import BaseFileGraph
|
||||
from ..component_registry import R
|
||||
from ...schema import FileLink, FileNode
|
||||
from ...schema.file_node import FileFrontMatter
|
||||
|
||||
|
||||
_TYPED_FRONTMATTER_FIELDS = {"title", "description", "tags"}
|
||||
_LINK_FIELDS = {"path", "anchor", "predicate"}
|
||||
|
||||
|
||||
@R.register("neo4j")
|
||||
class Neo4jFileGraph(BaseFileGraph):
|
||||
"""Neo4j-backed file graph; trusts ``FileLink.path`` for adjacency.
|
||||
|
||||
Connection params (constructor kwargs):
|
||||
uri: bolt URL, e.g. ``bolt://localhost:7687``
|
||||
user: auth user (default ``neo4j``)
|
||||
password: auth password
|
||||
database: target db name (default ``neo4j``)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
uri: str = "bolt://localhost:7687",
|
||||
user: str = "neo4j",
|
||||
password: str = "neo4j",
|
||||
database: str = "neo4j",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self._uri: str = uri
|
||||
self._user: str = user
|
||||
self._password: str = password
|
||||
self._database: str = database
|
||||
self._driver = None
|
||||
|
||||
# -- Lifecycle ---------------------------------------------------------
|
||||
|
||||
async def _start(self) -> None:
|
||||
await super()._start()
|
||||
try:
|
||||
from neo4j import AsyncGraphDatabase
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"Neo4jFileGraph requires the neo4j driver. " "Install with `pip install neo4j`.",
|
||||
) from e
|
||||
self._driver = AsyncGraphDatabase.driver(
|
||||
self._uri,
|
||||
auth=(self._user, self._password),
|
||||
)
|
||||
async with self._session() as session:
|
||||
await session.run(
|
||||
"CREATE CONSTRAINT file_path_unique IF NOT EXISTS " "FOR (f:File) REQUIRE f.path IS UNIQUE",
|
||||
)
|
||||
self.logger.info(
|
||||
f"Neo4jFileGraph '{self.store_name}' connected at " f"{self._uri}/{self._database}",
|
||||
)
|
||||
|
||||
async def _close(self) -> None:
|
||||
if self._driver is not None:
|
||||
await self._driver.close()
|
||||
self._driver = None
|
||||
await super()._close()
|
||||
|
||||
def _session(self):
|
||||
assert self._driver is not None, "Neo4jFileGraph not started"
|
||||
return self._driver.session(database=self._database)
|
||||
|
||||
# -- Node CRUD ---------------------------------------------------------
|
||||
|
||||
async def upsert_node(self, node: FileNode) -> None:
|
||||
props = self._node_props(node)
|
||||
# Trust link.path; only emit LINKS for links with non-empty path.
|
||||
link_payload = [
|
||||
{
|
||||
"idx": i,
|
||||
"anchor": link.anchor,
|
||||
"predicate": link.predicate,
|
||||
"target": link.path,
|
||||
}
|
||||
for i, link in enumerate(node.links)
|
||||
if link.path
|
||||
]
|
||||
async with self._session() as session:
|
||||
await session.execute_write(
|
||||
self._upsert_node_tx,
|
||||
node.path,
|
||||
props,
|
||||
link_payload,
|
||||
)
|
||||
# Late-arriving target: restore in-links from other nodes whose
|
||||
# links resolve here. Separate session call (not in tx) so it
|
||||
# doesn't block the upsert ack.
|
||||
await self._restore_inlinks(node.path)
|
||||
|
||||
@staticmethod
|
||||
async def _upsert_node_tx(tx, path, props, links):
|
||||
await tx.run(
|
||||
"MERGE (f:File {path: $path}) SET f += $props",
|
||||
path=path,
|
||||
props=props,
|
||||
)
|
||||
await tx.run(
|
||||
"MATCH (f:File {path: $path})-[r:LINKS]->() DELETE r",
|
||||
path=path,
|
||||
)
|
||||
# MERGE only when the target node exists. ``OPTIONAL MATCH`` +
|
||||
# ``WHERE t IS NOT NULL`` lets us batch this in one pass per link.
|
||||
for link in links:
|
||||
await tx.run(
|
||||
"""
|
||||
MATCH (s:File {path: $src})
|
||||
OPTIONAL MATCH (t:File {path: $dst})
|
||||
WITH s, t WHERE t IS NOT NULL
|
||||
MERGE (s)-[r:LINKS {idx: $idx}]->(t)
|
||||
SET r.anchor = $anchor, r.predicate = $predicate
|
||||
""",
|
||||
src=path,
|
||||
dst=link["target"],
|
||||
idx=link["idx"],
|
||||
anchor=link["anchor"],
|
||||
predicate=link["predicate"],
|
||||
)
|
||||
|
||||
async def _restore_inlinks(self, path: str) -> None:
|
||||
"""Walk other nodes' links, add LINKS to ``path`` where ``link.path``
|
||||
matches. Uses ``links_json`` on each node — no resolution logic."""
|
||||
async with self._session() as session:
|
||||
rec = await session.run(
|
||||
"""
|
||||
MATCH (f:File)
|
||||
WHERE f.path <> $path
|
||||
RETURN f.path AS p, f.links_json AS l
|
||||
""",
|
||||
path=path,
|
||||
)
|
||||
rows = [dict(r) async for r in rec]
|
||||
for row in rows:
|
||||
try:
|
||||
links = json.loads(row.get("l") or "[]")
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
for i, link in enumerate(links):
|
||||
if not isinstance(link, dict) or link.get("path") != path:
|
||||
continue
|
||||
async with self._session() as session:
|
||||
await session.run(
|
||||
"""
|
||||
MATCH (s:File {path: $src})
|
||||
MATCH (t:File {path: $dst})
|
||||
MERGE (s)-[r:LINKS {idx: $idx}]->(t)
|
||||
SET r.anchor = $anchor, r.predicate = $predicate
|
||||
""",
|
||||
src=row["p"],
|
||||
dst=path,
|
||||
idx=i,
|
||||
anchor=link.get("anchor"),
|
||||
predicate=link.get("predicate"),
|
||||
)
|
||||
|
||||
async def delete_node(self, path: str) -> FileNode | None:
|
||||
node = await self.get_node(path)
|
||||
if node is None:
|
||||
return None
|
||||
async with self._session() as session:
|
||||
await session.run(
|
||||
"MATCH (f:File {path: $path}) DETACH DELETE f",
|
||||
path=path,
|
||||
)
|
||||
return node
|
||||
|
||||
async def get_node(self, path: str) -> FileNode | None:
|
||||
async with self._session() as session:
|
||||
rec = await session.run(
|
||||
"MATCH (f:File {path: $path}) RETURN f LIMIT 1",
|
||||
path=path,
|
||||
)
|
||||
row = await rec.single()
|
||||
return self._row_to_node(row["f"]) if row else None
|
||||
|
||||
async def iter_nodes(self) -> AsyncIterator[tuple[str, FileNode]]:
|
||||
async with self._session() as session:
|
||||
rec = await session.run("MATCH (f:File) RETURN f")
|
||||
async for row in rec:
|
||||
node = self._row_to_node(row["f"])
|
||||
yield node.path, node
|
||||
|
||||
# -- Link access -------------------------------------------------------
|
||||
|
||||
async def get_outlinks(self, path: str) -> list[tuple[FileNode, FileLink]]:
|
||||
async with self._session() as session:
|
||||
rec = await session.run(
|
||||
"""
|
||||
MATCH (s:File {path: $path})-[r:LINKS]->(t:File)
|
||||
RETURN t, r ORDER BY r.idx ASC
|
||||
""",
|
||||
path=path,
|
||||
)
|
||||
rows = [dict(row) async for row in rec]
|
||||
return [(self._row_to_node(r["t"]), self._rel_to_link(r["r"], r["t"])) for r in rows]
|
||||
|
||||
async def get_inlinks(self, path: str) -> list[tuple[FileNode, FileLink]]:
|
||||
async with self._session() as session:
|
||||
rec = await session.run(
|
||||
"""
|
||||
MATCH (s:File)-[r:LINKS]->(t:File {path: $path})
|
||||
RETURN s, r ORDER BY s.path ASC, r.idx ASC
|
||||
""",
|
||||
path=path,
|
||||
)
|
||||
rows = [dict(row) async for row in rec]
|
||||
# ``rel_to_link`` needs the target's path to populate FileLink.path.
|
||||
return [(self._row_to_node(r["s"]), self._rel_to_link(r["r"], target_path=path)) for r in rows]
|
||||
|
||||
# -- Internal: row ↔ schema marshaling ---------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _node_props(node: FileNode) -> dict[str, Any]:
|
||||
fm = node.front_matter
|
||||
extras = dict(fm.__pydantic_extra__ or {})
|
||||
return {
|
||||
"path": node.path,
|
||||
"st_mtime": float(node.st_mtime),
|
||||
"title": fm.title or "",
|
||||
"description": fm.description or "",
|
||||
"tags": list(fm.tags or []),
|
||||
"links_json": json.dumps(
|
||||
[link.model_dump(exclude_none=True) for link in node.links],
|
||||
ensure_ascii=False,
|
||||
),
|
||||
"extra_json": json.dumps(extras, ensure_ascii=False, sort_keys=True),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _row_to_node(row) -> FileNode:
|
||||
d = dict(row)
|
||||
try:
|
||||
extras = json.loads(d.get("extra_json") or "{}")
|
||||
except json.JSONDecodeError:
|
||||
extras = {}
|
||||
try:
|
||||
links_raw = json.loads(d.get("links_json") or "[]")
|
||||
except json.JSONDecodeError:
|
||||
links_raw = []
|
||||
links: list[FileLink] = []
|
||||
for link in links_raw:
|
||||
if not isinstance(link, dict):
|
||||
continue
|
||||
# Defensive: strip any keys the schema doesn't recognise
|
||||
# (e.g. legacy fields from prior schema versions).
|
||||
clean = {k: v for k, v in link.items() if k in _LINK_FIELDS}
|
||||
try:
|
||||
links.append(FileLink(**clean))
|
||||
except Exception:
|
||||
continue
|
||||
fm_kwargs: dict[str, Any] = {
|
||||
"title": d.get("title", "") or "",
|
||||
"description": d.get("description", "") or "",
|
||||
"tags": d.get("tags") or None,
|
||||
}
|
||||
fm_kwargs.update(
|
||||
{k: v for k, v in extras.items() if k not in _TYPED_FRONTMATTER_FIELDS},
|
||||
)
|
||||
return FileNode(
|
||||
path=d["path"],
|
||||
st_mtime=float(d.get("st_mtime", 0.0)),
|
||||
links=links,
|
||||
chunk_ids=[],
|
||||
front_matter=FileFrontMatter(**fm_kwargs),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _rel_to_link(rel, target_path: Any = None) -> FileLink:
|
||||
"""Reconstitute a ``FileLink`` from a Neo4j relationship.
|
||||
|
||||
The relationship row carries ``anchor`` and ``predicate``; the
|
||||
target's ``path`` comes from the matched ``File`` node (passed
|
||||
in by callers that have it on hand). For ``get_outlinks`` it's
|
||||
the row's target node; for ``get_inlinks`` it's the path the
|
||||
caller queried for. Either way, FileLink.path is set so the
|
||||
link stays "safe by construction".
|
||||
"""
|
||||
d = dict(rel)
|
||||
if hasattr(target_path, "get"):
|
||||
# neo4j Node passed in
|
||||
path = target_path.get("path", "")
|
||||
else:
|
||||
path = target_path or ""
|
||||
return FileLink(
|
||||
path=path,
|
||||
anchor=d.get("anchor"),
|
||||
predicate=d.get("predicate"),
|
||||
)
|
||||
|
|
@ -17,7 +17,6 @@ class BaseFileParser(BaseComponent):
|
|||
super().__init__(**kwargs)
|
||||
self.working_dir = self.app_context.app_config.working_dir if self.app_context is not None else ""
|
||||
|
||||
|
||||
def _get_relative_path(self, path: str | Path) -> str:
|
||||
"""Get path relative to working_dir."""
|
||||
file_path = Path(path).absolute()
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ class DefaultFileParser(BaseFileParser):
|
|||
data = {}
|
||||
|
||||
front_matter = FileFrontMatter(**data)
|
||||
remaining = text[end_idx + 4:].lstrip("\n")
|
||||
remaining = text[end_idx + 4 :].lstrip("\n")
|
||||
return front_matter, remaining
|
||||
|
||||
async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
|
||||
|
|
@ -72,18 +72,25 @@ class DefaultFileParser(BaseFileParser):
|
|||
if content_bytes[end - 1] == ord(b"\n"):
|
||||
end_line -= 1
|
||||
|
||||
chunks.append(FileChunk(
|
||||
path=rel_path,
|
||||
start_line=start_line,
|
||||
end_line=end_line,
|
||||
text=chunk_text,
|
||||
).set_hash_id())
|
||||
chunks.append(
|
||||
FileChunk(
|
||||
path=rel_path,
|
||||
start_line=start_line,
|
||||
end_line=end_line,
|
||||
text=chunk_text,
|
||||
).set_hash_id(),
|
||||
)
|
||||
|
||||
if end >= len(content_bytes):
|
||||
break
|
||||
start += step
|
||||
|
||||
return FileNode(path=rel_path,
|
||||
st_mtime=stat.st_mtime,
|
||||
front_matter=front_matter,
|
||||
chunk_ids=[c.id for c in chunks]), chunks
|
||||
return (
|
||||
FileNode(
|
||||
path=rel_path,
|
||||
st_mtime=stat.st_mtime,
|
||||
front_matter=front_matter,
|
||||
chunk_ids=[c.id for c in chunks],
|
||||
),
|
||||
chunks,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -16,25 +16,20 @@ from pathlib import Path
|
|||
from typing import Any
|
||||
|
||||
import frontmatter
|
||||
from mistletoe.block_token import (
|
||||
CodeFence,
|
||||
Document,
|
||||
Heading,
|
||||
List,
|
||||
ListItem,
|
||||
SetextHeading,
|
||||
Table,
|
||||
TableRow,
|
||||
)
|
||||
from mistletoe.markdown_renderer import BlankLine, MarkdownRenderer
|
||||
|
||||
|
||||
from .base_file_parser import BaseFileParser
|
||||
from ..component_registry import R
|
||||
from ...schema import FileChunk, FileEdge, FileFrontMatter, FileNode
|
||||
from ..file_graph import BaseFileGraph
|
||||
from ...enumeration import ComponentEnum
|
||||
from ...schema import (
|
||||
FileChunk,
|
||||
FileLink,
|
||||
FileFrontMatter,
|
||||
FileNode,
|
||||
)
|
||||
from ...utils import hash_text
|
||||
|
||||
_PART_RESERVE = 18 # worst-case "[Part NNN/NNN]\n\n" prefix
|
||||
|
||||
from ...utils.wikilink_resolver import text_to_links
|
||||
|
||||
# -- AST node + helpers ---------------------------------------------------
|
||||
|
||||
|
|
@ -63,7 +58,7 @@ class MdNode:
|
|||
desc_toc: str = ""
|
||||
|
||||
|
||||
def _heading_text(node: Any, renderer: MarkdownRenderer) -> str:
|
||||
def _heading_text(node: Any, renderer) -> str:
|
||||
"""Heading text without `#` markers (for outline)."""
|
||||
rendered = renderer.render(node).rstrip("\n")
|
||||
if rendered.startswith("#"):
|
||||
|
|
@ -71,14 +66,15 @@ def _heading_text(node: Any, renderer: MarkdownRenderer) -> str:
|
|||
return rendered.split("\n", 1)[0].strip()
|
||||
|
||||
|
||||
def _dedup_edges(edges: list[FileEdge]) -> list[FileEdge]:
|
||||
def _dedup_links(links: list[FileLink]) -> list[FileLink]:
|
||||
"""Drop links with identical (path, predicate, anchor); preserve order."""
|
||||
seen: set[tuple] = set()
|
||||
out: list[FileEdge] = []
|
||||
for e in edges:
|
||||
key = (e.link, e.predicate)
|
||||
out: list[FileLink] = []
|
||||
for link in links:
|
||||
key = (link.path, link.predicate, link.anchor)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
out.append(e)
|
||||
out.append(link)
|
||||
return out
|
||||
|
||||
|
||||
|
|
@ -132,41 +128,75 @@ class LinkedFileParser(BaseFileParser):
|
|||
"""Markdown parser: frontmatter + wikilink edges + full-skeleton chunks."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
encoding: str = "utf-8",
|
||||
chunk_chars: int = 2000,
|
||||
embed_toc: bool = True,
|
||||
**kwargs,
|
||||
self,
|
||||
encoding: str = "utf-8",
|
||||
chunk_chars: int = 2000,
|
||||
embed_toc: bool = True,
|
||||
file_graph: str = "default",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.encoding = encoding
|
||||
self.chunk_chars = max(100, chunk_chars)
|
||||
self.embed_toc = embed_toc
|
||||
self._file_graph_name: str = file_graph
|
||||
|
||||
def _resolve_file_graph(self) -> BaseFileGraph | None:
|
||||
"""Lazily fetch the configured file_graph from app_context.
|
||||
|
||||
Lazy (rather than ``_start``) so the parser doesn't impose a
|
||||
component start-order constraint, and so tests can construct
|
||||
the parser without a graph wired up.
|
||||
"""
|
||||
if self.app_context is None:
|
||||
return None
|
||||
graphs = self.app_context.components.get(ComponentEnum.FILE_GRAPH, {})
|
||||
graph = graphs.get(self._file_graph_name)
|
||||
if graph is None:
|
||||
return None
|
||||
if not isinstance(graph, BaseFileGraph):
|
||||
raise TypeError(
|
||||
f"Expected BaseFileGraph, got {type(graph).__name__}",
|
||||
)
|
||||
return graph
|
||||
|
||||
async def parse(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]:
|
||||
from mistletoe.markdown_renderer import MarkdownRenderer
|
||||
from mistletoe.block_token import Document
|
||||
|
||||
file_path = Path(path)
|
||||
post = frontmatter.loads(file_path.read_text(encoding=self.encoding))
|
||||
absolute = str(file_path.absolute())
|
||||
|
||||
chunks: list[FileChunk] = []
|
||||
if post.content and post.content.strip():
|
||||
with MarkdownRenderer() as renderer:
|
||||
tree = self._build_tree(Document(post.content), renderer)
|
||||
chunks = self._chunk_node(tree, "", "", absolute, renderer)
|
||||
chunks = self._chunk_node(tree, "", "", str(file_path), renderer)
|
||||
|
||||
links: list[FileLink] = []
|
||||
graph = self._resolve_file_graph()
|
||||
if graph is not None:
|
||||
links = _dedup_links(await text_to_links(graph, post.content))
|
||||
|
||||
node = FileNode(
|
||||
path=absolute,
|
||||
path=str(file_path),
|
||||
st_mtime=file_path.stat().st_mtime,
|
||||
chunk_ids=[chunk.id for chunk in chunks],
|
||||
edges=_dedup_edges(FileEdge.from_text(post.content)),
|
||||
links=links,
|
||||
front_matter=FileFrontMatter(**dict(post.metadata)),
|
||||
)
|
||||
return node, chunks
|
||||
|
||||
def _build_tree(self, doc: Any, renderer: MarkdownRenderer) -> MdNode:
|
||||
def _build_tree(self, doc: Any, renderer) -> MdNode:
|
||||
"""Heading-level stack folds mistletoe's flat children into nested
|
||||
sections; non-headings attach as ``body`` to the current section
|
||||
(or root before the first heading)."""
|
||||
from mistletoe.markdown_renderer import BlankLine
|
||||
from mistletoe.block_token import (
|
||||
Heading,
|
||||
SetextHeading,
|
||||
)
|
||||
|
||||
root = MdNode(kind="root", start_line=1, end_line=1)
|
||||
stack: list[MdNode] = [root]
|
||||
for child in doc.children or []:
|
||||
|
|
@ -189,18 +219,27 @@ class LinkedFileParser(BaseFileParser):
|
|||
rendered = renderer.render(child).rstrip("\n")
|
||||
if not rendered:
|
||||
continue
|
||||
stack[-1].children.append(MdNode(
|
||||
kind="body", block=child, text=rendered,
|
||||
start_line=line, end_line=line + rendered.count("\n"),
|
||||
))
|
||||
stack[-1].children.append(
|
||||
MdNode(
|
||||
kind="body",
|
||||
block=child,
|
||||
text=rendered,
|
||||
start_line=line,
|
||||
end_line=line + rendered.count("\n"),
|
||||
),
|
||||
)
|
||||
_finalize(root)
|
||||
return root
|
||||
|
||||
# -- Recursive chunker ------------------------------------------------
|
||||
|
||||
def _chunk_node(
|
||||
self, node: MdNode, before: str, after: str,
|
||||
path: str, renderer: MarkdownRenderer,
|
||||
self,
|
||||
node: MdNode,
|
||||
before: str,
|
||||
after: str,
|
||||
path: str,
|
||||
renderer,
|
||||
) -> list[FileChunk]:
|
||||
"""Try the whole subtree; on overflow split (leaf) or descend.
|
||||
``before``/``after`` are TOC fragments that bracket each emitted
|
||||
|
|
@ -216,8 +255,16 @@ class LinkedFileParser(BaseFileParser):
|
|||
else:
|
||||
before_self = before
|
||||
if len(node.text) <= self.chunk_chars:
|
||||
return [self._make_chunk(before_self, node.text, after,
|
||||
node.start_line, node.end_line, path)]
|
||||
return [
|
||||
self._make_chunk(
|
||||
before_self,
|
||||
node.text,
|
||||
after,
|
||||
node.start_line,
|
||||
node.end_line,
|
||||
path,
|
||||
),
|
||||
]
|
||||
if node.kind == "body":
|
||||
return self._split_leaf(node, before, after, path, renderer)
|
||||
after_inside = _toc_join(node.desc_toc, after)
|
||||
|
|
@ -229,32 +276,65 @@ class LinkedFileParser(BaseFileParser):
|
|||
for c in node.children:
|
||||
if c.kind == "section":
|
||||
if run:
|
||||
chunks.extend(self._chunk_body_run(
|
||||
run, before_self, after_inside, path, renderer))
|
||||
chunks.extend(
|
||||
self._chunk_body_run(
|
||||
run,
|
||||
before_self,
|
||||
after_inside,
|
||||
path,
|
||||
renderer,
|
||||
),
|
||||
)
|
||||
run = []
|
||||
remaining = "\n\n".join(sub_tocs[sec_idx + 1:])
|
||||
chunks.extend(self._chunk_node(
|
||||
c, accumulated, _toc_join(remaining, after), path, renderer))
|
||||
remaining = "\n\n".join(sub_tocs[sec_idx + 1 :])
|
||||
chunks.extend(
|
||||
self._chunk_node(
|
||||
c,
|
||||
accumulated,
|
||||
_toc_join(remaining, after),
|
||||
path,
|
||||
renderer,
|
||||
),
|
||||
)
|
||||
accumulated = _toc_join(accumulated, sub_tocs[sec_idx])
|
||||
sec_idx += 1
|
||||
else:
|
||||
run.append(c)
|
||||
if run:
|
||||
chunks.extend(self._chunk_body_run(
|
||||
run, before_self, after_inside, path, renderer))
|
||||
chunks.extend(
|
||||
self._chunk_body_run(
|
||||
run,
|
||||
before_self,
|
||||
after_inside,
|
||||
path,
|
||||
renderer,
|
||||
),
|
||||
)
|
||||
return chunks
|
||||
|
||||
def _chunk_body_run(
|
||||
self, run: list[MdNode], before: str, after: str,
|
||||
path: str, renderer: MarkdownRenderer,
|
||||
self,
|
||||
run: list[MdNode],
|
||||
before: str,
|
||||
after: str,
|
||||
path: str,
|
||||
renderer,
|
||||
) -> list[FileChunk]:
|
||||
"""Greedy-pack consecutive body siblings under the same TOC slot.
|
||||
No ``[Part X/N]`` markers — distinct blocks, not a leaf split.
|
||||
Oversized single body recurses to ``_split_leaf``."""
|
||||
composite_size = sum(len(b.text) for b in run) + 2 * max(0, len(run) - 1)
|
||||
if composite_size <= self.chunk_chars:
|
||||
return [self._make_chunk(before, "\n\n".join(b.text for b in run), after,
|
||||
run[0].start_line, run[-1].end_line, path)]
|
||||
return [
|
||||
self._make_chunk(
|
||||
before,
|
||||
"\n\n".join(b.text for b in run),
|
||||
after,
|
||||
run[0].start_line,
|
||||
run[-1].end_line,
|
||||
path,
|
||||
),
|
||||
]
|
||||
|
||||
chunks: list[FileChunk] = []
|
||||
bucket: list[MdNode] = []
|
||||
|
|
@ -264,9 +344,16 @@ class LinkedFileParser(BaseFileParser):
|
|||
nonlocal bucket, bucket_chars
|
||||
if not bucket:
|
||||
return
|
||||
chunks.append(self._make_chunk(
|
||||
before, "\n\n".join(b.text for b in bucket), after,
|
||||
bucket[0].start_line, bucket[-1].end_line, path))
|
||||
chunks.append(
|
||||
self._make_chunk(
|
||||
before,
|
||||
"\n\n".join(b.text for b in bucket),
|
||||
after,
|
||||
bucket[0].start_line,
|
||||
bucket[-1].end_line,
|
||||
path,
|
||||
),
|
||||
)
|
||||
bucket = []
|
||||
bucket_chars = 0
|
||||
|
||||
|
|
@ -287,9 +374,19 @@ class LinkedFileParser(BaseFileParser):
|
|||
# -- Leaf splitters: build (text, start, end) units, hand off to packer
|
||||
|
||||
def _split_leaf(
|
||||
self, body: MdNode, before: str, after: str,
|
||||
path: str, renderer: MarkdownRenderer,
|
||||
self,
|
||||
body: MdNode,
|
||||
before: str,
|
||||
after: str,
|
||||
path: str,
|
||||
renderer,
|
||||
) -> list[FileChunk]:
|
||||
from mistletoe.block_token import (
|
||||
CodeFence,
|
||||
List,
|
||||
Table,
|
||||
)
|
||||
|
||||
block = body.block
|
||||
if isinstance(block, Table):
|
||||
return self._split_table(body, before, after, path)
|
||||
|
|
@ -300,9 +397,15 @@ class LinkedFileParser(BaseFileParser):
|
|||
return self._split_lines(body, before, after, path)
|
||||
|
||||
def _split_table(
|
||||
self, body: MdNode, before: str, after: str, path: str,
|
||||
self,
|
||||
body: MdNode,
|
||||
before: str,
|
||||
after: str,
|
||||
path: str,
|
||||
) -> list[FileChunk]:
|
||||
"""Repeat header + separator on every chunk."""
|
||||
from mistletoe.block_token import TableRow
|
||||
|
||||
lines = body.text.split("\n")
|
||||
header, data = "\n".join(lines[:2]), lines[2:]
|
||||
rows = [r for r in (body.block.children or []) if isinstance(r, TableRow)]
|
||||
|
|
@ -312,11 +415,21 @@ class LinkedFileParser(BaseFileParser):
|
|||
return rows[i].line_number if i < len(rows) and rows[i].line_number else base + i
|
||||
|
||||
units = [(text, line_of(i), line_of(i)) for i, text in enumerate(data)]
|
||||
return self._emit_packed(units, before, after, path,
|
||||
joiner="\n", wrap=f"{header}\n{{inner}}")
|
||||
return self._emit_packed(
|
||||
units,
|
||||
before,
|
||||
after,
|
||||
path,
|
||||
joiner="\n",
|
||||
wrap=f"{header}\n{{inner}}",
|
||||
)
|
||||
|
||||
def _split_code(
|
||||
self, body: MdNode, before: str, after: str, path: str,
|
||||
self,
|
||||
body: MdNode,
|
||||
before: str,
|
||||
after: str,
|
||||
path: str,
|
||||
) -> list[FileChunk]:
|
||||
"""Repeat fence opener + closer on every chunk."""
|
||||
code = body.block
|
||||
|
|
@ -327,17 +440,28 @@ class LinkedFileParser(BaseFileParser):
|
|||
if not raw:
|
||||
return []
|
||||
start = body.start_line + 1
|
||||
units = [(indent + ln, start + i, start + i)
|
||||
for i, ln in enumerate(raw.split("\n"))]
|
||||
return self._emit_packed(units, before, after, path, joiner="\n",
|
||||
wrap=f"{opener}\n{{inner}}\n{fence}",
|
||||
allow_empty=True)
|
||||
units = [(indent + ln, start + i, start + i) for i, ln in enumerate(raw.split("\n"))]
|
||||
return self._emit_packed(
|
||||
units,
|
||||
before,
|
||||
after,
|
||||
path,
|
||||
joiner="\n",
|
||||
wrap=f"{opener}\n{{inner}}\n{fence}",
|
||||
allow_empty=True,
|
||||
)
|
||||
|
||||
def _split_list(
|
||||
self, body: MdNode, before: str, after: str,
|
||||
path: str, renderer: MarkdownRenderer,
|
||||
self,
|
||||
body: MdNode,
|
||||
before: str,
|
||||
after: str,
|
||||
path: str,
|
||||
renderer,
|
||||
) -> list[FileChunk]:
|
||||
"""Pack list items; oversized items emit alone (overflow accepted)."""
|
||||
from mistletoe.block_token import ListItem
|
||||
|
||||
items = [c for c in (body.block.children or []) if isinstance(c, ListItem)]
|
||||
if not items:
|
||||
return self._split_lines(body, before, after, path)
|
||||
|
|
@ -348,28 +472,43 @@ class LinkedFileParser(BaseFileParser):
|
|||
continue
|
||||
line = it.line_number or body.start_line
|
||||
units.append((text, line, line + text.count("\n")))
|
||||
return self._emit_packed(units, before, after, path,
|
||||
joiner="\n", wrap="{inner}")
|
||||
return self._emit_packed(
|
||||
units,
|
||||
before,
|
||||
after,
|
||||
path,
|
||||
joiner="\n",
|
||||
wrap="{inner}",
|
||||
)
|
||||
|
||||
def _split_lines(
|
||||
self, body: MdNode, before: str, after: str, path: str,
|
||||
self,
|
||||
body: MdNode,
|
||||
before: str,
|
||||
after: str,
|
||||
path: str,
|
||||
) -> list[FileChunk]:
|
||||
"""Last-resort line-greedy split for paragraphs / quotes / html."""
|
||||
start = body.start_line
|
||||
units = [(line, start + i, start + i)
|
||||
for i, line in enumerate(body.text.split("\n"))]
|
||||
return self._emit_packed(units, before, after, path,
|
||||
joiner="\n", wrap="{inner}")
|
||||
units = [(line, start + i, start + i) for i, line in enumerate(body.text.split("\n"))]
|
||||
return self._emit_packed(
|
||||
units,
|
||||
before,
|
||||
after,
|
||||
path,
|
||||
joiner="\n",
|
||||
wrap="{inner}",
|
||||
)
|
||||
|
||||
def _emit_packed(
|
||||
self,
|
||||
units: list[tuple[str, int, int]],
|
||||
before: str,
|
||||
after: str,
|
||||
path: str,
|
||||
joiner: str,
|
||||
wrap: str,
|
||||
allow_empty: bool = False,
|
||||
self,
|
||||
units: list[tuple[str, int, int]],
|
||||
before: str,
|
||||
after: str,
|
||||
path: str,
|
||||
joiner: str,
|
||||
wrap: str,
|
||||
allow_empty: bool = False,
|
||||
) -> list[FileChunk]:
|
||||
"""Greedy-pack units into ``wrap`` envelopes; emit each piece.
|
||||
|
||||
|
|
@ -379,7 +518,7 @@ class LinkedFileParser(BaseFileParser):
|
|||
``[Part X/N]`` markers; single pieces don't.
|
||||
"""
|
||||
envelope = len(wrap.replace("{inner}", ""))
|
||||
budget = max(64, self.chunk_chars - envelope - _PART_RESERVE)
|
||||
budget = max(64, self.chunk_chars - envelope)
|
||||
sep_len = len(joiner)
|
||||
|
||||
parts: list[tuple[str, int, int]] = []
|
||||
|
|
@ -410,9 +549,15 @@ class LinkedFileParser(BaseFileParser):
|
|||
return [
|
||||
self._make_chunk(
|
||||
before,
|
||||
f"[Part {idx}/{total}]\n\n{wrap.replace('{inner}', inner)}"
|
||||
if total > 1 else wrap.replace("{inner}", inner),
|
||||
after, s, e, path,
|
||||
(
|
||||
f"[Part {idx}/{total}]\n\n{wrap.replace('{inner}', inner)}"
|
||||
if total > 1
|
||||
else wrap.replace("{inner}", inner)
|
||||
),
|
||||
after,
|
||||
s,
|
||||
e,
|
||||
path,
|
||||
)
|
||||
for idx, (inner, s, e) in enumerate(parts, 1)
|
||||
]
|
||||
|
|
@ -420,13 +565,13 @@ class LinkedFileParser(BaseFileParser):
|
|||
# -- Emit -------------------------------------------------------------
|
||||
|
||||
def _make_chunk(
|
||||
self,
|
||||
before: str,
|
||||
content: str,
|
||||
after: str,
|
||||
start_line: int,
|
||||
end_line: int,
|
||||
path: str,
|
||||
self,
|
||||
before: str,
|
||||
content: str,
|
||||
after: str,
|
||||
start_line: int,
|
||||
end_line: int,
|
||||
path: str,
|
||||
) -> FileChunk:
|
||||
"""Build one ``FileChunk`` — text is ``before + content + after``
|
||||
when ``embed_toc``, otherwise just ``content``."""
|
||||
|
|
|
|||
|
|
@ -12,11 +12,11 @@ class BaseFileStore(BaseComponent):
|
|||
component_type = ComponentEnum.FILE_STORE
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store_name: str,
|
||||
embedding_model: str = "default",
|
||||
keyword_index: str = "default",
|
||||
**kwargs,
|
||||
self,
|
||||
store_name: str,
|
||||
embedding_model: str = "default",
|
||||
keyword_index: str = "default",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
if not re.match(r"^[a-zA-Z0-9_]+$", store_name):
|
||||
|
|
@ -46,15 +46,13 @@ class BaseFileStore(BaseComponent):
|
|||
self.keyword_index = None
|
||||
await self.dump_file_nodes()
|
||||
|
||||
async def load_file_nodes(self):
|
||||
...
|
||||
async def load_file_nodes(self): ...
|
||||
|
||||
async def dump_file_nodes(self):
|
||||
...
|
||||
async def dump_file_nodes(self): ...
|
||||
|
||||
async def upsert_file(
|
||||
self,
|
||||
file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
|
||||
self,
|
||||
file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
|
||||
) -> None:
|
||||
"""Upsert a file and its chunks into the store."""
|
||||
|
||||
|
|
|
|||
|
|
@ -30,8 +30,10 @@ class LocalFileStore(BaseFileStore):
|
|||
async def _start(self) -> None:
|
||||
await super()._start()
|
||||
await self._load_jsonl(self.chunks_path, self.file_chunks, FileChunk, "id")
|
||||
self.logger.info(f"LocalFileStore '{self.store_name}' ready: "
|
||||
f"{len(self.file_nodes)} nodes, {len(self.file_chunks)} chunks")
|
||||
self.logger.info(
|
||||
f"LocalFileStore '{self.store_name}' ready: "
|
||||
f"{len(self.file_nodes)} nodes, {len(self.file_chunks)} chunks"
|
||||
)
|
||||
|
||||
async def _close(self) -> None:
|
||||
await self._dump_jsonl(self.chunks_path, list(self.file_chunks.values()))
|
||||
|
|
@ -71,8 +73,8 @@ class LocalFileStore(BaseFileStore):
|
|||
await self._dump_jsonl(self.nodes_path, list(self.file_nodes.values()))
|
||||
|
||||
async def upsert_file(
|
||||
self,
|
||||
file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
|
||||
self,
|
||||
file: tuple[FileNode, list[FileChunk]] | list[tuple[FileNode, list[FileChunk]]],
|
||||
) -> None:
|
||||
if isinstance(file, tuple):
|
||||
file = [file]
|
||||
|
|
|
|||
|
|
@ -17,16 +17,16 @@ class BaseFileWatcher(BaseComponent):
|
|||
component_type = ComponentEnum.FILE_WATCHER
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
watch_paths: list[str] | str,
|
||||
suffix_filters: list[str] | None = None,
|
||||
recursive: bool = True,
|
||||
force_polling: bool = True,
|
||||
debounce: int = 2000,
|
||||
poll_delay_ms: int = 2000,
|
||||
file_store: str = "default",
|
||||
file_parser: str = "default",
|
||||
**kwargs,
|
||||
self,
|
||||
watch_paths: list[str] | str,
|
||||
suffix_filters: list[str] | None = None,
|
||||
recursive: bool = True,
|
||||
force_polling: bool = True,
|
||||
debounce: int = 2000,
|
||||
poll_delay_ms: int = 2000,
|
||||
file_store: str = "default",
|
||||
file_parser: str = "default",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
watch_paths = [watch_paths] if isinstance(watch_paths, str) else watch_paths
|
||||
|
|
|
|||
|
|
@ -39,13 +39,13 @@ class LiteFileWatcher(BaseFileWatcher):
|
|||
try:
|
||||
logger.info(f"Watching: {valid_paths}")
|
||||
async for changes in awatch(
|
||||
*valid_paths,
|
||||
watch_filter=self.watch_filter,
|
||||
recursive=self.recursive,
|
||||
force_polling=self.force_polling,
|
||||
debounce=self.debounce,
|
||||
poll_delay_ms=self.poll_delay_ms,
|
||||
stop_event=self._stop_event,
|
||||
*valid_paths,
|
||||
watch_filter=self.watch_filter,
|
||||
recursive=self.recursive,
|
||||
force_polling=self.force_polling,
|
||||
debounce=self.debounce,
|
||||
poll_delay_ms=self.poll_delay_ms,
|
||||
stop_event=self._stop_event,
|
||||
):
|
||||
if self._stop_event.is_set():
|
||||
break
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ class BaseJob(BaseComponent):
|
|||
for raw in self.step_configs:
|
||||
config = raw if isinstance(raw, ComponentConfig) else ComponentConfig(**raw)
|
||||
if not config.backend:
|
||||
raise ValueError(f"Step is missing the required 'backend' field")
|
||||
raise ValueError("Step is missing the required 'backend' field")
|
||||
step_cls = R.get(ComponentEnum.STEP, config.backend)
|
||||
if not step_cls:
|
||||
raise ValueError(
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ class BaseKeywordIndex(BaseComponent):
|
|||
"""Initialize tokenizer and load existing index if available."""
|
||||
if self.app_context is None:
|
||||
from ..tokenizer import RegexTokenizer
|
||||
|
||||
self.tokenizer = RegexTokenizer(filter_stopwords=False)
|
||||
else:
|
||||
self.tokenizer = self.get_component(ComponentEnum.TOKENIZER, self.tokenizer_name)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ class DocMeta(TypedDict):
|
|||
len: Number of tokens in the document.
|
||||
token_ids: Set of unique token IDs present in the document.
|
||||
"""
|
||||
|
||||
len: int
|
||||
token_ids: set[int]
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,6 @@ Lives next to `runtime_context.py` because both are about the BaseStep
|
|||
interface — the response side, specifically. Used by every Step that
|
||||
returns a JSON-shaped payload (memory_*, sync, the three memory
|
||||
services).
|
||||
|
||||
Was previously at `reme2/mcp/steps/_common.py`, which leaked an MCP
|
||||
dependency into `reme2/memory/` services that legitimately need to
|
||||
serialize their results — the moved location breaks that cycle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
|
|||
|
|
@ -52,4 +52,4 @@ class RegexTokenizer(BaseTokenizer):
|
|||
tokens = [t for t in tokens if t not in self._stopwords]
|
||||
|
||||
result.append(tokens)
|
||||
return result
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ def _expand_env_vars(value: Any) -> Any:
|
|||
typos don't silently produce empty connection strings.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
|
||||
def repl(m: re.Match) -> str:
|
||||
name = m.group(1)
|
||||
default = m.group(2)
|
||||
|
|
@ -31,6 +32,7 @@ def _expand_env_vars(value: Any) -> Any:
|
|||
return default
|
||||
raise ValueError(f"Config references undefined env var: {name}")
|
||||
return v
|
||||
|
||||
return _ENV_VAR_RE.sub(repl, value)
|
||||
if isinstance(value, dict):
|
||||
return {k: _expand_env_vars(v) for k, v in value.items()}
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@ ReMe重构
|
|||
a. ✅ vault_root → working_dir @sen
|
||||
2. file_parser
|
||||
a. 抽象基类 parse: @jinli
|
||||
ⅰ. 输入是path:相对路径
|
||||
ⅰ. 输入是path:相对路径
|
||||
ⅱ. 输出是FileMetadata & list[FileChunks] & list[FileEdge]
|
||||
b. default parser 兼容老方案 @jinli
|
||||
b. default parser 兼容老方案 @jinli
|
||||
ⅰ. 带overlap的chunking策略 ,不输出FileEdge
|
||||
c. markdown parser @sen
|
||||
ⅰ. 根据markdown ast做chunk,不需要overlap
|
||||
|
|
@ -27,11 +27,11 @@ ReMe重构
|
|||
ⅳ. 【核心】检索机制 vector bm25 graph 如何进行融合
|
||||
4. file_watcher @jinli
|
||||
a. 抽象基类
|
||||
ⅰ. on_start:
|
||||
ⅰ. on_start:
|
||||
1. file_store 的start 在前,加载graph,file_watcher在后,递归扫描目录
|
||||
a. 通过ms_time对比graph,on_change 进行改动
|
||||
ⅱ. on_change:
|
||||
1. 更新/增加:
|
||||
1. 更新/增加:
|
||||
a. delete_chunks_by_path 更新数据库
|
||||
b. upate_chunks_by_path 更新数据库
|
||||
c. 更新graph
|
||||
|
|
@ -44,14 +44,14 @@ MemorySchema
|
|||
ⅰ. title
|
||||
ⅱ. desc
|
||||
ⅲ. tags
|
||||
ⅳ.
|
||||
ⅳ.
|
||||
2. memory文件结构目录
|
||||
a. MEMORY.md
|
||||
b. msg/files -> daily/YYYYMMDD/YYYYMMDD.md + xxxx.md
|
||||
ⅰ. YYYYMMDD.md
|
||||
1. xxx -> xxxx.md
|
||||
2. xxx -> xxxd.md
|
||||
ⅱ.
|
||||
ⅱ.
|
||||
c. daily -> topic/topic_l1/topic_l1.md + xxx.md + topic_l2
|
||||
d. proactive
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ class ComponentEnum(str, Enum):
|
|||
|
||||
FILE_STORE = "file_store"
|
||||
|
||||
FILE_GRAPH = "file_graph"
|
||||
|
||||
FILE_WATCHER = "file_watcher"
|
||||
|
||||
KEYWORD_INDEX = "keyword_index"
|
||||
|
|
|
|||
|
|
@ -1,39 +1,31 @@
|
|||
# reme2.mcp
|
||||
|
||||
Agent-facing MCP interface layer. Exposes the markdown vault under
|
||||
`reme2/` (file_store + watcher + memory services) as MCP tools for
|
||||
`claude-code` and other MCP clients.
|
||||
MCP transport layer. Exposes the agent-facing tools registered in
|
||||
`reme2.memory` over MCP for `claude-code` and other MCP clients.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
reme2/mcp/
|
||||
├── __init__.py
|
||||
├── server.py MCP server bootstrap; defaults to ../config/service.yaml
|
||||
└── steps/ @R.register MCP step shells
|
||||
├── memory_retriever.py memory_search + memory_graph_search
|
||||
├── memory_lint.py read-only Maintainer projection (lint findings)
|
||||
└── sync.py hot-path event-folder upsert
|
||||
├── server.py MCP server bootstrap; defaults to ../config/service.yaml
|
||||
└── test/ end-to-end profile smoke tests
|
||||
```
|
||||
|
||||
The 12 `memory_*` primitives (create/update/property_update/rename/
|
||||
delete/archive/get/list/links/backlinks/resolve_wikilink/count_tokens)
|
||||
live one layer down in `reme2/memory/memory_toolkit.py` — each is a
|
||||
single `BaseStep` subclass with two class methods: `execute()` for the
|
||||
MCP path (this layer) and a same-named method for the agent toolkit
|
||||
path that the Ingestor's ReActAgent consumes. Importing
|
||||
`reme2.mcp.steps` triggers all `@R.register` registrations.
|
||||
|
||||
The MCP layer's job is to **project** existing primitives as MCP tools
|
||||
— it owns no business logic. Everything else lives outside the
|
||||
transport boundary so memory services don't form an import cycle:
|
||||
That's it. Every `@R.register` step the agent invokes lives one layer
|
||||
down in `reme2/memory/` — there are no MCP-specific shells, because
|
||||
none of the steps depend on MCP transport details. Importing
|
||||
`reme2.memory` triggers all `@R.register` registrations.
|
||||
|
||||
| Concern | Location |
|
||||
|---|---|
|
||||
| Memory File System primitives | `reme2/component/file_store/`, `file_watcher/`, `file_parser/` |
|
||||
| Three memory services | `reme2/memory/` — Retriever / Ingestor / Maintainer |
|
||||
| Three memory services | `reme2/memory/{retriever,ingestor,maintainer}.py` |
|
||||
| Engine API (pure, schema-free) | `reme2/memory/memory_io.py` |
|
||||
| Schema-bound tools (BaseStep + agent toolkit) | `reme2/memory/memory_toolkit.py` |
|
||||
| Schema-bound write/read tools | `reme2/memory/memory_toolkit.py` |
|
||||
| Search step shells | `reme2/memory/memory_search.py` |
|
||||
| Lint step shell | `reme2/memory/memory_lint.py` |
|
||||
| Hot-path event upsert | `reme2/memory/sync.py` |
|
||||
| Memory schema (4 axes + presets + parser) | `reme2/memory/schema/` |
|
||||
| Path templates + name disambiguation | `reme2/utils/vault_paths.py` |
|
||||
| Step response serialization | `reme2/component/runtime_response.py` |
|
||||
|
|
@ -44,9 +36,8 @@ reme2.mcp → reme2.memory (incl. memory.schema) → reme2.component / reme2
|
|||
```
|
||||
|
||||
The Ingestor (`reme2/memory/ingestor.py`) self-registers its `ingest`
|
||||
MCP face — there's no shell for it under `steps/`. Topic creation lives
|
||||
inside the Ingestor's R-M-W loop; there is no separate `topic_create`
|
||||
tool.
|
||||
MCP face. Topic creation lives inside the Ingestor's R-M-W loop; there
|
||||
is no separate `topic_create` tool.
|
||||
|
||||
## Run
|
||||
|
||||
|
|
@ -91,12 +82,12 @@ Configs live in `reme2/config/`:
|
|||
|
||||
### Tools exposed (expert profile)
|
||||
|
||||
| Tool | Path | Purpose |
|
||||
| Tool | Source | Purpose |
|
||||
|---|---|---|
|
||||
| `sync` | steps/sync.py | Hot-path event-folder upsert (idempotent per `(date, name)`). |
|
||||
| `sync` | reme2/memory/sync.py | Hot-path event-folder upsert (idempotent per `(date, name)`). |
|
||||
| `ingest` | reme2/memory/ingestor.py | Cold-path LLM-driven distillation; owns topic creation. |
|
||||
| `memory_search` / `memory_graph_search` | steps/memory_retriever.py | V+K hybrid + optional graph BFS. |
|
||||
| `memory_lint` | steps/memory_lint.py | Read-only projection of Maintainer's lint findings. |
|
||||
| `memory_search` / `memory_graph_search` | reme2/memory/memory_search.py | V+K hybrid + optional graph BFS. |
|
||||
| `memory_lint` | reme2/memory/memory_lint.py | Read-only projection of Maintainer's lint findings. |
|
||||
| `memory_get` / `memory_list` / `memory_links` / `memory_backlinks` / `memory_resolve_wikilink` | reme2/memory/memory_toolkit.py | Read primitives. |
|
||||
| `memory_create` / `memory_update` / `memory_property_update` / `memory_rename` / `memory_delete` / `memory_archive` | reme2/memory/memory_toolkit.py | Raw write primitives (prefer `ingest` / `sync`). |
|
||||
| `memory_count_tokens` | reme2/memory/memory_toolkit.py | Token estimation. |
|
||||
|
|
|
|||
|
|
@ -1,20 +1,19 @@
|
|||
"""reme2.mcp — MCP interface layer.
|
||||
"""reme2.mcp — MCP transport layer.
|
||||
|
||||
The Agent-facing surface: server entrypoint + step shells that wrap
|
||||
the three services in `reme2.memory` (Retriever, Ingestor, Maintainer)
|
||||
plus the hot-write primitive (`sync`) and the raw `memory_*` write/read
|
||||
tools that bypass services and land directly on the Memory File System.
|
||||
Just the MCP server bootstrap (config loading, env-var overrides,
|
||||
sidecar HTTP). All agent-facing tools live in ``reme2.memory`` and
|
||||
register themselves there; this package only wires them into the MCP
|
||||
transport via the ``mcp`` service component.
|
||||
|
||||
This package depends on `reme2.memory`, `reme2.utils`, `reme2.component`
|
||||
— never the reverse. The Memory schema (the typed shape of every
|
||||
frontmatter) lives under `reme2.memory.schema/` so the services that
|
||||
own validation can use it without an import cycle through this
|
||||
transport layer.
|
||||
Dependency direction is strict: ``reme2.mcp → reme2.memory`` (and
|
||||
through it ``reme2.component`` / ``reme2.utils``). Importing
|
||||
``reme2.memory`` triggers all ``@R.register`` decorators for
|
||||
``memory_*`` / ``sync`` / ``memory_search`` / ``memory_lint`` /
|
||||
``ingest`` / ``maintainer`` / ``hybrid``.
|
||||
|
||||
Sub-packages:
|
||||
steps/ - all @R.register MCP step shells (memory_toolkit,
|
||||
memory_retriever, memory_lint, sync).
|
||||
server.py - MCP server bootstrap (defaults to ../config/service.yaml).
|
||||
Files:
|
||||
server.py — MCP server bootstrap (defaults to ../config/service.yaml).
|
||||
test/ — end-to-end profile smoke tests.
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ from pathlib import Path
|
|||
|
||||
# Make this work whether invoked as `python -m reme2.mcp.server` (repo
|
||||
# root already on sys.path) or `python reme2/mcp/server.py` (only this
|
||||
# file's dir is on sys.path — without this, `import reme2.mcp.steps`
|
||||
# would fail with ModuleNotFoundError).
|
||||
# file's dir is on sys.path — without this, `import reme2.memory` would
|
||||
# fail with ModuleNotFoundError).
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_REPO_ROOT))
|
||||
|
|
@ -32,10 +32,7 @@ load_dotenv(_REPO_ROOT / ".env")
|
|||
# Eagerly import side-effect modules so all @R.register() decorators run
|
||||
# before Application introspects the registry.
|
||||
import reme2 # noqa: E402,F401
|
||||
import reme2.mcp.steps # noqa: E402,F401
|
||||
import reme2.memory # noqa: E402,F401 -- registers retriever + maintainer
|
||||
import reme2.memory.ingestor # noqa: E402,F401
|
||||
import reme2.memory.summarizer # noqa: E402,F401
|
||||
import reme2.memory # noqa: E402,F401 -- registers every agent-facing tool
|
||||
import reme2.component.service.mcp_service # noqa: E402,F401
|
||||
|
||||
from reme2.application import Application # noqa: E402
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
"""MCP step shells — the @R.register classes the model invokes as MCP tools.
|
||||
|
||||
Three groups:
|
||||
|
||||
Hot-write primitives that bypass services and land on MFS directly:
|
||||
sync — idempotent event-folder upsert (event log)
|
||||
memory_toolkit — memory_create / update / property_update /
|
||||
rename / delete / archive / get / list /
|
||||
links / backlinks / resolve_wikilink /
|
||||
count_tokens. Each step lives in
|
||||
`reme2.memory.memory_toolkit` and exposes
|
||||
BOTH an `execute()` (this MCP path) and a
|
||||
same-named class method (the agent toolkit
|
||||
path used by the Ingestor's ReActAgent).
|
||||
|
||||
Service delegates (thin shells over the three memory services):
|
||||
memory_retriever — memory_search + memory_graph_search; delegate
|
||||
to the Retriever component
|
||||
(`reme2.memory.retriever`).
|
||||
memory_lint — read-only projection of the Maintainer's lint
|
||||
findings (`reme2.memory.maintainer`).
|
||||
|
||||
Topic creation is owned by the Ingestor (`reme2.memory.ingestor`) — its
|
||||
LLM-driven R-M-W loop decides when a new topic is warranted, applies the
|
||||
schema preset, and routes the write through `memory_create`. There is no
|
||||
separate `topic_create` MCP tool.
|
||||
|
||||
The Ingestor's MCP face (`ingest`) is registered from
|
||||
`reme2.memory.ingestor`. Importing this package triggers all the step
|
||||
registrations hosted here and in `reme2.memory.memory_toolkit`.
|
||||
"""
|
||||
|
||||
from ...memory import memory_toolkit # noqa: F401 -- triggers @R.register for memory_*
|
||||
from . import memory_lint # noqa: F401 -- triggers @R.register for memory_lint
|
||||
from . import memory_retriever # noqa: F401 -- memory_search / memory_graph_search
|
||||
from .sync import Sync
|
||||
|
||||
__all__ = [
|
||||
"Sync",
|
||||
]
|
||||
|
|
@ -22,7 +22,7 @@ from ._helpers import AppContext, make_context
|
|||
|
||||
|
||||
SUITES = {
|
||||
"expert": test_expert.CHECKS,
|
||||
"expert": test_expert.CHECKS,
|
||||
"service": test_service.CHECKS,
|
||||
}
|
||||
|
||||
|
|
@ -35,8 +35,7 @@ async def _run_suite(profile: str, checks: list[tuple[str, callable]]) -> tuple[
|
|||
passed = 0
|
||||
try:
|
||||
ctx, tmp = await make_context(profile)
|
||||
print(f" bootstrapped: vault={ctx.vault}, jobs={len(ctx.jobs)}, "
|
||||
f"file_store={len(ctx.file_store)}")
|
||||
print(f" bootstrapped: vault={ctx.vault}, jobs={len(ctx.jobs)}, " f"file_store={len(ctx.file_store)}")
|
||||
for label, fn in checks:
|
||||
try:
|
||||
summary = await fn(ctx)
|
||||
|
|
@ -69,7 +68,7 @@ async def _main() -> int:
|
|||
for profile, (passed, total) in summary.items():
|
||||
marker = "✓" if passed == total else "✗"
|
||||
print(f" {marker} {profile}: {passed}/{total}")
|
||||
failed += (total - passed)
|
||||
failed += total - passed
|
||||
return 0 if failed == 0 else failed
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -24,10 +24,7 @@ if str(_REPO_ROOT) not in sys.path:
|
|||
# Eager-import side-effect modules so all @R.register() decorators run
|
||||
# before Application introspects the registry.
|
||||
import reme2 # noqa: E402,F401
|
||||
import reme2.mcp.steps # noqa: E402,F401
|
||||
import reme2.memory # noqa: E402,F401 -- registers retriever + maintainer
|
||||
import reme2.memory.ingestor # noqa: E402,F401
|
||||
import reme2.memory.summarizer # noqa: E402,F401
|
||||
import reme2.memory # noqa: E402,F401 -- registers every agent-facing tool
|
||||
|
||||
from reme2.application import Application # noqa: E402
|
||||
from reme2.config import parse_args # noqa: E402
|
||||
|
|
@ -35,7 +32,7 @@ from reme2.config import parse_args # noqa: E402
|
|||
|
||||
CONFIG_DIR = _REPO_ROOT / "reme2" / "config"
|
||||
PROFILES = {
|
||||
"expert": CONFIG_DIR / "expert.yaml",
|
||||
"expert": CONFIG_DIR / "expert.yaml",
|
||||
"service": CONFIG_DIR / "service.yaml",
|
||||
}
|
||||
|
||||
|
|
@ -106,8 +103,7 @@ async def wait_for_index(watcher, expected_min: int, timeout_s: float = 15.0) ->
|
|||
last = now
|
||||
await asyncio.sleep(0.25)
|
||||
raise RuntimeError(
|
||||
f"watcher did not reach >= {expected_min} files within {timeout_s}s "
|
||||
f"(last seen: {last})"
|
||||
f"watcher did not reach >= {expected_min} files within {timeout_s}s " f"(last seen: {last})",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -156,7 +152,9 @@ async def make_context(profile: str) -> tuple[AppContext, Path]:
|
|||
vault.mkdir()
|
||||
seed_vault(vault)
|
||||
app = await build_app(PROFILES[profile], vault)
|
||||
await wait_for_index(app.context.components["file_watcher"]["default"],
|
||||
expected_min=SEED_FILE_COUNT)
|
||||
await wait_for_index(
|
||||
app.context.components["file_watcher"]["default"],
|
||||
expected_min=SEED_FILE_COUNT,
|
||||
)
|
||||
jobs = sorted(app.context.jobs.keys())
|
||||
return AppContext(app=app, vault=vault, jobs=jobs), tmp
|
||||
|
|
|
|||
|
|
@ -75,19 +75,29 @@ async def check_memory_list(ctx: AppContext) -> str:
|
|||
|
||||
|
||||
async def check_memory_search(ctx: AppContext) -> str:
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_search", query="collaborates", max_results=3, min_score=0.0,
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_search",
|
||||
query="collaborates",
|
||||
max_results=3,
|
||||
min_score=0.0,
|
||||
),
|
||||
)
|
||||
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
|
||||
assert len(hits) > 0, r
|
||||
return f"{len(hits)} hits"
|
||||
|
||||
|
||||
async def check_memory_graph_search(ctx: AppContext) -> str:
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_graph_search", query="Alice", max_results=5, min_score=0.0,
|
||||
graph_depth=1,
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_graph_search",
|
||||
query="Alice",
|
||||
max_results=5,
|
||||
min_score=0.0,
|
||||
graph_depth=1,
|
||||
),
|
||||
)
|
||||
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
|
||||
assert len(hits) > 0, r
|
||||
# At least one result should carry a graph_hop annotation
|
||||
|
|
@ -118,9 +128,12 @@ async def check_memory_resolve_wikilink(ctx: AppContext) -> str:
|
|||
|
||||
|
||||
async def check_memory_count_tokens(ctx: AppContext) -> str:
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_count_tokens", text="hello world from the smoke test",
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_count_tokens",
|
||||
text="hello world from the smoke test",
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict) and isinstance(r.get("tokens"), int), r
|
||||
assert r["tokens"] > 0, r
|
||||
return f"text → {r['tokens']} tokens"
|
||||
|
|
@ -142,18 +155,20 @@ async def check_memory_lint(ctx: AppContext) -> str:
|
|||
|
||||
|
||||
async def check_sync_create(ctx: AppContext) -> str:
|
||||
r = decode(await ctx.app.run_job(
|
||||
"sync",
|
||||
name="suite-event",
|
||||
description="full-profile suite",
|
||||
content="## ops\n- ran the suite\n",
|
||||
topics=["[[Alice]]"],
|
||||
tags=["suite"],
|
||||
materials=[
|
||||
{"filename": "raw-prompt.md", "content": "# user prompt\n\nrun suite\n"},
|
||||
{"filename": "tool-output.txt", "content": "exit=0\n"},
|
||||
],
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"sync",
|
||||
name="suite-event",
|
||||
description="full-profile suite",
|
||||
content="## ops\n- ran the suite\n",
|
||||
topics=["[[Alice]]"],
|
||||
tags=["suite"],
|
||||
materials=[
|
||||
{"filename": "raw-prompt.md", "content": "# user prompt\n\nrun suite\n"},
|
||||
{"filename": "tool-output.txt", "content": "exit=0\n"},
|
||||
],
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict), r
|
||||
assert r.get("created") is True and r.get("action") == "created", r
|
||||
materials = r.get("materials", [])
|
||||
|
|
@ -176,17 +191,19 @@ async def check_sync_create(ctx: AppContext) -> str:
|
|||
|
||||
|
||||
async def check_sync_append(ctx: AppContext) -> str:
|
||||
r = decode(await ctx.app.run_job(
|
||||
"sync",
|
||||
name="suite-event",
|
||||
content="## follow-up\n- second pass\n",
|
||||
topics=["[[Bob]]"], # union with [[Alice]]
|
||||
tags=["follow-up"],
|
||||
materials=[
|
||||
{"filename": "tool-output.txt", "content": "second run\n"}, # collision
|
||||
{"filename": "summary.md", "content": "# summary\n"},
|
||||
],
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"sync",
|
||||
name="suite-event",
|
||||
content="## follow-up\n- second pass\n",
|
||||
topics=["[[Bob]]"], # union with [[Alice]]
|
||||
tags=["follow-up"],
|
||||
materials=[
|
||||
{"filename": "tool-output.txt", "content": "second run\n"}, # collision
|
||||
{"filename": "summary.md", "content": "# summary\n"},
|
||||
],
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict), r
|
||||
assert r.get("created") is False and r.get("action") == "appended", r
|
||||
appended = r.get("materials", [])
|
||||
|
|
@ -204,11 +221,18 @@ async def check_sync_append(ctx: AppContext) -> str:
|
|||
async def check_sync_refuse_distilled(ctx: AppContext) -> str:
|
||||
index_path = str(getattr(ctx, "_suite_event_index"))
|
||||
await ctx.app.run_job(
|
||||
"memory_property_update", path=index_path, key="status", value="distilled",
|
||||
"memory_property_update",
|
||||
path=index_path,
|
||||
key="status",
|
||||
value="distilled",
|
||||
)
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"sync",
|
||||
name="suite-event",
|
||||
content="should be refused",
|
||||
),
|
||||
)
|
||||
r = decode(await ctx.app.run_job(
|
||||
"sync", name="suite-event", content="should be refused",
|
||||
))
|
||||
assert isinstance(r, dict) and "error" in r, r
|
||||
assert r.get("status") == "distilled", r
|
||||
assert r.get("suggested_name"), r
|
||||
|
|
@ -220,20 +244,22 @@ async def check_sync_refuse_distilled(ctx: AppContext) -> str:
|
|||
|
||||
async def check_memory_create(ctx: AppContext) -> str:
|
||||
target = ctx.abs_path("topics", "Carol", "Carol.md")
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_create",
|
||||
path=target,
|
||||
metadata={
|
||||
"title": "Carol",
|
||||
"lifecycle": "evolving",
|
||||
"scope": "class",
|
||||
"source": "curated",
|
||||
"role": "profile",
|
||||
"category": "profile",
|
||||
"tags": ["person"],
|
||||
},
|
||||
content="# Carol\n\nKnows [[Alice]].\n",
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_create",
|
||||
path=target,
|
||||
metadata={
|
||||
"title": "Carol",
|
||||
"lifecycle": "evolving",
|
||||
"scope": "class",
|
||||
"source": "curated",
|
||||
"role": "profile",
|
||||
"category": "profile",
|
||||
"tags": ["person"],
|
||||
},
|
||||
content="# Carol\n\nKnows [[Alice]].\n",
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict) and r.get("created") is True, r
|
||||
assert "error" not in r, r
|
||||
assert Path(target).is_file(), target
|
||||
|
|
@ -243,12 +269,14 @@ async def check_memory_create(ctx: AppContext) -> str:
|
|||
|
||||
async def check_memory_update(ctx: AppContext) -> str:
|
||||
carol = ctx.abs_path("topics", "Carol", "Carol.md")
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_update",
|
||||
path=carol,
|
||||
old_string="Knows [[Alice]].",
|
||||
new_string="Knows [[Alice]] and [[Bob]].",
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_update",
|
||||
path=carol,
|
||||
old_string="Knows [[Alice]].",
|
||||
new_string="Knows [[Alice]] and [[Bob]].",
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict) and r.get("replaced", 0) >= 1, r
|
||||
assert "Bob" in Path(carol).read_text(encoding="utf-8"), "edit not on disk"
|
||||
return f"body edit applied (replaced={r['replaced']})"
|
||||
|
|
@ -256,22 +284,30 @@ async def check_memory_update(ctx: AppContext) -> str:
|
|||
|
||||
async def check_memory_property_update(ctx: AppContext) -> str:
|
||||
carol = ctx.abs_path("topics", "Carol", "Carol.md")
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_property_update", path=carol, key="confidence", value="✅",
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_property_update",
|
||||
path=carol,
|
||||
key="confidence",
|
||||
value="✅",
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict) and "error" not in r, r
|
||||
assert r.get("key") == "confidence" and r.get("value") == "✅", r
|
||||
assert "confidence: ✅" in Path(carol).read_text(encoding="utf-8"), \
|
||||
"property write not on disk"
|
||||
assert "confidence: ✅" in Path(carol).read_text(encoding="utf-8"), "property write not on disk"
|
||||
return "set confidence=✅"
|
||||
|
||||
|
||||
async def check_memory_rename(ctx: AppContext) -> str:
|
||||
src = ctx.abs_path("topics", "Carol", "Carol.md")
|
||||
dst = ctx.abs_path("topics", "Carol", "Carol-renamed.md")
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_rename", old_path=src, new_path=dst,
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_rename",
|
||||
old_path=src,
|
||||
new_path=dst,
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict) and "error" not in r, r
|
||||
assert r.get("new_path") and Path(r["new_path"]).is_file(), r
|
||||
assert not Path(src).exists(), f"old path still on disk: {src}"
|
||||
|
|
@ -291,8 +327,7 @@ async def check_memory_archive(ctx: AppContext) -> str:
|
|||
|
||||
|
||||
async def check_memory_delete(ctx: AppContext) -> str:
|
||||
target = getattr(ctx, "_carol_archived", None) \
|
||||
or ctx.abs_path("topics", "Carol", "Carol-renamed.md")
|
||||
target = getattr(ctx, "_carol_archived", None) or ctx.abs_path("topics", "Carol", "Carol-renamed.md")
|
||||
r = decode(await ctx.app.run_job("memory_delete", path=target))
|
||||
assert isinstance(r, dict) and r.get("deleted") is True, r
|
||||
assert not Path(target).exists(), target
|
||||
|
|
@ -306,14 +341,20 @@ async def check_schema_path_template_refuses(ctx: AppContext) -> str:
|
|||
"""memory_create rejects paths outside topics/{X}/{Y}.md,
|
||||
events/{date}/{name}/..., or Archive/..."""
|
||||
target = ctx.abs_path("notes", "freeform.md") # outside any template
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_create",
|
||||
path=target,
|
||||
metadata={"title": "freeform",
|
||||
"lifecycle": "evolving", "scope": "class",
|
||||
"source": "curated", "role": "concept"},
|
||||
content="should be refused",
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_create",
|
||||
path=target,
|
||||
metadata={
|
||||
"title": "freeform",
|
||||
"lifecycle": "evolving",
|
||||
"scope": "class",
|
||||
"source": "curated",
|
||||
"role": "concept",
|
||||
},
|
||||
content="should be refused",
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict), r
|
||||
assert "error" in r and "template" in r["error"].lower(), r
|
||||
assert not Path(target).exists(), "file shouldn't have been written"
|
||||
|
|
@ -323,15 +364,21 @@ async def check_schema_path_template_refuses(ctx: AppContext) -> str:
|
|||
async def check_schema_path_template_force_bypass(ctx: AppContext) -> str:
|
||||
"""force=True bypasses the template gate."""
|
||||
target = ctx.abs_path("notes", "forced.md")
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_create",
|
||||
path=target,
|
||||
metadata={"title": "forced",
|
||||
"lifecycle": "evolving", "scope": "class",
|
||||
"source": "curated", "role": "concept"},
|
||||
content="forced through",
|
||||
force=True,
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_create",
|
||||
path=target,
|
||||
metadata={
|
||||
"title": "forced",
|
||||
"lifecycle": "evolving",
|
||||
"scope": "class",
|
||||
"source": "curated",
|
||||
"role": "concept",
|
||||
},
|
||||
content="forced through",
|
||||
force=True,
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict) and r.get("created") is True, r
|
||||
assert Path(target).is_file(), target
|
||||
return "force=True bypassed"
|
||||
|
|
@ -342,12 +389,14 @@ async def check_schema_status_skip_refused(ctx: AppContext) -> str:
|
|||
Trying distilled → active is reverse — must refuse."""
|
||||
event_index = getattr(ctx, "_suite_event_index", None)
|
||||
assert event_index is not None, "suite-event index not staged"
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_property_update",
|
||||
path=str(event_index),
|
||||
key="status",
|
||||
value="active",
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_property_update",
|
||||
path=str(event_index),
|
||||
key="status",
|
||||
value="active",
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict), r
|
||||
assert "error" in r and "transition" in r["error"].lower(), r
|
||||
assert r.get("prior") == "distilled", r
|
||||
|
|
@ -358,12 +407,14 @@ async def check_schema_status_invalid_value(ctx: AppContext) -> str:
|
|||
"""Random string for status is refused before any state-machine check."""
|
||||
event_index = getattr(ctx, "_suite_event_index", None)
|
||||
assert event_index is not None, "suite-event index not staged"
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_property_update",
|
||||
path=str(event_index),
|
||||
key="status",
|
||||
value="bogus",
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_property_update",
|
||||
path=str(event_index),
|
||||
key="status",
|
||||
value="bogus",
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict), r
|
||||
assert "error" in r and "invalid" in r["error"].lower(), r
|
||||
return "refused status='bogus'"
|
||||
|
|
@ -374,20 +425,27 @@ async def check_schema_status_force_bypass(ctx: AppContext) -> str:
|
|||
intentionally needs to step outside conventions."""
|
||||
event_index = getattr(ctx, "_suite_event_index", None)
|
||||
assert event_index is not None, "suite-event index not staged"
|
||||
r = decode(await ctx.app.run_job(
|
||||
"memory_property_update",
|
||||
path=str(event_index),
|
||||
key="status",
|
||||
value="active",
|
||||
force=True,
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_property_update",
|
||||
path=str(event_index),
|
||||
key="status",
|
||||
value="active",
|
||||
force=True,
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict), r
|
||||
assert "error" not in r, r
|
||||
# restore for any downstream checks
|
||||
decode(await ctx.app.run_job(
|
||||
"memory_property_update",
|
||||
path=str(event_index), key="status", value="distilled", force=True,
|
||||
))
|
||||
decode(
|
||||
await ctx.app.run_job(
|
||||
"memory_property_update",
|
||||
path=str(event_index),
|
||||
key="status",
|
||||
value="distilled",
|
||||
force=True,
|
||||
),
|
||||
)
|
||||
return "force=True bypassed"
|
||||
|
||||
|
||||
|
|
@ -398,28 +456,28 @@ async def check_schema_status_force_bypass(ctx: AppContext) -> str:
|
|||
# rely on side effects from earlier ones (e.g. sync.append needs
|
||||
# sync.create to have run).
|
||||
CHECKS: list[tuple[str, callable]] = [
|
||||
("registry", check_registry),
|
||||
("memory_get", check_memory_get),
|
||||
("memory_list", check_memory_list),
|
||||
("memory_search", check_memory_search),
|
||||
("memory_graph_search", check_memory_graph_search),
|
||||
("memory_links", check_memory_links),
|
||||
("memory_backlinks", check_memory_backlinks),
|
||||
("memory_resolve_wikilink", check_memory_resolve_wikilink),
|
||||
("memory_count_tokens", check_memory_count_tokens),
|
||||
("memory_lint", check_memory_lint),
|
||||
("sync.create", check_sync_create),
|
||||
("sync.append", check_sync_append),
|
||||
("sync.refuse_distilled", check_sync_refuse_distilled),
|
||||
("memory_create", check_memory_create),
|
||||
("memory_update", check_memory_update),
|
||||
("memory_property_update", check_memory_property_update),
|
||||
("memory_rename", check_memory_rename),
|
||||
("memory_archive", check_memory_archive),
|
||||
("memory_delete", check_memory_delete),
|
||||
("schema.path_template_refuses", check_schema_path_template_refuses),
|
||||
("schema.path_template_force", check_schema_path_template_force_bypass),
|
||||
("schema.status_skip_refused", check_schema_status_skip_refused),
|
||||
("schema.status_invalid_value", check_schema_status_invalid_value),
|
||||
("schema.status_force", check_schema_status_force_bypass),
|
||||
("registry", check_registry),
|
||||
("memory_get", check_memory_get),
|
||||
("memory_list", check_memory_list),
|
||||
("memory_search", check_memory_search),
|
||||
("memory_graph_search", check_memory_graph_search),
|
||||
("memory_links", check_memory_links),
|
||||
("memory_backlinks", check_memory_backlinks),
|
||||
("memory_resolve_wikilink", check_memory_resolve_wikilink),
|
||||
("memory_count_tokens", check_memory_count_tokens),
|
||||
("memory_lint", check_memory_lint),
|
||||
("sync.create", check_sync_create),
|
||||
("sync.append", check_sync_append),
|
||||
("sync.refuse_distilled", check_sync_refuse_distilled),
|
||||
("memory_create", check_memory_create),
|
||||
("memory_update", check_memory_update),
|
||||
("memory_property_update", check_memory_property_update),
|
||||
("memory_rename", check_memory_rename),
|
||||
("memory_archive", check_memory_archive),
|
||||
("memory_delete", check_memory_delete),
|
||||
("schema.path_template_refuses", check_schema_path_template_refuses),
|
||||
("schema.path_template_force", check_schema_path_template_force_bypass),
|
||||
("schema.status_skip_refused", check_schema_status_skip_refused),
|
||||
("schema.status_invalid_value", check_schema_status_invalid_value),
|
||||
("schema.status_force", check_schema_status_force_bypass),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -31,9 +31,7 @@ async def check_registry(ctx: AppContext) -> str:
|
|||
missing = [j for j in EXPECTED_JOBS if j not in ctx.jobs]
|
||||
assert not missing, f"missing jobs: {missing}"
|
||||
extras = [j for j in ctx.jobs if j not in EXPECTED_JOBS]
|
||||
assert not extras, (
|
||||
f"curated profile leaked extra jobs: {extras} — keep the surface tight"
|
||||
)
|
||||
assert not extras, f"curated profile leaked extra jobs: {extras} — keep the surface tight"
|
||||
return f"{len(ctx.jobs)} jobs registered"
|
||||
|
||||
|
||||
|
|
@ -41,9 +39,14 @@ async def check_registry(ctx: AppContext) -> str:
|
|||
|
||||
|
||||
async def check_retrieve_basic(ctx: AppContext) -> str:
|
||||
r = decode(await ctx.app.run_job(
|
||||
"retrieve", query="Alice Bob", max_results=5, min_score=0.0,
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"retrieve",
|
||||
query="Alice Bob",
|
||||
max_results=5,
|
||||
min_score=0.0,
|
||||
),
|
||||
)
|
||||
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
|
||||
assert len(hits) > 0, r
|
||||
return f"{len(hits)} hits"
|
||||
|
|
@ -51,13 +54,15 @@ async def check_retrieve_basic(ctx: AppContext) -> str:
|
|||
|
||||
async def check_retrieve_anchored(ctx: AppContext) -> str:
|
||||
"""Wikilink-anchored mode — `[[Project X]]` in the query seeds BFS."""
|
||||
r = decode(await ctx.app.run_job(
|
||||
"retrieve",
|
||||
query="What touches [[Project X]]?",
|
||||
max_results=5,
|
||||
min_score=0.0,
|
||||
graph_depth=1,
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"retrieve",
|
||||
query="What touches [[Project X]]?",
|
||||
max_results=5,
|
||||
min_score=0.0,
|
||||
graph_depth=1,
|
||||
),
|
||||
)
|
||||
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
|
||||
assert len(hits) > 0, r
|
||||
paths = {h.get("path") for h in hits if isinstance(h, dict)}
|
||||
|
|
@ -70,14 +75,16 @@ async def check_retrieve_anchored(ctx: AppContext) -> str:
|
|||
async def check_retrieve_topic_seeded(ctx: AppContext) -> str:
|
||||
"""Topic-rooted mode — explicit `seeds=[...]` instead of inline wikilink."""
|
||||
project_x = ctx.abs_path("topics", "Project X", "Project X.md")
|
||||
r = decode(await ctx.app.run_job(
|
||||
"retrieve",
|
||||
query="collaborates",
|
||||
max_results=5,
|
||||
min_score=0.0,
|
||||
seeds=[project_x],
|
||||
graph_depth=1,
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"retrieve",
|
||||
query="collaborates",
|
||||
max_results=5,
|
||||
min_score=0.0,
|
||||
seeds=[project_x],
|
||||
graph_depth=1,
|
||||
),
|
||||
)
|
||||
hits = r if isinstance(r, list) else (r.get("chunks") if isinstance(r, dict) else [])
|
||||
assert len(hits) > 0, r
|
||||
return f"{len(hits)} hits (seeded at Project X)"
|
||||
|
|
@ -87,19 +94,21 @@ async def check_retrieve_topic_seeded(ctx: AppContext) -> str:
|
|||
|
||||
|
||||
async def check_remember_log_create(ctx: AppContext) -> str:
|
||||
r = decode(await ctx.app.run_job(
|
||||
"remember",
|
||||
mode="log",
|
||||
name="curated-event",
|
||||
description="curated-profile suite",
|
||||
content="## ops\n- ran the curated suite\n",
|
||||
topics=["[[Alice]]"],
|
||||
tags=["curated"],
|
||||
materials=[
|
||||
{"filename": "raw-prompt.md", "content": "# user prompt\n\nrun curated suite\n"},
|
||||
{"filename": "tool-output.txt", "content": "exit=0\n"},
|
||||
],
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"remember",
|
||||
mode="log",
|
||||
name="curated-event",
|
||||
description="curated-profile suite",
|
||||
content="## ops\n- ran the curated suite\n",
|
||||
topics=["[[Alice]]"],
|
||||
tags=["curated"],
|
||||
materials=[
|
||||
{"filename": "raw-prompt.md", "content": "# user prompt\n\nrun curated suite\n"},
|
||||
{"filename": "tool-output.txt", "content": "exit=0\n"},
|
||||
],
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict), r
|
||||
assert r.get("created") is True and r.get("action") == "created", r
|
||||
materials = r.get("materials", [])
|
||||
|
|
@ -118,17 +127,19 @@ async def check_remember_log_create(ctx: AppContext) -> str:
|
|||
|
||||
|
||||
async def check_remember_log_append(ctx: AppContext) -> str:
|
||||
r = decode(await ctx.app.run_job(
|
||||
"remember",
|
||||
mode="log",
|
||||
name="curated-event",
|
||||
content="## follow-up\n- second pass\n",
|
||||
topics=["[[Bob]]"],
|
||||
tags=["follow-up"],
|
||||
materials=[
|
||||
{"filename": "tool-output.txt", "content": "second run\n"}, # collision
|
||||
],
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"remember",
|
||||
mode="log",
|
||||
name="curated-event",
|
||||
content="## follow-up\n- second pass\n",
|
||||
topics=["[[Bob]]"],
|
||||
tags=["follow-up"],
|
||||
materials=[
|
||||
{"filename": "tool-output.txt", "content": "second run\n"}, # collision
|
||||
],
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict), r
|
||||
assert r.get("created") is False and r.get("action") == "appended", r
|
||||
appended = r.get("materials", [])
|
||||
|
|
@ -150,9 +161,14 @@ async def check_remember_log_refuse_distilled(ctx: AppContext) -> str:
|
|||
# Give the watcher a moment to re-parse before sync re-reads frontmatter.
|
||||
await wait_for_index(ctx.watcher, expected_min=len(ctx.file_store))
|
||||
|
||||
r = decode(await ctx.app.run_job(
|
||||
"remember", mode="log", name="curated-event", content="should be refused",
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"remember",
|
||||
mode="log",
|
||||
name="curated-event",
|
||||
content="should be refused",
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict) and "error" in r, r
|
||||
assert r.get("status") == "distilled", r
|
||||
assert r.get("suggested_name"), r
|
||||
|
|
@ -164,26 +180,28 @@ async def check_remember_log_refuse_distilled(ctx: AppContext) -> str:
|
|||
|
||||
async def check_remember_distill_degraded(ctx: AppContext) -> str:
|
||||
target = ctx.abs_path("topics", "curated-ingested", "curated-ingested.md")
|
||||
r = decode(await ctx.app.run_job(
|
||||
"remember",
|
||||
# mode defaults to "distill"
|
||||
content="# curated-ingested\n\nproduced by the curated test suite.\n",
|
||||
target_path=target,
|
||||
metadata={
|
||||
"title": "curated-ingested",
|
||||
"lifecycle": "evolving",
|
||||
"scope": "class",
|
||||
"source": "curated",
|
||||
"role": "concept",
|
||||
"category": "concept",
|
||||
},
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"remember",
|
||||
# mode defaults to "distill"
|
||||
content="# curated-ingested\n\nproduced by the curated test suite.\n",
|
||||
target_path=target,
|
||||
metadata={
|
||||
"title": "curated-ingested",
|
||||
"lifecycle": "evolving",
|
||||
"scope": "class",
|
||||
"source": "curated",
|
||||
"role": "concept",
|
||||
"category": "concept",
|
||||
},
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict), r
|
||||
applied = r.get("applied") or []
|
||||
assert len(applied) == 1 and applied[0].get("ok") is True, r
|
||||
assert r.get("used_llm") is False, "expected degraded path (no LLM)"
|
||||
assert Path(target).is_file(), target
|
||||
return f"applied=1, used_llm=False"
|
||||
return "applied=1, used_llm=False"
|
||||
|
||||
|
||||
# ---------- maintain (lint + decay sweep) -----------------------------
|
||||
|
|
@ -206,9 +224,13 @@ async def check_maintain_dry_run(ctx: AppContext) -> str:
|
|||
async def check_maintain_targeted(ctx: AppContext) -> str:
|
||||
"""target_prefix narrows scan; events/ subtree should yield the
|
||||
one event folder created earlier in this suite."""
|
||||
r = decode(await ctx.app.run_job(
|
||||
"maintain", target_prefix="events/", dry_run=True,
|
||||
))
|
||||
r = decode(
|
||||
await ctx.app.run_job(
|
||||
"maintain",
|
||||
target_prefix="events/",
|
||||
dry_run=True,
|
||||
),
|
||||
)
|
||||
assert isinstance(r, dict), r
|
||||
assert r["scanned"] >= 1, r # at least the suite's own event folder
|
||||
return f"scanned={r['scanned']} under events/"
|
||||
|
|
@ -218,14 +240,14 @@ async def check_maintain_targeted(ctx: AppContext) -> str:
|
|||
|
||||
|
||||
CHECKS: list[tuple[str, callable]] = [
|
||||
("registry", check_registry),
|
||||
("retrieve.basic", check_retrieve_basic),
|
||||
("retrieve.anchored", check_retrieve_anchored),
|
||||
("retrieve.topic_seeded", check_retrieve_topic_seeded),
|
||||
("remember.log_create", check_remember_log_create),
|
||||
("remember.log_append", check_remember_log_append),
|
||||
("remember.log_refuse_distilled", check_remember_log_refuse_distilled),
|
||||
("remember.distill_degraded", check_remember_distill_degraded),
|
||||
("maintain.dry_run", check_maintain_dry_run),
|
||||
("maintain.targeted", check_maintain_targeted),
|
||||
("registry", check_registry),
|
||||
("retrieve.basic", check_retrieve_basic),
|
||||
("retrieve.anchored", check_retrieve_anchored),
|
||||
("retrieve.topic_seeded", check_retrieve_topic_seeded),
|
||||
("remember.log_create", check_remember_log_create),
|
||||
("remember.log_append", check_remember_log_append),
|
||||
("remember.log_refuse_distilled", check_remember_log_refuse_distilled),
|
||||
("remember.distill_degraded", check_remember_distill_degraded),
|
||||
("maintain.dry_run", check_maintain_dry_run),
|
||||
("maintain.targeted", check_maintain_targeted),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,22 +1,41 @@
|
|||
"""Memory subsystem — the three services on top of the core engine.
|
||||
"""Memory subsystem — agent-facing services + tools on top of the core engine.
|
||||
|
||||
Per the architecture blueprint:
|
||||
Three services (read / cold-write / treatment) plus the agent-callable
|
||||
write/read tools that operate on the Memory File System. Every
|
||||
``@R.register`` step a host agent might invoke lives here, regardless
|
||||
of whether it's reached via MCP, HTTP, or direct Python.
|
||||
|
||||
- retriever.py Read service. V + K + Graph BFS fusion + intent
|
||||
routing. Registered as a Step (`backend: hybrid`);
|
||||
MCP step shells in `reme2.mcp.steps.memory_retriever`
|
||||
instantiate it on demand.
|
||||
- ingestor.py Cold-write service. LLM-driven R-M-W curator.
|
||||
Triggered on explicit handoff (task end / SessionEnd).
|
||||
- maintainer.py Treatment service. Background Merge / Split / Decay /
|
||||
Lint, woken by cron or thresholds.
|
||||
- summarizer.py Auxiliary used by the services.
|
||||
Services
|
||||
retriever.py Read service — V + K + Graph BFS fusion + intent
|
||||
routing. Registered as ``hybrid``; consumed by
|
||||
the search shells below.
|
||||
ingestor.py Cold-write service — LLM-driven R-M-W curator
|
||||
(also registers an ``ingest`` MCP face).
|
||||
maintainer.py Treatment service — Merge / Split / Decay / Lint,
|
||||
woken by cron or thresholds.
|
||||
summarizer.py Auxiliary used by the services.
|
||||
|
||||
Hot-write MCP step shells (sync, memory_*) live in
|
||||
`reme2.mcp.steps`, NOT here — they bypass services and write MFS
|
||||
directly. Importing this package triggers @R.register on the three
|
||||
services so configs that name them resolve at boot.
|
||||
Tools (each ``@R.register`` exposes a step the agent invokes by name)
|
||||
memory_io.py Pure engine API (no schema, no Step boilerplate).
|
||||
memory_toolkit.py 12 ``memory_*`` primitives — get / list / links /
|
||||
backlinks / resolve_wikilink / count_tokens +
|
||||
create / update / property_update / rename /
|
||||
delete / archive.
|
||||
memory_search.py ``memory_search`` + ``memory_graph_search`` —
|
||||
shells over the Retriever service.
|
||||
memory_lint.py ``memory_lint`` — narrowed projection of the
|
||||
Maintainer's lint pass.
|
||||
sync.py ``sync`` — hot-path event-folder upsert
|
||||
(deterministic, no LLM).
|
||||
|
||||
Importing this package triggers every ``@R.register`` so configs that
|
||||
name these tools resolve at boot.
|
||||
"""
|
||||
|
||||
from . import retriever # noqa: F401 -- runs @R.register("hybrid")
|
||||
from . import maintainer # noqa: F401 -- runs @R.register("maintainer")
|
||||
from . import retriever # noqa: F401 -- @R.register("hybrid")
|
||||
from . import ingestor # noqa: F401 -- @R.register("ingestor")
|
||||
from . import maintainer # noqa: F401 -- @R.register("maintainer")
|
||||
from . import memory_toolkit # noqa: F401 -- @R.register("memory_*") x12
|
||||
from . import memory_search # noqa: F401 -- @R.register("memory_search", "memory_graph_search")
|
||||
from . import memory_lint # noqa: F401 -- @R.register("memory_lint")
|
||||
from . import sync # noqa: F401 -- @R.register("sync")
|
||||
|
|
|
|||
|
|
@ -107,9 +107,12 @@ class Ingestor(BaseStep):
|
|||
return
|
||||
if mode != "distill":
|
||||
self.context.response.success = False
|
||||
_set_answer(self.context, {
|
||||
"error": f"unknown mode {mode!r}; expected 'log' or 'distill'",
|
||||
})
|
||||
_set_answer(
|
||||
self.context,
|
||||
{
|
||||
"error": f"unknown mode {mode!r}; expected 'log' or 'distill'",
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
content: str = self.context.get("content", "") or ""
|
||||
|
|
@ -182,18 +185,22 @@ class Ingestor(BaseStep):
|
|||
supported. Useful for tests and bootstrap scripts."""
|
||||
result = IngestResult(used_llm=False)
|
||||
if not target_path:
|
||||
result.failed.append({
|
||||
"op": "create",
|
||||
"ok": False,
|
||||
"error": "degraded path: target_path is required when no LLM is configured",
|
||||
})
|
||||
result.failed.append(
|
||||
{
|
||||
"op": "create",
|
||||
"ok": False,
|
||||
"error": "degraded path: target_path is required when no LLM is configured",
|
||||
}
|
||||
)
|
||||
return result
|
||||
path = Path(target_path)
|
||||
if not path.is_absolute():
|
||||
path = self._working_dir() / path
|
||||
ok, payload = create_file(
|
||||
self.file_store, path,
|
||||
metadata=metadata, content=content,
|
||||
self.file_store,
|
||||
path,
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
)
|
||||
bucket = result.applied if ok else result.failed
|
||||
bucket.append({"op": "create", "ok": ok, "path": str(path), "result": payload})
|
||||
|
|
@ -203,9 +210,8 @@ class Ingestor(BaseStep):
|
|||
"""Hot-path event-folder upsert — same code path as the standalone
|
||||
`sync` step. Lazy-instantiated so we don't pay the construction
|
||||
cost on every distill call."""
|
||||
from ..mcp.steps.sync import Sync
|
||||
from .sync import Sync
|
||||
|
||||
if getattr(self, "_sync_step", None) is None:
|
||||
self._sync_step = Sync(app_context=self.app_context)
|
||||
await self._sync_step(self.context)
|
||||
|
||||
|
|
|
|||
|
|
@ -70,7 +70,6 @@ from pydantic import BaseModel, Field
|
|||
from ..component import R
|
||||
from ..component.base_step import BaseStep
|
||||
from ..component.runtime_response import _set_answer
|
||||
from ..enumeration import ComponentEnum
|
||||
from . import memory_io
|
||||
from .schema import parse_frontmatter
|
||||
|
||||
|
|
@ -171,12 +170,12 @@ class FileSignal(BaseModel):
|
|||
"""
|
||||
|
||||
path: str
|
||||
relpath: str = "" # path relative to working_dir, "" if outside
|
||||
lifecycle: str = "" # streaming / evolving / frozen
|
||||
scope: str = "" # instance / class
|
||||
source: str = "" # auto / curated / derived
|
||||
role: str = "" # observation / claim / question / ...
|
||||
category: str = "" # legacy field, kept for migration windows
|
||||
relpath: str = "" # path relative to working_dir, "" if outside
|
||||
lifecycle: str = "" # streaming / evolving / frozen
|
||||
scope: str = "" # instance / class
|
||||
source: str = "" # auto / curated / derived
|
||||
role: str = "" # observation / claim / question / ...
|
||||
category: str = "" # legacy field, kept for migration windows
|
||||
status: str = ""
|
||||
age_days: int = 0
|
||||
metadata: dict = Field(default_factory=dict)
|
||||
|
|
@ -335,22 +334,24 @@ class Maintainer(BaseStep):
|
|||
pass
|
||||
if target_prefix and not relpath.startswith(target_prefix):
|
||||
continue
|
||||
fm = meta.metadata or {}
|
||||
fm = meta.front_matter.model_dump()
|
||||
age_seconds = max(0.0, now - (meta.st_mtime or now))
|
||||
parsed, _ = parse_frontmatter(fm)
|
||||
signals.append(FileSignal(
|
||||
path=path,
|
||||
relpath=relpath,
|
||||
lifecycle=str(parsed.lifecycle.value) if parsed else "",
|
||||
scope=str(parsed.scope.value) if parsed else "",
|
||||
source=str(parsed.source.value) if parsed else "",
|
||||
role=str(parsed.role.value) if parsed else "",
|
||||
category=str(fm.get("category") or ""),
|
||||
status=str(fm.get("status") or ""),
|
||||
age_days=int(age_seconds // 86400),
|
||||
metadata=fm,
|
||||
declared_topics=list(fm.get("topics") or []),
|
||||
))
|
||||
signals.append(
|
||||
FileSignal(
|
||||
path=path,
|
||||
relpath=relpath,
|
||||
lifecycle=str(parsed.lifecycle.value) if parsed else "",
|
||||
scope=str(parsed.scope.value) if parsed else "",
|
||||
source=str(parsed.source.value) if parsed else "",
|
||||
role=str(parsed.role.value) if parsed else "",
|
||||
category=str(fm.get("category") or ""),
|
||||
status=str(fm.get("status") or ""),
|
||||
age_days=int(age_seconds // 86400),
|
||||
metadata=fm,
|
||||
declared_topics=list(fm.get("topics") or []),
|
||||
),
|
||||
)
|
||||
return signals
|
||||
|
||||
# -- proposers ----------------------------------------------------------
|
||||
|
|
@ -361,30 +362,41 @@ class Maintainer(BaseStep):
|
|||
for sig in signals:
|
||||
for link in sig.declared_topics:
|
||||
if not memory_io.resolve_wikilink(self.file_store, link)["exists"]:
|
||||
out.append(LintFinding(
|
||||
path=sig.path, kind="broken_wikilink",
|
||||
detail=f"unresolved wikilink {link!r}",
|
||||
))
|
||||
out.append(
|
||||
LintFinding(
|
||||
path=sig.path,
|
||||
kind="broken_wikilink",
|
||||
detail=f"unresolved wikilink {link!r}",
|
||||
),
|
||||
)
|
||||
# Memory schema check: tolerant parse, surface every error
|
||||
# the parser collected. Empty `errors` ↔ valid frontmatter.
|
||||
_, errors = parse_frontmatter(sig.metadata)
|
||||
for err in errors:
|
||||
out.append(LintFinding(
|
||||
path=sig.path, kind="schema_violation",
|
||||
detail=f"Memory: {err}"[:240],
|
||||
))
|
||||
out.append(
|
||||
LintFinding(
|
||||
path=sig.path,
|
||||
kind="schema_violation",
|
||||
detail=f"Memory: {err}"[:240],
|
||||
),
|
||||
)
|
||||
# Stem collisions: the engine API exposes the ambiguous-stem map.
|
||||
ambig = memory_io.find_collisions(self.file_store)
|
||||
for stem, paths in ambig.items():
|
||||
for p in paths:
|
||||
out.append(LintFinding(
|
||||
path=p, kind="stem_collision",
|
||||
detail=f"stem {stem!r} also claimed by {[x for x in paths if x != p]}",
|
||||
))
|
||||
out.append(
|
||||
LintFinding(
|
||||
path=p,
|
||||
kind="stem_collision",
|
||||
detail=f"stem {stem!r} also claimed by {[x for x in paths if x != p]}",
|
||||
),
|
||||
)
|
||||
return out
|
||||
|
||||
def _propose_decay(
|
||||
self, signals: list[FileSignal], decay_days: int,
|
||||
self,
|
||||
signals: list[FileSignal],
|
||||
decay_days: int,
|
||||
) -> list[DecayOp]:
|
||||
"""Distilled streaming memories past the freshness window.
|
||||
|
||||
|
|
@ -401,14 +413,18 @@ class Maintainer(BaseStep):
|
|||
continue
|
||||
if sig.age_days < decay_days:
|
||||
continue
|
||||
out.append(DecayOp(
|
||||
path=sig.path, age_days=sig.age_days,
|
||||
reason=f"streaming {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)",
|
||||
))
|
||||
out.append(
|
||||
DecayOp(
|
||||
path=sig.path,
|
||||
age_days=sig.age_days,
|
||||
reason=f"streaming {sig.status!r} for {sig.age_days}d (≥{decay_days}d window)",
|
||||
),
|
||||
)
|
||||
return out
|
||||
|
||||
async def _propose_enrich(
|
||||
self, signals: list[FileSignal],
|
||||
self,
|
||||
signals: list[FileSignal],
|
||||
) -> list[EnrichOp]:
|
||||
"""Bare wikilinks → typed via inline-bracketed Dataview wrap.
|
||||
|
||||
|
|
@ -418,13 +434,13 @@ class Maintainer(BaseStep):
|
|||
2. Read body, ask the LLM (one call per file) to assign one
|
||||
of `ALLOWED_PREDICATES` to each bare target — or 'skip'.
|
||||
3. For each accepted (target, predicate), locate each bare
|
||||
occurrence span via `FileEdge.from_text` and build a unique
|
||||
occurrence span via `iter_links` and build a unique
|
||||
context window snippet.
|
||||
4. Emit EnrichOp(path, target, predicate, old_string, new_string,
|
||||
confidence, reason). The apply step calls memory_update.
|
||||
|
||||
Idempotency: re-running won't re-enrich already-typed edges
|
||||
(filter is `predicate is None`). The OOV-tolerant FileEdge
|
||||
Idempotency: re-running won't re-enrich already-typed links
|
||||
(filter is `predicate is None`). The OOV-tolerant FileLink
|
||||
validator means a malformed LLM output collapses to None at the
|
||||
next parse, surfacing it again next sweep — bounded retries
|
||||
avoid infinite re-enrichment loops.
|
||||
|
|
@ -432,7 +448,8 @@ class Maintainer(BaseStep):
|
|||
return []
|
||||
|
||||
async def _propose_discover(
|
||||
self, signals: list[FileSignal],
|
||||
self,
|
||||
signals: list[FileSignal],
|
||||
) -> list[DiscoverOp]:
|
||||
"""Discover edges absent from body — append to `## Relations`.
|
||||
|
||||
|
|
@ -453,7 +470,9 @@ class Maintainer(BaseStep):
|
|||
return []
|
||||
|
||||
async def _propose_merge(
|
||||
self, signals: list[FileSignal], threshold: float,
|
||||
self,
|
||||
signals: list[FileSignal],
|
||||
threshold: float,
|
||||
) -> list[MergeOp]:
|
||||
"""Cluster near-duplicate topics → MergeOps (LLM-assisted).
|
||||
|
||||
|
|
@ -468,7 +487,9 @@ class Maintainer(BaseStep):
|
|||
return []
|
||||
|
||||
async def _propose_split(
|
||||
self, signals: list[FileSignal], token_threshold: int,
|
||||
self,
|
||||
signals: list[FileSignal],
|
||||
token_threshold: int,
|
||||
) -> list[SplitOp]:
|
||||
"""Topics over the token threshold → SplitOps (LLM-assisted).
|
||||
|
||||
|
|
@ -483,7 +504,8 @@ class Maintainer(BaseStep):
|
|||
# -- conflict resolution ------------------------------------------------
|
||||
|
||||
def _resolve_conflicts(
|
||||
self, proposed: list[BaseModel],
|
||||
self,
|
||||
proposed: list[BaseModel],
|
||||
) -> tuple[list[BaseModel], list[BaseModel]]:
|
||||
"""Apply the conflict matrix; return (kept_plan, dropped).
|
||||
|
||||
|
|
@ -588,7 +610,8 @@ class Maintainer(BaseStep):
|
|||
# -- apply --------------------------------------------------------------
|
||||
|
||||
async def _apply(
|
||||
self, plan: list[BaseModel],
|
||||
self,
|
||||
plan: list[BaseModel],
|
||||
) -> tuple[list[dict], list[dict]]:
|
||||
"""Execute the plan in fixed order. Each phase yields audit dicts.
|
||||
|
||||
|
|
@ -613,11 +636,13 @@ class Maintainer(BaseStep):
|
|||
result = await self._apply_one(op)
|
||||
applied.append({"op": op.op, **result}) # type: ignore[attr-defined]
|
||||
except Exception as e:
|
||||
failed.append({
|
||||
"op": op.op, # type: ignore[attr-defined]
|
||||
"payload": _dump(op),
|
||||
"error": f"{type(e).__name__}: {e}",
|
||||
})
|
||||
failed.append(
|
||||
{
|
||||
"op": op.op, # type: ignore[attr-defined]
|
||||
"payload": _dump(op),
|
||||
"error": f"{type(e).__name__}: {e}",
|
||||
}
|
||||
)
|
||||
return applied, failed
|
||||
|
||||
async def _apply_one(self, op: BaseModel) -> dict:
|
||||
|
|
@ -626,28 +651,44 @@ class Maintainer(BaseStep):
|
|||
return {"path": op.path, "kind": op.kind, "noop": True}
|
||||
if isinstance(op, EnrichOp):
|
||||
# TODO: wire to memory_io.update_body once apply-side is enabled.
|
||||
return {"path": op.path, "target": op.target, "predicate": op.predicate,
|
||||
"status": "pending_apply",
|
||||
"would": "memory_update wraps bare [[X]] as [predicate:: [[X]]]"}
|
||||
return {
|
||||
"path": op.path,
|
||||
"target": op.target,
|
||||
"predicate": op.predicate,
|
||||
"status": "pending_apply",
|
||||
"would": "memory_update wraps bare [[X]] as [predicate:: [[X]]]",
|
||||
}
|
||||
if isinstance(op, DiscoverOp):
|
||||
# TODO: append Dataview lines under ## Relations (create heading
|
||||
# if absent), then call file_store invalidation.
|
||||
return {"path": op.path, "edges_added": len(op.edges),
|
||||
"status": "pending_apply",
|
||||
"would": "append predicate:: [[Y]] under ## Relations heading"}
|
||||
return {
|
||||
"path": op.path,
|
||||
"edges_added": len(op.edges),
|
||||
"status": "pending_apply",
|
||||
"would": "append predicate:: [[Y]] under ## Relations heading",
|
||||
}
|
||||
if isinstance(op, DecayOp):
|
||||
# TODO: wire to MemoryArchive once we're ready to actually
|
||||
# move files from a cron context. For now, surface intent.
|
||||
return {"path": op.path, "status": "pending_apply",
|
||||
"would": "flip status=archived + move to archive_dir"}
|
||||
return {
|
||||
"path": op.path,
|
||||
"status": "pending_apply",
|
||||
"would": "flip status=archived + move to archive_dir",
|
||||
}
|
||||
if isinstance(op, MergeOp):
|
||||
return {"canonical": op.canonical, "sources": op.sources,
|
||||
"status": "pending_apply",
|
||||
"would": "merge bodies + rewrite incoming wikilinks + archive sources"}
|
||||
return {
|
||||
"canonical": op.canonical,
|
||||
"sources": op.sources,
|
||||
"status": "pending_apply",
|
||||
"would": "merge bodies + rewrite incoming wikilinks + archive sources",
|
||||
}
|
||||
if isinstance(op, SplitOp):
|
||||
return {"source": op.source, "sections": len(op.sections),
|
||||
"status": "pending_apply",
|
||||
"would": "extract sections + replace with [[…]] stubs"}
|
||||
return {
|
||||
"source": op.source,
|
||||
"sections": len(op.sections),
|
||||
"status": "pending_apply",
|
||||
"would": "extract sections + replace with [[…]] stubs",
|
||||
}
|
||||
raise TypeError(f"unknown op type: {type(op).__name__}")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,16 +3,21 @@
|
|||
The .md files are the SSOT (per `structure.md` §"核心引擎"). The engine
|
||||
is layered:
|
||||
|
||||
Memory File System → Watcher & Parser → Projections (vector / FTS / graph)
|
||||
Memory File System → Watcher & Parser → Projections (vector / FTS)
|
||||
(write entry) (incremental) (read entry, derived)
|
||||
|
||||
This module is the **single public API surface** over that engine. Every
|
||||
consumer — MCP step shells, the three memory services (Retriever,
|
||||
Ingestor, Maintainer), and the agent toolkit (`memory_toolkit`) — talks
|
||||
to the engine through these functions, not by reaching into
|
||||
`BaseFileStore` directly. That keeps `file_store` an implementation
|
||||
detail (could be local sqlite, remote, etc.) and gives the layering one
|
||||
place to evolve.
|
||||
consumer — agent-facing steps, the three memory services (Retriever,
|
||||
Ingestor, Maintainer), and the toolkit (`memory_toolkit`) — talks to
|
||||
the engine through these functions, not by reaching into
|
||||
``BaseFileStore`` directly.
|
||||
|
||||
Layering note. The slim ``BaseFileStore`` interface only owns the
|
||||
search projections (vector / FTS) plus atomic file upserts/deletes.
|
||||
Iteration, single-node lookup, and the wikilink graph live HERE — we
|
||||
walk ``LocalFileStore._nodes`` directly (engine-layer peer) and compute
|
||||
links on-the-fly from each ``FileNode.links``. There is no precomputed
|
||||
graph index; the graph is a derivation of the SSoT.
|
||||
|
||||
Naming convention:
|
||||
- Verb-first: `get_file`, `create_file`, `search_vector`.
|
||||
|
|
@ -21,11 +26,6 @@ Naming convention:
|
|||
- Pure-disk writes (`update_body`, `update_meta`, `delete_file`,
|
||||
`archive_file`) don't take `file_store` — they hit the filesystem
|
||||
and the watcher picks them up. The asymmetry is honest.
|
||||
|
||||
Vault-convention helpers — wikilink resolution, graph-walk ranking,
|
||||
chunk filtering, stem collisions — live here on top of the slim
|
||||
`BaseFileStore` (which only manages graph + chunks). The engine itself
|
||||
remains domain-agnostic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -38,21 +38,85 @@ from pathlib import Path
|
|||
|
||||
import frontmatter
|
||||
|
||||
from ..schema import ChunkFilter, FileChunk, FileNode, extract_wikilinks
|
||||
from ..schema.file_edge import _WIKILINK_RE
|
||||
from ..component.file_store.base_file_store import BaseFileStore
|
||||
from ..schema import ChunkFilter, FileChunk, FileLink, FileNode, extract_wikilinks
|
||||
from ..schema.file_link import _WIKILINK_RE
|
||||
from ..utils.wikilink_resolver import (
|
||||
resolve_wikilink as _resolve_wikilink,
|
||||
wikilink_candidates,
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Internal helpers — engine-layer access to the concrete node index
|
||||
# ===========================================================================
|
||||
#
|
||||
# ``BaseFileStore`` only declares the search/upsert contract; iteration
|
||||
# and single-node lookup live on the concrete impl (``LocalFileStore``).
|
||||
# memory_io is a peer at the engine layer, so reaching into ``_nodes``
|
||||
# is intentional — every iteration / graph walk funnels through here so
|
||||
# the day the engine grows a public ``iter_nodes()``, this is the only
|
||||
# place to swap.
|
||||
|
||||
|
||||
def _nodes(file_store: BaseFileStore) -> dict[str, FileNode]:
|
||||
"""The concrete in-memory ``{path: FileNode}`` index."""
|
||||
return file_store._nodes # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def _meta(node: FileNode) -> dict:
|
||||
"""Full frontmatter dict for a node — typed fields (title/description/
|
||||
tags) merged with any ``extra=allow`` extras."""
|
||||
return node.front_matter.model_dump()
|
||||
|
||||
|
||||
def _get_outlinks(
|
||||
file_store: BaseFileStore,
|
||||
path: str,
|
||||
) -> list[tuple[FileNode, FileLink]]:
|
||||
"""Resolved outgoing links from ``path`` — ``[(target_node, link), ...]``."""
|
||||
node = _nodes(file_store).get(path)
|
||||
if node is None:
|
||||
return []
|
||||
out: list[tuple[FileNode, FileLink]] = []
|
||||
for link in node.links:
|
||||
target = _resolve_wikilink(file_store, link.path)
|
||||
if target is None:
|
||||
continue
|
||||
target_node = _nodes(file_store).get(target)
|
||||
if target_node is not None:
|
||||
out.append((target_node, link))
|
||||
return out
|
||||
|
||||
|
||||
def _get_inlinks(
|
||||
file_store: BaseFileStore,
|
||||
path: str,
|
||||
) -> list[tuple[FileNode, FileLink]]:
|
||||
"""Resolved incoming links to ``path`` — linear scan over all nodes."""
|
||||
out: list[tuple[FileNode, FileLink]] = []
|
||||
for src_node in _nodes(file_store).values():
|
||||
for link in src_node.links:
|
||||
target = _resolve_wikilink(file_store, link.path)
|
||||
if target == path:
|
||||
out.append((src_node, link))
|
||||
return out
|
||||
|
||||
|
||||
def _filter_to_dict(chunk_filter: ChunkFilter | None) -> dict:
|
||||
"""Serialize ``ChunkFilter`` for the search engine's ``search_filter`` arg."""
|
||||
if chunk_filter is None:
|
||||
return {}
|
||||
return chunk_filter.model_dump(mode="json")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Section 1 — MFS Reads
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
async def get_file(
|
||||
file_store,
|
||||
file_store: BaseFileStore,
|
||||
path: str,
|
||||
*,
|
||||
include_chunks: bool = False,
|
||||
|
|
@ -62,14 +126,16 @@ async def get_file(
|
|||
On-disk frontmatter is the source of truth — the file_store cache may
|
||||
lag a write that hasn't been picked up by the watcher yet.
|
||||
"""
|
||||
node = file_store.get_node_by_path(path)
|
||||
node = await file_store.get_node_by_path(path) # type: ignore[attr-defined]
|
||||
result: dict = {"path": path, "exists": False}
|
||||
if node is not None:
|
||||
result.update({
|
||||
"exists": True,
|
||||
"metadata": node.metadata,
|
||||
"link": [e.model_dump(exclude_none=True) for e in node.edges],
|
||||
})
|
||||
result.update(
|
||||
{
|
||||
"exists": True,
|
||||
"metadata": _meta(node),
|
||||
"link": [link.model_dump(exclude_none=True) for link in node.links],
|
||||
}
|
||||
)
|
||||
|
||||
file_path = Path(path)
|
||||
if file_path.is_file():
|
||||
|
|
@ -80,13 +146,13 @@ async def get_file(
|
|||
result["metadata"] = dict(post.metadata)
|
||||
|
||||
if include_chunks:
|
||||
chunks = await file_store.get_chunks_by_path(path)
|
||||
chunks = await file_store.get_chunks_by_path(path) # type: ignore[attr-defined]
|
||||
result["chunks"] = [c.model_dump(exclude_none=True) for c in chunks]
|
||||
return result
|
||||
|
||||
|
||||
def list_files(
|
||||
file_store,
|
||||
file_store: BaseFileStore,
|
||||
*,
|
||||
path_prefix: str | None = None,
|
||||
tags: list[str] | None = None,
|
||||
|
|
@ -97,14 +163,14 @@ def list_files(
|
|||
metadata_filter = metadata or {}
|
||||
tag_filter = tags or []
|
||||
items: list[dict] = []
|
||||
for path, node in file_store.nodes.items():
|
||||
for path, node in _nodes(file_store).items():
|
||||
if path_prefix and not path.startswith(path_prefix):
|
||||
continue
|
||||
md = node.metadata or {}
|
||||
md = _meta(node)
|
||||
if metadata_filter and any(md.get(k) != v for k, v in metadata_filter.items()):
|
||||
continue
|
||||
if tag_filter:
|
||||
file_tags = set(md.get("tags", []) or [])
|
||||
file_tags = set(md.get("tags") or [])
|
||||
if not all(t in file_tags for t in tag_filter):
|
||||
continue
|
||||
items.append({"path": path, "metadata": md})
|
||||
|
|
@ -113,32 +179,32 @@ def list_files(
|
|||
return {"items": items, "count": len(items)}
|
||||
|
||||
|
||||
def _edge_to_dict(node, edge) -> dict:
|
||||
def _link_to_dict(node: FileNode, link: FileLink) -> dict:
|
||||
return {
|
||||
"path": node.path,
|
||||
"metadata": node.metadata,
|
||||
"predicate": edge.predicate,
|
||||
"anchor": edge.anchor,
|
||||
"metadata": _meta(node),
|
||||
"predicate": link.predicate,
|
||||
"anchor": link.anchor,
|
||||
}
|
||||
|
||||
|
||||
def get_links(file_store, path: str) -> dict:
|
||||
"""Files that `path` links TO (resolved). Each entry carries the typed-edge predicate."""
|
||||
def get_links(file_store: BaseFileStore, path: str) -> dict:
|
||||
"""Files that `path` links TO (resolved). Each entry carries the typed-link predicate."""
|
||||
return {
|
||||
"path": path,
|
||||
"links": [_edge_to_dict(m, e) for m, e in file_store.get_links(path)],
|
||||
"links": [_link_to_dict(m, link) for m, link in _get_outlinks(file_store, path)],
|
||||
}
|
||||
|
||||
|
||||
def get_backlinks(file_store, path: str) -> dict:
|
||||
"""Files that link TO `path`. Each entry carries the typed-edge predicate."""
|
||||
def get_backlinks(file_store: BaseFileStore, path: str) -> dict:
|
||||
"""Files that link TO `path`. Each entry carries the typed-link predicate."""
|
||||
return {
|
||||
"path": path,
|
||||
"backlinks": [_edge_to_dict(m, e) for m, e in file_store.get_backlinks(path)],
|
||||
"backlinks": [_link_to_dict(m, link) for m, link in _get_inlinks(file_store, path)],
|
||||
}
|
||||
|
||||
|
||||
def resolve_wikilink(file_store, wikilink: str) -> dict:
|
||||
def resolve_wikilink(file_store: BaseFileStore, wikilink: str) -> dict:
|
||||
"""Resolve a `[[target]]` wikilink with full ambiguity context.
|
||||
|
||||
Returns:
|
||||
|
|
@ -177,9 +243,9 @@ def resolve_wikilink(file_store, wikilink: str) -> dict:
|
|||
}
|
||||
|
||||
|
||||
def iter_files(file_store) -> Iterator[tuple[str, FileNode]]:
|
||||
def iter_files(file_store: BaseFileStore) -> Iterator[tuple[str, FileNode]]:
|
||||
"""Walk every indexed (path, FileNode). Used by Maintainer scans."""
|
||||
return iter(file_store.nodes.items())
|
||||
return iter(_nodes(file_store).items())
|
||||
|
||||
|
||||
async def count_tokens(
|
||||
|
|
@ -234,7 +300,7 @@ def _replace_wikilink_targets(text: str, mapping: dict[str, str]) -> str:
|
|||
|
||||
|
||||
def create_file(
|
||||
file_store,
|
||||
file_store: BaseFileStore,
|
||||
path: Path,
|
||||
*,
|
||||
metadata: dict,
|
||||
|
|
@ -328,7 +394,7 @@ def update_meta(path: Path | str, *, key: str, value) -> tuple[bool, dict]:
|
|||
|
||||
|
||||
def rename_file(
|
||||
file_store,
|
||||
file_store: BaseFileStore,
|
||||
working_dir: Path | str,
|
||||
*,
|
||||
old_path: Path | str,
|
||||
|
|
@ -349,8 +415,7 @@ def rename_file(
|
|||
if conflicts:
|
||||
return False, {
|
||||
"error": (
|
||||
f"stem `[[{new_p.stem}]]` would resolve ambiguously "
|
||||
f"to {len(conflicts) + 1} paths after this rename"
|
||||
f"stem `[[{new_p.stem}]]` would resolve ambiguously " f"to {len(conflicts) + 1} paths after this rename"
|
||||
),
|
||||
"conflicts": conflicts,
|
||||
"hint": (
|
||||
|
|
@ -375,7 +440,7 @@ def rename_file(
|
|||
except ValueError:
|
||||
pass
|
||||
|
||||
referring_paths = [m.path for m, _ in file_store.get_backlinks(str(old_p))]
|
||||
referring_paths = [m.path for m, _ in _get_inlinks(file_store, str(old_p))]
|
||||
|
||||
new_p.parent.mkdir(parents=True, exist_ok=True)
|
||||
old_p.rename(new_p)
|
||||
|
|
@ -469,43 +534,50 @@ def archive_file(
|
|||
|
||||
|
||||
async def search_vector(
|
||||
file_store,
|
||||
file_store: BaseFileStore,
|
||||
query: str,
|
||||
*,
|
||||
limit: int,
|
||||
chunk_filter: ChunkFilter | None = None,
|
||||
) -> list[FileChunk]:
|
||||
"""Vector similarity over the chunk-level Vector projection."""
|
||||
return await file_store.vector_search(query, limit, chunk_filter)
|
||||
return await file_store.vector_search(query, limit, _filter_to_dict(chunk_filter))
|
||||
|
||||
|
||||
async def search_keyword(
|
||||
file_store,
|
||||
file_store: BaseFileStore,
|
||||
query: str,
|
||||
*,
|
||||
limit: int,
|
||||
chunk_filter: ChunkFilter | None = None,
|
||||
) -> list[FileChunk]:
|
||||
"""FTS5 keyword search over the chunk-level keyword projection."""
|
||||
return await file_store.keyword_search(query, limit, chunk_filter)
|
||||
return await file_store.keyword_search(query, limit, _filter_to_dict(chunk_filter))
|
||||
|
||||
|
||||
async def get_chunks(file_store, paths: Iterable[str]) -> list[FileChunk]:
|
||||
async def get_chunks(
|
||||
file_store: BaseFileStore,
|
||||
paths: Iterable[str],
|
||||
) -> list[FileChunk]:
|
||||
"""Batch fetch chunks across many paths."""
|
||||
return await file_store.get_chunks_by_paths(paths)
|
||||
out: list[FileChunk] = []
|
||||
for p in paths:
|
||||
out.extend(await file_store.get_chunks_by_path(p)) # type: ignore[attr-defined]
|
||||
return out
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Section 4 — Graph helpers (BFS / scoring / collisions)
|
||||
# ===========================================================================
|
||||
#
|
||||
# Layered on top of the slim `BaseFileStore` (which only owns the
|
||||
# graph + chunks). These are vault conventions: folder-note preference,
|
||||
# wikilink-uniqueness gates, BFS for memory_graph_search.
|
||||
# All graph traversal walks ``FileNode.links`` directly via ``_get_outlinks``
|
||||
# / ``_get_inlinks``. There's no precomputed adjacency index — every BFS
|
||||
# resolves wikilinks per call. Cheap enough for typical vault sizes;
|
||||
# revisit if the maintainer's lint pass becomes a hotspot.
|
||||
|
||||
|
||||
def expand_neighbors(
|
||||
file_store,
|
||||
file_store: BaseFileStore,
|
||||
seeds: Iterable[str],
|
||||
*,
|
||||
depth: int = 1,
|
||||
|
|
@ -518,10 +590,11 @@ def expand_neighbors(
|
|||
if depth < 0:
|
||||
raise ValueError(f"depth must be >= 0, got {depth}")
|
||||
|
||||
nodes = _nodes(file_store)
|
||||
seen: dict[str, int] = {}
|
||||
frontier: deque[tuple[str, int]] = deque()
|
||||
for path in seeds:
|
||||
if path in file_store and path not in seen:
|
||||
if path in nodes and path not in seen:
|
||||
seen[path] = 0
|
||||
frontier.append((path, 0))
|
||||
|
||||
|
|
@ -532,9 +605,9 @@ def expand_neighbors(
|
|||
|
||||
neighbors: list[str] = []
|
||||
if direction in ("out", "both"):
|
||||
neighbors.extend(m.path for m, _ in file_store.get_links(path))
|
||||
neighbors.extend(m.path for m, _ in _get_outlinks(file_store, path))
|
||||
if direction in ("in", "both"):
|
||||
neighbors.extend(m.path for m, _ in file_store.get_backlinks(path))
|
||||
neighbors.extend(m.path for m, _ in _get_inlinks(file_store, path))
|
||||
|
||||
if per_node_cap is not None and len(neighbors) > per_node_cap:
|
||||
neighbors = neighbors[:per_node_cap]
|
||||
|
|
@ -549,7 +622,7 @@ def expand_neighbors(
|
|||
|
||||
|
||||
def subgraph_score(
|
||||
file_store,
|
||||
file_store: BaseFileStore,
|
||||
seeds: Iterable[str],
|
||||
*,
|
||||
decay: float = 0.5,
|
||||
|
|
@ -561,12 +634,16 @@ def subgraph_score(
|
|||
if not (0.0 <= decay <= 1.0):
|
||||
raise ValueError(f"decay must be in [0, 1], got {decay}")
|
||||
hops = expand_neighbors(
|
||||
file_store, seeds, depth=depth, direction=direction, per_node_cap=per_node_cap,
|
||||
file_store,
|
||||
seeds,
|
||||
depth=depth,
|
||||
direction=direction,
|
||||
per_node_cap=per_node_cap,
|
||||
)
|
||||
return {path: decay ** hop for path, hop in hops.items()}
|
||||
return {path: decay**hop for path, hop in hops.items()}
|
||||
|
||||
|
||||
def extract_anchors(file_store, text: str) -> list[str]:
|
||||
def extract_anchors(file_store: BaseFileStore, text: str) -> list[str]:
|
||||
"""Pull `[[X]]` anchors from `text` and resolve each (deduped)."""
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
|
|
@ -578,7 +655,10 @@ def extract_anchors(file_store, text: str) -> list[str]:
|
|||
return out
|
||||
|
||||
|
||||
def collisions_after_create(file_store, proposed_path: str | Path) -> list[str]:
|
||||
def collisions_after_create(
|
||||
file_store: BaseFileStore,
|
||||
proposed_path: str | Path,
|
||||
) -> list[str]:
|
||||
"""Existing paths that would conflict with adding `proposed_path`.
|
||||
|
||||
Folder-note rule: if `proposed_path`'s parent dir name == its stem,
|
||||
|
|
@ -591,23 +671,26 @@ def collisions_after_create(file_store, proposed_path: str | Path) -> list[str]:
|
|||
proposed_abs = str(p.resolve())
|
||||
is_folder_note = p.parent.name == stem
|
||||
|
||||
folder_hits = [
|
||||
path for path in file_store.nodes
|
||||
if Path(path).stem == stem and Path(path).parent.name == stem
|
||||
and path != proposed_abs
|
||||
]
|
||||
stem_hits = [
|
||||
path for path in file_store.get_paths_by_stem(stem)
|
||||
if path != proposed_abs
|
||||
]
|
||||
folder_hits: list[str] = []
|
||||
stem_hits: list[str] = []
|
||||
for path in _nodes(file_store):
|
||||
if path == proposed_abs:
|
||||
continue
|
||||
path_obj = Path(path)
|
||||
if path_obj.stem != stem:
|
||||
continue
|
||||
if path_obj.parent.name == stem:
|
||||
folder_hits.append(path)
|
||||
else:
|
||||
stem_hits.append(path)
|
||||
|
||||
if is_folder_note:
|
||||
return folder_hits
|
||||
return folder_hits + [sp for sp in stem_hits if sp not in folder_hits]
|
||||
return folder_hits + stem_hits
|
||||
|
||||
|
||||
def make_filter(
|
||||
file_store,
|
||||
file_store: BaseFileStore,
|
||||
*,
|
||||
paths: list[str] | None = None,
|
||||
tags: list[str] | None = None,
|
||||
|
|
@ -617,17 +700,13 @@ def make_filter(
|
|||
cf = ChunkFilter(paths=paths, tags=tags, exclude_paths=exclude_paths)
|
||||
if cf.is_empty():
|
||||
return cf
|
||||
cf.resolved_paths = {
|
||||
p for p, n in file_store.nodes.items() if cf.match_metadata(p, n.metadata)
|
||||
}
|
||||
cf.resolved_paths = {p for p, n in _nodes(file_store).items() if cf.match_metadata(p, _meta(n))}
|
||||
return cf
|
||||
|
||||
|
||||
def find_collisions(file_store) -> dict[str, list[str]]:
|
||||
def find_collisions(file_store: BaseFileStore) -> dict[str, list[str]]:
|
||||
"""Every stem that resolves to >1 path. Used by Maintainer.lint."""
|
||||
out: dict[str, list[str]] = {}
|
||||
for stem in file_store._stems:
|
||||
cands = wikilink_candidates(file_store, stem)
|
||||
if len(cands) > 1:
|
||||
out[stem] = cands
|
||||
return out
|
||||
by_stem: dict[str, list[str]] = {}
|
||||
for path in _nodes(file_store):
|
||||
by_stem.setdefault(Path(path).stem, []).append(path)
|
||||
return {s: ps for s, ps in by_stem.items() if len(ps) > 1}
|
||||
|
|
|
|||
|
|
@ -12,11 +12,10 @@ agent doesn't need to know about decay/merge/split knobs.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from ...component import R
|
||||
from ...component.base_step import BaseStep
|
||||
from ...component.runtime_response import _set_answer
|
||||
from ...enumeration import ComponentEnum
|
||||
from ...memory.maintainer import Maintainer
|
||||
from ..component import R
|
||||
from ..component.base_step import BaseStep
|
||||
from ..component.runtime_response import _set_answer
|
||||
from ..enumeration import ComponentEnum
|
||||
|
||||
|
||||
@R.register("memory_lint")
|
||||
|
|
@ -57,14 +56,18 @@ class MemoryLint(BaseStep):
|
|||
# action lives in `proposed` (LintFindings never get applied or
|
||||
# dropped). Reshape into a focused response.
|
||||
import json
|
||||
|
||||
raw = self.context.response.answer
|
||||
audit = json.loads(raw) if isinstance(raw, str) else (raw or {})
|
||||
findings = audit.get("proposed") or []
|
||||
|
||||
_set_answer(self.context, {
|
||||
"scanned": audit.get("scanned", 0),
|
||||
"findings": findings,
|
||||
"target_prefix": target_prefix,
|
||||
"ran_at": audit.get("ran_at", ""),
|
||||
})
|
||||
_set_answer(
|
||||
self.context,
|
||||
{
|
||||
"scanned": audit.get("scanned", 0),
|
||||
"findings": findings,
|
||||
"target_prefix": target_prefix,
|
||||
"ran_at": audit.get("ran_at", ""),
|
||||
},
|
||||
)
|
||||
self.context.response.success = True
|
||||
|
|
@ -1,26 +1,26 @@
|
|||
"""Memory retriever steps — thin MCP-facing wrappers over `BaseRetriever`.
|
||||
"""Memory search step shells — thin wrappers over the Retriever service.
|
||||
|
||||
Per the architecture blueprint, retrieval policy (V+K hybrid + graph
|
||||
BFS fusion + ranking + intent routing) is the **Retriever service**
|
||||
(`reme2.memory.retriever`), registered as `ComponentEnum.RETRIEVER`.
|
||||
These steps are the MCP projection: they translate `RuntimeContext`
|
||||
(paths/tags/exclude_paths filter, per-call knob overrides) into the
|
||||
retriever's call surface, then serialize results for the API response
|
||||
(joining file metadata onto each chunk so callers don't have to fire a
|
||||
`memory_get` per hit).
|
||||
BFS fusion + ranking + intent routing) lives in the **Retriever
|
||||
service** (`reme2.memory.retriever`), registered as
|
||||
``ComponentEnum.RETRIEVER``. The two steps here are the agent-facing
|
||||
projection: they translate ``RuntimeContext`` (paths/tags/exclude_paths
|
||||
filter, per-call knob overrides) into the retriever's call surface,
|
||||
then serialize results for the response (joining file metadata onto
|
||||
each chunk so callers don't have to fire a ``memory_get`` per hit).
|
||||
|
||||
Direct primary-key file ops (read/list/links/backlinks) live in
|
||||
`memory_io` — those aren't retrieval.
|
||||
``memory_io`` — those aren't retrieval.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ...component import R
|
||||
from ...component.base_step import BaseStep
|
||||
from ...component.runtime_response import _set_answer
|
||||
from ...enumeration import ComponentEnum
|
||||
from ...memory import memory_io
|
||||
from ...memory.retriever import BaseRetriever, HybridRetriever
|
||||
from ..component import R
|
||||
from ..component.base_step import BaseStep
|
||||
from ..component.runtime_response import _set_answer
|
||||
from ..enumeration import ComponentEnum
|
||||
from . import memory_io
|
||||
from .retriever import BaseRetriever, HybridRetriever
|
||||
|
||||
|
||||
# Per-shell singleton cache: instantiating the retriever is cheap (it
|
||||
|
|
@ -61,9 +61,16 @@ def _resolve_retriever(step: BaseStep) -> BaseRetriever:
|
|||
cls = HybridRetriever
|
||||
|
||||
knob_keys = (
|
||||
"vector_weight", "graph_weight", "graph_depth", "graph_decay",
|
||||
"graph_direction", "graph_mode", "graph_per_path_cap",
|
||||
"candidate_multiplier", "anchor_expand", "file_store",
|
||||
"vector_weight",
|
||||
"graph_weight",
|
||||
"graph_depth",
|
||||
"graph_decay",
|
||||
"graph_direction",
|
||||
"graph_mode",
|
||||
"graph_per_path_cap",
|
||||
"candidate_multiplier",
|
||||
"anchor_expand",
|
||||
"file_store",
|
||||
)
|
||||
init_kwargs = {k: step.kwargs[k] for k in knob_keys if k in step.kwargs}
|
||||
init_kwargs["app_context"] = step.app_context
|
||||
|
|
@ -84,9 +91,9 @@ def _serialize_chunk(chunk, file_store, extras: dict | None = None) -> dict:
|
|||
`extras` lets the caller attach step-specific fields (e.g. `graph_hop`).
|
||||
"""
|
||||
item = chunk.model_dump(exclude_none=True, exclude={"embedding"})
|
||||
node = file_store.get_node_by_path(chunk.path)
|
||||
node = file_store._nodes.get(chunk.path) # engine-layer peer access
|
||||
if node is not None:
|
||||
item["file_metadata"] = node.metadata
|
||||
item["file_metadata"] = node.front_matter.model_dump()
|
||||
item["file_st_mtime"] = node.st_mtime
|
||||
else:
|
||||
item["file_metadata"] = None
|
||||
|
|
@ -46,9 +46,9 @@ from ..enumeration import ComponentEnum
|
|||
|
||||
_STATUS_STATES = ("active", "distilled", "archived")
|
||||
_STATUS_TRANSITIONS: dict[str, set[str]] = {
|
||||
"active": {"active", "distilled"},
|
||||
"active": {"active", "distilled"},
|
||||
"distilled": {"distilled", "archived"},
|
||||
"archived": {"archived"},
|
||||
"archived": {"archived"},
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -59,10 +59,7 @@ def validate_status_transition(prior, requested) -> str | None:
|
|||
if requested is None:
|
||||
return None # delete operation
|
||||
if requested not in _STATUS_STATES:
|
||||
return (
|
||||
f"invalid status {requested!r}; must be one of "
|
||||
f"{list(_STATUS_STATES)}"
|
||||
)
|
||||
return f"invalid status {requested!r}; must be one of " f"{list(_STATUS_STATES)}"
|
||||
if prior in _STATUS_STATES and requested not in _STATUS_TRANSITIONS[prior]:
|
||||
return (
|
||||
f"status transition {prior!r} → {requested!r} not allowed; "
|
||||
|
|
@ -167,9 +164,12 @@ def create_file_with_schema(
|
|||
),
|
||||
}
|
||||
return memory_io.create_file(
|
||||
file_store, path,
|
||||
metadata=metadata, content=content,
|
||||
overwrite=overwrite, force=force,
|
||||
file_store,
|
||||
path,
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
overwrite=overwrite,
|
||||
force=force,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -336,9 +336,12 @@ class MemoryCreate(BaseStep):
|
|||
|
||||
target = Path(path)
|
||||
ok, payload = create_file_with_schema(
|
||||
self.file_store, target,
|
||||
metadata=metadata, content=content,
|
||||
overwrite=overwrite, force=force,
|
||||
self.file_store,
|
||||
target,
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
overwrite=overwrite,
|
||||
force=force,
|
||||
)
|
||||
self.context.response.success = ok
|
||||
if ok:
|
||||
|
|
@ -357,7 +360,8 @@ class MemoryCreate(BaseStep):
|
|||
gate both fire unless `force=True`."""
|
||||
target = Path(path)
|
||||
ok, payload = create_file_with_schema(
|
||||
self.file_store, target,
|
||||
self.file_store,
|
||||
target,
|
||||
metadata=dict(metadata or {}),
|
||||
content=content,
|
||||
overwrite=overwrite,
|
||||
|
|
@ -403,8 +407,10 @@ class MemoryRename(BaseStep):
|
|||
working_dir = Path(self.file_store.working_dir or Path.cwd()).resolve()
|
||||
|
||||
ok, payload = memory_io.rename_file(
|
||||
self.file_store, working_dir,
|
||||
old_path=old_path, new_path=new_path,
|
||||
self.file_store,
|
||||
working_dir,
|
||||
old_path=old_path,
|
||||
new_path=new_path,
|
||||
)
|
||||
self.context.response.success = ok
|
||||
_set_answer(self.context, payload)
|
||||
|
|
@ -414,8 +420,10 @@ class MemoryRename(BaseStep):
|
|||
vr = getattr(self.file_store, "working_dir", None)
|
||||
working_dir = Path(vr).resolve() if vr else Path.cwd()
|
||||
ok, payload = memory_io.rename_file(
|
||||
self.file_store, working_dir,
|
||||
old_path=old_path, new_path=new_path,
|
||||
self.file_store,
|
||||
working_dir,
|
||||
old_path=old_path,
|
||||
new_path=new_path,
|
||||
)
|
||||
return _tool_response("memory_rename", ok, payload, audit=self.audit)
|
||||
|
||||
|
|
@ -477,7 +485,10 @@ class MemoryUpdate(BaseStep):
|
|||
replace_all: bool = bool(self.context.get("replace_all", False))
|
||||
assert path, "path is required"
|
||||
ok, payload = memory_io.update_body(
|
||||
path, old_string=old_string, new_string=new_string, replace_all=replace_all,
|
||||
path,
|
||||
old_string=old_string,
|
||||
new_string=new_string,
|
||||
replace_all=replace_all,
|
||||
)
|
||||
self.context.response.success = ok
|
||||
_set_answer(self.context, payload)
|
||||
|
|
@ -491,7 +502,10 @@ class MemoryUpdate(BaseStep):
|
|||
) -> ToolResponse:
|
||||
"""Edit-style content update: replace `old_string` with `new_string`."""
|
||||
ok, payload = memory_io.update_body(
|
||||
path, old_string=old_string, new_string=new_string, replace_all=replace_all,
|
||||
path,
|
||||
old_string=old_string,
|
||||
new_string=new_string,
|
||||
replace_all=replace_all,
|
||||
)
|
||||
return _tool_response("memory_update", ok, payload, audit=self.audit)
|
||||
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ Two surfaces:
|
|||
standard Step lifecycle + `self.file_store` property + per-instance
|
||||
configuration via constructor kwargs. The default `execute()` dispatch
|
||||
just runs `graph_search` so the retriever can also be used directly as
|
||||
a step inside a job pipeline; the MCP step shells in
|
||||
`reme2.mcp.steps.memory_retriever` instead bypass `execute` and call
|
||||
a step inside a job pipeline; the search step shells in
|
||||
`reme2.memory.memory_search` instead bypass `execute` and call
|
||||
`search` / `graph_search` directly so they own the result-serialization
|
||||
shape.
|
||||
"""
|
||||
|
|
@ -183,7 +183,10 @@ class HybridRetriever(BaseRetriever):
|
|||
results = k_results[:max_results]
|
||||
else:
|
||||
results = self._merge_vk(
|
||||
v_results, k_results, self.vector_weight, text_weight,
|
||||
v_results,
|
||||
k_results,
|
||||
self.vector_weight,
|
||||
text_weight,
|
||||
)[:max_results]
|
||||
elif fs.embedding_model:
|
||||
results = await memory_io.search_vector(fs, query, limit=max_results, chunk_filter=chunk_filter)
|
||||
|
|
@ -287,7 +290,7 @@ class HybridRetriever(BaseRetriever):
|
|||
hops = memory_io.expand_neighbors(fs, seed_paths, depth=gd, direction=gdr)
|
||||
else:
|
||||
hops = {}
|
||||
graph_scores: dict[str, float] = {p: gdc ** h for p, h in hops.items()}
|
||||
graph_scores: dict[str, float] = {p: gdc**h for p, h in hops.items()}
|
||||
|
||||
# 4. Pull graph-only chunks (paths in expansion but not in V∪K).
|
||||
# Skip in 'boost' mode — boost only re-ranks V∪K, never adds candidates.
|
||||
|
|
|
|||
|
|
@ -76,14 +76,14 @@ class Role(StrEnum):
|
|||
Adding a new role is a one-line change here plus (optionally) a preset.
|
||||
"""
|
||||
|
||||
OBSERVATION = "observation" # what happened (events live here)
|
||||
CLAIM = "claim" # an assertion needing confidence (thesis, model)
|
||||
QUESTION = "question" # an open inquiry needing an answer
|
||||
PROFILE = "profile" # entity description (person, org, system)
|
||||
CONCEPT = "concept" # abstract idea / definition (company, sector, concept)
|
||||
METHOD = "method" # procedure / how-to
|
||||
REFERENCE = "reference" # pointer to external thing (tool, paper, code)
|
||||
FUNDAMENTALS = "fundamentals" # foundational data / baseline facts
|
||||
OBSERVATION = "observation" # what happened (events live here)
|
||||
CLAIM = "claim" # an assertion needing confidence (thesis, model)
|
||||
QUESTION = "question" # an open inquiry needing an answer
|
||||
PROFILE = "profile" # entity description (person, org, system)
|
||||
CONCEPT = "concept" # abstract idea / definition (company, sector, concept)
|
||||
METHOD = "method" # procedure / how-to
|
||||
REFERENCE = "reference" # pointer to external thing (tool, paper, code)
|
||||
FUNDAMENTALS = "fundamentals" # foundational data / baseline facts
|
||||
|
||||
|
||||
class Status(StrEnum):
|
||||
|
|
@ -206,6 +206,7 @@ class MemoryFileNode(FileNode):
|
|||
the parsed object without polluting this base schema. `populate_by_name`
|
||||
lets `originSessionId` (legacy camelCase) populate `origin_session_id`.
|
||||
"""
|
||||
|
||||
# -- Identity / common metadata ----------------------------------------
|
||||
|
||||
title: str = Field(default="")
|
||||
|
|
@ -274,8 +275,7 @@ class MemoryFileNode(FileNode):
|
|||
def _enforce_role_conditionals(self):
|
||||
if self.role is Role.CLAIM and self.confidence is None:
|
||||
raise ValueError(
|
||||
"role='claim' requires explicit confidence "
|
||||
"(one of ⏳ / ✅ / ❌)"
|
||||
"role='claim' requires explicit confidence " "(one of ⏳ / ✅ / ❌)",
|
||||
)
|
||||
return self
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ Zero LLM cost.
|
|||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from datetime import date as date_type, datetime, timezone
|
||||
|
|
@ -42,8 +41,8 @@ from pydantic import ValidationError
|
|||
|
||||
from reme2.component import R
|
||||
from reme2.component.base_step import BaseStep
|
||||
from reme2.memory.memory_io import collisions_after_create, create_file
|
||||
from reme2.memory.schema import EVENT_PRESET, MemoryFileNode
|
||||
from .memory_io import collisions_after_create, create_file
|
||||
from .schema import EVENT_PRESET, MemoryFileNode
|
||||
|
||||
|
||||
_SAFE_FILENAME_RE = re.compile(r"^[a-zA-Z0-9._-]+$")
|
||||
|
|
@ -213,10 +212,7 @@ class Sync(BaseStep):
|
|||
"""Filenames in `folder` excluding the index, sorted for stability."""
|
||||
if not folder.is_dir():
|
||||
return []
|
||||
return sorted(
|
||||
entry.name for entry in folder.iterdir()
|
||||
if entry.is_file() and entry.name != index_filename
|
||||
)
|
||||
return sorted(entry.name for entry in folder.iterdir() if entry.is_file() and entry.name != index_filename)
|
||||
|
||||
@staticmethod
|
||||
def _union(prior: list, incoming: list) -> list:
|
||||
|
|
@ -262,20 +258,31 @@ class Sync(BaseStep):
|
|||
await self._append(target, content, materials, topics, tags)
|
||||
else:
|
||||
await self._create(
|
||||
target, name, description, content, materials, topics, tags,
|
||||
on_date, origin_session_id,
|
||||
target,
|
||||
name,
|
||||
description,
|
||||
content,
|
||||
materials,
|
||||
topics,
|
||||
tags,
|
||||
on_date,
|
||||
origin_session_id,
|
||||
)
|
||||
|
||||
async def _create(
|
||||
self, target: Path, name: str, description: str, content: str,
|
||||
materials: list[dict], topics: list[str], tags: list[str],
|
||||
on_date, origin_session_id,
|
||||
self,
|
||||
target: Path,
|
||||
name: str,
|
||||
description: str,
|
||||
content: str,
|
||||
materials: list[dict],
|
||||
topics: list[str],
|
||||
tags: list[str],
|
||||
on_date,
|
||||
origin_session_id,
|
||||
) -> None:
|
||||
today = date_type.today().isoformat()
|
||||
on_date_str = (
|
||||
on_date.isoformat() if isinstance(on_date, date_type)
|
||||
else (on_date or today)
|
||||
)
|
||||
on_date_str = on_date.isoformat() if isinstance(on_date, date_type) else (on_date or today)
|
||||
# Start from EVENT_PRESET (4 axes + status + legacy `category`),
|
||||
# layer caller-supplied identity fields on top.
|
||||
metadata: dict = {
|
||||
|
|
@ -291,16 +298,20 @@ class Sync(BaseStep):
|
|||
metadata["originSessionId"] = origin_session_id
|
||||
|
||||
try:
|
||||
MemoryFileNode.model_validate({
|
||||
"path": str(target.resolve()),
|
||||
"st_mtime": 0.0,
|
||||
**metadata,
|
||||
})
|
||||
MemoryFileNode.model_validate(
|
||||
{
|
||||
"path": str(target.resolve()),
|
||||
"st_mtime": 0.0,
|
||||
**metadata,
|
||||
}
|
||||
)
|
||||
except ValidationError as e:
|
||||
self._set_error({
|
||||
"error": "MemoryFileNode schema validation failed",
|
||||
"details": e.errors(include_context=False, include_url=False),
|
||||
})
|
||||
self._set_error(
|
||||
{
|
||||
"error": "MemoryFileNode schema validation failed",
|
||||
"details": e.errors(include_context=False, include_url=False),
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
graph = self.file_store
|
||||
|
|
@ -308,32 +319,38 @@ class Sync(BaseStep):
|
|||
if conflicts:
|
||||
taken = {Path(p).stem for p in graph.nodes}
|
||||
suggested_name = _next_suffixed_stem(taken, name)
|
||||
self._set_error({
|
||||
"error": (
|
||||
f"stem `[[{name}]]` would resolve ambiguously "
|
||||
f"to {len(conflicts) + 1} paths after this create"
|
||||
),
|
||||
"conflicts": conflicts,
|
||||
"suggested_name": suggested_name,
|
||||
"hint": (
|
||||
f"retry with name='{suggested_name}', or pick a "
|
||||
f"semantic qualifier (e.g. '{name}-followup')."
|
||||
),
|
||||
})
|
||||
self._set_error(
|
||||
{
|
||||
"error": (
|
||||
f"stem `[[{name}]]` would resolve ambiguously "
|
||||
f"to {len(conflicts) + 1} paths after this create"
|
||||
),
|
||||
"conflicts": conflicts,
|
||||
"suggested_name": suggested_name,
|
||||
"hint": (
|
||||
f"retry with name='{suggested_name}', or pick a "
|
||||
f"semantic qualifier (e.g. '{name}-followup')."
|
||||
),
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
material_filenames = [m["filename"] for m in materials]
|
||||
index_body = self._emit_body(content, material_filenames)
|
||||
ok, payload = create_file(
|
||||
self.file_store, target,
|
||||
metadata=metadata, content=index_body,
|
||||
self.file_store,
|
||||
target,
|
||||
metadata=metadata,
|
||||
content=index_body,
|
||||
)
|
||||
if not ok:
|
||||
self._set_error({
|
||||
"path": str(target.resolve()),
|
||||
"error": payload.get("error", "create failed"),
|
||||
"details": payload,
|
||||
})
|
||||
self._set_error(
|
||||
{
|
||||
"path": str(target.resolve()),
|
||||
"error": payload.get("error", "create failed"),
|
||||
"details": payload,
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
material_paths: list[str] = []
|
||||
|
|
@ -348,19 +365,25 @@ class Sync(BaseStep):
|
|||
continue
|
||||
material_paths.append(str(material_path.resolve()))
|
||||
|
||||
self._set_ok({
|
||||
"path": str(target.resolve()),
|
||||
"category": "event",
|
||||
"status": "active",
|
||||
"topics": topics,
|
||||
"materials": material_paths,
|
||||
"created": True,
|
||||
"action": "created",
|
||||
})
|
||||
self._set_ok(
|
||||
{
|
||||
"path": str(target.resolve()),
|
||||
"category": "event",
|
||||
"status": "active",
|
||||
"topics": topics,
|
||||
"materials": material_paths,
|
||||
"created": True,
|
||||
"action": "created",
|
||||
}
|
||||
)
|
||||
|
||||
async def _append(
|
||||
self, target: Path, content: str, materials: list[dict],
|
||||
topics: list[str], tags: list[str],
|
||||
self,
|
||||
target: Path,
|
||||
content: str,
|
||||
materials: list[dict],
|
||||
topics: list[str],
|
||||
tags: list[str],
|
||||
) -> None:
|
||||
# Read current frontmatter + body.
|
||||
try:
|
||||
|
|
@ -379,15 +402,17 @@ class Sync(BaseStep):
|
|||
taken = {Path(p).stem for p in graph.nodes}
|
||||
base = target.stem
|
||||
suggested = _next_suffixed_stem(taken, base)
|
||||
self._set_error({
|
||||
"path": str(target.resolve()),
|
||||
"error": (
|
||||
f"event `{base}` already exists with status={status!r}; "
|
||||
f"pick a new name to start a fresh thread"
|
||||
),
|
||||
"status": status,
|
||||
"suggested_name": suggested,
|
||||
})
|
||||
self._set_error(
|
||||
{
|
||||
"path": str(target.resolve()),
|
||||
"error": (
|
||||
f"event `{base}` already exists with status={status!r}; "
|
||||
f"pick a new name to start a fresh thread"
|
||||
),
|
||||
"status": status,
|
||||
"suggested_name": suggested,
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
folder = target.parent
|
||||
|
|
@ -420,10 +445,7 @@ class Sync(BaseStep):
|
|||
if content.strip():
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
update_section = f"## Update — {ts}\n\n{content.rstrip()}\n"
|
||||
narrative = (
|
||||
f"{narrative.rstrip()}\n\n{update_section}"
|
||||
if narrative else update_section
|
||||
)
|
||||
narrative = f"{narrative.rstrip()}\n\n{update_section}" if narrative else update_section
|
||||
|
||||
all_filenames_sorted = sorted(set(existing_filenames))
|
||||
new_body = self._emit_body(narrative, all_filenames_sorted)
|
||||
|
|
@ -434,17 +456,21 @@ class Sync(BaseStep):
|
|||
meta["updated"] = date_type.today().isoformat()
|
||||
|
||||
try:
|
||||
MemoryFileNode.model_validate({
|
||||
"path": str(target.resolve()),
|
||||
"st_mtime": 0.0,
|
||||
**meta,
|
||||
})
|
||||
MemoryFileNode.model_validate(
|
||||
{
|
||||
"path": str(target.resolve()),
|
||||
"st_mtime": 0.0,
|
||||
**meta,
|
||||
}
|
||||
)
|
||||
except ValidationError as e:
|
||||
self._set_error({
|
||||
"path": str(target.resolve()),
|
||||
"error": "MemoryFileNode schema validation failed on append",
|
||||
"details": e.errors(include_context=False, include_url=False),
|
||||
})
|
||||
self._set_error(
|
||||
{
|
||||
"path": str(target.resolve()),
|
||||
"error": "MemoryFileNode schema validation failed on append",
|
||||
"details": e.errors(include_context=False, include_url=False),
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
new_post = frontmatter.Post(new_body, **meta)
|
||||
|
|
@ -454,12 +480,14 @@ class Sync(BaseStep):
|
|||
self._set_error({"path": str(target.resolve()), "error": f"write failed: {e}"})
|
||||
return
|
||||
|
||||
self._set_ok({
|
||||
"path": str(target.resolve()),
|
||||
"category": "event",
|
||||
"status": "active",
|
||||
"topics": meta["topics"],
|
||||
"materials": new_material_paths,
|
||||
"created": False,
|
||||
"action": "appended",
|
||||
})
|
||||
self._set_ok(
|
||||
{
|
||||
"path": str(target.resolve()),
|
||||
"category": "event",
|
||||
"status": "active",
|
||||
"topics": meta["topics"],
|
||||
"materials": new_material_paths,
|
||||
"created": False,
|
||||
"action": "appended",
|
||||
}
|
||||
)
|
||||
|
|
@ -5,7 +5,7 @@ from .as_msg_stat import AsBlockStat, AsMsgStat
|
|||
from .emb_node import EmbNode
|
||||
from .chunk_filter import ChunkFilter
|
||||
from .file_chunk import FileChunk
|
||||
from .file_edge import FileEdge, extract_wikilinks
|
||||
from .file_link import FileLink, extract_wikilinks, iter_links
|
||||
from .file_node import FileFrontMatter, FileNode
|
||||
from .request import Request
|
||||
from .response import Response
|
||||
|
|
@ -20,11 +20,12 @@ __all__ = [
|
|||
"EmbNode",
|
||||
"ChunkFilter",
|
||||
"FileChunk",
|
||||
"FileEdge",
|
||||
"FileLink",
|
||||
"FileFrontMatter",
|
||||
"FileNode",
|
||||
"Request",
|
||||
"Response",
|
||||
"StreamChunk",
|
||||
"extract_wikilinks",
|
||||
"iter_links",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -12,14 +12,14 @@ class EmbNode(BaseModel):
|
|||
embedding: np.ndarray | None = Field(default=None)
|
||||
metadata: dict = Field(default_factory=dict)
|
||||
|
||||
@field_validator('embedding', mode='before')
|
||||
@field_validator("embedding", mode="before")
|
||||
@classmethod
|
||||
def validate_embedding(cls, v):
|
||||
if v is None:
|
||||
return v
|
||||
return np.array(v, dtype=np.float16)
|
||||
|
||||
@field_serializer('embedding')
|
||||
@field_serializer("embedding")
|
||||
def serialize_embedding(self, v: np.ndarray | None, _info):
|
||||
if v is None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -15,5 +15,6 @@ class FileChunk(EmbNode):
|
|||
|
||||
def set_hash_id(self):
|
||||
from ..utils import hash_text
|
||||
|
||||
self.id = hash_text(" ".join([self.path, str(self.start_line), str(self.end_line), self.text]))
|
||||
return self
|
||||
return self
|
||||
|
|
|
|||
|
|
@ -1,224 +0,0 @@
|
|||
"""FileEdge — typed wikilink edge between vault files.
|
||||
|
||||
Single source of truth for the edge **schema** and the **inline parser**
|
||||
(`FileEdge.from_text`) that recovers edges from body text. Edges live
|
||||
exclusively in body text (frontmatter is not walked); the predicate
|
||||
vocabulary is **open** — any identifier-shaped token
|
||||
(`[A-Za-z][A-Za-z0-9_]*`) the parser sees is preserved verbatim on
|
||||
`FileEdge.predicate`. Vocabulary curation, if any, is the maintainer's
|
||||
job, not the schema's.
|
||||
|
||||
## Inline forms recognised by `FileEdge.from_text`
|
||||
|
||||
[[X]] bare wikilink → predicate=None
|
||||
extends:: [[X]] line-level Dataview → predicate="extends"
|
||||
[extends:: [[X]]] inline-bracketed → predicate="extends"
|
||||
|
||||
Multi-target — every wikilink under one typed context inherits its
|
||||
predicate (any separator works, not just commas):
|
||||
|
||||
extends:: [[A]], [[B]] line-level multi → 2 edges, both "extends"
|
||||
extends:: [[A]] and [[B]] prose-style multi → 2 edges, both "extends"
|
||||
[concerns:: [[A]], [[B]]] inline multi → 2 edges, both "concerns"
|
||||
extends:: [[A#s1]], [[B#s2]] multi w/ anchors → anchors preserved per link
|
||||
|
||||
Context precedence is **inline-bracketed > line-level > bare** —
|
||||
a wikilink inside a `[predicate:: …]` envelope is typed by that
|
||||
envelope even if the line happens to start `predicate:: …`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
# -- Regexes (module-private) ---------------------------------------------
|
||||
|
||||
# Wikilink. The optional `!` embed marker and `|alias` are matched but
|
||||
# not captured — both are presentational and dropped from the edge.
|
||||
# `target` is the file part; `anchor` is the optional `#…` suffix; we
|
||||
# rejoin them into a single `link` string at edge-construction time.
|
||||
_WIKILINK_RE = re.compile(
|
||||
r"""
|
||||
(?:!)?
|
||||
\[\[
|
||||
(?P<target>[^\]\|\#\n]+?)
|
||||
(?:\#(?P<anchor>[^\]\|\n]+))?
|
||||
(?:\|[^\]\n]+)?
|
||||
\]\]
|
||||
""",
|
||||
re.VERBOSE,
|
||||
)
|
||||
|
||||
# Line-level Dataview field. Anchored MULTILINE; allows leading bullet
|
||||
# (`-`/`*`/`+`) so `- extends:: [[X]]` works inside Markdown lists.
|
||||
_DATAVIEW_LINE_RE = re.compile(
|
||||
r"^[ \t]*(?:[-*+][ \t]+)?(?P<predicate>[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P<value>.+?)\s*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# Inline-bracketed field opener: `[predicate::`. The matching `]` is
|
||||
# located by depth-counted scan because the value may contain
|
||||
# `[[wikilink]]` whose inner brackets are part of the value.
|
||||
_INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P<predicate>[A-Za-z][A-Za-z0-9_]*)\s*::\s*")
|
||||
|
||||
|
||||
def _iter_inline_fields(text: str) -> list[tuple[int, int, str]]:
|
||||
"""Find inline-bracketed `[predicate:: …]` field spans by depth scan.
|
||||
|
||||
Returns a list of ``(start, end, predicate)`` triples. Newlines
|
||||
terminate the scan: an inline field that spans a line break is
|
||||
treated as malformed (matches Dataview semantics) and skipped.
|
||||
"""
|
||||
out: list[tuple[int, int, str]] = []
|
||||
for m in _INLINE_FIELD_OPEN_RE.finditer(text):
|
||||
depth = 1 # the outer '[' was the regex's first character
|
||||
i = m.end()
|
||||
n = len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c == "\n":
|
||||
break
|
||||
if c == "[":
|
||||
depth += 1
|
||||
elif c == "]":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
out.append((m.start(), i + 1, m.group("predicate")))
|
||||
break
|
||||
i += 1
|
||||
return out
|
||||
|
||||
|
||||
# -- Schema ---------------------------------------------------------------
|
||||
|
||||
|
||||
class FileEdge(BaseModel):
|
||||
"""2-field minimal edge model: ``link`` + ``predicate``.
|
||||
|
||||
``link`` preserves the wikilink as written, including any ``#anchor``
|
||||
suffix (e.g. ``"X"`` or ``"X#sec"``). The presentational ``|alias``
|
||||
and ``!`` embed prefix are discarded by the parser. ``path`` and
|
||||
``anchor`` are derived ``@property``s that split ``link`` on the
|
||||
first ``#`` for callers that want either part without re-splitting
|
||||
the string themselves.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
link: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"Wikilink as written, preserving any '#anchor' suffix "
|
||||
"(e.g. 'X' or 'X#sec'). Display alias and embed prefix discarded."
|
||||
),
|
||||
)
|
||||
predicate: str | None = Field(
|
||||
default=None,
|
||||
description="Typed-edge predicate (Dataview-style). None for bare [[X]].",
|
||||
)
|
||||
|
||||
@property
|
||||
def path(self) -> str:
|
||||
"""File-part of ``link`` (before any ``#anchor``).
|
||||
|
||||
For ``"X"`` returns ``"X"``; for ``"X#sec"`` returns ``"X"``;
|
||||
for ``"topics/Foo#bar"`` returns ``"topics/Foo"``. Always
|
||||
returns a non-empty string because ``link`` is required and the
|
||||
regex never matches an empty target.
|
||||
"""
|
||||
return self.link.split("#", 1)[0].strip()
|
||||
|
||||
@property
|
||||
def anchor(self) -> str | None:
|
||||
"""Heading or block anchor parsed from ``link`` (text after first ``#``).
|
||||
|
||||
Returns ``None`` if the link has no anchor or the anchor is empty.
|
||||
"""
|
||||
if "#" not in self.link:
|
||||
return None
|
||||
tail = self.link.split("#", 1)[1].strip()
|
||||
return tail or None
|
||||
|
||||
@classmethod
|
||||
def _from_match(cls, wm: re.Match, *, predicate: str | None) -> FileEdge:
|
||||
target = wm.group("target").strip()
|
||||
anchor = wm.group("anchor")
|
||||
link = f"{target}#{anchor.strip()}" if anchor else target
|
||||
return cls(link=link, predicate=predicate)
|
||||
|
||||
@classmethod
|
||||
def from_text(cls, text: str) -> list[FileEdge]:
|
||||
"""Extract all edges from body text in source order.
|
||||
|
||||
Single pass: every wikilink in the text becomes one edge, and
|
||||
its ``predicate`` is decided by the surrounding context with
|
||||
precedence **inline-bracketed > line-level > bare**:
|
||||
|
||||
* ``[predicate:: [[X]]]`` → ``predicate="predicate"``
|
||||
* ``predicate:: [[X]]`` → ``predicate="predicate"`` (line-level Dataview)
|
||||
* ``[[X]]`` → ``predicate=None`` (bare)
|
||||
|
||||
No consumed-span bookkeeping, no second sort: ``finditer``
|
||||
already yields wikilinks in source order, and per-position
|
||||
classification is unambiguous.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
# Inline-bracketed `[predicate:: ...]` envelopes need a
|
||||
# depth-counted scan (regex can't match balanced `[[…]]` inside).
|
||||
inline_spans = _iter_inline_fields(text)
|
||||
|
||||
return [
|
||||
cls._from_match(wm, predicate=_predicate_for(text, wm.start(), inline_spans))
|
||||
for wm in _WIKILINK_RE.finditer(text)
|
||||
]
|
||||
|
||||
|
||||
def _predicate_for(
|
||||
text: str,
|
||||
pos: int,
|
||||
inline_spans: list[tuple[int, int, str]],
|
||||
) -> str | None:
|
||||
"""Resolve the predicate governing a wikilink at offset ``pos``.
|
||||
|
||||
Checks the two typed-edge contexts in precedence order; falls
|
||||
through to ``None`` (bare) when neither applies.
|
||||
"""
|
||||
# 1. Inline-bracketed envelope `[predicate:: …]` containing pos.
|
||||
for field_start, field_end, predicate in inline_spans:
|
||||
if field_start <= pos < field_end:
|
||||
return predicate
|
||||
|
||||
# 2. Line-level `predicate:: value` whose value range covers pos.
|
||||
line_start = text.rfind("\n", 0, pos) + 1
|
||||
line_end = text.find("\n", pos)
|
||||
if line_end == -1:
|
||||
line_end = len(text)
|
||||
m = _DATAVIEW_LINE_RE.match(text[line_start:line_end])
|
||||
if m and line_start + m.start("value") <= pos:
|
||||
return m.group("predicate")
|
||||
|
||||
# 3. Bare wikilink — no predicate.
|
||||
return None
|
||||
|
||||
|
||||
# -- Public utility (target-only fast path) -------------------------------
|
||||
|
||||
|
||||
def extract_wikilinks(text: str) -> list[str]:
|
||||
"""Flat list of wikilink **file targets** in body text (no dedup).
|
||||
|
||||
Returns just the file part of each wikilink (before any ``#anchor``)
|
||||
because callers feed the result to `resolve_wikilink`, which matches
|
||||
against vault file stems / paths and would not recognise an anchor
|
||||
suffix. Single regex pass — cheaper than `FileEdge.from_text` when
|
||||
callers don't need predicates (e.g. ingestor's auto-discovery hint,
|
||||
memory_io anchor resolution).
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
return [m.group("target").strip() for m in _WIKILINK_RE.finditer(text)]
|
||||
192
reme2/schema/file_link.py
Normal file
192
reme2/schema/file_link.py
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
"""FileLink — typed wikilink between vault files.
|
||||
|
||||
One type, two states (the value of ``path`` distinguishes them):
|
||||
|
||||
* **pre-resolution** — ``path`` holds the raw wikilink target as
|
||||
written, e.g. ``"Foo"`` or ``"topics/Bar"``. Produced by
|
||||
``iter_links(text)``: a pure regex pass over body text, no graph
|
||||
access.
|
||||
* **resolved** — ``path`` holds the vault-relative path
|
||||
file_graph stores, e.g. ``"topics/Foo/Foo.md"``. Produced by
|
||||
``utils.wikilink_resolver.resolve_links(graph, links)`` (or the
|
||||
one-shot ``text_to_links(graph, text)``), which expands stem
|
||||
ambiguity into one ``FileLink`` per candidate path.
|
||||
|
||||
file_graph trusts ``link.path`` directly for adjacency: it only ever
|
||||
stores resolved links. The pre-resolution form is internal pipeline
|
||||
plumbing.
|
||||
|
||||
## Inline forms recognised by ``iter_links``
|
||||
|
||||
[[X]] bare wikilink → predicate=None
|
||||
extends:: [[X]] line-level Dataview → predicate="extends"
|
||||
[extends:: [[X]]] inline-bracketed → predicate="extends"
|
||||
|
||||
Multi-target — every wikilink under one typed context inherits its
|
||||
predicate (any separator works, not just commas):
|
||||
|
||||
extends:: [[A]], [[B]] line-level multi → 2 links, both "extends"
|
||||
extends:: [[A]] and [[B]] prose-style multi → 2 links, both "extends"
|
||||
[concerns:: [[A]], [[B]]] inline multi → 2 links, both "concerns"
|
||||
extends:: [[A#s1]], [[B#s2]] multi w/ anchors → anchors preserved per link
|
||||
|
||||
Context precedence is **inline-bracketed > line-level > bare** —
|
||||
a wikilink inside a ``[predicate:: …]`` envelope is typed by that
|
||||
envelope even if the line happens to start ``predicate:: …``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
# -- Regexes (module-private) ---------------------------------------------
|
||||
|
||||
_WIKILINK_RE = re.compile(
|
||||
r"""
|
||||
(?:!)?
|
||||
\[\[
|
||||
(?P<target>[^\]\|\#\n]+?)
|
||||
(?:\#(?P<anchor>[^\]\|\n]+))?
|
||||
(?:\|[^\]\n]+)?
|
||||
\]\]
|
||||
""",
|
||||
re.VERBOSE,
|
||||
)
|
||||
|
||||
_DATAVIEW_LINE_RE = re.compile(
|
||||
r"^[ \t]*(?:[-*+][ \t]+)?(?P<predicate>[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P<value>.+?)\s*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
_INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P<predicate>[A-Za-z][A-Za-z0-9_]*)\s*::\s*")
|
||||
|
||||
|
||||
def _iter_inline_fields(text: str) -> list[tuple[int, int, str]]:
|
||||
"""Find inline-bracketed ``[predicate:: …]`` field spans by depth scan."""
|
||||
out: list[tuple[int, int, str]] = []
|
||||
for m in _INLINE_FIELD_OPEN_RE.finditer(text):
|
||||
depth = 1
|
||||
i = m.end()
|
||||
n = len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c == "\n":
|
||||
break
|
||||
if c == "[":
|
||||
depth += 1
|
||||
elif c == "]":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
out.append((m.start(), i + 1, m.group("predicate")))
|
||||
break
|
||||
i += 1
|
||||
return out
|
||||
|
||||
|
||||
def _predicate_for(
|
||||
text: str,
|
||||
pos: int,
|
||||
inline_spans: list[tuple[int, int, str]],
|
||||
) -> str | None:
|
||||
"""Resolve the predicate governing a wikilink at offset ``pos``."""
|
||||
for field_start, field_end, predicate in inline_spans:
|
||||
if field_start <= pos < field_end:
|
||||
return predicate
|
||||
line_start = text.rfind("\n", 0, pos) + 1
|
||||
line_end = text.find("\n", pos)
|
||||
if line_end == -1:
|
||||
line_end = len(text)
|
||||
m = _DATAVIEW_LINE_RE.match(text[line_start:line_end])
|
||||
if m and line_start + m.start("value") <= pos:
|
||||
return m.group("predicate")
|
||||
return None
|
||||
|
||||
|
||||
# -- Schema ---------------------------------------------------------------
|
||||
|
||||
|
||||
class FileLink(BaseModel):
|
||||
"""Typed wikilink — ``(path, anchor, predicate)``.
|
||||
|
||||
Single type for both states (see module docstring): ``path`` is
|
||||
a raw wikilink target before resolution, a vault-relative resolved
|
||||
path after.
|
||||
|
||||
Fields:
|
||||
path wikilink target. Raw target text in pre-resolution
|
||||
form (e.g. ``"Foo"``, ``"topics/Bar"``);
|
||||
vault-relative resolved path in stored form
|
||||
(e.g. ``"topics/Foo/Foo.md"``).
|
||||
anchor heading or block anchor (text after ``#`` in the
|
||||
wikilink). Pass-through across resolution.
|
||||
predicate Dataview-style typed-link predicate; ``None`` for bare.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
path: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"Wikilink target. Pre-resolution: the raw target as written "
|
||||
"(e.g. 'Foo'). Resolved: the vault-relative path file_graph "
|
||||
"stores. Stem ambiguity is resolved BEFORE construction of "
|
||||
"the resolved form by emitting one FileLink per candidate."
|
||||
),
|
||||
)
|
||||
anchor: str | None = Field(
|
||||
default=None,
|
||||
description="Heading or block anchor (text after '#'). None if absent.",
|
||||
)
|
||||
predicate: str | None = Field(
|
||||
default=None,
|
||||
description="Typed-link predicate (Dataview-style). None for bare [[X]].",
|
||||
)
|
||||
|
||||
|
||||
# -- Public extraction utilities ------------------------------------------
|
||||
|
||||
|
||||
def iter_links(text: str) -> list[FileLink]:
|
||||
"""Extract every wikilink in ``text`` as a pre-resolution ``FileLink``.
|
||||
|
||||
Each emitted ``FileLink`` has ``path`` set to the raw wikilink
|
||||
target as written (the file portion only — the ``#anchor`` lives
|
||||
in its own field). Predicate is decided by surrounding context
|
||||
with precedence **inline-bracketed > line-level > bare**.
|
||||
|
||||
Pure function: no graph access. Pass the result through
|
||||
``utils.wikilink_resolver.resolve_links`` (or one-shot
|
||||
``text_to_links``) to get the resolved form file_graph stores.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
inline_spans = _iter_inline_fields(text)
|
||||
links: list[FileLink] = []
|
||||
for wm in _WIKILINK_RE.finditer(text):
|
||||
target = wm.group("target").strip()
|
||||
anchor_raw = wm.group("anchor")
|
||||
anchor = anchor_raw.strip() if anchor_raw else ""
|
||||
links.append(
|
||||
FileLink(
|
||||
path=target,
|
||||
anchor=anchor or None,
|
||||
predicate=_predicate_for(text, wm.start(), inline_spans),
|
||||
),
|
||||
)
|
||||
return links
|
||||
|
||||
|
||||
def extract_wikilinks(text: str) -> list[str]:
|
||||
"""Flat list of wikilink **file targets** in body text (no dedup).
|
||||
|
||||
Returns just the file part of each wikilink (before any ``#anchor``).
|
||||
Single regex pass — cheaper than ``iter_links`` when callers don't
|
||||
need predicates (e.g. ``extract_anchors`` for query seeding in the
|
||||
retriever).
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
return [m.group("target").strip() for m in _WIKILINK_RE.finditer(text)]
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from .file_edge import FileEdge
|
||||
from .file_link import FileLink
|
||||
|
||||
|
||||
class FileFrontMatter(BaseModel):
|
||||
|
|
@ -18,6 +18,6 @@ class FileFrontMatter(BaseModel):
|
|||
class FileNode(BaseModel):
|
||||
path: str = Field(default=...)
|
||||
st_mtime: float = Field(default=...)
|
||||
edges: list[FileEdge] = Field(default_factory=list)
|
||||
links: list[FileLink] = Field(default_factory=list)
|
||||
chunk_ids: list[str] = Field(default_factory=list)
|
||||
front_matter: FileFrontMatter = Field(default_factory=FileFrontMatter)
|
||||
|
|
|
|||
|
|
@ -34,7 +34,8 @@ def keyword_score(query: str, text: str) -> float:
|
|||
|
||||
|
||||
def filter_chunks(
|
||||
chunks: list[FileChunk], chunk_filter: ChunkFilter | None,
|
||||
chunks: list[FileChunk],
|
||||
chunk_filter: ChunkFilter | None,
|
||||
) -> list[FileChunk]:
|
||||
"""Restrict `chunks` to those whose path passes the (compiled) filter."""
|
||||
if chunk_filter is None or chunk_filter.resolved_paths is None:
|
||||
|
|
|
|||
|
|
@ -1,56 +1,285 @@
|
|||
"""Wikilink resolution — pure functions over a `BaseFileStore`'s graph.
|
||||
"""Wikilink resolver — vault convention over ``BaseFileGraph``.
|
||||
|
||||
Wikilink resolution is a vault convention (path/stem forms, folder-note
|
||||
preference) layered on top of the engine's plain `dict[path, FileNode]`.
|
||||
Keeping it out of `BaseFileStore` keeps the engine domain-agnostic.
|
||||
Stem-form wikilinks like ``[[Foo]]`` are a vault convention. The
|
||||
file_graph engine doesn't know about them — it only stores nodes and
|
||||
trusts ``FileLink.path`` for adjacency. This module bridges the gap,
|
||||
applying:
|
||||
|
||||
* **folder-note rule** — ``[[X]]`` prefers ``topics/X/X.md`` over a
|
||||
sibling ``topics/X.md`` when both exist
|
||||
* **stem ambiguity** — ``[[Foo]]`` matching multiple paths is handled
|
||||
by **emitting multiple ``FileLink`` records, one per candidate**
|
||||
(file_graph then has unambiguous adjacency)
|
||||
* **path-form passthrough** — ``[[foo/bar]]`` and ``[[foo/bar.md]]``
|
||||
resolve directly against the graph (vault-relative paths; no
|
||||
working_dir absolutization)
|
||||
|
||||
All paths are vault-relative — ``graph.iter_nodes()`` returns the
|
||||
key form file_graph stores, and emitted ``FileLink.path`` matches.
|
||||
|
||||
All functions are stateless — they walk ``graph.iter_nodes()`` per
|
||||
call. For batch operations (``resolve_links`` over many links) the
|
||||
stem index is built once and reused.
|
||||
|
||||
Seven entry points mapped to call sites:
|
||||
|
||||
resolve single link → path | None
|
||||
used by: ``extract_anchors``, ``memory_resolve_wikilink``
|
||||
candidates stem → [path] (folder-note ordered first)
|
||||
used by: ``memory_resolve_wikilink`` (ambiguity report)
|
||||
collisions all stems with >1 path
|
||||
used by: ``maintainer.lint``
|
||||
collisions_for paths conflicting with a proposed new path
|
||||
used by: ``memory_create``, ``sync`` (preflight)
|
||||
extract_anchors parse [[X]] from text + resolve, dedup
|
||||
used by: ``retriever`` (query anchor seeds)
|
||||
resolve_links ``[FileLink]`` (raw path) → ``[FileLink]`` (resolved)
|
||||
core resolution; stem ambiguity expands
|
||||
text_to_links one-shot: ``iter_links(text)`` + ``resolve_links``
|
||||
used by: parser pipeline (before ``upsert_node``)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from reme2.component.file_store.base_file_store import BaseFileStore
|
||||
from ..component.file_graph.base_file_graph import BaseFileGraph
|
||||
from ..schema import FileLink
|
||||
from ..schema.file_link import extract_wikilinks, iter_links
|
||||
from .logger_utils import get_logger
|
||||
|
||||
|
||||
def wikilink_candidates(store: BaseFileStore, target: str) -> list[str]:
|
||||
"""Paths `[[target]]` would resolve to (folder-note hit wins).
|
||||
_logger = get_logger()
|
||||
|
||||
Folder-note convention: `topics/X/X.md` is the cluster head for
|
||||
bare `[[X]]`, beating any sibling that just shares the stem.
|
||||
|
||||
# =========================================================================
|
||||
# Internal helpers
|
||||
# =========================================================================
|
||||
|
||||
|
||||
async def _build_stem_index(graph: BaseFileGraph) -> dict[str, list[str]]:
|
||||
"""Walk once, group paths by stem. Used by batch hot paths."""
|
||||
out: dict[str, list[str]] = {}
|
||||
async for path, _ in graph.iter_nodes():
|
||||
out.setdefault(Path(path).stem, []).append(path)
|
||||
return out
|
||||
|
||||
|
||||
def _split_link(link: str) -> tuple[str, str]:
|
||||
"""Return ``(target, anchor)``. Anchor empty if no ``#``.
|
||||
|
||||
For raw link strings supplied by external callers (which may still
|
||||
arrive in ``[[A#section]]`` form). Pre-extracted ``FileLink``
|
||||
records already carry ``path`` and ``anchor`` separately.
|
||||
"""
|
||||
folder_hits = sorted(
|
||||
p for p in store._stems.get(target, ()) if Path(p).parent.name == target
|
||||
)
|
||||
if not link:
|
||||
return "", ""
|
||||
if "#" not in link:
|
||||
return link.strip(), ""
|
||||
target_raw, anchor_raw = link.split("#", 1)
|
||||
return target_raw.strip(), anchor_raw.strip()
|
||||
|
||||
|
||||
def _is_path_form(target: str) -> bool:
|
||||
return "/" in target or target.endswith(".md")
|
||||
|
||||
|
||||
def _filter_stem_candidates(stem: str, paths: list[str]) -> list[str]:
|
||||
"""Apply folder-note rule: if any folder-note exists, ONLY folder-notes
|
||||
are candidates; otherwise all sibling-stems are candidates."""
|
||||
if not paths:
|
||||
return []
|
||||
folder_hits = sorted(p for p in paths if Path(p).parent.name == stem)
|
||||
if folder_hits:
|
||||
return folder_hits
|
||||
return store.get_paths_by_stem(target)
|
||||
return sorted(paths)
|
||||
|
||||
|
||||
def resolve_wikilink(store: BaseFileStore, wikilink: str) -> str | None:
|
||||
"""Resolve a `[[target]]` to one absolute path.
|
||||
# =========================================================================
|
||||
# Public API — single-shot lookups
|
||||
# =========================================================================
|
||||
|
||||
- Path form (`topics/X/X` or `topics/X/X.md`) → relative to `working_dir`.
|
||||
- Stem form (`X`) → folder-note preference, else unique stem hit.
|
||||
- Ambiguous stems return None and log a warning.
|
||||
|
||||
async def resolve(graph: BaseFileGraph, link: str) -> str | None:
|
||||
"""Resolve a single wikilink to **one** vault-relative path, or None.
|
||||
|
||||
Returns None if dangling or ambiguous (with warning on ambiguity).
|
||||
Use ``resolve_links`` for the multi-link expansion semantics.
|
||||
"""
|
||||
target = wikilink.strip()
|
||||
target, _ = _split_link(link)
|
||||
if not target:
|
||||
return None
|
||||
|
||||
if "/" in target or target.endswith(".md"):
|
||||
if store.working_dir is None:
|
||||
return None
|
||||
if _is_path_form(target):
|
||||
candidate = target if target.endswith(".md") else f"{target}.md"
|
||||
abs_candidate = str((store.working_dir / candidate).resolve())
|
||||
return abs_candidate if abs_candidate in store.nodes else None
|
||||
return candidate if await graph.get_node(candidate) else None
|
||||
|
||||
candidates = wikilink_candidates(store, target)
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
if len(candidates) > 1:
|
||||
store.logger.warning(
|
||||
f"Wikilink [[{target}]] is ambiguous, candidates: {candidates}",
|
||||
paths = [p async for p, _ in graph.iter_nodes() if Path(p).stem == target]
|
||||
candidates_for_stem = _filter_stem_candidates(target, paths)
|
||||
if len(candidates_for_stem) == 1:
|
||||
return candidates_for_stem[0]
|
||||
if len(candidates_for_stem) > 1:
|
||||
_logger.warning(
|
||||
f"Wikilink [[{target}]] is ambiguous, " f"candidates: {candidates_for_stem}",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def candidates(graph: BaseFileGraph, stem: str) -> list[str]:
|
||||
"""All paths a ``[[stem]]`` could match. Folder-note hits ordered first."""
|
||||
folder_hits: list[str] = []
|
||||
stem_hits: list[str] = []
|
||||
async for path, _ in graph.iter_nodes():
|
||||
p = Path(path)
|
||||
if p.stem != stem:
|
||||
continue
|
||||
if p.parent.name == stem:
|
||||
folder_hits.append(path)
|
||||
else:
|
||||
stem_hits.append(path)
|
||||
if folder_hits:
|
||||
return sorted(folder_hits)
|
||||
return sorted(stem_hits)
|
||||
|
||||
|
||||
async def collisions(graph: BaseFileGraph) -> dict[str, list[str]]:
|
||||
"""Every stem that resolves to >1 path. Used by maintainer.lint."""
|
||||
stem_index = await _build_stem_index(graph)
|
||||
return {stem: sorted(paths) for stem, paths in stem_index.items() if len(paths) > 1}
|
||||
|
||||
|
||||
async def collisions_for(
|
||||
graph: BaseFileGraph,
|
||||
proposed_path: str | Path,
|
||||
) -> list[str]:
|
||||
"""Existing paths that would conflict with adding ``proposed_path``.
|
||||
|
||||
``proposed_path`` is treated as vault-relative (matching the form
|
||||
``graph.iter_nodes()`` returns). Folder-note rule: when the
|
||||
proposed path is itself a folder-note (parent dir name == stem),
|
||||
only colliding folder-notes are returned. Otherwise all paths
|
||||
sharing the stem are returned.
|
||||
"""
|
||||
p = Path(proposed_path)
|
||||
stem = p.stem
|
||||
proposed_str = str(p)
|
||||
is_folder_note = p.parent.name == stem
|
||||
|
||||
folder_hits: list[str] = []
|
||||
stem_hits: list[str] = []
|
||||
async for path, _ in graph.iter_nodes():
|
||||
if path == proposed_str:
|
||||
continue
|
||||
path_obj = Path(path)
|
||||
if path_obj.stem != stem:
|
||||
continue
|
||||
if path_obj.parent.name == stem:
|
||||
folder_hits.append(path)
|
||||
else:
|
||||
stem_hits.append(path)
|
||||
|
||||
if is_folder_note:
|
||||
return sorted(folder_hits)
|
||||
return sorted(folder_hits) + sorted(stem_hits)
|
||||
|
||||
|
||||
async def extract_anchors(graph: BaseFileGraph, text: str) -> list[str]:
|
||||
"""Pull ``[[X]]`` from ``text``, resolve each, dedup in source order.
|
||||
|
||||
Uses single-target ``resolve`` semantics — ambiguous stems return
|
||||
no anchor. (For multi-link expansion at *write* time, see
|
||||
``resolve_links``; ``extract_anchors`` is for read-time seeding
|
||||
where a single deterministic target is wanted.)
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for raw in extract_wikilinks(text):
|
||||
hit = await resolve(graph, raw)
|
||||
if hit is not None and hit not in seen:
|
||||
seen.add(hit)
|
||||
out.append(hit)
|
||||
return out
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Public API — parser pipeline
|
||||
# =========================================================================
|
||||
|
||||
|
||||
async def resolve_links(
|
||||
graph: BaseFileGraph,
|
||||
links: Iterable[FileLink],
|
||||
) -> list[FileLink]:
|
||||
"""Resolve pre-resolution ``FileLink`` records against the graph.
|
||||
|
||||
Each input link has ``path`` holding a raw wikilink target (the
|
||||
extractor form). Returns FileLinks with ``path`` rewritten to the
|
||||
vault-relative resolved path file_graph stores. ``anchor`` and
|
||||
``predicate`` pass through unchanged.
|
||||
|
||||
Stem expansion semantics — one input link produces zero, one, or
|
||||
many output links:
|
||||
|
||||
* path-form, target indexed → 1 link
|
||||
* path-form, dangling → 0 links
|
||||
* stem-form, 1 folder-note hit → 1 link
|
||||
* stem-form, N folder-note hits → N links (one per)
|
||||
* stem-form, 0 folder-notes, 1 sibling → 1 link
|
||||
* stem-form, 0 folder-notes, N siblings → N links (one per)
|
||||
* stem-form, dangling → 0 links
|
||||
|
||||
Build the stem index lazily: only paid if at least one input is
|
||||
stem-form.
|
||||
"""
|
||||
link_list = list(links)
|
||||
if not link_list:
|
||||
return []
|
||||
|
||||
stem_index: dict[str, list[str]] | None = None
|
||||
out: list[FileLink] = []
|
||||
for link in link_list:
|
||||
target = link.path
|
||||
if not target:
|
||||
continue
|
||||
|
||||
if _is_path_form(target):
|
||||
candidate = target if target.endswith(".md") else f"{target}.md"
|
||||
if await graph.get_node(candidate) is None:
|
||||
continue
|
||||
out.append(
|
||||
FileLink(
|
||||
path=candidate,
|
||||
anchor=link.anchor,
|
||||
predicate=link.predicate,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
# Stem form — may expand into multiple links.
|
||||
if stem_index is None:
|
||||
stem_index = await _build_stem_index(graph)
|
||||
for chosen in _filter_stem_candidates(target, stem_index.get(target, [])):
|
||||
out.append(
|
||||
FileLink(
|
||||
path=chosen,
|
||||
anchor=link.anchor,
|
||||
predicate=link.predicate,
|
||||
),
|
||||
)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
async def text_to_links(
|
||||
graph: BaseFileGraph,
|
||||
text: str,
|
||||
) -> list[FileLink]:
|
||||
"""One-shot: extract wikilinks from ``text`` and resolve to safe links.
|
||||
|
||||
Equivalent to ``await resolve_links(graph, iter_links(text))``.
|
||||
The parser pipeline's single-call entry point.
|
||||
"""
|
||||
return await resolve_links(graph, iter_links(text))
|
||||
|
|
|
|||
|
|
@ -26,20 +26,25 @@ def main() -> None:
|
|||
)
|
||||
ap.add_argument("path", help="Path to a markdown file.")
|
||||
ap.add_argument(
|
||||
"--chunk-chars", type=int, default=2000,
|
||||
help="Max characters per chunk content (default: 2000). "
|
||||
"Excludes TOC skeleton when embed_toc is on.",
|
||||
"--chunk-chars",
|
||||
type=int,
|
||||
default=2000,
|
||||
help="Max characters per chunk content (default: 2000). " "Excludes TOC skeleton when embed_toc is on.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--no-toc", action="store_true",
|
||||
"--no-toc",
|
||||
action="store_true",
|
||||
help="Disable the full-doc TOC skeleton wrap; chunks become plain content.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--show-edges", action="store_true",
|
||||
help="Print extracted FileEdges before chunks.",
|
||||
"--show-edges",
|
||||
action="store_true",
|
||||
help="Print extracted FileLinks before chunks.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--preview", type=int, default=0,
|
||||
"--preview",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Truncate each chunk to N chars in output (0 = full text).",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
|
@ -59,12 +64,12 @@ def main() -> None:
|
|||
sizes = [len(c.text) for c in chunks]
|
||||
print(f"chars min/avg/max: {min(sizes)} / {sum(sizes)//len(sizes)} / {max(sizes)}")
|
||||
if args.show_edges:
|
||||
print(f"\nedges ({len(node.edges)}):")
|
||||
for e in node.edges:
|
||||
print(f"\nlinks ({len(node.links)}):")
|
||||
for link in node.links:
|
||||
print(
|
||||
f" → {e.link}"
|
||||
+ (f" predicate={e.predicate}" if e.predicate else "")
|
||||
+ (f" anchor={e.anchor}" if e.anchor else "")
|
||||
f" → {link.path}"
|
||||
+ (f" predicate={link.predicate}" if link.predicate else "")
|
||||
+ (f" anchor={link.anchor}" if link.anchor else ""),
|
||||
)
|
||||
|
||||
for i, c in enumerate(chunks):
|
||||
|
|
|
|||
|
|
@ -1,348 +0,0 @@
|
|||
"""FileEdge unit tests — body-only edge extraction.
|
||||
|
||||
Covers the three legal inline forms (bare / line-level Dataview /
|
||||
inline-bracketed Dataview), multi-target expansion, dedup against
|
||||
typed wrappers, open-vocabulary predicate pass-through, the explicit
|
||||
decision that frontmatter is no longer walked for links, and the
|
||||
2-field schema (`link` + `predicate`) with `anchor` as a derived
|
||||
property and `alias` / `embed` discarded at parse time.
|
||||
"""
|
||||
|
||||
from reme2.schema.file_edge import FileEdge, extract_wikilinks
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Bare wikilinks
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_bare_wikilink():
|
||||
edges = FileEdge.from_text("see [[X]]")
|
||||
assert len(edges) == 1
|
||||
assert edges[0].link == "X"
|
||||
assert edges[0].predicate is None
|
||||
assert edges[0].anchor is None
|
||||
|
||||
|
||||
def test_anchor_alias_embed_collapse_into_link():
|
||||
"""`![[X#sec|Alias]]` → link='X#sec' (alias and embed dropped)."""
|
||||
edges = FileEdge.from_text("![[X#sec|Alias]]")
|
||||
assert len(edges) == 1
|
||||
assert edges[0].link == "X#sec"
|
||||
# anchor is a derived property parsed from link.
|
||||
assert edges[0].anchor == "sec"
|
||||
|
||||
|
||||
def test_alias_only_drops_to_link():
|
||||
edges = FileEdge.from_text("[[X|Alias]]")
|
||||
assert len(edges) == 1
|
||||
assert edges[0].link == "X"
|
||||
assert edges[0].anchor is None
|
||||
|
||||
|
||||
def test_embed_only_drops_to_link():
|
||||
edges = FileEdge.from_text("![[X]]")
|
||||
assert len(edges) == 1
|
||||
assert edges[0].link == "X"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Line-level Dataview
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_line_level_field():
|
||||
edges = FileEdge.from_text("extends:: [[Source Topic]]")
|
||||
assert len(edges) == 1
|
||||
assert edges[0].link == "Source Topic"
|
||||
assert edges[0].predicate == "extends"
|
||||
|
||||
|
||||
def test_line_level_multi_target():
|
||||
edges = FileEdge.from_text("concerns:: [[A]], [[B]], [[C]]")
|
||||
assert [(e.link, e.predicate) for e in edges] == [
|
||||
("A", "concerns"),
|
||||
("B", "concerns"),
|
||||
("C", "concerns"),
|
||||
]
|
||||
|
||||
|
||||
def test_line_level_with_bullet():
|
||||
edges = FileEdge.from_text("- extends:: [[X]]\n * concerns:: [[Y]]")
|
||||
assert [(e.link, e.predicate) for e in edges] == [
|
||||
("X", "extends"),
|
||||
("Y", "concerns"),
|
||||
]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Multi-edge cases — many wikilinks under one or several typed contexts
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_multi_target_with_anchors_preserves_each():
|
||||
"""Each comma-separated target keeps its own anchor in `link`."""
|
||||
edges = FileEdge.from_text("extends:: [[A#sec1]], [[B#sec2]], [[C]]")
|
||||
assert [(e.link, e.anchor, e.predicate) for e in edges] == [
|
||||
("A#sec1", "sec1", "extends"),
|
||||
("B#sec2", "sec2", "extends"),
|
||||
("C", None, "extends"),
|
||||
]
|
||||
|
||||
|
||||
def test_multi_target_non_comma_separator_still_typed():
|
||||
"""Wikilinks anywhere in the value range (not just comma-separated)
|
||||
inherit the line's predicate. Useful for prose-style fields."""
|
||||
edges = FileEdge.from_text("extends:: [[A]] and also [[B]]")
|
||||
assert [(e.link, e.predicate) for e in edges] == [
|
||||
("A", "extends"),
|
||||
("B", "extends"),
|
||||
]
|
||||
|
||||
|
||||
def test_multi_dataview_lines_each_multi_target():
|
||||
"""Multiple Dataview lines each with multi-target → all edges typed
|
||||
by their respective line's predicate."""
|
||||
edges = FileEdge.from_text(
|
||||
"extends:: [[A]], [[B]]\nrelates:: [[C]], [[D]]"
|
||||
)
|
||||
assert [(e.link, e.predicate) for e in edges] == [
|
||||
("A", "extends"),
|
||||
("B", "extends"),
|
||||
("C", "relates"),
|
||||
("D", "relates"),
|
||||
]
|
||||
|
||||
|
||||
def test_inline_bracketed_then_bare_on_same_line():
|
||||
"""Inline-bracketed governs only the wikilinks inside its brackets;
|
||||
a trailing bare wikilink on the same line stays bare."""
|
||||
edges = FileEdge.from_text("[ext:: [[A]]] then [[B]]")
|
||||
assert [(e.link, e.predicate) for e in edges] == [
|
||||
("A", "ext"),
|
||||
("B", None),
|
||||
]
|
||||
|
||||
|
||||
def test_mid_line_dataview_like_not_typed():
|
||||
"""``predicate::`` only counts at line start (modulo bullet) —
|
||||
a `predicate::` mid-line is just prose, so its wikilinks are bare."""
|
||||
edges = FileEdge.from_text("[ext:: [[A]]] and concerns:: [[B]]")
|
||||
assert [(e.link, e.predicate) for e in edges] == [
|
||||
("A", "ext"),
|
||||
("B", None), # `concerns::` mid-line is not Dataview
|
||||
]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Inline-bracketed Dataview
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_inline_bracketed():
|
||||
edges = FileEdge.from_text("This [extends:: [[Y]]] something else.")
|
||||
assert len(edges) == 1
|
||||
assert edges[0].link == "Y"
|
||||
assert edges[0].predicate == "extends"
|
||||
|
||||
|
||||
def test_inline_bracketed_multi_target():
|
||||
edges = FileEdge.from_text("[concerns:: [[A]], [[B]]]")
|
||||
assert [(e.link, e.predicate) for e in edges] == [
|
||||
("A", "concerns"),
|
||||
("B", "concerns"),
|
||||
]
|
||||
|
||||
|
||||
def test_inline_bracketed_skips_cross_line():
|
||||
# A `[predicate:: ...]` that spans a newline is malformed → the inner
|
||||
# wikilink falls back to bare; the unmatched `[` does not eat tail text.
|
||||
edges = FileEdge.from_text("[extends:: [[X]]\nbad]")
|
||||
assert len(edges) == 1
|
||||
assert edges[0].link == "X"
|
||||
assert edges[0].predicate is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Dedup: a wikilink inside a typed wrapper should not double-emit
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_inline_bracketed_does_not_double_emit():
|
||||
edges = FileEdge.from_text("see [extends:: [[X]]] again.")
|
||||
assert len(edges) == 1
|
||||
assert edges[0].predicate == "extends"
|
||||
|
||||
|
||||
def test_line_level_value_does_not_double_emit():
|
||||
edges = FileEdge.from_text("extends:: [[X]]")
|
||||
assert len(edges) == 1
|
||||
|
||||
|
||||
def test_typed_and_bare_coexist_for_same_target():
|
||||
edges = FileEdge.from_text("extends:: [[X]]\nFree text mentioning [[X]] again.")
|
||||
links_preds = sorted(((e.link, e.predicate or "") for e in edges))
|
||||
assert links_preds == [("X", ""), ("X", "extends")]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Open-vocabulary predicates — any identifier-shaped token passes through
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_arbitrary_predicate_preserved():
|
||||
edges = FileEdge.from_text("anything_goes:: [[X]]")
|
||||
assert len(edges) == 1
|
||||
assert edges[0].link == "X"
|
||||
assert edges[0].predicate == "anything_goes"
|
||||
|
||||
|
||||
def test_inline_arbitrary_predicate_preserved():
|
||||
edges = FileEdge.from_text("[wat:: [[X]]]")
|
||||
assert len(edges) == 1
|
||||
assert edges[0].predicate == "wat"
|
||||
|
||||
|
||||
def test_predicate_must_be_identifier_shaped():
|
||||
# A leading digit fails the regex `[A-Za-z][A-Za-z0-9_]*` so the line is
|
||||
# not recognised as a Dataview field — the wikilink falls back to bare.
|
||||
edges = FileEdge.from_text("123bad:: [[X]]")
|
||||
assert len(edges) == 1
|
||||
assert edges[0].link == "X"
|
||||
assert edges[0].predicate is None
|
||||
|
||||
|
||||
def test_file_edge_accepts_any_predicate_string():
|
||||
e = FileEdge(link="X", predicate="totally_made_up")
|
||||
assert e.predicate == "totally_made_up"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Frontmatter is NOT walked for links — explicit regression
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_frontmatter_links_block_is_ignored():
|
||||
# Even if a YAML-shaped string sits at the top of body, FileEdge.from_text
|
||||
# only operates on body. We pass body text directly here, so this test
|
||||
# asserts the API surface no longer accepts a metadata dict.
|
||||
import inspect
|
||||
|
||||
sig = inspect.signature(FileEdge.from_text)
|
||||
assert list(sig.parameters.keys()) == ["text"], (
|
||||
"FileEdge.from_text should accept body text only — frontmatter walk removed"
|
||||
)
|
||||
|
||||
|
||||
def test_no_frontmatter_walker_exported():
|
||||
from reme2.schema import file_edge as fe
|
||||
|
||||
for removed in (
|
||||
"parse_wikilinks_from_metadata",
|
||||
"extract_wikilinks_from_metadata",
|
||||
"extract_inline_fields",
|
||||
"extract_typed_edges",
|
||||
"InlineField",
|
||||
):
|
||||
assert not hasattr(fe, removed), (
|
||||
f"{removed} should have been removed when YAML edges were dropped"
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# FileEdge schema
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_file_edge_extra_forbid():
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
FileEdge(link="X", target="X") # type: ignore[call-arg]
|
||||
|
||||
|
||||
def test_file_edge_minimal_field_set():
|
||||
"""Stored fields are just `link` and `predicate`. `path` and `anchor`
|
||||
are `@property`s (not stored fields) so they shouldn't appear in
|
||||
`model_dump()`. `target` / `alias` / `embed` were dropped entirely."""
|
||||
e = FileEdge(link="X")
|
||||
dumped = e.model_dump()
|
||||
assert set(dumped.keys()) == {"link", "predicate"}
|
||||
for removed in ("target", "alias", "embed", "anchor", "path"):
|
||||
assert removed not in dumped
|
||||
|
||||
|
||||
def test_anchor_property_parses_from_link():
|
||||
"""`anchor` is derived from `link`, not stored separately."""
|
||||
assert FileEdge(link="X").anchor is None
|
||||
assert FileEdge(link="X#sec").anchor == "sec"
|
||||
assert FileEdge(link="X#sec#more").anchor == "sec#more"
|
||||
# Empty anchor is treated as no anchor.
|
||||
assert FileEdge(link="X#").anchor is None
|
||||
assert FileEdge(link="X# ").anchor is None
|
||||
|
||||
|
||||
def test_path_property_parses_from_link():
|
||||
"""`path` is derived from `link` — file part before any `#anchor`."""
|
||||
assert FileEdge(link="X").path == "X"
|
||||
assert FileEdge(link="X#sec").path == "X"
|
||||
assert FileEdge(link="topics/Foo").path == "topics/Foo"
|
||||
assert FileEdge(link="topics/Foo#bar").path == "topics/Foo"
|
||||
# Multiple '#' — only first splits; the rest live in anchor.
|
||||
assert FileEdge(link="X#a#b").path == "X"
|
||||
assert FileEdge(link="X#a#b").anchor == "a#b"
|
||||
# Empty anchor → path is still the full prefix.
|
||||
assert FileEdge(link="X#").path == "X"
|
||||
|
||||
|
||||
def test_path_anchor_roundtrip_via_link():
|
||||
"""Reconstructing `link` from `path` + `anchor` yields the original."""
|
||||
for link in ("X", "X#sec", "topics/Foo", "topics/Foo#bar", "X#a#b"):
|
||||
e = FileEdge(link=link)
|
||||
rebuilt = e.path if not e.anchor else f"{e.path}#{e.anchor}"
|
||||
assert rebuilt == link, f"{link!r} → {rebuilt!r}"
|
||||
|
||||
|
||||
def test_anchor_is_not_constructor_arg():
|
||||
"""Since `anchor` and `path` are properties, passing them to the
|
||||
constructor should fail (extra='forbid')."""
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
FileEdge(link="X", anchor="sec") # type: ignore[call-arg]
|
||||
with pytest.raises(ValidationError):
|
||||
FileEdge(link="X", path="X") # type: ignore[call-arg]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Back-compat: extract_wikilinks returns flat target list (used by ingestor)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_extract_wikilinks_flat_targets():
|
||||
targets = extract_wikilinks("see [[X]] and extends:: [[Y]] and [extends:: [[Z]]]")
|
||||
assert targets == ["X", "Y", "Z"]
|
||||
|
||||
|
||||
def test_extract_wikilinks_strips_anchor():
|
||||
"""`extract_wikilinks` returns just the file part — anchor stripped
|
||||
so callers can feed it to `resolve_wikilink`."""
|
||||
targets = extract_wikilinks("see [[X#sec]] and ![[Y#a|alias]]")
|
||||
assert targets == ["X", "Y"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Source ordering stability
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_edges_sorted_by_source_position():
|
||||
body = (
|
||||
"intro [[First]] then\n"
|
||||
"extends:: [[Second]]\n"
|
||||
"tail [[Third]]\n"
|
||||
)
|
||||
edges = FileEdge.from_text(body)
|
||||
assert [e.link for e in edges] == ["First", "Second", "Third"]
|
||||
345
tests/test_file_link.py
Normal file
345
tests/test_file_link.py
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
"""FileLink unit tests — body-only wikilink extraction.
|
||||
|
||||
Covers:
|
||||
* three legal inline forms (bare / line-level Dataview / inline-bracketed
|
||||
Dataview) and multi-target expansion within each
|
||||
* dedup against typed wrappers (no double-emit when a wikilink lives
|
||||
inside a typed envelope)
|
||||
* open-vocabulary predicate pass-through and the identifier-shape gate
|
||||
* the explicit decision that frontmatter is no longer walked for links
|
||||
(extraction takes body text only)
|
||||
* the 3-field schema: ``(path, anchor, predicate)`` — all real fields,
|
||||
extra='forbid'
|
||||
|
||||
Pre-resolution form: ``iter_links(text)`` returns ``FileLink`` records with
|
||||
``path`` set to the raw wikilink target as written. The resolver
|
||||
(``utils.wikilink_resolver``) rewrites ``path`` to the vault-relative
|
||||
resolved path; here we only test the extractor.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from reme2.schema import FileLink
|
||||
from reme2.schema.file_link import extract_wikilinks, iter_links
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Bare wikilinks
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_bare_wikilink():
|
||||
links = iter_links("see [[X]]")
|
||||
assert len(links) == 1
|
||||
assert links[0].path == "X"
|
||||
assert links[0].predicate is None
|
||||
assert links[0].anchor is None
|
||||
|
||||
|
||||
def test_anchor_split_from_target():
|
||||
"""``[[X#sec]]`` → path='X', anchor='sec' (separate field, not derived)."""
|
||||
links = iter_links("![[X#sec|Alias]]")
|
||||
assert len(links) == 1
|
||||
assert links[0].path == "X"
|
||||
assert links[0].anchor == "sec"
|
||||
|
||||
|
||||
def test_alias_only_dropped():
|
||||
links = iter_links("[[X|Alias]]")
|
||||
assert len(links) == 1
|
||||
assert links[0].path == "X"
|
||||
assert links[0].anchor is None
|
||||
|
||||
|
||||
def test_embed_only_dropped():
|
||||
links = iter_links("![[X]]")
|
||||
assert len(links) == 1
|
||||
assert links[0].path == "X"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Line-level Dataview
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_line_level_field():
|
||||
links = iter_links("extends:: [[Source Topic]]")
|
||||
assert len(links) == 1
|
||||
assert links[0].path == "Source Topic"
|
||||
assert links[0].predicate == "extends"
|
||||
|
||||
|
||||
def test_line_level_multi_target():
|
||||
links = iter_links("concerns:: [[A]], [[B]], [[C]]")
|
||||
assert [(link.path, link.predicate) for link in links] == [
|
||||
("A", "concerns"),
|
||||
("B", "concerns"),
|
||||
("C", "concerns"),
|
||||
]
|
||||
|
||||
|
||||
def test_line_level_with_bullet():
|
||||
links = iter_links("- extends:: [[X]]\n * concerns:: [[Y]]")
|
||||
assert [(link.path, link.predicate) for link in links] == [
|
||||
("X", "extends"),
|
||||
("Y", "concerns"),
|
||||
]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Multi-link cases — many wikilinks under one or several typed contexts
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_multi_target_with_anchors_preserves_each():
|
||||
"""Each comma-separated target keeps its own anchor as a separate field."""
|
||||
links = iter_links("extends:: [[A#sec1]], [[B#sec2]], [[C]]")
|
||||
assert [(link.path, link.anchor, link.predicate) for link in links] == [
|
||||
("A", "sec1", "extends"),
|
||||
("B", "sec2", "extends"),
|
||||
("C", None, "extends"),
|
||||
]
|
||||
|
||||
|
||||
def test_multi_target_non_comma_separator_still_typed():
|
||||
"""Wikilinks anywhere in the value range (not just comma-separated)
|
||||
inherit the line's predicate. Useful for prose-style fields."""
|
||||
links = iter_links("extends:: [[A]] and also [[B]]")
|
||||
assert [(link.path, link.predicate) for link in links] == [
|
||||
("A", "extends"),
|
||||
("B", "extends"),
|
||||
]
|
||||
|
||||
|
||||
def test_multi_dataview_lines_each_multi_target():
|
||||
"""Multiple Dataview lines each with multi-target → all links typed
|
||||
by their respective line's predicate."""
|
||||
links = iter_links(
|
||||
"extends:: [[A]], [[B]]\nrelates:: [[C]], [[D]]",
|
||||
)
|
||||
assert [(link.path, link.predicate) for link in links] == [
|
||||
("A", "extends"),
|
||||
("B", "extends"),
|
||||
("C", "relates"),
|
||||
("D", "relates"),
|
||||
]
|
||||
|
||||
|
||||
def test_inline_bracketed_then_bare_on_same_line():
|
||||
"""Inline-bracketed governs only the wikilinks inside its brackets;
|
||||
a trailing bare wikilink on the same line stays bare."""
|
||||
links = iter_links("[ext:: [[A]]] then [[B]]")
|
||||
assert [(link.path, link.predicate) for link in links] == [
|
||||
("A", "ext"),
|
||||
("B", None),
|
||||
]
|
||||
|
||||
|
||||
def test_mid_line_dataview_like_not_typed():
|
||||
"""``predicate::`` only counts at line start (modulo bullet) —
|
||||
a ``predicate::`` mid-line is just prose, so its wikilinks are bare."""
|
||||
links = iter_links("[ext:: [[A]]] and concerns:: [[B]]")
|
||||
assert [(link.path, link.predicate) for link in links] == [
|
||||
("A", "ext"),
|
||||
("B", None), # `concerns::` mid-line is not Dataview
|
||||
]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Inline-bracketed Dataview
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_inline_bracketed():
|
||||
links = iter_links("This [extends:: [[Y]]] something else.")
|
||||
assert len(links) == 1
|
||||
assert links[0].path == "Y"
|
||||
assert links[0].predicate == "extends"
|
||||
|
||||
|
||||
def test_inline_bracketed_multi_target():
|
||||
links = iter_links("[concerns:: [[A]], [[B]]]")
|
||||
assert [(link.path, link.predicate) for link in links] == [
|
||||
("A", "concerns"),
|
||||
("B", "concerns"),
|
||||
]
|
||||
|
||||
|
||||
def test_inline_bracketed_skips_cross_line():
|
||||
# A `[predicate:: ...]` that spans a newline is malformed → the inner
|
||||
# wikilink falls back to bare; the unmatched `[` does not eat tail text.
|
||||
links = iter_links("[extends:: [[X]]\nbad]")
|
||||
assert len(links) == 1
|
||||
assert links[0].path == "X"
|
||||
assert links[0].predicate is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Dedup: a wikilink inside a typed wrapper should not double-emit
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_inline_bracketed_does_not_double_emit():
|
||||
links = iter_links("see [extends:: [[X]]] again.")
|
||||
assert len(links) == 1
|
||||
assert links[0].predicate == "extends"
|
||||
|
||||
|
||||
def test_line_level_value_does_not_double_emit():
|
||||
links = iter_links("extends:: [[X]]")
|
||||
assert len(links) == 1
|
||||
|
||||
|
||||
def test_typed_and_bare_coexist_for_same_target():
|
||||
links = iter_links("extends:: [[X]]\nFree text mentioning [[X]] again.")
|
||||
paths_preds = sorted(((link.path, link.predicate or "") for link in links))
|
||||
assert paths_preds == [("X", ""), ("X", "extends")]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Open-vocabulary predicates — any identifier-shaped token passes through
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_arbitrary_predicate_preserved():
|
||||
links = iter_links("anything_goes:: [[X]]")
|
||||
assert len(links) == 1
|
||||
assert links[0].path == "X"
|
||||
assert links[0].predicate == "anything_goes"
|
||||
|
||||
|
||||
def test_inline_arbitrary_predicate_preserved():
|
||||
links = iter_links("[wat:: [[X]]]")
|
||||
assert len(links) == 1
|
||||
assert links[0].predicate == "wat"
|
||||
|
||||
|
||||
def test_predicate_must_be_identifier_shaped():
|
||||
# A leading digit fails the regex `[A-Za-z][A-Za-z0-9_]*` so the line is
|
||||
# not recognised as a Dataview field — the wikilink falls back to bare.
|
||||
links = iter_links("123bad:: [[X]]")
|
||||
assert len(links) == 1
|
||||
assert links[0].path == "X"
|
||||
assert links[0].predicate is None
|
||||
|
||||
|
||||
def test_file_link_accepts_any_predicate_string():
|
||||
link = FileLink(path="X", predicate="totally_made_up")
|
||||
assert link.predicate == "totally_made_up"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Frontmatter is NOT walked for links — explicit regression
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_iter_links_takes_body_text_only():
|
||||
"""``iter_links`` operates on body text — no frontmatter walking."""
|
||||
sig = inspect.signature(iter_links)
|
||||
assert list(sig.parameters.keys()) == ["text"], "iter_links should accept body text only — frontmatter walk removed"
|
||||
|
||||
|
||||
def test_no_frontmatter_walker_exported():
|
||||
from reme2.schema import file_link as fl
|
||||
|
||||
for removed in (
|
||||
"parse_wikilinks_from_metadata",
|
||||
"extract_wikilinks_from_metadata",
|
||||
"extract_inline_fields",
|
||||
"extract_typed_edges",
|
||||
"InlineField",
|
||||
):
|
||||
assert not hasattr(fl, removed), f"{removed} should have been removed when YAML links were dropped"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# FileLink schema
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_file_link_extra_forbid():
|
||||
with pytest.raises(ValidationError):
|
||||
FileLink(path="X", target="X") # type: ignore[call-arg]
|
||||
|
||||
|
||||
def test_file_link_field_set():
|
||||
"""Stored fields are ``(path, anchor, predicate)`` — no others."""
|
||||
link = FileLink(path="X")
|
||||
dumped = link.model_dump()
|
||||
assert set(dumped.keys()) == {"path", "anchor", "predicate"}
|
||||
assert dumped == {"path": "X", "anchor": None, "predicate": None}
|
||||
|
||||
|
||||
def test_file_link_dump_excludes_none_when_asked():
|
||||
link = FileLink(path="X")
|
||||
assert link.model_dump(exclude_none=True) == {"path": "X"}
|
||||
|
||||
|
||||
def test_file_link_full_construction():
|
||||
link = FileLink(path="topics/Foo.md", anchor="sec", predicate="extends")
|
||||
assert link.path == "topics/Foo.md"
|
||||
assert link.anchor == "sec"
|
||||
assert link.predicate == "extends"
|
||||
|
||||
|
||||
def test_file_link_path_required():
|
||||
with pytest.raises(ValidationError):
|
||||
FileLink() # type: ignore[call-arg]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Anchor extraction edge cases — anchor is a real field, captured by the
|
||||
# regex's named group (not derived from path string-splitting)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_anchor_extraction_edge_cases():
|
||||
"""``[[X]]`` → no anchor; ``[[X#sec]]`` → anchor='sec'.
|
||||
The wikilink regex requires the anchor capture to be one or more
|
||||
chars, so a literal ``[[X#]]`` doesn't match the regex at all (the
|
||||
trailing ``#`` makes it invalid syntax). Whitespace-only anchors
|
||||
are treated as no anchor (stripped to empty → None)."""
|
||||
assert iter_links("[[X]]")[0].anchor is None
|
||||
assert iter_links("[[X#sec]]")[0].anchor == "sec"
|
||||
# `[[X#]]` is not a valid wikilink — anchor group requires 1+ chars.
|
||||
assert iter_links("[[X#]]") == []
|
||||
# `[[X# ]]` matches but strips to empty → anchor=None.
|
||||
assert iter_links("[[X# ]]")[0].anchor is None
|
||||
|
||||
|
||||
def test_anchor_inside_pipe_alias_still_extracted():
|
||||
"""Anchor is captured before the alias pipe."""
|
||||
links = iter_links("[[topics/Foo#sec|Display]]")
|
||||
assert links[0].path == "topics/Foo"
|
||||
assert links[0].anchor == "sec"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Back-compat: extract_wikilinks returns flat target list (used by retriever)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_extract_wikilinks_flat_targets():
|
||||
targets = extract_wikilinks("see [[X]] and extends:: [[Y]] and [extends:: [[Z]]]")
|
||||
assert targets == ["X", "Y", "Z"]
|
||||
|
||||
|
||||
def test_extract_wikilinks_strips_anchor():
|
||||
"""``extract_wikilinks`` returns just the file part — anchor stripped
|
||||
so callers can feed it to ``resolve``."""
|
||||
targets = extract_wikilinks("see [[X#sec]] and ![[Y#a|alias]]")
|
||||
assert targets == ["X", "Y"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Source ordering stability
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_links_sorted_by_source_position():
|
||||
body = "intro [[First]] then\n" "extends:: [[Second]]\n" "tail [[Third]]\n"
|
||||
links = iter_links(body)
|
||||
assert [link.path for link in links] == ["First", "Second", "Third"]
|
||||
|
|
@ -45,19 +45,7 @@ def _all_headings(text: str) -> list[str]:
|
|||
|
||||
|
||||
def test_tree_groups_under_headings():
|
||||
txt = (
|
||||
"# Top\n"
|
||||
"para1\n"
|
||||
"\n"
|
||||
"## Sub A\n"
|
||||
"para2\n"
|
||||
"\n"
|
||||
"### Deeper\n"
|
||||
"para3\n"
|
||||
"\n"
|
||||
"## Sub B\n"
|
||||
"para4\n"
|
||||
)
|
||||
txt = "# Top\n" "para1\n" "\n" "## Sub A\n" "para2\n" "\n" "### Deeper\n" "para3\n" "\n" "## Sub B\n" "para4\n"
|
||||
from mistletoe.block_token import Document
|
||||
from mistletoe.markdown_renderer import MarkdownRenderer
|
||||
|
||||
|
|
@ -142,23 +130,13 @@ def test_every_chunk_lists_every_doc_heading():
|
|||
expected_headings = {"# Doc", "## Section A", "### Subsection", "## Section B"}
|
||||
for c in chunks:
|
||||
present = set(_all_headings(c.text))
|
||||
assert expected_headings <= present, (
|
||||
f"chunk missing headings {expected_headings - present}: {c.text!r}"
|
||||
)
|
||||
assert expected_headings <= present, f"chunk missing headings {expected_headings - present}: {c.text!r}"
|
||||
|
||||
|
||||
def test_owner_section_holds_chunk_content():
|
||||
"""The chunk's content sits directly under its owner heading — not
|
||||
under any other section's heading."""
|
||||
txt = (
|
||||
"# Doc\n"
|
||||
"\n"
|
||||
"## A\n"
|
||||
"alpha alpha alpha alpha here\n"
|
||||
"\n"
|
||||
"## B\n"
|
||||
"bravo bravo bravo bravo here\n"
|
||||
)
|
||||
txt = "# Doc\n" "\n" "## A\n" "alpha alpha alpha alpha here\n" "\n" "## B\n" "bravo bravo bravo bravo here\n"
|
||||
chunks = _parser(60)._chunk(txt, "/x.md")
|
||||
a_chunk = next(c for c in chunks if "alpha" in c.text)
|
||||
b_chunk = next(c for c in chunks if "bravo" in c.text)
|
||||
|
|
@ -383,15 +361,7 @@ def test_paragraph_line_split_keeps_skeleton():
|
|||
def test_embed_toc_off_strips_skeleton():
|
||||
"""With embed_toc=False, chunks contain only their own content —
|
||||
no full-doc heading skeleton wrapping them."""
|
||||
txt = (
|
||||
"# Doc\n"
|
||||
"\n"
|
||||
"## A\n"
|
||||
"para A long content here\n"
|
||||
"\n"
|
||||
"## B\n"
|
||||
"para B long content here\n"
|
||||
)
|
||||
txt = "# Doc\n" "\n" "## A\n" "para A long content here\n" "\n" "## B\n" "para B long content here\n"
|
||||
chunks = _parser(60, embed_toc=False)._chunk(txt, "/x.md")
|
||||
a = next(c for c in chunks if "para A" in c.text)
|
||||
b = next(c for c in chunks if "para B" in c.text)
|
||||
|
|
@ -456,9 +426,9 @@ def test_body_run_budget_excludes_toc():
|
|||
txt = (
|
||||
"# Top\n"
|
||||
"## Sub long heading title here\n"
|
||||
"abcdefghij abcdefghij abcdefghij\n" # 32 chars body
|
||||
"abcdefghij abcdefghij abcdefghij\n" # 32 chars body
|
||||
"\n"
|
||||
"klmnopqrst klmnopqrst klmnopqrst\n" # 32 chars body
|
||||
"klmnopqrst klmnopqrst klmnopqrst\n" # 32 chars body
|
||||
)
|
||||
# 80 chars budget covers the joined body (32+2+32=66) but is
|
||||
# smaller than body+TOC under old (counting) semantics (~120).
|
||||
|
|
@ -590,10 +560,7 @@ def test_paragraph_split_marks_parts():
|
|||
"delta line four with extra padding text here\n"
|
||||
)
|
||||
chunks = _parser(100)._chunk(txt, "/x.md")
|
||||
para_chunks = [
|
||||
c for c in chunks
|
||||
if any(t in c.text for t in ("alpha", "beta", "gamma", "delta"))
|
||||
]
|
||||
para_chunks = [c for c in chunks if any(t in c.text for t in ("alpha", "beta", "gamma", "delta"))]
|
||||
assert len(para_chunks) >= 2
|
||||
n = len(para_chunks)
|
||||
for i, c in enumerate(para_chunks, 1):
|
||||
|
|
@ -602,15 +569,7 @@ def test_paragraph_split_marks_parts():
|
|||
|
||||
def test_single_piece_leaf_has_no_part_marker():
|
||||
"""When a leaf block fits in one piece, no [Part] prefix is added."""
|
||||
txt = (
|
||||
"# Doc\n"
|
||||
"\n"
|
||||
"## T\n"
|
||||
"\n"
|
||||
"| a | b |\n"
|
||||
"|---|---|\n"
|
||||
"| 1 | 2 |\n"
|
||||
)
|
||||
txt = "# Doc\n" "\n" "## T\n" "\n" "| a | b |\n" "|---|---|\n" "| 1 | 2 |\n"
|
||||
chunks = _parser(500)._chunk(txt, "/x.md")
|
||||
assert len(chunks) == 1
|
||||
assert "[Part" not in chunks[0].text
|
||||
|
|
@ -642,11 +601,11 @@ def test_part_marker_absent_for_body_run_packing():
|
|||
|
||||
def test_chunk_line_ranges_track_source():
|
||||
txt = (
|
||||
"# Top\n" # line 1
|
||||
"intro\n" # line 2
|
||||
"# Top\n" # line 1
|
||||
"intro\n" # line 2
|
||||
"\n"
|
||||
"## Sub\n" # line 4
|
||||
"body\n" # line 5
|
||||
"## Sub\n" # line 4
|
||||
"body\n" # line 5
|
||||
)
|
||||
chunks = _parser(500)._chunk(txt, "/x.md")
|
||||
assert len(chunks) == 1
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue