From 2e8a27ec5f45f8bfd83f0e441741ff64d9fc9364 Mon Sep 17 00:00:00 2001 From: huangsen Date: Fri, 15 May 2026 16:19:18 +0800 Subject: [PATCH] refactor(parser): rename wikilink_resolver to link_parser module BREAKING CHANGE: renamed `utils.wikilink_resolver` to `utils.link_parser` --- .../file_parser/linked_file_parser.py | 2 +- reme2/schema/file_link.py | 15 +- reme2/steps/crud/download.py | 35 +- reme2/steps/crud/list.py | 85 ++- reme2/steps/crud/property/delete.py | 22 +- reme2/steps/crud/property/read.py | 22 +- reme2/steps/crud/property/update.py | 46 +- reme2/steps/crud/stat.py | 31 +- reme2/steps/crud/upload.py | 22 +- reme2/steps/graph/traverse.py | 57 ++- reme2/steps/lint/__init__.py | 24 + reme2/steps/lint/collisions.py | 55 ++ reme2/steps/lint/dangling.py | 65 +++ reme2/steps/lint/orphans.py | 62 +++ reme2/steps/lint/schema.py | 67 +++ reme2/utils/link_parser.py | 162 ++++++ reme2/utils/path_resolver.py | 190 +++++++ reme2/utils/wikilink_resolver.py | 482 ------------------ 18 files changed, 825 insertions(+), 619 deletions(-) create mode 100644 reme2/steps/lint/__init__.py create mode 100644 reme2/steps/lint/collisions.py create mode 100644 reme2/steps/lint/dangling.py create mode 100644 reme2/steps/lint/orphans.py create mode 100644 reme2/steps/lint/schema.py create mode 100644 reme2/utils/link_parser.py create mode 100644 reme2/utils/path_resolver.py delete mode 100644 reme2/utils/wikilink_resolver.py diff --git a/reme2/component/file_parser/linked_file_parser.py b/reme2/component/file_parser/linked_file_parser.py index 1052c130..d559fc57 100644 --- a/reme2/component/file_parser/linked_file_parser.py +++ b/reme2/component/file_parser/linked_file_parser.py @@ -29,7 +29,7 @@ from ...schema import ( FileNode, ) from ...utils import hash_text -from ...utils.wikilink_resolver import text_to_links +from ...utils.link_parser import text_to_links # -- AST node + helpers --------------------------------------------------- diff --git a/reme2/schema/file_link.py b/reme2/schema/file_link.py index 7dab44e4..f7ae600c 100644 --- a/reme2/schema/file_link.py +++ b/reme2/schema/file_link.py @@ -7,11 +7,12 @@ One type, two states (the value of ``path`` distinguishes them): * **resolved** — ``path`` holds the vault-relative path file_graph stores, e.g. ``"topics/Foo/Foo.md"``. -The extractor (``utils.wikilink_resolver.iter_links``) produces the -pre-resolution form from body text. The resolver -(``utils.wikilink_resolver.resolve_links``, or one-shot -``text_to_links``) rewrites ``path`` to the resolved form, expanding -stem ambiguity into one ``FileLink`` per candidate. +The extractor (``utils.link_parser.iter_links``) produces the +pre-resolution form from body text. The batch resolver +(``utils.link_parser.resolve_links``, or one-shot ``text_to_links``) +delegates to ``utils.path_resolver.resolve`` to rewrite ``path`` to +the resolved form, expanding short-path ambiguity into one +``FileLink`` per candidate. file_graph trusts ``link.path`` directly for adjacency: it only ever stores resolved links. The pre-resolution form is internal pipeline @@ -47,8 +48,8 @@ class FileLink(BaseModel): description=( "Wikilink target. Pre-resolution: the raw target as written " "(e.g. 'Foo'). Resolved: the vault-relative path file_graph " - "stores. Stem ambiguity is resolved BEFORE construction of " - "the resolved form by emitting one FileLink per candidate." + "stores. Short-path ambiguity is resolved BEFORE construction " + "of the resolved form by emitting one FileLink per candidate." ), ) anchor: str | None = Field( diff --git a/reme2/steps/crud/download.py b/reme2/steps/crud/download.py index a79e9eb1..4422e041 100644 --- a/reme2/steps/crud/download.py +++ b/reme2/steps/crud/download.py @@ -8,9 +8,10 @@ The temp root is lazy and session-scoped — created on first download, left for the OS to clean up at process exit. Each download lands in its own subdirectory so concurrent agents don't trample each other. -Also exports ``resolve_path`` — the shared helper for turning -a vault-relative or absolute path into an absolute on-disk path. -``upload`` and ``list`` import it from here. +``path`` accepts a short link or a vault-relative path; resolution +goes through ``path_resolver.resolve_to_absolute``. Short-link +ambiguity is surfaced as ``error="ambiguous"`` with the candidate +list — the download is **not** executed in that case. """ from __future__ import annotations @@ -26,6 +27,7 @@ from ..runtime_response import _set_answer, _tool_response from ...component import R from ...enumeration import ComponentEnum +from ...utils import path_resolver _TEMP_ROOT: Path | None = None @@ -39,20 +41,6 @@ def _get_temp_root() -> Path: return _TEMP_ROOT -def resolve_path(file_store, path: str) -> Path: - """Compose the absolute on-disk path for relative entry. - - Public so sibling steps (``upload``, ``list``, event tools) can - reuse the same path-resolution rule. Absolute paths pass through; - relative paths join under ``file_store.working_dir``. - """ - working_dir = getattr(file_store, "working_dir", None) or "." - p = Path(path) - if p.is_absolute(): - return p.resolve() - return (Path(working_dir) / p).resolve() - - @R.register("file_download") class FileDownload(BaseStep): """Copy a vault file to a session temp dir; return the local path.""" @@ -65,18 +53,23 @@ class FileDownload(BaseStep): assert self.context is not None path: str = self.context.get("path", "") or "" assert path, "path is required" - payload = self._download(path) + payload = await self._download(path) self.context.response.success = "error" not in payload _set_answer(self.context, payload) async def file_download(self, path: str) -> ToolResponse: """Copy a vault file to session temp dir; return the local path.""" - payload = self._download(path) + payload = await self._download(path) ok = "error" not in payload return _tool_response("file_download", ok, payload, audit=self.audit) - def _download(self, path: str) -> dict: - src = resolve_path(self.file_store, path) + async def _download(self, path: str) -> dict: + try: + src = await path_resolver.resolve_to_absolute(self.file_store, path) + except path_resolver.PathAmbiguous as e: + return {"path": path, "error": "ambiguous", "candidates": e.candidates} + except path_resolver.PathNotFound: + return {"path": path, "error": "not found"} if not src.is_file(): return {"path": path, "error": "not found"} dst_dir = Path(tempfile.mkdtemp(prefix="dl-", dir=_get_temp_root())) diff --git a/reme2/steps/crud/list.py b/reme2/steps/crud/list.py index a6e920f9..499e1e6a 100644 --- a/reme2/steps/crud/list.py +++ b/reme2/steps/crud/list.py @@ -1,20 +1,22 @@ -"""``file_list`` — enumerate vault files with optional frontmatter filters. +"""``file_list`` — enumerate vault files under a directory. -Walks the file_graph in one shot via ``get_nodes(None)`` — no filesystem -scan, no per-file frontmatter parse, no per-file graph round-trip. +Walks the file_graph in one shot via ``get_nodes(None)`` — no +filesystem scan, no per-file frontmatter parse, no per-file graph +round-trip. -Filters compose: - path_prefix — prefix match against the indexed path string - tags — every requested tag must be present in ``tags`` - metadata — frontmatter must equal each ``{key: value}`` pair - -The ``metadata`` filter sees the full frontmatter dict — typed fields -(title / description / tags) merged with any ``extra=allow`` extras — -so agents can filter on schema-known keys or arbitrary extras alike. +Parameters: + path — directory to list under (vault-relative or absolute). + Empty = working_dir root. Short links are not + meaningful for directories. + limit — cap the number of returned items. + recursive — descend into subdirectories. Default False = direct + children only. """ from __future__ import annotations +from pathlib import Path + from agentscope.tool import ToolResponse from ..base_step import BaseStep @@ -22,43 +24,31 @@ from ..runtime_response import _set_answer, _tool_response from ...component import R from ...enumeration import ComponentEnum +from ...utils import path_resolver -def _matches( - path: str, - md: dict, - *, - path_prefix: str | None, - tags: list[str], - metadata: dict, -) -> bool: - if path_prefix and not path.startswith(path_prefix): - return False - if metadata and any(md.get(k) != v for k, v in metadata.items()): - return False - if tags: - file_tags = set(md.get("tags") or []) - if not all(t in file_tags for t in tags): - return False - return True +def _under(node_abs: Path, target_dir: Path, *, recursive: bool) -> bool: + if recursive: + return node_abs == target_dir or target_dir in node_abs.parents + return node_abs.parent == target_dir async def _list( file_store, *, - path_prefix: str | None, - tags: list[str], - metadata: dict, + path: str, + recursive: bool, limit: int, ) -> dict: if not file_store.file_graph: return {"items": [], "count": 0} + target_dir = path_resolver.to_absolute(file_store, path or ".") items: list[dict] = [] for node in await file_store.file_graph.get_nodes(): - md = node.front_matter.model_dump() - if not _matches(node.path, md, path_prefix=path_prefix, tags=tags, metadata=metadata): + node_abs = path_resolver.to_absolute(file_store, node.path) + if not _under(node_abs, target_dir, recursive=recursive): continue - items.append({"path": node.path, "metadata": md}) + items.append({"path": node.path, "metadata": node.front_matter.model_dump()}) if len(items) >= limit: break return {"items": items, "count": len(items)} @@ -66,7 +56,7 @@ async def _list( @R.register("file_list") class FileList(BaseStep): - """Enumerate vault files with optional frontmatter filters.""" + """Enumerate vault files under a directory.""" component_type = ComponentEnum.STEP @@ -76,26 +66,31 @@ class FileList(BaseStep): assert self.context is not None result = await _list( self.file_store, - path_prefix=self.context.get("prefix") or self.context.get("path_prefix"), - tags=self.context.get("tags") or [], - metadata=self.context.get("metadata") or {}, + path=self.context.get("path") or "", + recursive=bool(self.context.get("recursive", False)), limit=int(self.context.get("limit") or 100), ) _set_answer(self.context, result) async def file_list( self, - prefix: str | None = None, - tags: list[str] | None = None, - metadata: dict | None = None, + path: str = "", limit: int = 100, + recursive: bool = False, ) -> ToolResponse: - """List vault files. Filters: path prefix, frontmatter tags / fields.""" + """List vault files under ``path``. + + Args: + path: directory to list (relative to working_dir or absolute). + Empty = working_dir root. + limit: cap the number of returned items. Default 100. + recursive: descend into subdirectories. Default False = + direct children only. + """ result = await _list( self.file_store, - path_prefix=prefix, - tags=tags or [], - metadata=metadata or {}, + path=path, + recursive=recursive, limit=limit, ) return _tool_response("file_list", True, result, audit=self.audit) diff --git a/reme2/steps/crud/property/delete.py b/reme2/steps/crud/property/delete.py index c1fc9c6c..0bcea44d 100644 --- a/reme2/steps/crud/property/delete.py +++ b/reme2/steps/crud/property/delete.py @@ -5,6 +5,10 @@ Returns both ``deleted`` (keys that were present and removed) and a no-op happened. The file is rewritten only when at least one key is actually removed — calling delete with all-missing keys is a zero-side-effect read. + +``path`` accepts a short link or vault-relative path. Short-link +ambiguity is reported via ``error="ambiguous"`` with the candidate +list — the delete is **not** executed in that case. """ from __future__ import annotations @@ -12,17 +16,21 @@ from __future__ import annotations import frontmatter from agentscope.tool import ToolResponse -from ..download import resolve_path - from ...base_step import BaseStep from ...runtime_response import _set_answer, _tool_response from ....component import R from ....enumeration import ComponentEnum +from ....utils import path_resolver -def _delete(file_store, path: str, keys: list[str]) -> dict: - target = resolve_path(file_store, path) +async def _delete(file_store, path: str, keys: list[str]) -> dict: + try: + target = await path_resolver.resolve_to_absolute(file_store, path) + except path_resolver.PathAmbiguous as e: + return {"path": path, "error": "ambiguous", "candidates": e.candidates} + except path_resolver.PathNotFound: + return {"path": path, "error": "not found"} if not target.is_file(): return {"path": path, "error": "not found"} if not keys: @@ -57,12 +65,12 @@ class PropertyDelete(BaseStep): async def execute(self): assert self.context is not None - path: str = self.context.get("path") or self.context.get("path") or "" + path: str = self.context.get("path") or "" assert path, "path is required" keys = self.context.get("keys") or [] if isinstance(keys, str): keys = [keys] - payload = _delete(self.file_store, path, list(keys)) + payload = await _delete(self.file_store, path, list(keys)) self.context.response.success = "error" not in payload _set_answer(self.context, payload) @@ -74,6 +82,6 @@ class PropertyDelete(BaseStep): """Remove the listed ``keys`` from the frontmatter at ``path``.""" if isinstance(keys, str): keys = [keys] - payload = _delete(self.file_store, path, list(keys)) + payload = await _delete(self.file_store, path, list(keys)) ok = "error" not in payload return _tool_response("property:delete", ok, payload, audit=self.audit) diff --git a/reme2/steps/crud/property/read.py b/reme2/steps/crud/property/read.py index 57105e23..3169751b 100644 --- a/reme2/steps/crud/property/read.py +++ b/reme2/steps/crud/property/read.py @@ -3,6 +3,10 @@ Cheap structured read — frontmatter only, no body. Use ``crud:read`` when you need the body too. Returns ``{exists: false}`` when the target doesn't exist; otherwise ``{exists: true, frontmatter: {...}}``. + +``path`` accepts a short link or vault-relative path. Short-link +ambiguity is reported via ``error="ambiguous"`` with the candidate +list — the read is **not** executed in that case. """ from __future__ import annotations @@ -10,17 +14,21 @@ from __future__ import annotations import frontmatter from agentscope.tool import ToolResponse -from ..download import resolve_path - from ...base_step import BaseStep from ...runtime_response import _set_answer, _tool_response from ....component import R from ....enumeration import ComponentEnum +from ....utils import path_resolver -def _read(file_store, path: str) -> dict: - target = resolve_path(file_store, path) +async def _read(file_store, path: str) -> dict: + try: + target = await path_resolver.resolve_to_absolute(file_store, path) + except path_resolver.PathAmbiguous as e: + return {"path": path, "error": "ambiguous", "candidates": e.candidates} + except path_resolver.PathNotFound: + return {"path": path, "exists": False} if not target.is_file(): return {"path": path, "exists": False} raw = target.read_text(encoding="utf-8") @@ -38,14 +46,14 @@ class PropertyRead(BaseStep): async def execute(self): assert self.context is not None - path: str = self.context.get("path") or self.context.get("path") or "" + path: str = self.context.get("path") or "" assert path, "path is required" - payload = _read(self.file_store, path) + payload = await _read(self.file_store, path) self.context.response.success = payload.get("exists", False) _set_answer(self.context, payload) async def property_read(self, path: str) -> ToolResponse: """Return the frontmatter dict of the memory file at ``path``.""" - payload = _read(self.file_store, path) + payload = await _read(self.file_store, path) ok = payload.get("exists", False) return _tool_response("property:read", ok, payload, audit=self.audit) diff --git a/reme2/steps/crud/property/update.py b/reme2/steps/crud/property/update.py index b85bd322..bf771e0b 100644 --- a/reme2/steps/crud/property/update.py +++ b/reme2/steps/crud/property/update.py @@ -1,14 +1,16 @@ """``property:update`` — merge a patch into a memory file's frontmatter. -Patch semantics are merge-write: keys in ``patch`` overwrite existing -keys, keys not in ``patch`` are left untouched. To remove a key, -use ``property:delete`` — passing ``None`` here just sets the literal -None (which is rarely what you want). +Read-modify-write the YAML frontmatter; body content is untouched. +The watcher / parser pick up the change asynchronously. -Per the obsidian config convention the agent calls this with -arbitrary keyword arguments — ``path="My Note" status=done xx=xxx``. -The Step packs everything except ``path`` into the patch dict, so -the tool method accepts ``**fields`` directly. +Two input modes (mutually compatible — they merge in order): + * ``patch`` — explicit ``dict`` of frontmatter updates. + * ``**fields`` — free-form key/value pairs (obsidian convention + ``path=... key=val key=val``); merged on top of ``patch``. + +``path`` accepts a short link or vault-relative path. Short-link +ambiguity is reported via ``error="ambiguous"`` with the candidate +list — the update is **not** executed in that case. """ from __future__ import annotations @@ -16,25 +18,27 @@ from __future__ import annotations import frontmatter from agentscope.tool import ToolResponse -from ..download import resolve_path - from ...base_step import BaseStep from ...runtime_response import _set_answer, _tool_response from ....component import R from ....enumeration import ComponentEnum +from ....utils import path_resolver # Context keys that belong to plumbing (request envelope, response # slot, the path itself) and must never be promoted into a free-form # frontmatter patch. -_RESERVED_CONTEXT_KEYS = { - "path", "path", "patch", "response", "request", "data", -} +_RESERVED_CONTEXT_KEYS = {"path", "patch", "response", "request", "data"} -def _update(file_store, path: str, patch: dict) -> dict: - target = resolve_path(file_store, path) +async def _update(file_store, path: str, patch: dict) -> dict: + try: + target = await path_resolver.resolve_to_absolute(file_store, path) + except path_resolver.PathAmbiguous as e: + return {"path": path, "error": "ambiguous", "candidates": e.candidates} + except path_resolver.PathNotFound: + return {"path": path, "error": "not found"} if not target.is_file(): return {"path": path, "error": "not found"} if not patch: @@ -43,11 +47,7 @@ def _update(file_store, path: str, patch: dict) -> dict: post = frontmatter.loads(raw) post.metadata.update(patch) target.write_text(frontmatter.dumps(post), encoding="utf-8") - return { - "path": path, - "applied": dict(patch), - "frontmatter": dict(post.metadata), - } + return {"path": path, "updated": list(patch.keys())} @R.register("property:update") @@ -60,7 +60,7 @@ class PropertyUpdate(BaseStep): async def execute(self): assert self.context is not None - path: str = self.context.get("path") or self.context.get("path") or "" + path: str = self.context.get("path") or "" assert path, "path is required" patch = self.context.get("patch") if patch is None: @@ -71,7 +71,7 @@ class PropertyUpdate(BaseStep): k: v for k, v in self.context.data.items() if k not in _RESERVED_CONTEXT_KEYS } - payload = _update(self.file_store, path, dict(patch or {})) + payload = await _update(self.file_store, path, dict(patch or {})) self.context.response.success = "error" not in payload _set_answer(self.context, payload) @@ -88,6 +88,6 @@ class PropertyUpdate(BaseStep): merged.update(patch) if fields: merged.update(fields) - payload = _update(self.file_store, path, merged) + payload = await _update(self.file_store, path, merged) ok = "error" not in payload return _tool_response("property:update", ok, payload, audit=self.audit) diff --git a/reme2/steps/crud/stat.py b/reme2/steps/crud/stat.py index d27cfc10..fe070c13 100644 --- a/reme2/steps/crud/stat.py +++ b/reme2/steps/crud/stat.py @@ -13,24 +13,28 @@ Returns a uniform envelope: is ``"file"`` / ``"dir"`` (covers event workspace probes too). ``frontmatter`` is populated only for ``.md`` files and only when parsing succeeds — schema validity is a lint concern. + +``path`` accepts a short link, vault-relative path, or directory +path. File-style inputs go through ``path_resolver.resolve_to_absolute`` +so short-link ambiguity surfaces as ``error="ambiguous"`` (with +candidates) and the call is **not** executed. Directory paths and +non-indexed files fall back to a plain ``working_dir`` join. """ from __future__ import annotations import mimetypes from datetime import datetime -from pathlib import Path import frontmatter from agentscope.tool import ToolResponse -from .download import resolve_path - from ..base_step import BaseStep from ..runtime_response import _set_answer, _tool_response from ...component import R from ...enumeration import ComponentEnum +from ...utils import path_resolver def _iso(ts: float) -> str: @@ -38,15 +42,22 @@ def _iso(ts: float) -> str: return datetime.fromtimestamp(ts).isoformat() -def _stat(file_store, path: str) -> dict: - target = resolve_path(file_store, path) +async def _stat(file_store, path: str) -> dict: + try: + target = await path_resolver.resolve_to_absolute(file_store, path) + except path_resolver.PathAmbiguous as e: + return {"path": path, "error": "ambiguous", "candidates": e.candidates} + except path_resolver.PathNotFound: + # Directory or unindexed path — fall back to plain join. + target = path_resolver.to_absolute(file_store, path) + if not target.exists(): return {"path": path, "exists": False} st = target.stat() out: dict = { "path": path, - "path": str(target), + "absolute_path": str(target), "exists": True, "type": "dir" if target.is_dir() else "file", "mtime": _iso(st.st_mtime), @@ -78,8 +89,8 @@ class FileStat(BaseStep): assert self.context is not None path: str = self.context.get("path", "") or "" assert path, "path is required" - payload = _stat(self.file_store, path) - self.context.response.success = payload.get("exists", False) + payload = await _stat(self.file_store, path) + self.context.response.success = payload.get("exists", False) and "error" not in payload _set_answer(self.context, payload) async def file_stat(self, path: str) -> ToolResponse: @@ -89,6 +100,6 @@ class FileStat(BaseStep): ``size`` / ``mime`` / ``frontmatter`` are file-only; ``frontmatter`` is markdown-only. """ - payload = _stat(self.file_store, path) - ok = payload.get("exists", False) + payload = await _stat(self.file_store, path) + ok = payload.get("exists", False) and "error" not in payload return _tool_response("file_stat", ok, payload, audit=self.audit) diff --git a/reme2/steps/crud/upload.py b/reme2/steps/crud/upload.py index 551fc45d..c87bc56e 100644 --- a/reme2/steps/crud/upload.py +++ b/reme2/steps/crud/upload.py @@ -1,7 +1,6 @@ -"""``file_upload`` — copy a local file into the vault. +"""``file_upload`` — copy a local file into the vault at a given path. -Agent flow: agent has a file at some local path (a download result, a -freshly generated artifact, a user-supplied attachment), and wants it +Used to put materials (or any binary/text artifact) into the vault to live inside the vault at a known location. This is type-agnostic — text, binary, anything — so it's the entry point for non-markdown materials too. @@ -10,6 +9,11 @@ The watcher / parser pick up the new file asynchronously; this step just performs the copy. ``overwrite=True`` is the default since most upload flows are intentional replacements (re-uploading the same material after re-generation). + +``path`` **must be a vault-relative path** with a directory component +— short links and absolute paths are rejected. File creation +needs an unambiguous primary key, and short links can't promise that +without a graph entry. Use ``file_move`` to rename existing files. """ from __future__ import annotations @@ -20,19 +24,17 @@ from pathlib import Path from agentscope.tool import ToolResponse -from .download import resolve_path - from ..base_step import BaseStep from ..runtime_response import _set_answer, _tool_response from ...component import R from ...enumeration import ComponentEnum +from ...utils import path_resolver @R.register("file_upload") class FileUpload(BaseStep): - """Copy a local file into the vault. Watcher / parser register the - FileNode asynchronously.""" + """Copy ``local_path`` into the vault at ``path``.""" component_type = ComponentEnum.STEP @@ -57,10 +59,14 @@ class FileUpload(BaseStep): return _tool_response("file_upload", ok, payload, audit=self.audit) def _upload(self, local_path: str, path: str, overwrite: bool) -> dict: + if Path(path).is_absolute(): + return {"path": path, "error": "path must be vault-relative"} + if path_resolver.is_short_path(path): + return {"path": path, "error": "path must include a directory component"} src = Path(local_path) if not src.is_file(): return {"local_path": local_path, "error": "source not found"} - dst = resolve_path(self.file_store, path) + dst = path_resolver.to_absolute(self.file_store, path) if dst.exists() and not overwrite: return {"path": path, "error": "destination exists; pass overwrite=True"} dst.parent.mkdir(parents=True, exist_ok=True) diff --git a/reme2/steps/graph/traverse.py b/reme2/steps/graph/traverse.py index 95ce3f20..8a680c1c 100644 --- a/reme2/steps/graph/traverse.py +++ b/reme2/steps/graph/traverse.py @@ -21,6 +21,11 @@ data we already have. Direction vocabulary accepts both the obsidian convention (``forward`` / ``backward`` / ``both``) and the engine convention (``out`` / ``in`` / ``both``). + +Seeds accept short links / vault-relative paths and go through +``path_resolver.resolve`` so the BFS keys match the graph's stored +form. Short-link ambiguity surfaces as ``error="ambiguous"`` (with +candidates) and the BFS is **not** executed in that case. """ from __future__ import annotations @@ -30,14 +35,13 @@ from pathlib import Path from agentscope.tool import ToolResponse -from ..crud.download import resolve_path - from ..base_step import BaseStep from ..runtime_response import _set_answer, _tool_response from ...component import R from ...enumeration import ComponentEnum from ...schema import FileLink +from ...utils import path_resolver _FORWARD = {"out", "forward"} @@ -123,13 +127,24 @@ def _bfs( return results -def _normalize_seeds(file_store, raw) -> list[str]: - """Coerce a single path or list of paths into resolved absolute strings.""" +async def _normalize_seeds(file_store, raw) -> list[str]: + """Resolve seeds (short link / relative path) to vault-relative keys. + + Raises ``PathAmbiguous`` / ``PathNotFound`` from ``path_resolver``; + callers wrap to convert into a structured response. + """ if isinstance(raw, (str, Path)): items = [raw] else: items = list(raw or []) - return [str(resolve_path(file_store, str(p))) for p in items if p] + if file_store.file_graph is None: + raise path_resolver.PathNotFound("no file_graph configured") + out: list[str] = [] + for p in items: + if not p: + continue + out.append(await path_resolver.resolve(file_store.file_graph, str(p))) + return out @R.register("graph:traverse") @@ -151,14 +166,27 @@ class GraphTraverse(BaseStep): assert self.context is not None path = self.context.get("path") seeds_raw = self.context.get("seeds") if path is None else path - seeds = _normalize_seeds(self.file_store, seeds_raw) depth = int(self.context.get("depth") or self.context.get("max_depth") or 1) direction = (self.context.get("direction") or "forward").lower() predicate = self.context.get("predicate") - assert seeds, "path (or seeds) is required" assert direction in _VALID_DIRECTIONS, ( f"direction must be one of {sorted(_VALID_DIRECTIONS)}, got {direction!r}" ) + try: + seeds = await _normalize_seeds(self.file_store, seeds_raw) + except path_resolver.PathAmbiguous as e: + self.context.response.success = False + _set_answer(self.context, { + "error": "ambiguous", + "target": e.target, + "candidates": e.candidates, + }) + return + except path_resolver.PathNotFound as e: + self.context.response.success = False + _set_answer(self.context, {"error": "not found", "target": e.target}) + return + assert seeds, "path (or seeds) is required" outbound, inbound = await _build_indexes(self.file_store) results = _bfs(seeds, depth, direction, predicate, outbound, inbound) _set_answer(self.context, results) @@ -183,7 +211,20 @@ class GraphTraverse(BaseStep): assert direction in _VALID_DIRECTIONS, ( f"direction must be one of {sorted(_VALID_DIRECTIONS)}, got {direction!r}" ) - seeds = _normalize_seeds(self.file_store, path) + try: + seeds = await _normalize_seeds(self.file_store, path) + except path_resolver.PathAmbiguous as e: + return _tool_response( + "graph:traverse", False, + {"error": "ambiguous", "target": e.target, "candidates": e.candidates}, + audit=self.audit, + ) + except path_resolver.PathNotFound as e: + return _tool_response( + "graph:traverse", False, + {"error": "not found", "target": e.target}, + audit=self.audit, + ) assert seeds, "path is required" outbound, inbound = await _build_indexes(self.file_store) results = _bfs(seeds, depth, direction, predicate, outbound, inbound) diff --git a/reme2/steps/lint/__init__.py b/reme2/steps/lint/__init__.py new file mode 100644 index 00000000..eeae2fec --- /dev/null +++ b/reme2/steps/lint/__init__.py @@ -0,0 +1,24 @@ +"""Lint steps — atomic vault-health checks. + +Read-only diagnostics for maintainer / CLI / scheduled-job use; +agents typically don't need them in their per-call working set. + +Four atomic checks, each does one thing and returns pure data: + + lint:dangling — FileLinks pointing to non-existent nodes + lint:orphans — nodes with no inlinks AND no outlinks + lint:collisions — basenames resolving to >1 path (short-link + ambiguity) + lint:schema — nodes violating frontmatter schema + (missing required fields, invalid status) + +Maintainer compositions live in ``memory/maintainer.py``; this +package is the underlying primitives. Each step is also independently +MCP/agent callable for ad-hoc checks. Bind via +``memory.lint_toolkit.build_lint_toolkit``. +""" + +from . import dangling # noqa: F401 -- @R.register("lint:dangling") +from . import orphans # noqa: F401 -- @R.register("lint:orphans") +from . import collisions # noqa: F401 -- @R.register("lint:collisions") +from . import schema # noqa: F401 -- @R.register("lint:schema") diff --git a/reme2/steps/lint/collisions.py b/reme2/steps/lint/collisions.py new file mode 100644 index 00000000..eaed65b9 --- /dev/null +++ b/reme2/steps/lint/collisions.py @@ -0,0 +1,55 @@ +"""``lint:collisions`` — find basenames resolving to >1 path. + +Atomic vault-health check: groups every indexed path by its +filename + extension, surfaces any basename owned by more than one +path. Operates synchronously over the local file_store index — no +async graph round-trip needed. + +Returns: + + {count: int, groups: {: [, ...]}} + +Each path list is sorted for stable output across runs. +""" + +from __future__ import annotations + +from pathlib import Path + +from agentscope.tool import ToolResponse + +from ..base_step import BaseStep +from ..runtime_response import _set_answer, _tool_response + +from ...component import R +from ...enumeration import ComponentEnum + + +def _scan_collisions(file_store) -> dict[str, list[str]]: + by_name: dict[str, list[str]] = {} + for path in file_store.file_nodes: + by_name.setdefault(Path(path).name, []).append(path) + return {name: sorted(paths) for name, paths in by_name.items() if len(paths) > 1} + + +@R.register("lint:collisions") +class LintCollisions(BaseStep): + """List basenames resolving to >1 path (short-link ambiguity).""" + + component_type = ComponentEnum.STEP + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + groups = _scan_collisions(self.file_store) + _set_answer(self.context, {"count": len(groups), "groups": groups}) + + async def lint_collisions(self) -> ToolResponse: + """List basenames resolving to >1 path (short-link ambiguity).""" + groups = _scan_collisions(self.file_store) + return _tool_response( + "lint:collisions", True, + {"count": len(groups), "groups": groups}, + audit=self.audit, + ) diff --git a/reme2/steps/lint/dangling.py b/reme2/steps/lint/dangling.py new file mode 100644 index 00000000..98692628 --- /dev/null +++ b/reme2/steps/lint/dangling.py @@ -0,0 +1,65 @@ +"""``lint:dangling`` — find FileLinks pointing to non-existent nodes. + +Atomic vault-health check: walks every indexed node's ``links`` and +reports each edge whose ``path`` target isn't present in the index. +Pure dict iteration over ``file_store.file_nodes`` — no filesystem +walk, no graph round-trip. + +Returns: + + {count: int, findings: [ + {source, target, predicate, anchor}, + ... + ]} +""" + +from __future__ import annotations + +from agentscope.tool import ToolResponse + +from ..base_step import BaseStep +from ..runtime_response import _set_answer, _tool_response + +from ...component import R +from ...enumeration import ComponentEnum + + +def _scan_dangling(file_store) -> list[dict]: + """One entry per dangling edge.""" + nodes = file_store.file_nodes + out: list[dict] = [] + for src_path, node in nodes.items(): + for link in node.links: + if not link.path: + continue + if link.path not in nodes: + out.append({ + "source": src_path, + "target": link.path, + "predicate": link.predicate, + "anchor": link.anchor, + }) + return out + + +@R.register("lint:dangling") +class LintDangling(BaseStep): + """List every FileLink whose target is not in the graph.""" + + component_type = ComponentEnum.STEP + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + findings = _scan_dangling(self.file_store) + _set_answer(self.context, {"count": len(findings), "findings": findings}) + + async def lint_dangling(self) -> ToolResponse: + """List every FileLink whose target is not in the graph.""" + findings = _scan_dangling(self.file_store) + return _tool_response( + "lint:dangling", True, + {"count": len(findings), "findings": findings}, + audit=self.audit, + ) diff --git a/reme2/steps/lint/orphans.py b/reme2/steps/lint/orphans.py new file mode 100644 index 00000000..17b79614 --- /dev/null +++ b/reme2/steps/lint/orphans.py @@ -0,0 +1,62 @@ +"""``lint:orphans`` — find nodes with no inlinks AND no outlinks. + +Atomic vault-health check: a node is an orphan if its ``links`` is +empty AND no other node references it. O(N + E) — one pass to mark +every referenced path, one pass to filter unreferenced + linkless +nodes. + +Returns: + + {count: int, paths: [, ...]} + +The ``paths`` list is sorted for stable output across runs. +""" + +from __future__ import annotations + +from agentscope.tool import ToolResponse + +from ..base_step import BaseStep +from ..runtime_response import _set_answer, _tool_response + +from ...component import R +from ...enumeration import ComponentEnum + + +def _scan_orphans(file_store) -> list[str]: + nodes = file_store.file_nodes + referenced: set[str] = set() + for node in nodes.values(): + for link in node.links: + if link.path: + referenced.add(link.path) + out: list[str] = [] + for path, node in nodes.items(): + has_out = any(link.path for link in node.links) + has_in = path in referenced + if not has_out and not has_in: + out.append(path) + return sorted(out) + + +@R.register("lint:orphans") +class LintOrphans(BaseStep): + """List nodes with no inlinks AND no outlinks.""" + + component_type = ComponentEnum.STEP + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + orphans = _scan_orphans(self.file_store) + _set_answer(self.context, {"count": len(orphans), "paths": orphans}) + + async def lint_orphans(self) -> ToolResponse: + """List nodes with no inlinks AND no outlinks.""" + orphans = _scan_orphans(self.file_store) + return _tool_response( + "lint:orphans", True, + {"count": len(orphans), "paths": orphans}, + audit=self.audit, + ) diff --git a/reme2/steps/lint/schema.py b/reme2/steps/lint/schema.py new file mode 100644 index 00000000..cd312c52 --- /dev/null +++ b/reme2/steps/lint/schema.py @@ -0,0 +1,67 @@ +"""``lint:schema`` — find nodes whose frontmatter violates the memory schema. + +Atomic vault-health check: required-key presence + ``status`` enum +validity. Mirrors the validator that fires inside ``memory_create``; +keep ``_REQUIRED_META_KEYS`` and ``_VALID_STATUS`` in sync with the +path-template + status-state-machine rules in ``memory_toolkit``. + +Returns: + + {count: int, findings: [ + {path, errors: [, ...]}, + ... + ]} +""" + +from __future__ import annotations + +from agentscope.tool import ToolResponse + +from ..base_step import BaseStep +from ..runtime_response import _set_answer, _tool_response + +from ...component import R +from ...enumeration import ComponentEnum + + +_REQUIRED_META_KEYS = ("title", "lifecycle", "scope", "source", "role") +_VALID_STATUS = {"active", "distilled", "archived"} + + +def _scan_schema(file_store) -> list[dict]: + out: list[dict] = [] + for path, node in file_store.file_nodes.items(): + meta = node.front_matter.model_dump() + errs: list[str] = [] + missing = [k for k in _REQUIRED_META_KEYS if not meta.get(k)] + if missing: + errs.append(f"missing required: {missing}") + status = meta.get("status") + if status is not None and status not in _VALID_STATUS: + errs.append(f"invalid status: {status!r} (expected one of {sorted(_VALID_STATUS)})") + if errs: + out.append({"path": path, "errors": errs}) + return out + + +@R.register("lint:schema") +class LintSchema(BaseStep): + """List nodes whose frontmatter violates the memory schema.""" + + component_type = ComponentEnum.STEP + + audit: list[dict] | None = None + + async def execute(self): + assert self.context is not None + findings = _scan_schema(self.file_store) + _set_answer(self.context, {"count": len(findings), "findings": findings}) + + async def lint_schema(self) -> ToolResponse: + """List nodes whose frontmatter violates the memory schema.""" + findings = _scan_schema(self.file_store) + return _tool_response( + "lint:schema", True, + {"count": len(findings), "findings": findings}, + audit=self.audit, + ) diff --git a/reme2/utils/link_parser.py b/reme2/utils/link_parser.py new file mode 100644 index 00000000..09251b3a --- /dev/null +++ b/reme2/utils/link_parser.py @@ -0,0 +1,162 @@ +"""Link parser — wikilink syntax extraction from text. + +A **Link** is a textual reference to a vault file (``[[X]]`` and +typed/anchored variants). This module is the syntax layer: +text → ``FileLink`` records. + +``FileLink.path`` here is the **raw target as written** (no +``.md`` completion, no resolution). To turn the raw target into a +vault-relative key, call ``resolve_links`` (or ``text_to_links`` for +one-shot extract+resolve), which delegates to ``path_resolver``. + +## Inline forms recognised + + [[X]] bare wikilink → predicate=None + extends:: [[X]] line-level Dataview → predicate="extends" + [extends:: [[X]]] inline-bracketed → predicate="extends" + +Multi-target — every wikilink under one typed context inherits its +predicate (any separator works): + + extends:: [[A]], [[B]] → 2 links, both "extends" + [concerns:: [[A]] and [[B]]] → 2 links, both "concerns" + extends:: [[A#s1]], [[B#s2]] → anchors preserved per link + +Context precedence: **inline-bracketed > line-level > bare**. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable + +from ..component.file_graph.base_file_graph import BaseFileGraph +from ..schema import FileLink +from . import path_resolver + + +WIKILINK_RE = re.compile( + r""" + (?:!)? + \[\[ + (?P[^\]\|\#\n]+?) + (?:\#(?P[^\]\|\n]+))? + (?:\|[^\]\n]+)? + \]\] + """, + re.VERBOSE, +) + +_DATAVIEW_LINE_RE = re.compile( + r"^[ \t]*(?:[-*+][ \t]+)?(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P.+?)\s*$", + re.MULTILINE, +) + +_INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*") + + +def _iter_inline_fields(text: str) -> list[tuple[int, int, str]]: + """Find inline-bracketed ``[predicate:: …]`` field spans by depth scan.""" + out: list[tuple[int, int, str]] = [] + for m in _INLINE_FIELD_OPEN_RE.finditer(text): + depth = 1 + i = m.end() + n = len(text) + while i < n: + c = text[i] + if c == "\n": + break + if c == "[": + depth += 1 + elif c == "]": + depth -= 1 + if depth == 0: + out.append((m.start(), i + 1, m.group("predicate"))) + break + i += 1 + return out + + +def _predicate_for( + text: str, + pos: int, + inline_spans: list[tuple[int, int, str]], +) -> str | None: + """Resolve the predicate governing a wikilink at offset ``pos``.""" + for field_start, field_end, predicate in inline_spans: + if field_start <= pos < field_end: + return predicate + line_start = text.rfind("\n", 0, pos) + 1 + line_end = text.find("\n", pos) + if line_end == -1: + line_end = len(text) + m = _DATAVIEW_LINE_RE.match(text[line_start:line_end]) + if m and line_start + m.start("value") <= pos: + return m.group("predicate") + return None + + +def iter_links(text: str) -> list[FileLink]: + """Extract every wikilink in ``text`` as a pre-resolution ``FileLink``. + + ``FileLink.path`` is the raw target as written (no ``.md`` + completion, no graph lookup); ``anchor`` and ``predicate`` are + parsed from surrounding syntax. Pure function — no graph access. + Pass through ``resolve_links`` (or ``text_to_links``) to get the + vault-relative form. + """ + if not text: + return [] + inline_spans = _iter_inline_fields(text) + links: list[FileLink] = [] + for wm in WIKILINK_RE.finditer(text): + target = wm.group("target").strip() + anchor_raw = wm.group("anchor") + anchor = anchor_raw.strip() if anchor_raw else "" + links.append(FileLink( + path=target, + anchor=anchor or None, + predicate=_predicate_for(text, wm.start(), inline_spans), + )) + return links + + +async def resolve_links( + graph: BaseFileGraph, links: Iterable[FileLink], +) -> list[FileLink]: + """Resolve each ``FileLink.path`` via ``path_resolver``. + + Parser-pipeline semantics — short-path ambiguity **expands** into + one ``FileLink`` per candidate (so the body's wikilink is recorded + against every plausible target). Dangling links are dropped. + + Operation-time callers that need a single primary key should call + ``path_resolver.resolve`` directly and let ``PathAmbiguous`` + propagate. + """ + out: list[FileLink] = [] + for link in links: + if not link.path: + continue + try: + resolved = await path_resolver.resolve(graph, link.path) + out.append(FileLink( + path=resolved, + anchor=link.anchor, + predicate=link.predicate, + )) + except path_resolver.PathAmbiguous as e: + for c in e.candidates: + out.append(FileLink( + path=c, + anchor=link.anchor, + predicate=link.predicate, + )) + except path_resolver.PathNotFound: + continue + return out + + +async def text_to_links(graph: BaseFileGraph, text: str) -> list[FileLink]: + """One-shot ``iter_links`` + ``resolve_links`` for the parser pipeline.""" + return await resolve_links(graph, iter_links(text)) diff --git a/reme2/utils/path_resolver.py b/reme2/utils/path_resolver.py new file mode 100644 index 00000000..0964fe21 --- /dev/null +++ b/reme2/utils/path_resolver.py @@ -0,0 +1,190 @@ +"""Path resolver — single source of truth for vault path resolution. + +Vault paths come in two forms: + + * **relative path** — the canonical primary key for each file + (e.g. ``"topics/Alice/Alice.md"``). One path, one file. + * **short path** — a simplification with no directory component + (e.g. ``"Alice"``, ``"Alice.md"``). Convenient to type, but + **may map to multiple relative paths** when more than one file + shares the basename. + +Two layers of resolution: + + Layer 1 — vault key (graph-backed) + ``resolve(graph, path)`` — short path / relative path → the + canonical vault-relative primary key. Raises ``PathAmbiguous`` + on short-link multi-match, ``PathNotFound`` on miss. + + Layer 2 — disk path (file_store-backed) + ``to_absolute(file_store, relative_path)`` — vault-relative + path → absolute on-disk ``Path``. Pure ``working_dir`` join. + + Combined — file access + ``resolve_to_absolute(file_store, path)`` — Layer 1 + Layer 2. + The standard entry point for any operation that **accesses an + existing file** (read, update, delete). Short-link callers get + ambiguity errors; relative-path callers get plain join. + +For **file creation** callers must pass a vault-relative path +(use ``to_absolute`` directly) — short links are meaningless until a +file exists. + +This module is the bottom of the path stack — it knows nothing about +``FileLink`` or text syntax. Wikilink/Link concerns live in +``link_parser``, which calls into here. + +## Conventions applied here + + 1. Implicit ``.md`` — a path whose last segment has no extension + is completed to ``X.md``. ``image.png`` is left alone. + 2. Folder-note rule — when both ``X.md`` and ``X/X.md`` exist, + the folder-note (``X/X.md``) wins as the canonical resolution + for ``X``. Other basename collisions stay ambiguous. +""" + +from __future__ import annotations + +from pathlib import Path + +from ..component.file_graph.base_file_graph import BaseFileGraph + + +# =========================================================================== +# Exceptions +# =========================================================================== + + +class PathError(Exception): + """Base for path-resolution failures.""" + + +class PathNotFound(PathError): + """No node matches the given path.""" + + def __init__(self, target: str): + super().__init__(f"path not in vault: {target!r}") + self.target = target + + +class PathAmbiguous(PathError): + """A short path matches more than one relative path. + + Caller must qualify the path (add directory components) or pick + one of ``self.candidates`` explicitly. Step callers should surface + ``self.candidates`` to the user and abort the operation. + """ + + def __init__(self, target: str, candidates: list[str]): + super().__init__(f"path {target!r} is ambiguous: {candidates}") + self.target = target + self.candidates = list(candidates) + + +# =========================================================================== +# Predicates +# =========================================================================== + + +def is_short_path(path: str) -> bool: + """``True`` when ``path`` has no directory component (short form).""" + return bool(path) and "/" not in path + + +# =========================================================================== +# Internal helpers +# =========================================================================== + + +def _complete(path: str) -> str: + """Apply the implicit ``.md`` rule to ``path``.""" + if not path: + return path + last = path.rsplit("/", 1)[-1] + return path if "." in last else path + ".md" + + +def _filter_folder_note(basename: str, paths: list[str]) -> list[str]: + """Apply folder-note rule. Sorted for determinism.""" + if not paths: + return [] + stem = Path(basename).stem + folder_hits = sorted(p for p in paths if Path(p).parent.name == stem) + return folder_hits or sorted(paths) + + +# =========================================================================== +# Layer 1 — graph-backed vault key resolution +# =========================================================================== + + +async def resolve(graph: BaseFileGraph, path: str) -> str: + """Resolve ``path`` to a single vault-relative key. Raises on failure. + + Applies implicit ``.md`` completion before lookup. Dispatches by + shape: + * literal path (contains ``/``) → direct ``get_nodes`` lookup + * short path (no ``/``) → basename match + folder-note rule + + Raises: + ``PathNotFound`` — no matching node. + ``PathAmbiguous`` — short path with multiple matches. + """ + if not path: + raise PathNotFound(path) + target = _complete(path) + + if not is_short_path(target): + if await graph.get_nodes([target]): + return target + raise PathNotFound(target) + + matches = [n.path for n in await graph.get_nodes() if Path(n.path).name == target] + candidates = _filter_folder_note(target, matches) + if len(candidates) == 1: + return candidates[0] + if candidates: + raise PathAmbiguous(target, candidates) + raise PathNotFound(target) + + +# =========================================================================== +# Layer 2 — filesystem path composition +# =========================================================================== + + +def to_absolute(file_store, relative_path: str) -> Path: + """Compose absolute on-disk ``Path`` from a vault-relative path. + + ``relative_path`` should be vault-relative; absolute inputs pass + through unchanged (Python's ``Path / abs`` join). For paths that + may be short links, route through ``resolve_to_absolute`` instead + so ambiguity surfaces. + """ + working_dir = getattr(file_store, "working_dir", None) or "." + return (Path(working_dir) / relative_path).resolve() + + +# =========================================================================== +# Combined — file access +# =========================================================================== + + +async def resolve_to_absolute(file_store, path: str) -> Path: + """User path → absolute on-disk ``Path`` (the file-access entry point). + + Resolves short links / relative paths via ``resolve`` (graph), then + composes the absolute ``Path`` via ``to_absolute``. Any operation + that touches an existing vault file should go through here so + short-link ambiguity surfaces as ``PathAmbiguous`` (with candidates) + rather than silently picking a wrong file. + + Raises: + ``PathNotFound`` — no matching node. + ``PathAmbiguous`` — short path with multiple matches. + """ + graph = getattr(file_store, "file_graph", None) + if graph is None: + raise PathNotFound(path) + relative = await resolve(graph, path) + return to_absolute(file_store, relative) diff --git a/reme2/utils/wikilink_resolver.py b/reme2/utils/wikilink_resolver.py deleted file mode 100644 index 0904d1b0..00000000 --- a/reme2/utils/wikilink_resolver.py +++ /dev/null @@ -1,482 +0,0 @@ -"""Wikilink syntax + resolver — vault convention over ``BaseFileGraph``. - -This module is the single home for wikilink **syntax** (regex -extraction, predicate detection) and wikilink **resolution** (mapping -raw targets to vault-relative paths via the file_graph). - -The ``FileLink`` schema (see ``schema.file_link``) is just a typed -record; here we define how it's produced from text and resolved -against the graph. - -## Two-layer link semantics - - 1. Implicit extension — ``[[Foo]]`` has no extension on its last - segment, so it's completed to ``Foo.md`` (markdown is the - default vault content type). ``[[image.png]]`` already has an - extension; left as-is. Done in ``iter_links`` at extraction - time, so all FileLinks emerge with extension-bearing paths. - - 2. Short link — a path with no ``/`` (after implicit completion) - is matched against the basename of every node in the graph. - The folder-note rule applies: when both ``X.md`` and - ``X/X.md`` exist, the folder-note wins. Ambiguity expands into - multiple FileLink records (one per candidate path). Targets - containing ``/`` are treated as literal paths and looked up - directly. - -So ``[[Foo]]`` → ``Foo.md`` (implicit) → search basename ``Foo.md`` -across the vault, returning e.g. ``topics/Foo/Foo.md``; -``[[topics/Bar]]`` → ``topics/Bar.md`` (implicit) → literal lookup; -``[[image.png]]`` → ``image.png`` (no completion needed) → search -basename ``image.png``; ``[[topics/image.png]]`` → literal lookup. - -All paths are vault-relative — ``graph.iter_nodes()`` returns the -key form file_graph stores, and emitted ``FileLink.path`` matches. - -All resolver functions are stateless — they walk ``graph.iter_nodes()`` -per call. For batch operations (``resolve_links``) the basename index -is built once and reused. - -## Inline forms recognised by ``iter_links`` - - [[X]] bare wikilink → predicate=None - extends:: [[X]] line-level Dataview → predicate="extends" - [extends:: [[X]]] inline-bracketed → predicate="extends" - -Multi-target — every wikilink under one typed context inherits its -predicate (any separator works, not just commas): - - extends:: [[A]], [[B]] line-level multi → 2 links, both "extends" - extends:: [[A]] and [[B]] prose-style multi → 2 links, both "extends" - [concerns:: [[A]], [[B]]] inline multi → 2 links, both "concerns" - extends:: [[A#s1]], [[B#s2]] multi w/ anchors → anchors preserved per link - -Context precedence is **inline-bracketed > line-level > bare** — -a wikilink inside a ``[predicate:: …]`` envelope is typed by that -envelope even if the line happens to start ``predicate:: …``. - -## Entry points - - Extraction (no graph) - iter_links text → [FileLink] (path = target after implicit .md) - extract_wikilinks text → [str] (raw target list, no completion) - - Resolution (graph-backed) - resolve single link → path | None - used by: ``extract_anchors``, ``memory_resolve_wikilink`` - candidates target → [path] (folder-note ordered first) - used by: ``memory_resolve_wikilink`` (ambiguity report) - collisions all basenames with >1 path - used by: ``maintainer.lint`` - collisions_for paths conflicting with a proposed new path - used by: ``memory_create``, ``sync`` (preflight) - extract_anchors parse [[X]] from text + resolve, dedup - used by: ``retriever`` (query anchor seeds) - resolve_links ``[FileLink]`` (raw path) → ``[FileLink]`` (resolved) - core resolution; short-link ambiguity expands - text_to_links one-shot: ``iter_links(text)`` + ``resolve_links`` - used by: parser pipeline (before ``upsert_node``) -""" - -from __future__ import annotations - -import re -from collections.abc import Iterable -from pathlib import Path - -from ..component.file_graph.base_file_graph import BaseFileGraph -from ..schema import FileLink -from .logger_utils import get_logger - - -_logger = get_logger() - - -# ========================================================================= -# Wikilink syntax — regex extraction + predicate detection -# ========================================================================= - - -_WIKILINK_RE = re.compile( - r""" - (?:!)? - \[\[ - (?P[^\]\|\#\n]+?) - (?:\#(?P[^\]\|\n]+))? - (?:\|[^\]\n]+)? - \]\] - """, - re.VERBOSE, -) - -_DATAVIEW_LINE_RE = re.compile( - r"^[ \t]*(?:[-*+][ \t]+)?(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*(?P.+?)\s*$", - re.MULTILINE, -) - -_INLINE_FIELD_OPEN_RE = re.compile(r"\[(?P[A-Za-z][A-Za-z0-9_]*)\s*::\s*") - - -def _iter_inline_fields(text: str) -> list[tuple[int, int, str]]: - """Find inline-bracketed ``[predicate:: …]`` field spans by depth scan.""" - out: list[tuple[int, int, str]] = [] - for m in _INLINE_FIELD_OPEN_RE.finditer(text): - depth = 1 - i = m.end() - n = len(text) - while i < n: - c = text[i] - if c == "\n": - break - if c == "[": - depth += 1 - elif c == "]": - depth -= 1 - if depth == 0: - out.append((m.start(), i + 1, m.group("predicate"))) - break - i += 1 - return out - - -def _predicate_for( - text: str, - pos: int, - inline_spans: list[tuple[int, int, str]], -) -> str | None: - """Resolve the predicate governing a wikilink at offset ``pos``.""" - for field_start, field_end, predicate in inline_spans: - if field_start <= pos < field_end: - return predicate - line_start = text.rfind("\n", 0, pos) + 1 - line_end = text.find("\n", pos) - if line_end == -1: - line_end = len(text) - m = _DATAVIEW_LINE_RE.match(text[line_start:line_end]) - if m and line_start + m.start("value") <= pos: - return m.group("predicate") - return None - - -def _complete_path(target: str) -> str: - """Append ``.md`` if the last path segment has no extension. - - Implements the implicit-markdown rule: ``[[Foo]]`` → ``Foo.md``, - ``[[topics/Bar]]`` → ``topics/Bar.md``, but ``[[image.png]]`` is - left alone. ``"."`` in the last segment counts as "has extension". - """ - if not target: - return target - last = target.rsplit("/", 1)[-1] - if "." in last: - return target - return target + ".md" - - -def iter_links(text: str) -> list[FileLink]: - """Extract every wikilink in ``text`` as a pre-resolution ``FileLink``. - - Each emitted ``FileLink`` has ``path`` set to the wikilink target - with the implicit ``.md`` rule applied (so ``[[Foo]]`` emerges as - ``path="Foo.md"``); the ``#anchor`` lives in its own field. - Predicate is decided by surrounding context with precedence - **inline-bracketed > line-level > bare**. - - Pure function: no graph access. Pass the result through - ``resolve_links`` (or one-shot ``text_to_links``) to apply - short-link resolution and get the form file_graph stores. - """ - if not text: - return [] - inline_spans = _iter_inline_fields(text) - links: list[FileLink] = [] - for wm in _WIKILINK_RE.finditer(text): - target = wm.group("target").strip() - anchor_raw = wm.group("anchor") - anchor = anchor_raw.strip() if anchor_raw else "" - links.append(FileLink( - path=_complete_path(target), - anchor=anchor or None, - predicate=_predicate_for(text, wm.start(), inline_spans), - )) - return links - - -def extract_wikilinks(text: str) -> list[str]: - """Flat list of wikilink **file targets** in body text (no dedup). - - Returns just the file part of each wikilink **as written** — - no implicit ``.md`` completion (callers like ``resolve`` apply - completion themselves). Single regex pass — cheaper than - ``iter_links`` when callers don't need predicates. - """ - if not text: - return [] - return [m.group("target").strip() for m in _WIKILINK_RE.finditer(text)] - - -# ========================================================================= -# Resolution helpers (internal) -# ========================================================================= - - -async def _build_basename_index(graph: BaseFileGraph) -> dict[str, list[str]]: - """Walk once, group paths by basename (file name with extension). - - Used by short-link resolution batch hot paths. - """ - out: dict[str, list[str]] = {} - async for path, _ in graph.iter_nodes(): - out.setdefault(Path(path).name, []).append(path) - return out - - -def _split_link(link: str) -> tuple[str, str]: - """Return ``(target, anchor)``. Anchor empty if no ``#``. - - For raw link strings supplied by external callers (which may still - arrive in ``[[A#section]]`` form). Pre-extracted ``FileLink`` - records already carry ``path`` and ``anchor`` separately. - """ - if not link: - return "", "" - if "#" not in link: - return link.strip(), "" - target_raw, anchor_raw = link.split("#", 1) - return target_raw.strip(), anchor_raw.strip() - - -def _has_dir(target: str) -> bool: - """``True`` for literal-path targets (containing ``/``); - ``False`` for short links (basename only). - """ - return "/" in target - - -def _filter_short_link_candidates(basename: str, paths: list[str]) -> list[str]: - """Apply folder-note rule to short-link basename matches. - - If any path is a folder-note (parent dir name == file stem) the - folder-notes win as the only candidates; otherwise all matches - are returned. Sorted for determinism. - """ - if not paths: - return [] - stem = Path(basename).stem - folder_hits = sorted(p for p in paths if Path(p).parent.name == stem) - if folder_hits: - return folder_hits - return sorted(paths) - - -# ========================================================================= -# Public API — single-shot lookups -# ========================================================================= - - -async def resolve(graph: BaseFileGraph, link: str) -> str | None: - """Resolve a single wikilink to **one** vault-relative path, or None. - - Applies implicit ``.md`` completion, then dispatches: - * literal path (contains ``/``) → direct ``get_node`` lookup - * short link (no ``/``) → basename match + folder-note rule - - Returns None if dangling or ambiguous (with warning on ambiguity). - Use ``resolve_links`` for the multi-link expansion semantics. - """ - target, _ = _split_link(link) - if not target: - return None - target = _complete_path(target) - - if _has_dir(target): - return target if await graph.get_node(target) else None - - paths = [ - p async for p, _ in graph.iter_nodes() if Path(p).name == target - ] - candidates_for = _filter_short_link_candidates(target, paths) - if len(candidates_for) == 1: - return candidates_for[0] - if len(candidates_for) > 1: - _logger.warning( - f"Wikilink [[{target}]] is ambiguous, " - f"candidates: {candidates_for}", - ) - return None - - -async def candidates(graph: BaseFileGraph, target: str) -> list[str]: - """All vault paths a ``[[target]]`` could match. - - Applies implicit ``.md`` completion. For literal paths (with ``/``) - returns ``[target]`` if it exists else ``[]``. For short links - returns every node whose basename matches, with folder-note hits - ordered first. - """ - target = _complete_path(target) - if _has_dir(target): - return [target] if await graph.get_node(target) else [] - - stem = Path(target).stem - folder_hits: list[str] = [] - name_hits: list[str] = [] - async for path, _ in graph.iter_nodes(): - if Path(path).name != target: - continue - if Path(path).parent.name == stem: - folder_hits.append(path) - else: - name_hits.append(path) - if folder_hits: - return sorted(folder_hits) - return sorted(name_hits) - - -async def collisions(graph: BaseFileGraph) -> dict[str, list[str]]: - """Every basename that resolves to >1 path. Used by maintainer.lint. - - Reflects short-link ambiguity: ``[[X.md]]`` (or ``[[X]]``) hitting - multiple files in different directories. - """ - basename_index = await _build_basename_index(graph) - return { - name: sorted(paths) - for name, paths in basename_index.items() - if len(paths) > 1 - } - - -async def collisions_for( - graph: BaseFileGraph, proposed_path: str | Path, -) -> list[str]: - """Existing paths that would conflict with adding ``proposed_path``. - - ``proposed_path`` is vault-relative (matches ``graph.iter_nodes()``). - Folder-note rule: when the proposed path is itself a folder-note - (parent dir name == file stem), only colliding folder-notes are - returned. Otherwise all paths sharing the basename are returned. - """ - p = Path(proposed_path) - name = p.name - stem = p.stem - proposed_str = str(p) - is_folder_note = p.parent.name == stem - - folder_hits: list[str] = [] - name_hits: list[str] = [] - async for path, _ in graph.iter_nodes(): - if path == proposed_str: - continue - path_obj = Path(path) - if path_obj.name != name: - continue - if path_obj.parent.name == stem: - folder_hits.append(path) - else: - name_hits.append(path) - - if is_folder_note: - return sorted(folder_hits) - return sorted(folder_hits) + sorted(name_hits) - - -async def extract_anchors(graph: BaseFileGraph, text: str) -> list[str]: - """Pull ``[[X]]`` from ``text``, resolve each, dedup in source order. - - Uses single-target ``resolve`` semantics — ambiguous short links - return no anchor. (For multi-link expansion at *write* time, see - ``resolve_links``; ``extract_anchors`` is for read-time seeding - where a single deterministic target is wanted.) - """ - if not text: - return [] - seen: set[str] = set() - out: list[str] = [] - for raw in extract_wikilinks(text): - hit = await resolve(graph, raw) - if hit is not None and hit not in seen: - seen.add(hit) - out.append(hit) - return out - - -# ========================================================================= -# Public API — parser pipeline -# ========================================================================= - - -async def resolve_links( - graph: BaseFileGraph, links: Iterable[FileLink], -) -> list[FileLink]: - """Resolve pre-resolution ``FileLink`` records against the graph. - - Each input link has ``path`` already passed through - ``_complete_path`` (so it has an extension). Returns FileLinks - with ``path`` rewritten to the vault-relative resolved form - file_graph stores. ``anchor`` and ``predicate`` pass through - unchanged. - - Resolution dispatch: - * literal path (contains ``/``) → direct ``get_node`` lookup; - keep if exists, drop if dangling. - * short link (no ``/``) → basename match + folder-note - rule. Ambiguity expands into multiple FileLinks (one per - candidate path); dangling produces zero. - - Output cardinality per input link: - - * literal, target indexed → 1 link - * literal, dangling → 0 links - * short, 1 folder-note hit → 1 link - * short, N folder-note hits → N links (one per) - * short, 0 folder-notes, 1 basename hit → 1 link - * short, 0 folder-notes, N basename hits → N links (one per) - * short, dangling → 0 links - - Build the basename index lazily: only paid if at least one input - is a short link. - """ - link_list = list(links) - if not link_list: - return [] - - basename_index: dict[str, list[str]] | None = None - out: list[FileLink] = [] - for link in link_list: - target = link.path - if not target: - continue - - if _has_dir(target): - if await graph.get_node(target) is None: - continue - out.append(FileLink( - path=target, - anchor=link.anchor, - predicate=link.predicate, - )) - continue - - # Short link — may expand into multiple links. - if basename_index is None: - basename_index = await _build_basename_index(graph) - for chosen in _filter_short_link_candidates(target, basename_index.get(target, [])): - out.append(FileLink( - path=chosen, - anchor=link.anchor, - predicate=link.predicate, - )) - - return out - - -async def text_to_links( - graph: BaseFileGraph, text: str, -) -> list[FileLink]: - """One-shot: extract wikilinks from ``text`` and resolve to safe links. - - Equivalent to ``await resolve_links(graph, iter_links(text))``. - The parser pipeline's single-call entry point. - """ - return await resolve_links(graph, iter_links(text))