mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-22 00:32:49 +00:00
30971af5ec
Some checks are pending
Pre-commit / run (ubuntu-latest) (push) Waiting to run
BREAKING CHANGE: Split monolithic agent_toolkit into modular components: - memory_toolkit.py: memory_get, memory_create, memory_update_body, memory_update_meta, memory_search + schema policy helpers - file_toolkit.py: file_download, file_upload, file_delete, file_list, file_move + path resolution and session temp dir - graph_toolkit.py: graph_traverse + adjacency BFS - event_toolkit.py: event_open, event_complete (using memory schema gates) - lint_toolkit.py: check_* atomic primitives remain separate Agent toolkit now composes these modules instead of containing all logic. - Remove sync job from EXPECTED_JOBS since sync.py is no longer imported - Add event_open and event_complete jobs to test expectations - Update module imports to reference new toolkit structure - Maintain backward compatibility through composition layer
440 lines
15 KiB
Python
440 lines
15 KiB
Python
"""Memory File System engine API — minimal surface for the agent toolkit.
|
|
|
|
The .md files are the SSOT. The engine layer is:
|
|
|
|
Memory File System → Watcher & Parser → Projections (vector / FTS)
|
|
(write entry) (incremental) (read entry, derived)
|
|
|
|
This module is the single public surface over that engine for the
|
|
agent toolkit. ``BaseFileGraph`` deliberately doesn't expose "scan
|
|
everything" (only ``get_nodes(paths)``); when we need to enumerate
|
|
(file_list, collisions check, filter resolution) we walk the
|
|
filesystem directly. The graph is consulted only for adjacency
|
|
lookups around a known path.
|
|
|
|
Public surface (everything else has been removed):
|
|
|
|
Reads: get_file, list_files
|
|
Writes: create_file, update_body, update_meta, delete_file,
|
|
rename_file
|
|
Filter: make_filter
|
|
|
|
Pure-disk writes (``update_body`` / ``update_meta`` / ``delete_file``)
|
|
don't take ``file_store`` — they hit the filesystem and the watcher
|
|
picks them up. Engine-aware writes (``create_file`` / ``rename_file``)
|
|
take ``file_store`` because they consult the vault index for
|
|
collision/adjacency gates before writing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
|
|
import frontmatter
|
|
|
|
from ..component.file_store.base_file_store import BaseFileStore
|
|
from ..schema import ChunkFilter, FileNode
|
|
from ..utils.wikilink_resolver import _WIKILINK_RE
|
|
|
|
|
|
# ===========================================================================
|
|
# Internal helpers
|
|
# ===========================================================================
|
|
|
|
|
|
def _vault_root(file_store: BaseFileStore) -> Path | None:
|
|
"""Resolve the vault root from the file_store. ``working_dir`` is the
|
|
convention; falls back to ``working_path`` if set on the component."""
|
|
wd = getattr(file_store, "working_dir", None)
|
|
if wd:
|
|
return Path(wd).resolve()
|
|
wp = getattr(file_store, "working_path", None)
|
|
if wp:
|
|
return Path(wp).resolve()
|
|
return None
|
|
|
|
|
|
_SKIP_DIRS = {".git", ".obsidian", ".reme", ".reme2", "__pycache__", "Archive"}
|
|
|
|
|
|
def _walk_vault(
|
|
file_store: BaseFileStore,
|
|
*,
|
|
suffix: str = ".md",
|
|
) -> Iterator[Path]:
|
|
"""Yield every file under the vault matching ``suffix``.
|
|
|
|
Filesystem-as-SSOT: the .md files are authoritative for "what
|
|
exists". Hidden directories and ``Archive/`` are skipped (Archive
|
|
lives in the vault but isn't part of the active set).
|
|
"""
|
|
root = _vault_root(file_store)
|
|
if root is None or not root.is_dir():
|
|
return
|
|
for path in root.rglob(f"*{suffix}"):
|
|
if any(part in _SKIP_DIRS for part in path.relative_to(root).parts[:-1]):
|
|
continue
|
|
if path.is_file():
|
|
yield path.resolve()
|
|
|
|
|
|
def _walk_vault_meta(
|
|
file_store: BaseFileStore,
|
|
*,
|
|
suffix: str = ".md",
|
|
) -> Iterator[tuple[str, dict]]:
|
|
"""Walk the vault, yielding ``(absolute_path_str, frontmatter_dict)``.
|
|
|
|
Files that fail to parse get an empty dict — that's a lint concern,
|
|
not the listing concern.
|
|
"""
|
|
for abs_path in _walk_vault(file_store, suffix=suffix):
|
|
try:
|
|
raw = abs_path.read_text(encoding="utf-8")
|
|
meta = dict(frontmatter.loads(raw).metadata)
|
|
except Exception:
|
|
meta = {}
|
|
yield str(abs_path), meta
|
|
|
|
|
|
async def _get_node(file_store: BaseFileStore, path: str) -> FileNode | None:
|
|
"""Single-node fetch via the file_graph contract."""
|
|
if not file_store.file_graph:
|
|
return None
|
|
nodes = await file_store.file_graph.get_nodes([path])
|
|
return nodes[0] if nodes else None
|
|
|
|
|
|
def _replace_wikilink_targets(text: str, mapping: dict[str, str]) -> str:
|
|
"""Rewrite wikilink targets in raw text. Anchors / aliases / embed prefix kept."""
|
|
if not mapping:
|
|
return text
|
|
|
|
def sub(m: re.Match) -> str:
|
|
target_raw = m.group(1)
|
|
target = target_raw.strip()
|
|
if target in mapping:
|
|
return m.group(0).replace(target_raw, mapping[target], 1)
|
|
return m.group(0)
|
|
|
|
return _WIKILINK_RE.sub(sub, text)
|
|
|
|
|
|
def collisions_after_create(
|
|
file_store: BaseFileStore,
|
|
proposed_path: str | Path,
|
|
) -> list[str]:
|
|
"""Existing paths that would conflict with adding `proposed_path`.
|
|
|
|
Folder-note rule: if `proposed_path`'s parent dir name == its stem,
|
|
only colliding folder-notes are returned (siblings with the same
|
|
stem don't ambiguate). Otherwise both folder-notes AND stem hits
|
|
are returned.
|
|
|
|
Walks the filesystem since the engine contract has no "scan all
|
|
paths" API on the graph. Public so other write paths (sync.py)
|
|
can reuse the same gate.
|
|
"""
|
|
p = Path(proposed_path)
|
|
stem = p.stem
|
|
proposed_abs = str(p.resolve())
|
|
is_folder_note = p.parent.name == stem
|
|
|
|
folder_hits: list[str] = []
|
|
stem_hits: list[str] = []
|
|
for abs_path in _walk_vault(file_store):
|
|
path = str(abs_path)
|
|
if path == proposed_abs:
|
|
continue
|
|
if abs_path.stem != stem:
|
|
continue
|
|
if abs_path.parent.name == stem:
|
|
folder_hits.append(path)
|
|
else:
|
|
stem_hits.append(path)
|
|
|
|
if is_folder_note:
|
|
return folder_hits
|
|
return folder_hits + stem_hits
|
|
|
|
|
|
# ===========================================================================
|
|
# Reads
|
|
# ===========================================================================
|
|
|
|
|
|
async def get_file(file_store: BaseFileStore, path: str) -> dict:
|
|
"""Read frontmatter + body from disk; attach links from the graph.
|
|
|
|
On-disk frontmatter is the source of truth — the graph cache may
|
|
lag a write that the watcher hasn't picked up yet. The graph is
|
|
consulted only for the link list (adjacency).
|
|
"""
|
|
result: dict = {"path": path, "exists": False}
|
|
|
|
file_path = Path(path)
|
|
if file_path.is_file():
|
|
raw = file_path.read_text(encoding="utf-8")
|
|
post = frontmatter.loads(raw)
|
|
result["exists"] = True
|
|
result["content"] = post.content
|
|
result["metadata"] = dict(post.metadata)
|
|
|
|
node = await _get_node(file_store, path)
|
|
if node is not None:
|
|
result["link"] = [link.model_dump(exclude_none=True) for link in node.links]
|
|
if not result["exists"]:
|
|
result["exists"] = True
|
|
result.setdefault("metadata", node.front_matter.model_dump())
|
|
else:
|
|
result["link"] = []
|
|
return result
|
|
|
|
|
|
def list_files(
|
|
file_store: BaseFileStore,
|
|
*,
|
|
path_prefix: str | None = None,
|
|
tags: list[str] | None = None,
|
|
metadata: dict | None = None,
|
|
limit: int = 100,
|
|
) -> dict:
|
|
"""List vault files filtered by frontmatter exact-match, tags, and prefix.
|
|
|
|
Filesystem walk — the graph contract has no enumeration API.
|
|
``path_prefix`` matches the absolute path string.
|
|
"""
|
|
metadata_filter = metadata or {}
|
|
tag_filter = tags or []
|
|
items: list[dict] = []
|
|
for path, md in _walk_vault_meta(file_store):
|
|
if path_prefix and not path.startswith(path_prefix):
|
|
continue
|
|
if metadata_filter and any(md.get(k) != v for k, v in metadata_filter.items()):
|
|
continue
|
|
if tag_filter:
|
|
file_tags = set(md.get("tags") or [])
|
|
if not all(t in file_tags for t in tag_filter):
|
|
continue
|
|
items.append({"path": path, "metadata": md})
|
|
if len(items) >= limit:
|
|
break
|
|
return {"items": items, "count": len(items)}
|
|
|
|
|
|
# ===========================================================================
|
|
# Writes
|
|
# ===========================================================================
|
|
|
|
|
|
def create_file(
|
|
file_store: BaseFileStore,
|
|
path: Path,
|
|
*,
|
|
metadata: dict,
|
|
content: str,
|
|
overwrite: bool = False,
|
|
force: bool = False,
|
|
) -> tuple[bool, dict]:
|
|
"""Create a markdown file. Refuses when:
|
|
|
|
- file already exists (unless ``overwrite=True``)
|
|
- creating it would make ``[[stem]]`` ambiguous (unless ``force=True``)
|
|
"""
|
|
if path.exists() and not overwrite:
|
|
return False, {"path": str(path), "error": "file already exists"}
|
|
|
|
if not force:
|
|
conflicts = collisions_after_create(file_store, path)
|
|
if conflicts:
|
|
return False, {
|
|
"path": str(path),
|
|
"error": (
|
|
f"stem `[[{path.stem}]]` would resolve ambiguously "
|
|
f"to {len(conflicts) + 1} paths after this create"
|
|
),
|
|
"conflicts": conflicts,
|
|
"hint": (
|
|
f"either rename to a unique stem, or have callers "
|
|
f"link via the explicit-path form "
|
|
f"`[[{path.parent.name}/{path.stem}]]`; pass "
|
|
f"force=true only if you accept the ambiguity"
|
|
),
|
|
}
|
|
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
post = frontmatter.Post(content, **metadata)
|
|
path.write_text(frontmatter.dumps(post), encoding="utf-8")
|
|
return True, {"path": str(path), "created": True}
|
|
|
|
|
|
def update_body(
|
|
path: Path | str,
|
|
*,
|
|
old_string: str,
|
|
new_string: str,
|
|
replace_all: bool = False,
|
|
) -> tuple[bool, dict]:
|
|
"""Edit-style content update — replace ``old_string`` with ``new_string``."""
|
|
target = Path(path)
|
|
if not target.is_file():
|
|
return False, {"path": str(target), "error": "file not found"}
|
|
if not old_string:
|
|
return False, {
|
|
"path": str(target),
|
|
"error": "old_string is required (use create_file to write a new file)",
|
|
}
|
|
raw = target.read_text(encoding="utf-8")
|
|
occurrences = raw.count(old_string)
|
|
if occurrences == 0:
|
|
return False, {"path": str(target), "error": "old_string not found in file"}
|
|
if occurrences > 1 and not replace_all:
|
|
return False, {
|
|
"path": str(target),
|
|
"error": f"old_string appears {occurrences} times; pass replace_all=true to replace all",
|
|
"occurrences": occurrences,
|
|
}
|
|
if replace_all:
|
|
new_raw = raw.replace(old_string, new_string)
|
|
else:
|
|
new_raw = raw.replace(old_string, new_string, 1)
|
|
target.write_text(new_raw, encoding="utf-8")
|
|
return True, {
|
|
"path": str(target),
|
|
"replaced": occurrences if replace_all else 1,
|
|
}
|
|
|
|
|
|
def update_meta(path: Path | str, *, key: str, value) -> tuple[bool, dict]:
|
|
"""Update a single YAML frontmatter key. ``value=None`` deletes the key."""
|
|
target = Path(path)
|
|
if not target.is_file():
|
|
return False, {"path": str(target), "error": "file not found"}
|
|
raw = target.read_text(encoding="utf-8")
|
|
post = frontmatter.loads(raw)
|
|
if value is None:
|
|
post.metadata.pop(key, None)
|
|
else:
|
|
post.metadata[key] = value
|
|
target.write_text(frontmatter.dumps(post), encoding="utf-8")
|
|
return True, {"path": str(target), "key": key, "value": value}
|
|
|
|
|
|
def delete_file(path: Path | str) -> tuple[bool, dict]:
|
|
"""Delete a file. Watcher removes from store + graph asynchronously."""
|
|
target = Path(path)
|
|
if not target.exists():
|
|
return False, {"path": str(target), "error": "not found"}
|
|
target.unlink()
|
|
return True, {"path": str(target), "deleted": True}
|
|
|
|
|
|
def rename_file(
|
|
file_store: BaseFileStore,
|
|
working_dir: Path | str,
|
|
*,
|
|
old_path: Path | str,
|
|
new_path: Path | str,
|
|
) -> tuple[bool, dict]:
|
|
"""Rename a file and rewrite incoming wikilinks across the vault.
|
|
|
|
Walks the filesystem to find referencing files (the graph
|
|
contract has no source-path lookup for inbound edges, and the
|
|
rewrite is text-level anyway). Same-stem renames within the same
|
|
folder are no-op for link rewrite.
|
|
"""
|
|
old_p = Path(old_path).resolve()
|
|
new_p = Path(new_path).resolve()
|
|
|
|
if not old_p.is_file():
|
|
return False, {"old_path": str(old_p), "error": "old_path not found"}
|
|
if new_p.exists():
|
|
return False, {"new_path": str(new_p), "error": "new_path already exists"}
|
|
if old_p == new_p:
|
|
return False, {"error": "old_path and new_path are the same"}
|
|
|
|
conflicts = collisions_after_create(file_store, new_p)
|
|
if conflicts:
|
|
return False, {
|
|
"error": (
|
|
f"stem `[[{new_p.stem}]]` would resolve ambiguously "
|
|
f"to {len(conflicts) + 1} paths after this rename"
|
|
),
|
|
"conflicts": conflicts,
|
|
"hint": (
|
|
f"either rename to a unique stem (consider a "
|
|
f"domain-specific suffix), or have callers link via "
|
|
f"the explicit-path form `[[{new_p.parent.name}/{new_p.stem}]]`"
|
|
),
|
|
}
|
|
|
|
working_dir_p = Path(working_dir).resolve()
|
|
old_stem = old_p.stem
|
|
new_stem = new_p.stem
|
|
replacements: dict[str, str] = {}
|
|
if old_stem != new_stem:
|
|
replacements[old_stem] = new_stem
|
|
try:
|
|
old_rel = str(old_p.relative_to(working_dir_p).with_suffix(""))
|
|
new_rel = str(new_p.relative_to(working_dir_p).with_suffix(""))
|
|
if old_rel != new_rel:
|
|
replacements[old_rel] = new_rel
|
|
replacements[old_rel + ".md"] = new_rel + ".md"
|
|
except ValueError:
|
|
pass
|
|
|
|
new_p.parent.mkdir(parents=True, exist_ok=True)
|
|
old_p.rename(new_p)
|
|
|
|
updated_files: list[str] = []
|
|
write_errors: list[dict] = []
|
|
if replacements:
|
|
for abs_path in _walk_vault(file_store):
|
|
if abs_path == new_p:
|
|
continue
|
|
try:
|
|
raw = abs_path.read_text(encoding="utf-8")
|
|
new_raw = _replace_wikilink_targets(raw, replacements)
|
|
if new_raw != raw:
|
|
abs_path.write_text(new_raw, encoding="utf-8")
|
|
updated_files.append(str(abs_path))
|
|
except Exception as exc:
|
|
write_errors.append({"path": str(abs_path), "error": str(exc)})
|
|
|
|
return True, {
|
|
"old_path": str(old_p),
|
|
"new_path": str(new_p),
|
|
"stem_changed": old_stem != new_stem,
|
|
"replacements": replacements,
|
|
"updated_files": updated_files,
|
|
"write_errors": write_errors,
|
|
}
|
|
|
|
|
|
# ===========================================================================
|
|
# Filter helper
|
|
# ===========================================================================
|
|
|
|
|
|
def make_filter(
|
|
file_store: BaseFileStore,
|
|
*,
|
|
paths: list[str] | None = None,
|
|
tags: list[str] | None = None,
|
|
exclude_paths: list[str] | None = None,
|
|
) -> ChunkFilter | None:
|
|
"""Compile a ChunkFilter and resolve it against the vault.
|
|
|
|
Walks the filesystem to determine which paths match the filter's
|
|
metadata clauses (the engine contract has no path enumeration on
|
|
the graph). Empty filters skip the walk.
|
|
"""
|
|
cf = ChunkFilter(paths=paths, tags=tags, exclude_paths=exclude_paths)
|
|
if cf.is_empty():
|
|
return cf
|
|
cf.resolved_paths = {
|
|
p for p, md in _walk_vault_meta(file_store) if cf.match_metadata(p, md)
|
|
}
|
|
return cf
|