mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-22 00:32:49 +00:00
1852d05f7b
Add support for `paths=None` in `get_nodes` method to return all real nodes as the graph's "scan everything" entry point. When `paths` is explicitly passed as an empty list, returns empty list. Virtual placeholders are filtered out in both cases. BREAKING CHANGE: The `get_nodes` method signature changed from requiring a list of paths to accepting an optional list or None. refactor(memory): move runtime_response imports to steps package Move runtime_response imports from memory package to steps package to improve code organization and reduce circular dependencies. refactor(config): update obsidian configuration table structure Update the obsidian configuration markdown table to improve clarity and fix incorrect section headings. Change 'stat/list' to 'file' and 'return specific tag info' to 'tags' section with proper commands. chore(memory): remove deprecated toolkit modules Remove deprecated agent_toolkit.py, file_toolkit.py, and graph_toolkit.py modules as functionality has been moved to steps package. feat(steps): add new CRUD file operations Add new steps for file operations including file_download and file_list operations with proper path resolution and filtering capabilities.
434 lines
16 KiB
Python
434 lines
16 KiB
Python
"""Memory category — schema-bound markdown management + hybrid retrieval.
|
|
|
|
Five tools:
|
|
|
|
memory_get read structured (frontmatter + body + links)
|
|
memory_create schema-validated create
|
|
memory_update_body Edit-style body replacement
|
|
memory_update_meta frontmatter patch (status state machine enforced)
|
|
memory_search V+K hybrid retrieval
|
|
|
|
Plus ``memory_graph_search`` (MCP-only — no agent toolkit method, but
|
|
registered so HTTP/MCP callers can still tune graph fusion knobs).
|
|
|
|
Schema policy lives here — ``validate_status_transition``,
|
|
``validate_path_template``, ``_create_with_schema``, ``_update_status``
|
|
— and is reused by ``event_toolkit`` so event index files go through
|
|
the same gates as memory_create.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import frontmatter
|
|
from agentscope.tool import ToolResponse
|
|
|
|
from . import memory_io
|
|
from ..component import R
|
|
from ..component.base_step import BaseStep
|
|
from ..enumeration import ComponentEnum
|
|
from .retriever import BaseRetriever, HybridRetriever
|
|
from ..steps.runtime_response import _set_answer, _tool_response
|
|
|
|
|
|
# ===========================================================================
|
|
# Section 1 — Schema policy (status state machine + path templates)
|
|
# ===========================================================================
|
|
#
|
|
# Used by memory_create (path template), memory_update_meta (status
|
|
# state machine), and event_open (both, since the event index file
|
|
# follows the same memory schema). Pure helpers; the gates fire only
|
|
# when force=False.
|
|
|
|
|
|
_STATUS_STATES = ("active", "distilled", "archived")
|
|
_STATUS_TRANSITIONS: dict[str, set[str]] = {
|
|
"active": {"active", "distilled"},
|
|
"distilled": {"distilled", "archived"},
|
|
"archived": {"archived"},
|
|
}
|
|
|
|
|
|
def validate_status_transition(prior, requested) -> str | None:
|
|
"""Return error string if the requested status transition is invalid."""
|
|
if requested is None:
|
|
return None
|
|
if requested not in _STATUS_STATES:
|
|
return f"invalid status {requested!r}; must be one of {list(_STATUS_STATES)}"
|
|
if prior in _STATUS_STATES and requested not in _STATUS_TRANSITIONS[prior]:
|
|
return (
|
|
f"status transition {prior!r} → {requested!r} not allowed; "
|
|
f"state machine is single-direction "
|
|
f"active → distilled → archived"
|
|
)
|
|
return None
|
|
|
|
|
|
def validate_path_template(path: Path, working_dir: Path | str | None) -> str | None:
|
|
"""Return error string if `path` doesn't match an agent-facing template.
|
|
|
|
Allowed templates (relative to working_dir):
|
|
topics/{folder}/{name}.md — topic file
|
|
events/{date}/{name}/{filename} — event index OR sibling material
|
|
Archive/... — archive moves can land anywhere
|
|
"""
|
|
if working_dir is None:
|
|
return None
|
|
vault = Path(working_dir).resolve()
|
|
try:
|
|
rel = path.resolve().relative_to(vault)
|
|
except ValueError:
|
|
return f"path {path} is outside working_dir {vault}"
|
|
parts = rel.parts
|
|
if not parts:
|
|
return "path has no components relative to working_dir"
|
|
head = parts[0]
|
|
if head == "Archive":
|
|
return None
|
|
if head == "topics" and len(parts) >= 3:
|
|
return None
|
|
if head == "events" and len(parts) >= 4:
|
|
return None
|
|
return (
|
|
f"path {rel} doesn't match a known template — expected one of: "
|
|
f"topics/{{folder}}/{{name}}.md, "
|
|
f"events/{{date}}/{{name}}/{{filename}}, or Archive/..."
|
|
)
|
|
|
|
|
|
def update_status(path: Path | str, *, value, force: bool = False) -> tuple[bool, dict]:
|
|
"""Schema-aware status flip. Reads current status, validates the
|
|
transition, then delegates to ``memory_io.update_meta``."""
|
|
target = Path(path)
|
|
if not force:
|
|
prior = None
|
|
if target.is_file():
|
|
try:
|
|
prior = frontmatter.loads(
|
|
target.read_text(encoding="utf-8"),
|
|
).metadata.get("status")
|
|
except Exception:
|
|
prior = None
|
|
err = validate_status_transition(prior, value)
|
|
if err is not None:
|
|
return False, {
|
|
"path": str(target),
|
|
"key": "status",
|
|
"error": err,
|
|
"prior": prior,
|
|
"requested": value,
|
|
}
|
|
return memory_io.update_meta(target, key="status", value=value)
|
|
|
|
|
|
def create_with_schema(
|
|
file_store,
|
|
path: Path,
|
|
*,
|
|
metadata: dict,
|
|
content: str,
|
|
overwrite: bool = False,
|
|
force: bool = False,
|
|
) -> tuple[bool, dict]:
|
|
"""Schema-aware file create — path template gate then engine."""
|
|
if not force:
|
|
working_dir = getattr(file_store, "working_dir", None)
|
|
template_err = validate_path_template(path, working_dir)
|
|
if template_err is not None:
|
|
return False, {
|
|
"path": str(path),
|
|
"error": template_err,
|
|
"hint": (
|
|
"place topics under topics/{folder}/{name}.md and "
|
|
"events under events/{date}/{name}/...; pass "
|
|
"force=true only if you intentionally need a "
|
|
"non-template path"
|
|
),
|
|
}
|
|
return memory_io.create_file(
|
|
file_store, path,
|
|
metadata=metadata, content=content,
|
|
overwrite=overwrite, force=force,
|
|
)
|
|
|
|
|
|
# ===========================================================================
|
|
# Section 2 — Memory tools
|
|
# ===========================================================================
|
|
|
|
|
|
@R.register("memory_get")
|
|
class MemoryGet(BaseStep):
|
|
"""Read a single memory file (frontmatter + body, optional chunks)."""
|
|
|
|
audit: list[dict] | None = None
|
|
|
|
async def execute(self):
|
|
assert self.context is not None
|
|
path: str = self.context.get("path", "") or ""
|
|
include_chunks: bool = bool(self.context.get("include_chunks", False))
|
|
assert path, "path is required"
|
|
result = await memory_io.get_file(self.file_store, path, include_chunks=include_chunks)
|
|
_set_answer(self.context, result)
|
|
|
|
async def memory_get(self, path: str, include_chunks: bool = False) -> ToolResponse:
|
|
"""Read a single memory file (frontmatter + body, optional chunks)."""
|
|
result = await memory_io.get_file(self.file_store, path, include_chunks=include_chunks)
|
|
return _tool_response("memory_get", True, result, audit=self.audit)
|
|
|
|
|
|
@R.register("memory_create")
|
|
class MemoryCreate(BaseStep):
|
|
"""Create a markdown file. Path-template gate + wikilink-uniqueness
|
|
gate fire unless ``force=True``."""
|
|
|
|
audit: list[dict] | None = None
|
|
|
|
async def execute(self):
|
|
assert self.context is not None
|
|
path: str = self.context.get("path", "") or ""
|
|
metadata: dict = dict(self.context.get("metadata") or {})
|
|
content: str = self.context.get("content", "") or ""
|
|
overwrite: bool = bool(self.context.get("overwrite", False))
|
|
force: bool = bool(self.context.get("force", False))
|
|
assert path, "path is required"
|
|
target = Path(path)
|
|
ok, payload = create_with_schema(
|
|
self.file_store, target,
|
|
metadata=metadata, content=content,
|
|
overwrite=overwrite, force=force,
|
|
)
|
|
self.context.response.success = ok
|
|
if ok:
|
|
payload = {**payload, "path": str(target.resolve())}
|
|
_set_answer(self.context, payload)
|
|
|
|
async def memory_create(
|
|
self,
|
|
path: str,
|
|
metadata: dict | None = None,
|
|
content: str = "",
|
|
overwrite: bool = False,
|
|
force: bool = False,
|
|
) -> ToolResponse:
|
|
"""Create a markdown file. Path template + wikilink uniqueness
|
|
gates fire unless ``force=True``."""
|
|
target = Path(path)
|
|
ok, payload = create_with_schema(
|
|
self.file_store, target,
|
|
metadata=dict(metadata or {}), content=content,
|
|
overwrite=overwrite, force=force,
|
|
)
|
|
if ok:
|
|
payload = {**payload, "path": str(target.resolve())}
|
|
return _tool_response("memory_create", ok, payload, audit=self.audit)
|
|
|
|
|
|
@R.register("memory_update_body")
|
|
class MemoryUpdateBody(BaseStep):
|
|
"""Edit-style body update: replace ``old_string`` with ``new_string``.
|
|
Frontmatter is preserved verbatim."""
|
|
|
|
audit: list[dict] | None = None
|
|
|
|
async def execute(self):
|
|
assert self.context is not None
|
|
path: str = self.context.get("path", "") or ""
|
|
old_string: str = self.context.get("old_string", "") or ""
|
|
new_string: str = self.context.get("new_string", "") or ""
|
|
replace_all: bool = bool(self.context.get("replace_all", False))
|
|
assert path, "path is required"
|
|
ok, payload = memory_io.update_body(
|
|
path, old_string=old_string, new_string=new_string, replace_all=replace_all,
|
|
)
|
|
self.context.response.success = ok
|
|
_set_answer(self.context, payload)
|
|
|
|
async def memory_update_body(
|
|
self,
|
|
path: str,
|
|
old_string: str,
|
|
new_string: str,
|
|
replace_all: bool = False,
|
|
) -> ToolResponse:
|
|
"""Edit-style body update: replace ``old_string`` with ``new_string``."""
|
|
ok, payload = memory_io.update_body(
|
|
path, old_string=old_string, new_string=new_string, replace_all=replace_all,
|
|
)
|
|
return _tool_response("memory_update_body", ok, payload, audit=self.audit)
|
|
|
|
|
|
@R.register("memory_update_meta")
|
|
class MemoryUpdateMeta(BaseStep):
|
|
"""Frontmatter patch (merge). value=None deletes the key.
|
|
``status`` transitions go through the state-machine validator
|
|
unless ``force=True``."""
|
|
|
|
audit: list[dict] | None = None
|
|
|
|
async def execute(self):
|
|
assert self.context is not None
|
|
path: str = self.context.get("path", "") or ""
|
|
patch: dict = dict(self.context.get("patch") or {})
|
|
force: bool = bool(self.context.get("force", False))
|
|
assert path, "path is required"
|
|
ok, payload = await self._apply(path, patch, force)
|
|
self.context.response.success = ok
|
|
_set_answer(self.context, payload)
|
|
|
|
async def memory_update_meta(
|
|
self,
|
|
path: str,
|
|
patch: dict,
|
|
force: bool = False,
|
|
) -> ToolResponse:
|
|
"""Frontmatter patch (merge). value=None deletes the key."""
|
|
ok, payload = await self._apply(path, dict(patch or {}), force)
|
|
return _tool_response("memory_update_meta", ok, payload, audit=self.audit)
|
|
|
|
async def _apply(self, path: str, patch: dict, force: bool) -> tuple[bool, dict]:
|
|
results: dict[str, dict] = {}
|
|
all_ok = True
|
|
for key, value in patch.items():
|
|
if key == "status":
|
|
ok, payload = update_status(path, value=value, force=force)
|
|
else:
|
|
ok, payload = memory_io.update_meta(path, key=key, value=value)
|
|
results[key] = payload
|
|
if not ok:
|
|
all_ok = False
|
|
break
|
|
return all_ok, {"path": path, "applied": results}
|
|
|
|
|
|
# ===========================================================================
|
|
# Section 3 — Retrieval (memory_search + memory_graph_search)
|
|
# ===========================================================================
|
|
|
|
|
|
_RETRIEVER_CACHE: dict[int, BaseRetriever] = {}
|
|
|
|
|
|
def _resolve_retriever(step: BaseStep) -> BaseRetriever:
|
|
"""Get (or build) the retriever instance for this step."""
|
|
cached = _RETRIEVER_CACHE.get(id(step))
|
|
if cached is not None:
|
|
return cached
|
|
retriever = R.get(ComponentEnum.RETRIEVER, "hybrid")
|
|
if retriever is None:
|
|
retriever = HybridRetriever(app_context=step.app_context)
|
|
elif isinstance(retriever, type):
|
|
retriever = retriever(app_context=step.app_context)
|
|
_RETRIEVER_CACHE[id(step)] = retriever
|
|
return retriever
|
|
|
|
|
|
def _serialize_chunk(chunk, file_store, extras: dict | None = None) -> dict:
|
|
"""Flatten a FileChunk into a dict, joining file metadata."""
|
|
item = chunk.model_dump() if hasattr(chunk, "model_dump") else dict(chunk)
|
|
node = file_store.file_nodes.get(item.get("path"))
|
|
if node is not None:
|
|
meta = node.front_matter.model_dump()
|
|
item["file_metadata"] = meta
|
|
item["file_st_mtime"] = node.st_mtime
|
|
else:
|
|
item["file_metadata"] = {}
|
|
item["file_st_mtime"] = None
|
|
if extras:
|
|
item.update(extras)
|
|
return item
|
|
|
|
|
|
@R.register("memory_search")
|
|
class MemorySearch(BaseStep):
|
|
"""Pure-relevance retrieval (V + K hybrid). Delegates to the Retriever."""
|
|
|
|
audit: list[dict] | None = None
|
|
|
|
async def execute(self):
|
|
assert self.context is not None
|
|
query: str = self.context.get("query", "").strip()
|
|
min_score: float = self.context.get("min_score", 0.1)
|
|
max_results: int = self.context.get("max_results", 5)
|
|
assert query, "Query cannot be empty"
|
|
assert 0.0 <= min_score <= 1.0, f"min_score must be in [0,1], got {min_score}"
|
|
assert max_results > 0, f"max_results must be positive, got {max_results}"
|
|
chunk_filter = memory_io.make_filter(
|
|
self.file_store,
|
|
paths=self.context.get("paths") or None,
|
|
tags=self.context.get("tags") or None,
|
|
exclude_paths=self.context.get("exclude_paths") or None,
|
|
)
|
|
retriever = _resolve_retriever(self)
|
|
results = await retriever.search(
|
|
query=query, max_results=max_results, min_score=min_score, chunk_filter=chunk_filter,
|
|
)
|
|
payload = [_serialize_chunk(r, self.file_store) for r in results]
|
|
_set_answer(self.context, payload)
|
|
|
|
async def memory_search(
|
|
self,
|
|
query: str,
|
|
max_results: int = 5,
|
|
min_score: float = 0.1,
|
|
paths: list[str] | None = None,
|
|
tags: list[str] | None = None,
|
|
exclude_paths: list[str] | None = None,
|
|
) -> ToolResponse:
|
|
"""Pure-relevance retrieval (V + K hybrid)."""
|
|
chunk_filter = memory_io.make_filter(
|
|
self.file_store, paths=paths, tags=tags, exclude_paths=exclude_paths,
|
|
)
|
|
retriever = _resolve_retriever(self)
|
|
results = await retriever.search(
|
|
query=query, max_results=max_results, min_score=min_score, chunk_filter=chunk_filter,
|
|
)
|
|
payload = [_serialize_chunk(r, self.file_store) for r in results]
|
|
return _tool_response("memory_search", True, payload, audit=self.audit)
|
|
|
|
|
|
@R.register("memory_graph_search")
|
|
class MemoryGraphSearch(BaseStep):
|
|
"""V + K + graph BFS fusion. MCP-only — not bound to agent toolkit.
|
|
|
|
Per-call overrides for fusion knobs are forwarded if present in
|
|
the RuntimeContext."""
|
|
|
|
_OVERRIDE_KEYS = (
|
|
"vector_weight", "graph_weight", "graph_depth", "graph_decay",
|
|
"graph_direction", "graph_mode", "graph_per_path_cap", "anchor_expand",
|
|
)
|
|
|
|
async def execute(self):
|
|
assert self.context is not None
|
|
ctx = self.context
|
|
query: str = ctx.get("query", "").strip()
|
|
max_results: int = int(ctx.get("max_results", 5))
|
|
min_score: float = float(ctx.get("min_score", 0.0))
|
|
explicit_seeds: list[str] = list(ctx.get("seeds") or [])
|
|
assert query or explicit_seeds, "query or seeds must be provided"
|
|
assert max_results > 0
|
|
chunk_filter = memory_io.make_filter(
|
|
self.file_store,
|
|
paths=ctx.get("paths") or None,
|
|
tags=ctx.get("tags") or None,
|
|
exclude_paths=ctx.get("exclude_paths") or None,
|
|
)
|
|
overrides = {k: ctx.get(k) for k in self._OVERRIDE_KEYS if ctx.get(k) is not None}
|
|
retriever = _resolve_retriever(self)
|
|
results = await retriever.search(
|
|
query=query, max_results=max_results, min_score=min_score,
|
|
chunk_filter=chunk_filter, seeds=explicit_seeds or None, **overrides,
|
|
)
|
|
payload = [_serialize_chunk(r, self.file_store) for r in results]
|
|
_set_answer(self.context, payload)
|
|
|
|
|
|
MEMORY_TOOL_NAMES: tuple[str, ...] = (
|
|
"memory_get",
|
|
"memory_create",
|
|
"memory_update_body",
|
|
"memory_update_meta",
|
|
"memory_search",
|
|
)
|