From 3dee10d4f91c32f0af282b779105bed5ab0165a8 Mon Sep 17 00:00:00 2001 From: Sen Huang <48879559+ployts@users.noreply.github.com> Date: Fri, 26 Jun 2026 14:46:42 +0800 Subject: [PATCH] feat: add Claude Code plugin with auto-memory functionality (#297) * feat: add Claude Code plugin with auto-memory functionality * refactor(auto_memory): fix spacing in json parsing logic --- plugins/.claude-plugin/marketplace.json | 17 ++ plugins/README.md | 71 ++++++ plugins/reme/.claude-plugin/plugin.json | 13 ++ plugins/reme/.mcp.json | 8 + plugins/reme/hooks/auto_memory.py | 173 ++++++++++++++ plugins/reme/hooks/hooks.json | 16 ++ plugins/reme/skills/reme-memory/SKILL.md | 58 +++++ reme/components/agent_wrapper/__init__.py | 4 +- .../agent_wrapper/cc_agent_wrapper.py | 11 +- reme/config/default.yaml | 17 ++ reme/steps/evolve/__init__.py | 2 + reme/steps/evolve/auto_memory_cc.py | 183 +++++++++++++++ skills/claude_code_memory/SKILL.md | 4 - tests/integration/run_reme_e2e_local.py | 43 ++++ tests/integration/test_reme_e2e.py | 216 ++++++++++++++++++ 15 files changed, 827 insertions(+), 9 deletions(-) create mode 100644 plugins/.claude-plugin/marketplace.json create mode 100644 plugins/README.md create mode 100644 plugins/reme/.claude-plugin/plugin.json create mode 100644 plugins/reme/.mcp.json create mode 100755 plugins/reme/hooks/auto_memory.py create mode 100644 plugins/reme/hooks/hooks.json create mode 100644 plugins/reme/skills/reme-memory/SKILL.md create mode 100644 reme/steps/evolve/auto_memory_cc.py delete mode 100644 skills/claude_code_memory/SKILL.md create mode 100644 tests/integration/run_reme_e2e_local.py create mode 100644 tests/integration/test_reme_e2e.py diff --git a/plugins/.claude-plugin/marketplace.json b/plugins/.claude-plugin/marketplace.json new file mode 100644 index 00000000..e207daa4 --- /dev/null +++ b/plugins/.claude-plugin/marketplace.json @@ -0,0 +1,17 @@ +{ + "name": "reme-marketplace", + "owner": { + "name": "EconML team of Alibaba Tongyi Lab", + "url": "https://github.com/agentscope-ai/ReMe" + }, + "metadata": { + "description": "ReMe — file-native long-term memory for AI agents. Claude Code plugin." + }, + "plugins": [ + { + "name": "reme", + "source": "./reme", + "description": "Connect Claude Code to a running ReMe MCP server for file-native long-term memory: recall (digest) on demand, plus automatic background recording of each session via a Stop hook." + } + ] +} diff --git a/plugins/README.md b/plugins/README.md new file mode 100644 index 00000000..a295b294 --- /dev/null +++ b/plugins/README.md @@ -0,0 +1,71 @@ +# ReMe plugin for Claude Code + +Connect Claude Code to [ReMe](https://github.com/agentscope-ai/ReMe) — file-native long-term memory +for AI agents. The plugin gives the agent **recall** (read long-term memory) and **records every +session automatically** via a Stop hook. Consolidation of daily notes into long-term `digest/` +knowledge runs server-side in ReMe. + +## What you get + +- **MCP tools** from the `reme` server: `search`, `traverse`, `daily_list`, `frontmatter_read`, + `read`, `auto_memory_cc`, and more. +- **Stop hook** (`hooks/auto_memory.py`) — when a session ends it calls ReMe's server-side + `auto_memory_cc` tool in a detached background process, passing **only the session id**. The server + resolves that session's transcript on disk and records the durable facts into today's daily note. + Recording is fully automatic and asynchronous — the agent never records by hand, and stopping is + never delayed. Best-effort: if the server is down it logs and gives up silently. +- **Skill** `reme-memory` — recall long-term memory before answering (semantic `search`, topological + `traverse`, state `daily_list`/`frontmatter_read`, then `read` with citations), plus a server + status check. Recording is handled silently by the Stop hook. + +## Deployment model + +The plugin **connects to a shared HTTP MCP server you start once** — it does not spawn ReMe. One +server means one set of background watchers / dream cron across all your Claude Code windows. + +## Prerequisites + +1. Install ReMe (Python 3.11+): + + ```bash + pip install "reme-ai[core]" + ``` + +2. Configure model credentials in a `.env` (see `example.env`): + + ```bash + EMBEDDING_API_KEY=sk-xxx + EMBEDDING_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 + LLM_API_KEY=sk-xxx + LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 + ``` + +3. Start the ReMe MCP server (one time, leave it running): + + ```bash + reme start service.backend=mcp service.transport=streamable-http + ``` + + It serves `http://127.0.0.1:2333/mcp`. To use a different port, start with + `service.port=` and update the `url` in `.mcp.json` to match. + +## Install the plugin + +``` +/plugin marketplace add ./plugins +/plugin install reme@reme-marketplace +``` + +(Or point `/plugin marketplace add` at the GitHub repo + subpath once published.) Restart Claude Code, then +run `/mcp` to confirm the `reme` server and its tools are connected; the `reme-memory` skill can then +recall memory and report server health. + +## Notes + +- The plugin's MCP server URL lives in `plugins/reme/.mcp.json`. Keep it in sync with how you start + ReMe (host/port). The Stop hook reads this same file to find the server (override with `REME_HOST` + / `REME_PORT` env vars). +- The Stop hook needs `python3` on `PATH` and resolves transcripts under `~/.claude/projects` + (override the base with `CLAUDE_CONFIG_DIR`). It logs to `plugins/reme/logs/auto_memory_hook.log`. +- The MCP tool-name prefix (`mcp__reme__…`) may include the server segment depending on your Claude + Code version; the skill uses the `mcp__reme__*` wildcard so it works either way. diff --git a/plugins/reme/.claude-plugin/plugin.json b/plugins/reme/.claude-plugin/plugin.json new file mode 100644 index 00000000..0c7195f7 --- /dev/null +++ b/plugins/reme/.claude-plugin/plugin.json @@ -0,0 +1,13 @@ +{ + "name": "reme", + "version": "0.1.0", + "description": "File-native long-term memory for Claude Code, backed by a running ReMe MCP server. A memory skill recalls long-term knowledge on demand; recording is automatic via a Stop hook that records each session in the background.", + "author": { + "name": "EconML team of Alibaba Tongyi Lab", + "email": "jinli.yl@alibaba-inc.com" + }, + "homepage": "https://reme.agentscope.io/", + "repository": "https://github.com/agentscope-ai/ReMe", + "license": "Apache-2.0", + "keywords": ["memory", "reme", "mcp", "long-term-memory", "agent"] +} diff --git a/plugins/reme/.mcp.json b/plugins/reme/.mcp.json new file mode 100644 index 00000000..b5d271ee --- /dev/null +++ b/plugins/reme/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "reme": { + "type": "http", + "url": "http://127.0.0.1:2333/mcp" + } + } +} diff --git a/plugins/reme/hooks/auto_memory.py b/plugins/reme/hooks/auto_memory.py new file mode 100755 index 00000000..08431396 --- /dev/null +++ b/plugins/reme/hooks/auto_memory.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""ReMe Stop hook: fire-and-forget auto-memory for the current session. + +Claude Code runs this on the ``Stop`` event and feeds the hook payload as JSON +on stdin. We read only ``session_id`` from it and hand that to ReMe's server-side +``auto_memory_cc`` tool over the (already-running) MCP server — the server +resolves *this* session's transcript on disk and records the durable facts. No +messages are sent from here; the agent never has to record by hand. + +The actual run spins up an inner agent and can take a while, so we detach +(double-fork) and return immediately: stopping is never blocked. Any failure is +logged, never surfaced — recording is best-effort. +""" + +from __future__ import annotations + +import json +import os +import sys +import urllib.error +import urllib.request +from datetime import datetime + +# auto_memory drives an inner agent; give it room. The foreground process has +# already returned by the time this matters (we are detached), so a long ceiling +# is harmless. +_CALL_TIMEOUT = 600 + + +def _plugin_root() -> str: + return os.environ.get("CLAUDE_PLUGIN_ROOT") or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _server_url() -> str: + """ReMe MCP endpoint. Prefer the bundled .mcp.json so it stays in sync.""" + mcp_json = os.path.join(_plugin_root(), ".mcp.json") + try: + with open(mcp_json, encoding="utf-8") as f: + url = json.load(f)["mcpServers"]["reme"]["url"] + if url: + return url + except Exception: + pass + host = os.environ.get("REME_HOST", "127.0.0.1") + port = os.environ.get("REME_PORT", "2333") + return f"http://{host}:{port}/mcp" + + +def _log(session_id: str, status: str, detail: str = "") -> None: + try: + log_dir = os.path.join(_plugin_root(), "logs") + os.makedirs(log_dir, exist_ok=True) + stamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + line = f"{stamp} session={session_id} {status}" + if detail: + line += f" {detail}" + with open(os.path.join(log_dir, "auto_memory_hook.log"), "a", encoding="utf-8") as f: + f.write(line + "\n") + except Exception: + pass + + +def _post(url: str, body: dict, headers: dict) -> "urllib.request.addinfourl": + data = json.dumps(body).encode("utf-8") + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + return urllib.request.urlopen(req, timeout=_CALL_TIMEOUT) + + +def _read_jsonrpc(resp) -> dict | None: + """Return the JSON-RPC envelope from a JSON or text/event-stream response.""" + ctype = resp.headers.get("content-type", "") + body = resp.read().decode("utf-8", "replace") + if "text/event-stream" in ctype: + result = None + for line in body.splitlines(): + line = line.strip() + if not line.startswith("data:"): + continue + try: + obj = json.loads(line[len("data:") :].strip()) + except json.JSONDecodeError: + continue + if isinstance(obj, dict) and ("result" in obj or "error" in obj): + result = obj + return result + try: + return json.loads(body) + except json.JSONDecodeError: + return None + + +def _mcp_call(url: str, tool: str, arguments: dict) -> dict | None: + """Minimal MCP streamable-http client: initialize -> initialized -> tools/call.""" + base = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} + + # 1. initialize (captures the session id header) + init = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "reme-stop-hook", "version": "1.0"}, + }, + } + with _post(url, init, base) as resp: + mcp_session = resp.headers.get("mcp-session-id") + _read_jsonrpc(resp) + + headers = dict(base) + if mcp_session: + headers["mcp-session-id"] = mcp_session + + # 2. notifications/initialized (no id; 202 with empty body) + try: + with _post(url, {"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, headers) as resp: + resp.read() + except urllib.error.HTTPError: + pass + + # 3. tools/call + call = {"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": tool, "arguments": arguments}} + with _post(url, call, headers) as resp: + return _read_jsonrpc(resp) + + +def _daemonize() -> None: + """Double-fork + setsid so the (slow) call outlives the hook and is reaped by init.""" + if os.fork() > 0: + os._exit(0) # original process returns -> hook completes, Claude stops + os.setsid() + if os.fork() > 0: + os._exit(0) + devnull = os.open(os.devnull, os.O_RDWR) + for fd in (0, 1, 2): + os.dup2(devnull, fd) + + +def main() -> None: + """Entry point: read the hook payload from stdin and record the session.""" + try: + payload = json.loads(sys.stdin.read() or "{}") + except Exception: + payload = {} + + session_id = payload.get("session_id") or "" + if not session_id: + return # nothing to anchor a recording on + + # Detach before the slow agent run. Without fork() (e.g. Windows) we fall + # through and run inline — correct, just not async. + if hasattr(os, "fork"): + _daemonize() + + url = _server_url() + try: + result = _mcp_call(url, "auto_memory_cc", {"session_id": session_id}) + if result is None: + _log(session_id, "no-response") + elif "error" in result: + _log(session_id, "error", json.dumps(result["error"], ensure_ascii=False)[:500]) + else: + _log(session_id, "ok") + except urllib.error.URLError as exc: + # Server not running / unreachable — expected when ReMe isn't started. + _log(session_id, "unreachable", str(exc.reason)) + except Exception as exc: # noqa: BLE001 - best-effort, never surface + _log(session_id, "exception", repr(exc)[:500]) + + +if __name__ == "__main__": + main() diff --git a/plugins/reme/hooks/hooks.json b/plugins/reme/hooks/hooks.json new file mode 100644 index 00000000..988876c3 --- /dev/null +++ b/plugins/reme/hooks/hooks.json @@ -0,0 +1,16 @@ +{ + "description": "On stop, record this Claude Code session into ReMe long-term memory (background, async).", + "hooks": { + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "python3 \"${CLAUDE_PLUGIN_ROOT}/hooks/auto_memory.py\"", + "timeout": 30 + } + ] + } + ] + } +} diff --git a/plugins/reme/skills/reme-memory/SKILL.md b/plugins/reme/skills/reme-memory/SKILL.md new file mode 100644 index 00000000..15125e11 --- /dev/null +++ b/plugins/reme/skills/reme-memory/SKILL.md @@ -0,0 +1,58 @@ +--- +name: reme-memory +description: Use ReMe as file-native long-term memory in Claude Code. RECALL — search ReMe before answering questions about past conversations, preferences, project history, or decisions. +--- + +# ReMe Memory + +ReMe is the persistent, file-native memory layer for this agent. It stores conversations and +resources as Markdown files with frontmatter and `[[wikilinks]]`, and consolidates them into +long-term **digest** knowledge. Your job with this plugin is **recall**. + +The recall tools come from the `reme` MCP server (surfaced as `mcp__reme__…`): `search`, `traverse`, +`daily_list`, `frontmatter_read`, `read`. They are only available when the user has the server +running: + +``` +reme start service.backend=mcp service.transport=streamable-http +``` + +If the tools are missing, that server is not running — tell the user the command above instead of +guessing answers. + +## Recall (read long-term memory) + +Before answering questions about previous conversations, user preferences, project history, +decisions, or long-term context, recall from ReMe first. ReMe answers **three independent kinds of +question** — pick the mode the request needs; don't merge them into one call. Durable knowledge +lives under `digest/`, daily notes under `daily/`, external materials under `resource/`. + +1. **Semantic** (default — "what do we know about X?"): `search` with `query=""`, + `limit=5` (optional `min_score`). Hybrid vector + BM25 with one-hop wikilink expansion. +2. **Topological** ("what links to this node?"): `traverse` with `path=""`, `depth=1` + (raise to 2 only when needed), `direction=both` to walk the `[[wikilink]]` graph. +3. **State** ("what exists / what was recorded on ?"): `daily_list` with `date="YYYY-MM-DD"` + (empty = today) to list a day's notes, or `frontmatter_read` with a `path` to inspect one file's + frontmatter — structural lookup, no semantic matching. + +Then `read` the relevant hits by `path` (optionally `start_line`/`end_line`; prefer `digest/` paths +for durable knowledge) to pull the content behind a hit. Cite the workspace-relative paths you used. +If nothing useful comes back, say so plainly rather than guessing. + +## Server status + +To check ReMe is up: call `version` and `health_check`, then summarize the version and the health +snapshot (components, workspace). If the `mcp__reme__…` tools are not available at all, the server +is not running — tell the user to start it with the command above. The plugin connects at +`http://127.0.0.1:2333/mcp`; a different host/port must match the `url` in `plugins/reme/.mcp.json`. + +## Workspace model + +``` +daily/ lightly-processed memory: daily facts, conversation summaries +digest/ long-term consolidated knowledge (what recall mainly surfaces) +resource/ external raw materials +``` + +Consolidation of `daily/` into `digest/` and proactive interest extraction run **server-side** in +the ReMe process (background watchers + dream cron). The plugin does not drive them. diff --git a/reme/components/agent_wrapper/__init__.py b/reme/components/agent_wrapper/__init__.py index cd049bc6..1f084900 100644 --- a/reme/components/agent_wrapper/__init__.py +++ b/reme/components/agent_wrapper/__init__.py @@ -2,6 +2,6 @@ from .base_agent_wrapper import BaseAgentWrapper from .as_agent_wrapper import AsAgentWrapper -from .cc_agent_wrapper import CcAgentWrapper +from .cc_agent_wrapper import CcAgentWrapper, CcFileSessionStore -__all__ = ["BaseAgentWrapper", "AsAgentWrapper", "CcAgentWrapper"] +__all__ = ["BaseAgentWrapper", "AsAgentWrapper", "CcAgentWrapper", "CcFileSessionStore"] diff --git a/reme/components/agent_wrapper/cc_agent_wrapper.py b/reme/components/agent_wrapper/cc_agent_wrapper.py index e434b137..a67e50c0 100644 --- a/reme/components/agent_wrapper/cc_agent_wrapper.py +++ b/reme/components/agent_wrapper/cc_agent_wrapper.py @@ -19,8 +19,13 @@ if TYPE_CHECKING: from claude_agent_sdk.types import SessionKey, SessionStoreEntry, SessionStoreListEntry -class _CcFileSessionStore: - """File-backed Claude Code SessionStore rooted under the ReMe workspace.""" +class CcFileSessionStore: + """File-backed Claude Code SessionStore rooted under a given directory. + + Rooted under the ReMe workspace for the inner agent's own sessions, but the + same reader is reused (with a different root) to load an *outer* Claude Code + session's transcript by id — see ``AutoMemoryCCStep``. + """ def __init__(self, root: Path) -> None: self.root = root @@ -274,7 +279,7 @@ class CcAgentWrapper(BaseAgentWrapper): opts.env.setdefault("CLAUDE_CONFIG_DIR", str(claude_config_dir)) if opts.skills is not None: self._ensure_claude_skill_dir(claude_config_dir) - opts.session_store = opts.session_store or _CcFileSessionStore(self.session_path / "claude_code") + opts.session_store = opts.session_store or CcFileSessionStore(self.session_path / "claude_code") job_tools: list[str] = kwargs.get("job_tools", []) resolved_jobs = self._resolve_job_tools(job_tools) diff --git a/reme/config/default.yaml b/reme/config/default.yaml index cd692c9b..8bae8a49 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -136,6 +136,23 @@ jobs: steps: - backend: auto_memory_step + auto_memory_cc: + backend: base + description: "Auto-memory (Claude Code): record a Claude Code session into a daily note, resolved from its session_id" + parameters: + type: object + properties: + session_id: + type: string + description: "Claude Code session id; its transcript is resolved from disk" + memory_hint: + type: string + description: "optional hint" + required: + - session_id + steps: + - backend: auto_memory_cc_step + auto_resource: backend: base description: "Auto-resource: interpret resource files into daily notes" diff --git a/reme/steps/evolve/__init__.py b/reme/steps/evolve/__init__.py index c5f658a3..e2ab5fd6 100644 --- a/reme/steps/evolve/__init__.py +++ b/reme/steps/evolve/__init__.py @@ -2,12 +2,14 @@ from ._evolve import now from .auto_memory import AutoMemoryStep +from .auto_memory_cc import AutoMemoryCCStep from .auto_resource import AutoResourceStep from .dream import DreamExtractStep, DreamFinishStep, DreamIntegrateStep, DreamTopicsStep, ProactiveStep __all__ = [ "now", "AutoMemoryStep", + "AutoMemoryCCStep", "AutoResourceStep", "DreamExtractStep", "DreamFinishStep", diff --git a/reme/steps/evolve/auto_memory_cc.py b/reme/steps/evolve/auto_memory_cc.py new file mode 100644 index 00000000..b0ecd871 --- /dev/null +++ b/reme/steps/evolve/auto_memory_cc.py @@ -0,0 +1,183 @@ +"""auto_memory_cc — record a Claude Code session, resolved from its session_id. + +The ReMe plugin's Stop hook hands the server only a ``session_id`` (never the +messages), and it fires on *every* stop. Unlike :class:`AutoMemoryStep` — whose +callers have no session management, so it re-serializes ``Msg`` history into its +own dialog store — Claude Code already manages the session as a transcript on +disk. So this step manages everything through the :class:`CcFileSessionStore` +abstraction and avoids the ``Msg`` round-trip entirely: + +1. **load** the outer Claude Code session's transcript entries (Claude Code side). +2. **save** the *raw* entries into ReMe's own CC SessionStore — ``append`` dedups + by record ``uuid``, so this both copies the conversation into ReMe and tells + us the **increment** since the last stop. +3. render only that increment into plain ``{role, name, content}`` messages and + defer to :class:`AutoMemoryStep` for the daily-note write/merge. + +Both the read (Claude Code side) and the copy (ReMe side) use the same +file-backed SessionStore, just rooted at different directories. +""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Any + +from .auto_memory import AutoMemoryStep +from ...components import R +from ...components.agent_wrapper import CcFileSessionStore + +# Whole-message-drop when a user turn is only Claude-Code-injected boilerplate. +_INJECTED_TAGS = ( + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", +) +_TOOL_EXCERPT = 200 + + +@R.register("auto_memory_cc_step") +class AutoMemoryCCStep(AutoMemoryStep): + """Resolve a Claude Code session_id to its *new* turns, then reuse AutoMemoryStep.""" + + # Sub-directory under the session dir holding ReMe's copy of CC transcripts. + _CC_STORE_SUBDIR = "claude_code" + + async def execute(self): + assert self.context is not None + session_id: str = self.context.get("session_id", "") + cc_entries = await self._load_cc_session(session_id) + new_entries = await self._save_cc_session(session_id, cc_entries) + messages = self._entries_to_messages(new_entries) + self.logger.info( + f"[{self.name}] resolved Claude Code session session_id={session_id!r} " + f"transcript={len(cc_entries)} new_entries={len(new_entries)} messages={len(messages)}", + ) + self.context["messages"] = messages + await super().execute() + + # Claude Code owns the session (transcript + the CC SessionStore copy made in + # _save_cc_session); AutoMemoryStep's Msg-history dialog store does not apply. + async def _save_session_messages(self, session_id: str, messages) -> None: # noqa: D401 + return + + def _session_link(self, session_id: str) -> str: + return f"[[{self._session_dir()}/{self._CC_STORE_SUBDIR}/{session_id}.jsonl]]" + + # ----- session: Claude Code side <-> ReMe CC SessionStore ---------------- + + async def _load_cc_session(self, session_id: str) -> list[dict]: + """Load the outer Claude Code transcript entries (Claude Code side).""" + transcript_dir = self._resolve_transcript_dir(session_id) + if transcript_dir is None: + return [] + store = CcFileSessionStore(transcript_dir) + return await store.load({"session_id": session_id}) or [] + + async def _save_cc_session(self, session_id: str, cc_entries: list[dict]) -> list[dict]: + """Copy raw CC entries into ReMe's CC SessionStore; return the increment. + + Only identity-bearing entries are copied: every conversational entry + (user / assistant / attachment) carries a ``uuid``, while uuid-less rows + are CC control bookkeeping (queue operations, last-prompt) that would + otherwise re-copy on every stop. Dedup against the already-stored uuids + yields exactly the turns added since the previous stop. + """ + if not session_id: + return [] + store = self._reme_cc_store() + key = {"session_id": session_id} + cc_entries = [e for e in cc_entries if isinstance(e, dict) and e.get("uuid")] + existing = await store.load(key) or [] + seen = {e.get("uuid") for e in existing if isinstance(e, dict) and e.get("uuid")} + increment = [e for e in cc_entries if e.get("uuid") not in seen] + await store.append(key, increment) + return increment + + def _reme_cc_store(self) -> CcFileSessionStore: + root = self.file_store.workspace_path / self._session_dir() / self._CC_STORE_SUBDIR + return CcFileSessionStore(root) + + @staticmethod + def _projects_dir() -> Path: + base = Path(os.environ.get("CLAUDE_CONFIG_DIR") or "~/.claude").expanduser() + return base if base.name == "projects" else base / "projects" + + def _resolve_transcript_dir(self, session_id: str) -> Path | None: + """Return the project directory holding ``.jsonl`` (newest).""" + projects = self._projects_dir() + if not session_id or not projects.is_dir(): + return None + matches = list(projects.glob(f"*/{session_id}.jsonl")) + if not matches: + return None + matches.sort(key=lambda p: p.stat().st_mtime, reverse=True) + return matches[0].parent + + # ----- rendering: raw CC entries -> plain agent messages ----------------- + + @classmethod + def _entries_to_messages(cls, entries: list[dict]) -> list[dict[str, str]]: + messages: list[dict[str, str]] = [] + for record in entries: + if not isinstance(record, dict) or record.get("type") not in ("user", "assistant"): + continue + message = record.get("message") or {} + role = message.get("role") + if role not in ("user", "assistant"): + continue + text = cls._render_content(message.get("content", "")) + if not text or cls._is_injected_only(text): + continue + messages.append({"role": role, "name": role, "content": text}) + return messages + + @classmethod + def _render_content(cls, content: Any) -> str: + if isinstance(content, str): + return content.strip() + if not isinstance(content, list): + return "" + + parts: list[str] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "text": + if t := (block.get("text") or "").strip(): + parts.append(t) + elif btype == "tool_use": + name = block.get("name", "?") + try: + inp = json.dumps(block.get("input"), ensure_ascii=False)[:_TOOL_EXCERPT] + except (TypeError, ValueError): + inp = str(block.get("input"))[:_TOOL_EXCERPT] + parts.append(f"[tool {name}({inp})]") + elif btype == "tool_result": + inner = block.get("content") + excerpt = cls._render_content(inner) if isinstance(inner, list) else str(inner or "") + excerpt = excerpt.strip() + if len(excerpt) > _TOOL_EXCERPT: + excerpt = excerpt[:_TOOL_EXCERPT] + "..." + parts.append(f"[tool_result {excerpt}]") + # thinking blocks are private reasoning -> dropped + return "\n".join(p for p in parts if p).strip() + + @staticmethod + def _is_injected_only(text: str) -> bool: + stripped = text.strip() + if not stripped.startswith(_INJECTED_TAGS): + return False + remaining = re.sub(r"<([a-z-]+)>.*?", "", stripped, flags=re.DOTALL) + return len(remaining.strip()) < 16 diff --git a/skills/claude_code_memory/SKILL.md b/skills/claude_code_memory/SKILL.md deleted file mode 100644 index acba8a89..00000000 --- a/skills/claude_code_memory/SKILL.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -name: claude_code_memory -description: claude_code_memory ---- \ No newline at end of file diff --git a/tests/integration/run_reme_e2e_local.py b/tests/integration/run_reme_e2e_local.py new file mode 100644 index 00000000..81d2a82c --- /dev/null +++ b/tests/integration/run_reme_e2e_local.py @@ -0,0 +1,43 @@ +"""Run the ReMe e2e loop against the real ./.reme workspace (no temp dir). + +Same 5 stages as tests/integration/test_reme_e2e.py, but the workspace is +pinned to /.reme so the produced files survive for inspection. + + python tests/integration/run_reme_e2e_local.py +""" + +import asyncio +import sys +from pathlib import Path + +INTEGRATION_DIR = Path(__file__).resolve().parent +REPO_ROOT = INTEGRATION_DIR.parents[1] +sys.path.insert(0, str(INTEGRATION_DIR)) + +# pylint: disable=wrong-import-position +from _workspace_fixture import WorkspaceEnv, temp_chdir # noqa: E402 +import test_reme_e2e as e2e # noqa: E402 + +from reme.utils import load_env # noqa: E402 + + +async def main() -> None: + """Run the e2e loop against a local .reme workspace for manual inspection.""" + load_env() + workspace = REPO_ROOT / ".reme" + workspace.mkdir(parents=True, exist_ok=True) + env = WorkspaceEnv(workspace=workspace, workspace_dir=workspace) + env.clean() # wipe daily/digest/resource/metadata from any prior run + + print(f"[local] workspace = {workspace}") + with temp_chdir(REPO_ROOT): + reme = await env.make_reme() + try: + await e2e._run_loop(env, reme) # noqa: SLF001 # pylint: disable=protected-access + finally: + await env.close_all() + print(f"\n[local] done — inspect files under: {workspace}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/integration/test_reme_e2e.py b/tests/integration/test_reme_e2e.py new file mode 100644 index 00000000..77c2f3cd --- /dev/null +++ b/tests/integration/test_reme_e2e.py @@ -0,0 +1,216 @@ +"""End-to-end integration test for the full ReMe memory loop. + +Unlike the per-job tests in this directory (which each exercise one job in +isolation), this drives the *whole* lifecycle through the public +``ReMe.run_job`` surface, in the order a real deployment runs it: + + 1. provision — ``auto_memory`` : a conversation becomes a daily note + 2. provision — ``auto_memory`` : a follow-up updates the same note + 3. consolidate — ``auto_dream`` : today's daily notes become digest nodes + 4. recall — ``reindex`` + ``search`` : a distinctive fact is retrievable + 5. proactive — ``proactive`` : surfaced interest topics are readable + +The point is to prove the stages *compose*: a fact spoken in step 1 must +survive provisioning, consolidation, and indexing, and still come back out +of ``search`` in step 4. Assertions are deliberately lenient (``>= N`` hit +counts, "any stage that carried the fact") so the test asserts the loop is +intact without pinning the LLM's exact wording. + +Search runs BM25-only (the default ``file_store`` leaves ``embedding_store`` +empty), so this needs LLM_API_KEY but not an embedding key. Requires +LLM_API_KEY (and optionally LLM_BASE_URL / LLM_MODEL_NAME) in the +environment or a .env file at the repo root. Hits the real LLM API. +""" + +import asyncio +import sys +from pathlib import Path + +INTEGRATION_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(INTEGRATION_DIR)) + +# pylint: disable=wrong-import-position +from _workspace_fixture import workspace_env # noqa: E402 + +SESSION_ID = "project-meridian-sync" + +# A distinctive, self-contained topic — invented proper nouns ("Meridian", +# "WebTransport over QUIC") so a later search hit is unambiguously traceable +# back to this conversation rather than to model priors. +_TURN_1 = [ + { + "name": "user", + "role": "user", + "content": ("记一下 Project Meridian 的架构决定:实时协同改用 CRDT," "后端选 Yjs。今天 2026-06-20 定的。"), + }, + { + "name": "assistant", + "role": "assistant", + "content": "好的,已记录:Project Meridian 实时协同采用 CRDT,后端 Yjs,2026-06-20 决定。", + }, + { + "name": "user", + "role": "user", + "content": ( + "动机是旧的 OT (operational transform) 方案在离线编辑合并时冲突太多," + "CRDT 的最终一致性更适合多端离线场景。" + ), + }, +] + +_TURN_2 = [ + { + "name": "user", + "role": "user", + "content": ( + "Meridian 传输层更新:放弃 WebSocket,改走 WebTransport over QUIC。" + "原因是 WebSocket 的 head-of-line blocking 在弱网下让 CRDT update 批量延迟。" + ), + }, + { + "name": "assistant", + "role": "assistant", + "content": "明白,传输层从 WebSocket 切到 WebTransport (QUIC),解决队头阻塞。", + }, + { + "name": "user", + "role": "user", + "content": "下一步:2026-06-27 前完成 WebTransport 的 fallback 到 WebSocket 的降级逻辑。", + }, +] + +# Facts seeded across the two conversation turns; the loop must carry enough +# of them through to the final search. +_TOPIC_FACTS = ("Meridian", "CRDT", "Yjs", "WebTransport", "QUIC", "WebSocket") + + +def _read(p: Path) -> str: + return p.read_text(encoding="utf-8") + + +def _all_workspace_text(env) -> str: + """Concatenate every daily note + digest node — the consolidated memory.""" + parts = [_read(p) for p in env.daily_notes()] + parts += [_read(p) for p in env.digest_files()] + return "\n\n".join(parts) + + +async def _run_loop(env, reme) -> None: + """The 5-stage e2e body, factored out so a local driver can reuse it + against a non-temp workspace (see run_reme_e2e_local.py).""" + today = env.today + print("\n" + "=" * 70) + print("[setup] workspace_root =", env.workspace_dir) + print("[setup] today =", today) + print("=" * 70) + + # ---- 1. provision: CREATE a daily note from turn 1 ---------- + create = await reme.run_job( + "auto_memory", + messages=_TURN_1, + session_id=SESSION_ID, + ) + assert create.success is True, f"auto_memory CREATE failed: {create.answer!r}" + meta = create.metadata or {} + assert meta.get("created") is True, f"expected created=True, got {meta!r}" + note_rel = f"daily/{today}/{SESSION_ID}.md" + assert meta.get("path") == note_rel, f"unexpected note path: {meta!r}" + note_path = env.workspace_dir / note_rel + assert note_path.is_file(), f"daily note not written: {note_path}" + after_create = _read(note_path) + print(f"\n[1/5 provision-create] {note_path} ({len(after_create)} bytes)\n{after_create}") + for fact in ("CRDT", "Yjs"): + assert fact in after_create, f"CREATE dropped fact {fact!r}\n{after_create}" + + # ---- 2. provision: UPDATE the same note from turn 2 --------- + update = await reme.run_job( + "auto_memory", + messages=_TURN_2, + session_id=SESSION_ID, + ) + assert update.success is True, f"auto_memory UPDATE failed: {update.answer!r}" + umeta = update.metadata or {} + assert umeta.get("created") is False, f"expected created=False on UPDATE, got {umeta!r}" + assert umeta.get("path") == note_rel, f"UPDATE wrote a different note: {umeta!r}" + after_update = _read(note_path) + print(f"\n[2/5 provision-update] {note_path} ({len(after_update)} bytes)\n{after_update}") + # old facts survive, new facts land + assert "CRDT" in after_update, f"UPDATE dropped pre-existing fact\n{after_update}" + new_hits = [f for f in ("WebTransport", "QUIC", "WebSocket") if f in after_update] + print(f"[2/5] new transport facts landed: {new_hits}") + assert len(new_hits) >= 2, f"UPDATE only landed {new_hits!r}\n{after_update}" + + # ---- 3. consolidate: dream today's daily notes into digest -- + dream = await reme.run_job( + "auto_dream", + date=today, + hint="Integration e2e: preserve Project Meridian CRDT/Yjs/WebTransport facts.", + topic_count=3, + ) + assert dream.success is True, f"auto_dream failed: {dream.answer!r}\n{dream.metadata!r}" + dmeta = (dream.metadata or {}).get("dream") or {} + print(f"\n[3/5 consolidate] dream summary: {dmeta!r}") + assert dmeta.get("date") == today, f"dream ran for wrong date: {dmeta!r}" + assert dmeta.get("files_changed", 0) >= 1, f"dream changed no files: {dmeta!r}" + digest_paths = env.digest_files() + assert digest_paths, "consolidation produced no digest nodes" + consolidated = _all_workspace_text(env) + carried = [f for f in _TOPIC_FACTS if f in consolidated] + print(f"[3/5] facts present after consolidation: {carried}") + print(f"[3/5] digest nodes: {[str(p.relative_to(env.workspace_dir)) for p in digest_paths]}") + assert len(carried) >= 3, f"consolidation lost the topic; only {carried!r} survived" + + # ---- 4. recall: rebuild the index, then search it ----------- + reindex = await reme.run_job("reindex") + assert reindex.success is True, f"reindex failed: {reindex.answer!r}" + + hit_facts: list[str] = [] + searched: list[str] = [] + for query in ("Project Meridian 实时协同传输层", "CRDT Yjs WebTransport"): + result = await reme.run_job("search", query=query, limit=5) + assert result.success is True, f"search failed for {query!r}: {result.answer!r}" + answer = result.answer or "" + results_meta = (result.metadata or {}).get("results") or [] + print( + f"\n[4/5 recall] query={query!r} -> {len(results_meta)} hit(s)\n" f"{answer[:1500]}", + ) + searched.append(query) + hit_facts += [f for f in _TOPIC_FACTS if f in answer] + hit_facts = sorted(set(hit_facts)) + print(f"[4/5] facts recalled via search across {searched}: {hit_facts}") + assert hit_facts, ( + "search recalled none of the seeded facts — the provision->" "consolidate->index->search loop is broken" + ) + + # ---- 5. proactive: read the interests surfaced by the dream -- + proactive = await reme.run_job("proactive", date=today, include_content=True) + assert proactive.success is True, f"proactive failed: {proactive.answer!r}" + pmeta = proactive.metadata or {} + assert pmeta.get("path") == f"daily/{today}/interests.yaml", f"unexpected interests path: {pmeta!r}" + topics = pmeta.get("topics") or [] + print(f"\n[5/5 proactive] topics: {topics}") + assert topics, f"proactive surfaced no interest topics: {pmeta!r}" + + print("\n" + "=" * 70) + print("test_reme_e2e_full_loop passed") + print("=" * 70) + + +def test_reme_e2e_full_loop(): + """provision -> consolidate -> recall, end to end, via the public job API.""" + + async def run(): + with workspace_env() as env: + reme = await env.make_reme() + try: + await _run_loop(env, reme) + finally: + await env.close_all() + + asyncio.run(run()) + + +if __name__ == "__main__": + print("=== ReMe end-to-end integration test ===") + test_reme_e2e_full_loop() + print("\nIntegration test passed!")