From f0afcfd13a54d2e3aace845395abc7157c139133 Mon Sep 17 00:00:00 2001 From: celanwang Date: Mon, 26 Oct 2026 18:56:30 +0800 Subject: [PATCH] refactor(workspace): consolidate ownership and Codex integration --- AGENTS.md | 13 +- docs/en/configuration.md | 9 +- docs/zh/configuration.md | 4 + reme/application.py | 45 ++-- .../agent_wrapper/codex_agent_wrapper.py | 188 +++++++++------ .../agent_wrapper/codex_mcp_server.py | 224 +++++++++++------- .../agent_wrapper/loopback_job_server.py | 148 ------------ reme/components/application_context.py | 88 ++++--- reme/components/base_component.py | 26 +- reme/schema/workspace.py | 36 +-- reme/steps/base_step.py | 16 +- reme/steps/evolve/proactive/topics.py | 10 +- reme/steps/evolve/proactive/utils.py | 2 +- reme/steps/index/update_changes.py | 5 +- tests/unit/test_codex_agent_wrapper.py | 95 ++------ tests/unit/test_workspace_binding.py | 10 +- tests/unit/test_workspace_phase3.py | 176 ++++++++------ 17 files changed, 548 insertions(+), 547 deletions(-) delete mode 100644 reme/components/agent_wrapper/loopback_job_server.py diff --git a/AGENTS.md b/AGENTS.md index bb53cc0a..3836471e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,8 +132,12 @@ Treat Step instances as invocation-scoped: - `self.context.response.answer`, `success`, and `metadata` are request-scoped output. Because the same response travels through the Step chain, later Steps may consume metadata produced earlier, but it is not application-lifetime or durable storage. -- `self.app_context.metadata` holds in-memory state shared across Job/Step invocations for the life of one - `Application`, such as counters, tool-context state, session maps, or locks. +- `self.app_context.metadata` holds application-lifetime operational state shared across Job/Step invocations, + such as counters. `ApplicationContext.shared_components` owns genuinely shared components; its `components` + compatibility view also exposes default-workspace instances to legacy callers. +- Each `WorkspaceContext` owns its paths, stateful components, tool contexts, search budgets, agent session state, + and locks. `RuntimeContext.workspace` is the explicit binding used by workspace-bound execution; internal Steps + must not resolve a default workspace repeatedly or use the compatibility component view. - Workspace files or a dedicated Component/store hold durable state that must survive restart. Use narrow, namespaced keys in `app_context.metadata` and protect shared mutable values against concurrent access. The @@ -144,8 +148,9 @@ dedicated lifecycle, locking, or persistence, promote it to a typed context fiel Additional Step contracts: -- `Ref` dependencies resolve in this order: Step kwargs, current `RuntimeContext`, then the named application component. - The value is cached only on the current Step instance and cleared before each call. +- `Ref` dependencies resolve in this order: Step kwargs, current `RuntimeContext`, then the bound workspace for scoped + components or the application shared-component registry for shared components. The value is cached only on the + current Step instance and cleared before each call. - `input_mapping` and `output_mapping` copy keys within `RuntimeContext.data`; missing sources are ignored. - Dispatched Steps receive the current `RuntimeContext`, so their data and response are shared. - Base jobs convert uncaught Step errors into `Response(success=False)`; stream jobs emit an error chunk and always a diff --git a/docs/en/configuration.md b/docs/en/configuration.md index d681c786..5bb68d2b 100644 --- a/docs/en/configuration.md +++ b/docs/en/configuration.md @@ -100,8 +100,13 @@ External stateful backends that cannot isolate workspaces (currently the `neo4j` startup in multi-workspace configurations. The binding propagates through nested job calls and agent tool calls (`job_tools`) as a server-owned constraint -that models cannot override. Background and cron jobs bind through their job configuration, so extra workspaces -are watched by declaring explicit per-alias variants: +that models cannot override. For Codex with multiple workspaces, `job_tools` requires the resident HTTP service +with MCP enabled, and the selected jobs must be exposed by that service. Its STDIO bridge forwards to this service +without creating another Application. The multi-workspace one-shot CLI combination is not supported; `mcp_config` +retains its legacy meaning only for single-workspace Codex job tools. + +Background and cron jobs bind through their job configuration, so extra workspaces are watched by declaring +explicit per-alias variants: ```yaml jobs: diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index d2422797..019fbd72 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -102,6 +102,10 @@ Job 调用接受可选的 `workspace` 字段指定绑定的别名,例如 在多 workspace 配置下会在启动时被拒绝。 绑定会经嵌套 Job 调用和 agent 工具调用(`job_tools`)传播,作为模型无法覆盖的服务端约束。 +多 workspace 下的 Codex `job_tools` 需要启用 MCP 的常驻 HTTP 服务,且所选 Job 必须由该服务暴露; +其 STDIO bridge 转发到该服务,不会创建另一个 Application。多 workspace 与 one-shot CLI 的组合暂不支持; +`mcp_config` 仅在单 workspace 的 Codex job tools 中保留原有语义。 + Background 和 cron Job 通过 Job 配置绑定别名,额外 workspace 的 watch 通过显式配置按别名的变体实现: ```yaml diff --git a/reme/application.py b/reme/application.py index 0b4d4b68..76c6e88e 100644 --- a/reme/application.py +++ b/reme/application.py @@ -73,7 +73,6 @@ class Application(BaseComponent): """Instantiate declared components; workspace-scoped ones once per alias.""" multi_workspace = len(self.context.workspaces) > 1 for ctype, group in self.config.components.items(): - self.context.components[ctype] = {} for name, cfg in group.items(): label = f"Component '{name}'" backend_cls = self._backend_class(ctype, cfg, label) @@ -92,12 +91,8 @@ class Application(BaseComponent): workspace_alias=alias, ) workspace.components.setdefault(ctype, {})[name] = instance - # The default alias instance doubles as the plain lookup entry. - self.context.components[ctype][name] = next(iter(self.context.workspaces.values())).components[ - ctype - ][name] else: - self.context.components[ctype][name] = self._instantiate( + self.context.shared_components.setdefault(ctype, {})[name] = self._instantiate( ctype, cfg, label=label, @@ -166,16 +161,13 @@ class Application(BaseComponent): ) -> dict[_NodeKey, BaseComponent]: """One node per component instance; scoped components contribute one node per alias.""" nodes: dict[_NodeKey, BaseComponent] = {} - scoped_keys: set[tuple[str, str]] = set() for alias, workspace in self.context.workspaces.items(): for ctype, group in workspace.components.items(): for name, instance in group.items(): nodes[(ctype, name, alias)] = instance - scoped_keys.add((ctype, name)) - for ctype, group in self.context.components.items(): + for ctype, group in self.context.shared_components.items(): for name, comp in group.items(): - if (ctype, name) not in scoped_keys: - nodes[(ctype, name, "")] = comp + nodes[(ctype, name, "")] = comp if replacement is not None: nodes[replacement[0]] = replacement[1] return nodes @@ -297,11 +289,10 @@ class Application(BaseComponent): """Update an existing component by type/name; never creates missing components.""" async with self._component_mutation_lock: component_type = component_type_name(component_enum) - group = self.context.components.get(component_type) - if not group or name not in group: + component = self.context.get_component(component_type, name) + if component is None: raise KeyError(f"Component '{name}' not found in {component_type}") - component = group[name] # Validate fields without evaluating lazy properties before injection. for key in kwargs: if getattr_static(component, key, _UNSET) is _UNSET and not hasattr(component, key): @@ -317,7 +308,7 @@ class Application(BaseComponent): for group in workspace.components.values(): for instance in group.values(): instances[id(instance)] = instance - for group in self.context.components.values(): + for group in self.context.shared_components.values(): for component in group.values(): instances[id(component)] = component return list(instances.values()) @@ -365,19 +356,27 @@ class Application(BaseComponent): """ async with self._component_mutation_lock: component_type = component_type_name(component_enum) - group = self.context.components.get(component_type) config_group = self.config.components.get(component_type) - if not group or name not in group or config_group is None: - raise KeyError(f"Component '{name}' not found in {component_type}") - - old_component = group[name] - scoped_instances = self.context.workspace_components.get(component_type, {}).get(name) - if scoped_instances is not None and len(scoped_instances) > 1: + scoped_instances = { + alias: workspace.components[component_type][name] + for alias, workspace in self.context.workspaces.items() + if name in workspace.components.get(component_type, {}) + } + if len(scoped_instances) > 1: raise NotImplementedError( f"replace_component does not support workspace-scoped component " f"'{component_type}:{name}' in multi-workspace configurations", ) - alias = next(iter(scoped_instances)) if scoped_instances is not None else None + alias = next(iter(scoped_instances), None) + group = ( + self.context.workspaces[alias].components.get(component_type) + if alias is not None + else self.context.shared_components.get(component_type) + ) + if not group or name not in group or config_group is None: + raise KeyError(f"Component '{name}' not found in {component_type}") + + old_component = group[name] replacement_config = ( config.model_copy(deep=True) if isinstance(config, ComponentConfig) diff --git a/reme/components/agent_wrapper/codex_agent_wrapper.py b/reme/components/agent_wrapper/codex_agent_wrapper.py index f40c678d..078a8500 100644 --- a/reme/components/agent_wrapper/codex_agent_wrapper.py +++ b/reme/components/agent_wrapper/codex_agent_wrapper.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio from collections.abc import AsyncGenerator +from contextlib import suppress from dataclasses import dataclass from functools import partial import hashlib @@ -11,14 +12,16 @@ import json import os from pathlib import Path import sys +import tempfile from typing import Any, TYPE_CHECKING from .base_agent_wrapper import BaseAgentWrapper -from .loopback_job_server import LoopbackJobServer from ..base_component import _context_workspaces from ..component_registry import R +from ...constants import normalize_connect_host from ...enumeration import ChunkEnum from ...schema import StreamChunk, TokenUsage +from ...schema.workspace import WorkspaceAgentState if TYPE_CHECKING: from openai_codex import AsyncCodex, AsyncThread, CodexConfig, RunInput @@ -116,9 +119,7 @@ class CodexAgentWrapper(BaseAgentWrapper): self._release_threads = release_threads self._turn_lock = asyncio.Lock() self._thread_tool_contexts: dict[str, str] = {} - self._thread_owners: dict[str, set[str]] = {} - self._job_server: LoopbackJobServer | None = None - self._mcp_config_warned = False + self._mcp_snapshot_path: Path | None = None @property def session_path(self) -> Path: @@ -149,15 +150,47 @@ class CodexAgentWrapper(BaseAgentWrapper): relative_source = os.path.relpath(source, target.parent) target.symlink_to(relative_source, target_is_directory=True) - def _mcp_config_source_unused(self, kwargs: dict[str, Any]) -> None: - """Warn once that job_tools now execute in the parent via the loopback endpoint.""" + @property + def wrapper_session_path(self) -> Path: + """ReMe-owned session data, kept separate from a shared OAuth CODEX_HOME.""" + if self.app_context is None: + return self.workspace_path / "mem_session" / "codex" + return self.workspace_path / self.app_context.app_config.mem_session_dir / "codex" + + def _explicit_mcp_config(self, kwargs: dict[str, Any]) -> str | None: value = kwargs.get("mcp_config") if "mcp_config" in kwargs else self.mcp_config - if value is not None and not self._mcp_config_warned: - self._mcp_config_warned = True - self.logger.warning( - "mcp_config is ignored for job_tools: Codex job tools execute in the parent " - "process via the wrapper-owned loopback endpoint", - ) + if value is None: + return None + source = Path(str(value)).expanduser() + if source.suffix in {".yaml", ".yml", ".json"}: + if not source.is_absolute(): + source = self.workspace_path / source + return str(source.absolute()) + return str(value) + + def _effective_config_snapshot(self) -> Path: + """Create one private snapshot that remains valid for the client lifetime.""" + if self._mcp_snapshot_path is not None: + return self._mcp_snapshot_path + if self.app_context is None: + raise RuntimeError("Cannot snapshot MCP config without an app_context") + snapshot_dir = self.wrapper_session_path / "reme-mcp" + snapshot_dir.mkdir(parents=True, exist_ok=True) + fd, raw_path = tempfile.mkstemp(prefix="config-", suffix=".json", dir=snapshot_dir) + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(self.app_context.app_config.model_dump(mode="json"), stream) + except BaseException: + with suppress(OSError): + os.close(fd) + Path(raw_path).unlink(missing_ok=True) + raise + self._mcp_snapshot_path = Path(raw_path) + return self._mcp_snapshot_path + + def _mcp_config_source(self, kwargs: dict[str, Any]) -> str: + return self._explicit_mcp_config(kwargs) or str(self._effective_config_snapshot()) @staticmethod def _resolve_auth_config(auth_mode: str, api_key: str = "", base_url: str = "") -> _CodexAuthConfig: @@ -209,19 +242,32 @@ class CodexAgentWrapper(BaseAgentWrapper): names = ", ".join(invalid) raise TypeError(f"Codex client options must be configured on the wrapper: {names}") - async def _ensure_job_server(self) -> LoopbackJobServer: - """Start the wrapper-owned loopback job endpoint on first bridge use.""" - if self._job_server is None: - if self.app_context is None: - raise RuntimeError("Codex job_tools require an app_context") - self._job_server = LoopbackJobServer(self.app_context) - await self._job_server.start() - return self._job_server + def _resident_mcp_endpoint(self) -> str: + from ..service import HttpService, MCPService + + service = self.app_context.service if self.app_context is not None else None + if isinstance(service, HttpService) and service.mcp_enabled: + return f"http://{normalize_connect_host(service.host)}:{service.port}{service.mcp_path}" + if isinstance(service, MCPService) and service.transport == "streamable-http": + return f"http://{normalize_connect_host(service.host)}:{service.port}/mcp" + raise RuntimeError( + "Multi-workspace Codex job_tools require a resident HTTP MCP service; " + "one-shot CLI and other service backends are not supported for this combination", + ) + + @staticmethod + def _validate_resident_jobs(service, jobs) -> None: + unavailable = [ + job.name + for job in jobs + if not job.enabled or not job.enable_serve or (service.jobs is not None and job.name not in service.jobs) + ] + if unavailable: + raise ValueError(f"Codex job_tools must be exposed by the resident MCP service: {', '.join(unavailable)}") def _mcp_server_config(self, kwargs: dict[str, Any]) -> dict[str, Any] | None: from ..job import BackgroundJob, StreamJob - self._mcp_config_source_unused(kwargs) job_names = list(dict.fromkeys(kwargs.get("job_tools") or [])) if not job_names: return None @@ -229,26 +275,29 @@ class CodexAgentWrapper(BaseAgentWrapper): unsupported = [job.name for job in jobs if isinstance(job, (BackgroundJob, StreamJob))] if unsupported: raise TypeError(f"Codex job_tools must be non-stream request jobs: {', '.join(unsupported)}") - if self._job_server is None: - raise RuntimeError("Codex job endpoint is not started; call _ensure_job_server first") - alias = str(kwargs.get("workspace") or "") - args = [ - "-m", - "reme.components.agent_wrapper.codex_mcp_server", - "--endpoint", - self._job_server.url, - "--token", - self._job_server.token, - ] - if alias: - args.extend(["--workspace-alias", alias]) + workspaces = _context_workspaces(self.app_context) + multi_workspace = workspaces is not None and len(workspaces) > 1 + if multi_workspace: + if self._explicit_mcp_config(kwargs) is not None: + raise ValueError("mcp_config is only supported for single-workspace Codex job_tools") + endpoint = self._resident_mcp_endpoint() + self._validate_resident_jobs(self.app_context.service, jobs) + args = ["-m", "reme.components.agent_wrapper.codex_mcp_server", "--endpoint", endpoint] + else: + args = [ + "-m", + "reme.components.agent_wrapper.codex_mcp_server", + "--config", + self._mcp_config_source(kwargs), + "--workspace", + str(self.workspace_path), + ] for name in job_names: args.extend(["--job", name]) args.extend(["--tool-context-id", str(kwargs.get("tool_context_id") or "")]) injected_job_kwargs = dict(kwargs.get("injected_job_kwargs") or {}) - if workspace := kwargs.get("workspace"): - # The bridge injects the binding into nested job calls; models cannot override it. + if multi_workspace and (workspace := kwargs.get("workspace")): injected_job_kwargs["workspace"] = workspace if injected_job_kwargs: args.extend(["--injected-job-kwargs", json.dumps(injected_job_kwargs, sort_keys=True)]) @@ -288,6 +337,7 @@ class CodexAgentWrapper(BaseAgentWrapper): from openai_codex.types import Personality, ThreadSource, ThreadStartSource alias = str(kwargs.get("workspace") or "") + tool_contexts = self._tool_contexts(alias) resume = kwargs.get("resume") or "" session_id = kwargs.get("session_id") or "" if resume and session_id and resume != session_id: @@ -297,12 +347,9 @@ class CodexAgentWrapper(BaseAgentWrapper): if fork_session and not thread_id: raise ValueError("fork_session=True requires resume or session_id") if thread_id: - # Cross-workspace thread access is rejected before touching the SDK. await self._check_thread_ownership(thread_id, alias) - if kwargs.get("job_tools"): - await self._ensure_job_server() - if self._release_threads and self._thread_tool_contexts: - await self._prune_thread_tool_contexts(codex) + if self._release_threads and tool_contexts: + await self._prune_thread_tool_contexts(codex, alias) requested_tool_context = json.dumps( { "tool_context_id": str(kwargs.get("tool_context_id") or ""), @@ -311,9 +358,8 @@ class CodexAgentWrapper(BaseAgentWrapper): sort_keys=True, default=str, ) - guard_key = f"{alias}/{thread_id}" if alias else thread_id - if not fork_session and guard_key in self._thread_tool_contexts: - if requested_tool_context != self._thread_tool_contexts[guard_key]: + if not fork_session and thread_id in tool_contexts: + if requested_tool_context != tool_contexts[thread_id]: raise ValueError("tool_context_id and injected_job_kwargs cannot change when resuming a Codex thread") common = { @@ -348,19 +394,28 @@ class CodexAgentWrapper(BaseAgentWrapper): **common, ) if fork_session or not thread_id: - # Threads created under this binding are owned by this workspace. self._record_thread_ownership(alias, thread.id) - self._thread_tool_contexts[f"{alias}/{thread.id}" if alias else thread.id] = requested_tool_context + tool_contexts[thread.id] = requested_tool_context return thread # ----- Thread ownership (per-workspace, persisted) --------------------- - def _normalize_thread_alias(self, alias: str) -> str: - """Map a call alias to a concrete workspace alias ("" means the default).""" + def _agent_state(self, alias: str) -> WorkspaceAgentState | None: workspaces = _context_workspaces(self.app_context) if not workspaces: - return alias - return alias if alias in workspaces else next(iter(workspaces)) + return None + return workspaces[self._normalize_thread_alias(alias)].agent_state(self.name) + + def _tool_contexts(self, alias: str) -> dict[str, str]: + state = self._agent_state(alias) + return state.thread_tool_contexts if state is not None else self._thread_tool_contexts + + def _normalize_thread_alias(self, alias: str) -> str: + """Map an omitted alias to the default while preserving explicit aliases.""" + workspaces = _context_workspaces(self.app_context) + if workspaces and not alias: + return next(iter(workspaces)) + return alias def _thread_ownership_path(self, alias: str) -> Path | None: workspaces = _context_workspaces(self.app_context) @@ -372,9 +427,10 @@ class CodexAgentWrapper(BaseAgentWrapper): return workspace.mem_session_path / "codex" / "owned_threads.json" def _owned_thread_ids(self, alias: str) -> set[str]: - """Load (once) the persisted ids of threads this workspace created.""" - if alias in self._thread_owners: - return self._thread_owners[alias] + """Load the persisted ids into the owning workspace's agent state.""" + state = self._agent_state(alias) + if state is not None and state.owned_thread_ids is not None: + return state.owned_thread_ids ids: set[str] = set() path = self._thread_ownership_path(alias) if path is not None and path.is_file(): @@ -383,7 +439,8 @@ class CodexAgentWrapper(BaseAgentWrapper): ids = {str(thread_id) for thread_id in raw.get("thread_ids", [])} except (OSError, ValueError, AttributeError) as exc: self.logger.warning(f"Ignoring unreadable Codex thread ownership file {path}: {exc}") - self._thread_owners[alias] = ids + if state is not None: + state.owned_thread_ids = ids return ids def _record_thread_ownership(self, alias: str, thread_id: str) -> None: @@ -453,20 +510,19 @@ class CodexAgentWrapper(BaseAgentWrapper): return codex async def _close(self) -> None: - """Close the persistent app-server and the wrapper-owned job endpoint.""" + """Close the persistent app-server and clear session execution guards.""" async with self._turn_lock: codex, self._codex = self._codex, None self._thread_tool_contexts.clear() - try: - if codex is not None: - await codex.close() - finally: - if self._job_server is not None: - job_server, self._job_server = self._job_server, None - await job_server.close() + for workspace in (_context_workspaces(self.app_context) or {}).values(): + if state := workspace.agent_states.get(self.name): + state.thread_tool_contexts.clear() + if codex is not None: + await codex.close() - async def _prune_thread_tool_contexts(self, codex: AsyncCodex) -> None: + async def _prune_thread_tool_contexts(self, codex: AsyncCodex, alias: str = "") -> None: """Forget tool constraints only after the server confirms a thread is unloaded.""" + tool_contexts = self._tool_contexts(alias) try: from openai_codex.generated.v2_all import ThreadLoadedListResponse @@ -485,9 +541,9 @@ class CodexAgentWrapper(BaseAgentWrapper): except Exception as exc: # pylint: disable=broad-exception-caught self.logger.warning(f"Failed to list loaded Codex threads: {exc}") return - self._thread_tool_contexts = { - key: context for key, context in self._thread_tool_contexts.items() if key.rsplit("/", 1)[-1] in loaded_ids - } + retained = {thread_id: context for thread_id, context in tool_contexts.items() if thread_id in loaded_ids} + tool_contexts.clear() + tool_contexts.update(retained) async def _release_thread(self, codex: AsyncCodex, thread_id: str) -> None: """Unsubscribe so the app-server can unload the thread once it is idle.""" diff --git a/reme/components/agent_wrapper/codex_mcp_server.py b/reme/components/agent_wrapper/codex_mcp_server.py index dd0d6ad9..1cba8a9b 100644 --- a/reme/components/agent_wrapper/codex_mcp_server.py +++ b/reme/components/agent_wrapper/codex_mcp_server.py @@ -1,25 +1,27 @@ -"""Thin FastMCP STDIO bridge forwarding Codex job tools to the parent process. +"""Expose ReMe jobs to Codex over FastMCP STDIO. -This subprocess never builds a ReMe Application or any workspace store. It -asks the parent-owned loopback endpoint (LoopbackJobServer) for job metadata -and forwards every tool call — together with the server-owned workspace alias, -tool context id, and injected kwargs — back to the parent for execution, so -all workspace state has exactly one owner. +Multi-workspace callers forward to the resident MCP service without creating +an Application. Single-workspace callers retain the legacy child-service path. """ import argparse +import asyncio import json +from pathlib import Path from typing import Any -import httpx +from .base_agent_wrapper import BaseAgentWrapper +from ..client import MCPClient +from ...config import resolve_app_config +from ...reme import ReMe def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Forward selected ReMe jobs from the parent loopback endpoint") - parser.add_argument("--endpoint", required=True, help="Parent loopback endpoint base URL") - parser.add_argument("--token", required=True, help="Bearer token for the loopback endpoint") + parser = argparse.ArgumentParser(description="Expose selected ReMe jobs over FastMCP STDIO") + parser.add_argument("--config", default="default", help="ReMe config name or file path") + parser.add_argument("--workspace", help="Single-workspace directory for the legacy child service") + parser.add_argument("--endpoint", help="Resident streamable-HTTP MCP endpoint for forwarding") parser.add_argument("--job", dest="jobs", action="append", required=True, help="ReMe job name; repeat as needed") - parser.add_argument("--workspace-alias", default="", help="Workspace alias injected into every job call") parser.add_argument("--tool-context-id", default="", help="Context id injected into every job call") parser.add_argument( "--injected-job-kwargs", @@ -30,89 +32,145 @@ def _parse_args() -> argparse.Namespace: return parser.parse_args() -class _EndpointClient: - """Minimal client for the parent's loopback job server.""" +def _prepare_config( + config: dict[str, Any], + job_names: list[str], + tool_context_id: str = "", + injected_job_kwargs: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Configure the legacy single-workspace child Application for MCP STDIO.""" + selected = set(job_names) + jobs: dict[str, dict[str, Any]] = {} + for name, raw_job_config in (config.get("jobs") or {}).items(): + job_config = dict(raw_job_config) + if job_config.get("backend") in {"background", "cron"}: + continue + if name in selected: + job_config["enable_serve"] = True + jobs[name] = job_config - def __init__(self, endpoint: str, token: str, timeout: float) -> None: - self._client = httpx.Client( - base_url=endpoint, - headers={"Authorization": f"Bearer {token}"}, - timeout=timeout, - ) + missing = sorted(selected.difference(jobs)) + if missing: + raise KeyError(f"Codex job tools not found or not request jobs: {', '.join(missing)}") - def list_jobs(self) -> list[dict[str, Any]]: - """Fetch the servable job metadata from the parent endpoint.""" - response = self._client.get("/jobs") - response.raise_for_status() - return response.json()["jobs"] + service: dict[str, Any] = { + "backend": "mcp", + "transport": "stdio", + "jobs": job_names, + "tool_error_on_failure": True, + } + injected = dict(injected_job_kwargs or {}) + if tool_context_id: + injected["tool_context_id"] = tool_context_id + if injected: + service["injected_job_kwargs"] = injected - def call(self, job: str, kwargs: dict[str, Any], injected_job_kwargs: dict[str, Any] | None = None) -> str: - """Forward one tool call to the parent endpoint and unwrap its answer.""" - response = self._client.post( - "/call", - json={"job": job, "kwargs": kwargs, "injected_job_kwargs": injected_job_kwargs or {}}, - ) - if response.status_code != 200: - try: - detail = response.json().get("error") or response.text - except ValueError: - detail = response.text - raise RuntimeError(str(detail)) - payload = response.json() - if not payload.get("success"): - raise RuntimeError(str(payload.get("answer") or payload)) - return str(payload.get("answer") or "") - - def close(self) -> None: - """Close the underlying HTTP client.""" - self._client.close() + prepared = dict(config) + prepared["jobs"] = jobs + prepared["service"] = service + return prepared -def main() -> None: - """Fetch job metadata from the parent and serve forwarding tools over STDIO.""" +class _ResidentMCPClient: + """Reuse the standard ReMe MCP client for resident tool discovery and calls.""" + + def __init__(self, endpoint: str, timeout: float) -> None: + from fastmcp.client import StreamableHttpTransport + + self._client = MCPClient(transport=StreamableHttpTransport(url=endpoint), timeout=timeout) + + async def start(self) -> None: + """Connect to the resident service.""" + await self._client.start() + + async def list_jobs(self) -> list[dict[str, Any]]: + """Return the resident service's tool descriptors.""" + return await self._client.list_actions() + + async def call(self, job: str, kwargs: dict[str, Any], injected: dict[str, Any]) -> str: + """Call one resident tool with server-owned arguments enforced.""" + # pylint: disable=protected-access + payload = BaseAgentWrapper._merge_injected_job_kwargs(kwargs, injected) + async for text in self._client(action=job, **payload): + return text + return "" + + async def close(self) -> None: + """Close the resident connection.""" + await self._client.close() + + +def _injected_kwargs(args: argparse.Namespace) -> dict[str, Any]: + injected = json.loads(args.injected_job_kwargs) if args.injected_job_kwargs else {} + if not isinstance(injected, dict): + raise TypeError("--injected-job-kwargs must be a JSON object") + injected = dict(injected) + if args.tool_context_id: + injected["tool_context_id"] = args.tool_context_id + return injected + + +async def _run_forwarder(args: argparse.Namespace, injected: dict[str, Any]) -> None: from fastmcp import FastMCP from fastmcp.exceptions import ToolError from fastmcp.tools import FunctionTool - args = _parse_args() - injected_job_kwargs = json.loads(args.injected_job_kwargs) if args.injected_job_kwargs else {} - if not isinstance(injected_job_kwargs, dict): - raise TypeError("--injected-job-kwargs must be a JSON object") - injected = dict(injected_job_kwargs) - if args.workspace_alias: - injected["workspace"] = args.workspace_alias - if args.tool_context_id: - injected["tool_context_id"] = args.tool_context_id - - client = _EndpointClient(args.endpoint, args.token, args.tool_timeout) - metadata = {entry["name"]: entry for entry in client.list_jobs()} - missing = sorted(set(args.jobs) - set(metadata)) - if missing: - raise KeyError(f"Codex job tools not found on the parent endpoint: {', '.join(missing)}") - - server = FastMCP("reme-codex-bridge") - for name in args.jobs: - entry = metadata[name] - - def forward(_client=client, _name=name, _injected=None, **kwargs): - try: - return _client.call(_name, kwargs, _injected or injected) - except Exception as exc: # noqa: BLE001 - surface every failure as a tool error - raise ToolError(str(exc)) from exc - - server.add_tool( - FunctionTool( - name=name, - description=entry.get("description") or "", - parameters=entry.get("parameters") or {"type": "object", "properties": {}}, - fn=forward, - ), - ) - + client = _ResidentMCPClient(args.endpoint, args.tool_timeout) + await client.start() try: - server.run(transport="stdio") + metadata = {entry["name"]: entry for entry in await client.list_jobs()} + missing = sorted(set(args.jobs) - set(metadata)) + if missing: + raise KeyError(f"Codex job tools not found on the resident MCP service: {', '.join(missing)}") + + server = FastMCP("reme-codex-bridge") + for name in args.jobs: + entry = metadata[name] + + def make_forwarder(job_name: str): + async def forward(**kwargs): + try: + return await client.call(job_name, kwargs, injected) + except Exception as exc: # noqa: BLE001 - surface every failure as a tool error + raise ToolError(str(exc)) from exc + + return forward + + parameters = entry.get("inputSchema") or entry.get("parameters") + # pylint: disable=protected-access + parameters = BaseAgentWrapper._strip_injected_parameters(parameters, injected) + server.add_tool( + FunctionTool( + name=name, + description=entry.get("description") or "", + parameters=parameters or {"type": "object", "properties": {}}, + fn=make_forwarder(name), + ), + ) + await server.run_async(transport="stdio") finally: - client.close() + await client.close() + + +def main() -> None: + """Forward to a resident service or start the legacy single-workspace child service.""" + args = _parse_args() + injected = _injected_kwargs(args) + if args.endpoint: + asyncio.run(_run_forwarder(args, injected)) + return + if not args.workspace: + raise ValueError("--workspace is required when --endpoint is not supplied") + config = resolve_app_config( + config=args.config, + workspace_dir=str(Path(args.workspace).absolute()), + enable_logo=False, + log_to_console=False, + log_to_file=False, + log_config=False, + ) + config = _prepare_config(config, args.jobs, args.tool_context_id, injected) + ReMe(**config).run_app() if __name__ == "__main__": diff --git a/reme/components/agent_wrapper/loopback_job_server.py b/reme/components/agent_wrapper/loopback_job_server.py deleted file mode 100644 index fa242336..00000000 --- a/reme/components/agent_wrapper/loopback_job_server.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Wrapper-owned loopback job endpoint for agent-bridge subprocesses. - -The Codex job bridge used to spawn a full child Application per thread; every -thread then held its own workspace stores and became a second persistence -writer. This endpoint keeps job execution in the parent process instead: a -minimal HTTP/JSON server on 127.0.0.1 with an ephemeral port and a bearer -token, owned by one CodexAgentWrapper instance for its lifetime. Bridge -subprocesses are thin FastMCP forwarders that call this endpoint. -""" - -import asyncio -import json -import secrets -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from ..application_context import ApplicationContext - from ..job import BaseJob - -_REASON = { - 200: "OK", - 400: "Bad Request", - 401: "Unauthorized", - 404: "Not Found", - 500: "Internal Server Error", -} - - -class LoopbackJobServer: - """Serve this application's request jobs to trusted local subprocesses.""" - - def __init__(self, app_context: "ApplicationContext") -> None: - self._app_context = app_context - self.token = secrets.token_urlsafe(24) - self._server: asyncio.Server | None = None - - @property - def is_running(self) -> bool: - """Whether the endpoint is currently bound.""" - return self._server is not None - - @property - def url(self) -> str: - """Base URL of the bound endpoint.""" - if self._server is None: - raise RuntimeError("Loopback job server is not started") - port = self._server.sockets[0].getsockname()[1] - return f"http://127.0.0.1:{port}" - - async def start(self) -> None: - """Bind the endpoint; idempotent so lazy bridge creation can retry.""" - if self._server is None: - self._server = await asyncio.start_server(self._handle, "127.0.0.1", 0) - - async def close(self) -> None: - """Stop the endpoint and release the port.""" - server, self._server = self._server, None - if server is not None: - server.close() - await server.wait_closed() - - # ----- HTTP handling --------------------------------------------------- - - async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), timeout=30) - request_line, *header_lines = head.decode("latin-1").split("\r\n") - method, target, _version = request_line.split(" ", 2) - headers = {} - for line in header_lines: - if ":" in line: - key, value = line.split(":", 1) - headers[key.strip().lower()] = value.strip() - length = int(headers.get("content-length", 0) or 0) - body = await reader.readexactly(length) if length else b"" - status, payload = await self._dispatch(method, target, headers, body) - except Exception as exc: # pylint: disable=broad-exception-caught - status, payload = 500, {"error": f"{type(exc).__name__}: {exc}"} - data = json.dumps(payload).encode() - writer.write( - f"HTTP/1.1 {status} {_REASON.get(status, 'OK')}\r\n" - "content-type: application/json\r\n" - f"content-length: {len(data)}\r\n" - "connection: close\r\n\r\n".encode() + data, - ) - try: - await writer.drain() - finally: - writer.close() - - async def _dispatch( - self, - method: str, - target: str, - headers: dict[str, str], - body: bytes, - ) -> tuple[int, dict[str, Any]]: - if headers.get("authorization") != f"Bearer {self.token}": - return 401, {"error": "unauthorized"} - if method == "GET" and target == "/jobs": - return 200, {"jobs": self._job_metadata()} - if method == "POST" and target == "/call": - try: - request = json.loads(body or b"{}") - except json.JSONDecodeError as exc: - return 400, {"error": f"invalid JSON body: {exc}"} - return await self._call(request) - return 404, {"error": "not found"} - - # ----- Job surface ------------------------------------------------------- - - def _request_jobs(self) -> dict[str, "BaseJob"]: - """Return the application's non-stream, non-background jobs by name.""" - from ..job import BackgroundJob, StreamJob - - return { - name: job for name, job in self._app_context.jobs.items() if not isinstance(job, (StreamJob, BackgroundJob)) - } - - def _job_metadata(self) -> list[dict[str, Any]]: - """Describe every servable job for bridge tool registration.""" - return [ - {"name": name, "description": job.description, "parameters": job.parameters} - for name, job in sorted(self._request_jobs().items()) - ] - - async def _call(self, request: dict[str, Any]) -> tuple[int, dict[str, Any]]: - name = str(request.get("job") or "") - job = self._request_jobs().get(name) - if job is None: - return 404, {"error": f"job {name!r} not found or not a request job"} - - kwargs = dict(request.get("kwargs") or {}) - injected = dict(request.get("injected_job_kwargs") or {}) - conflicts = sorted(injected.keys() & kwargs.keys()) - if conflicts: - return 400, {"error": f"{', '.join(conflicts)} injected by the server and cannot be provided"} - kwargs.update(injected) - if workspace := request.get("workspace"): - kwargs["workspace"] = workspace - if tool_context_id := request.get("tool_context_id"): - kwargs["tool_context_id"] = tool_context_id - - try: - response = await job(**kwargs) - except ValueError as exc: - return 400, {"error": str(exc)} - return 200, {"success": response.success, "answer": response.answer} diff --git a/reme/components/application_context.py b/reme/components/application_context.py index e52475b6..e2b617be 100644 --- a/reme/components/application_context.py +++ b/reme/components/application_context.py @@ -1,5 +1,6 @@ """Application context: shared state container for components, jobs, and service.""" +from collections.abc import Iterator, MutableMapping from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Any @@ -12,65 +13,80 @@ if TYPE_CHECKING: from .service import BaseService -class ApplicationContext: - """Passive state container holding parsed config and wired components. +class _LegacyComponentView(MutableMapping): + """Compatibility mapping combining shared components with default-workspace instances.""" - The Application class performs the actual wiring (registry lookups and - component instantiation); this class only stores the results so that - components, jobs, and the service can find each other at runtime. - """ + def __init__(self, context: "ApplicationContext") -> None: + self._context = context + + def _default_components(self) -> dict: + return next(iter(self._context.workspaces.values())).components + + def __getitem__(self, key: str) -> dict: + if key in self._context.shared_components: + return self._context.shared_components[key] + return self._default_components()[key] + + def __setitem__(self, key: str, value: dict) -> None: + if value and all(getattr(component, "workspace_scoped", False) is True for component in value.values()): + self._default_components()[key] = value + self._context.shared_components.pop(key, None) + else: + self._context.shared_components[key] = value + self._default_components().pop(key, None) + + def __delitem__(self, key: str) -> None: + if key in self._context.shared_components: + del self._context.shared_components[key] + else: + del self._default_components()[key] + + def __iter__(self) -> Iterator[str]: + yield from self._context.shared_components + yield from (key for key in self._default_components() if key not in self._context.shared_components) + + def __len__(self) -> int: + return len(set(self._context.shared_components) | set(self._default_components())) + + +class ApplicationContext: + """Application-wide configuration, shared resources, and workspace registry.""" def __init__(self, *, registry: ComponentRegistry | None = None, **kwargs): - # Parse raw kwargs into a typed, validated config object. self.app_config: ApplicationConfig = ApplicationConfig(**kwargs) self.registry = registry or create_application_registry() - - # Ordered per-workspace path views; the first entry is the default workspace. self.workspaces: dict[str, WorkspaceContext] = { alias: WorkspaceContext.from_config(alias, path, self.app_config) for alias, path in self.app_config.workspaces.items() } - self.workspace: WorkspaceContext = next(iter(self.workspaces.values())) - # Populated by Application during initialization. + self.workspace: WorkspaceContext = next(iter(self.workspaces.values())) self.service: "BaseService | None" = None - self.components: dict[str, dict[str, "BaseComponent"]] = {} + self.shared_components: dict[str, dict[str, "BaseComponent"]] = {} + self._components = _LegacyComponentView(self) self.jobs: dict[str, "BaseJob"] = {} self.thread_pool: ThreadPoolExecutor | None = None - - # Application-lifetime shared state. Values remain available across Job and Step - # invocations while this Application is running, so Jobs may keep cross-call state here. - # This is in-memory state, not durable storage; use workspace files or a store when state - # must survive an Application restart. self.metadata: dict[str, Any] = {} @property - def workspace_components(self) -> dict[str, dict[str, dict[str, "BaseComponent"]]]: - """Compatibility view of scoped instances, ctype -> name -> alias. + def components(self) -> MutableMapping: + """Legacy component lookup; internal execution uses explicit shared or workspace ownership.""" + return self._components - The authoritative owner of these instances is each WorkspaceContext; - this view exists for transitional readers. - """ - grouped: dict[str, dict[str, dict[str, "BaseComponent"]]] = {} - for alias, workspace in self.workspaces.items(): - for ctype, group in workspace.components.items(): - for name, instance in group.items(): - grouped.setdefault(ctype, {}).setdefault(name, {})[alias] = instance - return grouped + @components.setter + def components(self, value: dict) -> None: + self.shared_components = {} + for ctype, group in value.items(): + self._components[ctype] = group def get_component(self, ctype: str, name: str, alias: str | None = None) -> "BaseComponent | None": - """Resolve a component, honoring workspace bindings for scoped components. - - Returns None when the component is not configured at all. Raises - KeyError when a workspace-scoped component has no instance for the - requested alias — a missing scoped resource must fail, never fall back. - """ + """Compatibility resolver for external callers that select a workspace by alias.""" alias = alias or next(iter(self.workspaces)) workspace = self.workspaces.get(alias) if workspace is None: raise KeyError(f"Unknown workspace alias {alias!r}") if name in workspace.components.get(ctype, {}): - return workspace.components[ctype][name] + return workspace.get_component(ctype, name) if any(name in ws.components.get(ctype, {}) for ws in self.workspaces.values()): raise KeyError(f"Component '{name}' in {ctype} has no instance for workspace alias {alias!r}") - return self.components.get(ctype, {}).get(name) + return self.shared_components.get(ctype, {}).get(name) diff --git a/reme/components/base_component.py b/reme/components/base_component.py index 1f704bfb..f4075639 100644 --- a/reme/components/base_component.py +++ b/reme/components/base_component.py @@ -221,20 +221,24 @@ class BaseComponent(ComponentMixin, ABC): setattr(self, attr, None) def _resolve_from_context(self, attr: str, dep: Dependency) -> None: - """Context-bound mode: look up the component from ``app_context.components``. - - Workspace-scoped dependencies resolve to the same-alias instance as the - dependent; a missing alias instance is an error, never a fallback. - """ + """Resolve scoped dependencies from their workspace and shared ones from the application.""" workspaces = getattr(self.app_context, "workspaces", None) workspace = workspaces.get(self.workspace_alias) if workspaces and self.workspace_alias is not None else None - scoped_group = workspace.components.get(dep.ctype) if workspace is not None else None - if scoped_group is not None: - target = scoped_group.get(dep.name) - if target is None: - raise ValueError(f"{dep.ctype} '{dep.name}' has no instance for workspace '{self.workspace_alias}'.") + if workspace is not None and dep.name in workspace.components.get(dep.ctype, {}): + target = workspace.get_component(dep.ctype, dep.name) + elif workspace is not None and any( + dep.name in candidate.components.get(dep.ctype, {}) for candidate in workspaces.values() + ): + raise ValueError(f"{dep.ctype} '{dep.name}' has no instance for workspace '{self.workspace_alias}'.") else: - target = self.app_context.components.get(dep.ctype, {}).get(dep.name) + shared = getattr(self.app_context, "shared_components", None) + if shared is None: + shared = self.app_context.components + target = shared.get(dep.ctype, {}).get(dep.name) + if target is None and workspaces: + default_workspace = next(iter(workspaces.values())) + if dep.name in default_workspace.components.get(dep.ctype, {}): + target = default_workspace.get_component(dep.ctype, dep.name) if target is not None: setattr(self, attr, target) elif dep.optional: diff --git a/reme/schema/workspace.py b/reme/schema/workspace.py index f30cb92b..30f3070c 100644 --- a/reme/schema/workspace.py +++ b/reme/schema/workspace.py @@ -9,17 +9,17 @@ if TYPE_CHECKING: from .application_config import ApplicationConfig +@dataclass +class WorkspaceAgentState: + """Session execution state owned by one agent wrapper within one workspace.""" + + thread_tool_contexts: dict[str, str] = field(default_factory=dict) + owned_thread_ids: set[str] | None = None + + @dataclass(frozen=True) class WorkspaceContext: - """Owns everything scoped to one workspace of an application. - - Built from ApplicationConfig when the application context is created. - Beyond the resolved paths, the workspace owns its stateful component - instances (stores, graphs, indexes, catalogs), in-memory execution state - (drafts, search dedup, call budgets), session ownership records, and - workspace-scoped locks. Shared, stateless clients stay on the - ApplicationContext; per-call bindings reference this object. - """ + """Owns the paths, stateful components, execution state, and locks of one workspace.""" alias: str workspace_path: Path @@ -29,17 +29,16 @@ class WorkspaceContext: resource_path: Path | None daily_path: Path | None digest_path: Path | None - # Workspace-scoped component instances keyed by component type then name. - # Values are BaseComponent instances; typed as Any to keep this module - # free of component imports (schema must not depend on components). components: dict[str, dict[str, Any]] = field(default_factory=dict) - # Drafts and search dedup state, keyed by tool_context_id. tool_contexts: dict[str, Any] = field(default_factory=dict) - # Search call budgets keyed by tool_context_id. search_call_budgets: dict[str, Any] = field(default_factory=dict) - # Workspace-scoped named asyncio locks (e.g. dream integration). + agent_states: dict[str, WorkspaceAgentState] = field(default_factory=dict) locks: dict[str, asyncio.Lock] = field(default_factory=dict) + def agent_state(self, wrapper_name: str) -> WorkspaceAgentState: + """Return the named wrapper's state within this workspace.""" + return self.agent_states.setdefault(wrapper_name, WorkspaceAgentState()) + @classmethod def from_config(cls, alias: str, workspace: str, config: "ApplicationConfig") -> "WorkspaceContext": """Resolve one alias of the config's workspaces mapping into absolute paths.""" @@ -59,6 +58,13 @@ class WorkspaceContext: digest_path=child(config.digest_dir), ) + def get_component(self, ctype: str, name: str) -> Any: + """Return a component owned by this workspace, never a default or shared substitute.""" + try: + return self.components[ctype][name] + except KeyError as exc: + raise KeyError(f"Component '{name}' in {ctype} has no instance for workspace '{self.alias}'") from exc + def storage_dirs(self) -> list[Path]: """Subdirectories the application materializes at startup, in configuration order.""" return [ diff --git a/reme/steps/base_step.py b/reme/steps/base_step.py index 15702fad..c70beb97 100644 --- a/reme/steps/base_step.py +++ b/reme/steps/base_step.py @@ -87,14 +87,16 @@ class Ref: if self.optional: return None raise RuntimeError(f"app_context is not set when resolving '{self.key}'") - if _context_workspaces(obj.app_context) is not None: + if (workspaces := _context_workspaces(obj.app_context)) is not None: context = obj.context bound = getattr(context, "workspace", None) if context is not None else None - comp = obj.app_context.get_component( - self.comp_enum, - name, - alias=bound.alias if bound is not None else None, - ) + is_scoped = any(name in workspace.components.get(self.comp_enum, {}) for workspace in workspaces.values()) + if is_scoped: + if bound is None: + raise RuntimeError(f"A workspace binding is required when resolving '{self.key}'") + comp = bound.get_component(self.comp_enum, name) + else: + comp = obj.app_context.shared_components.get(self.comp_enum, {}).get(name) else: # Test doubles and standalone-style contexts without the workspace layer. comp = obj.app_context.components.get(self.comp_enum, {}).get(name) @@ -159,6 +161,8 @@ class BaseStep(ComponentMixin, ABC): del self.__dict__[key] self.context = RuntimeContext.from_context(context, **kwargs) assert self.context is not None + if (workspaces := _context_workspaces(self.app_context)) is not None and self.context.workspace is None: + self.context.workspace = next(iter(workspaces.values())) if self.input_mapping: self.context.apply_mapping(self.input_mapping) result = await self.execute() diff --git a/reme/steps/evolve/proactive/topics.py b/reme/steps/evolve/proactive/topics.py index a22156cc..d49caa9b 100644 --- a/reme/steps/evolve/proactive/topics.py +++ b/reme/steps/evolve/proactive/topics.py @@ -343,11 +343,9 @@ class ProactiveTopicsStep(BaseStep): return [] bound = getattr(self.context, "workspace", None) if self.context is not None else None if _context_workspaces(self.app_context) is not None: - catalog = self.app_context.get_component( - ComponentEnum.FILE_CATALOG, - "digest", - alias=bound.alias if bound is not None else None, - ) + if bound is None: + raise RuntimeError("A workspace binding is required to resolve the digest catalog") + catalog = bound.components.get(ComponentEnum.FILE_CATALOG, {}).get("digest") else: # Test doubles and standalone-style contexts without the workspace layer. catalog = self.app_context.components.get(ComponentEnum.FILE_CATALOG, {}).get("digest") @@ -368,7 +366,7 @@ class ProactiveTopicsStep(BaseStep): name = self.kwargs.get("as_embedding", "default") if self.app_context is None: return None - return self.app_context.components.get(ComponentEnum.AS_EMBEDDING, {}).get(name) + return self.app_context.shared_components.get(ComponentEnum.AS_EMBEDDING, {}).get(name) async def _semantic_verdict( self, diff --git a/reme/steps/evolve/proactive/utils.py b/reme/steps/evolve/proactive/utils.py index a4a5b0ef..8839cc33 100644 --- a/reme/steps/evolve/proactive/utils.py +++ b/reme/steps/evolve/proactive/utils.py @@ -681,7 +681,7 @@ def resolve_agent_wrapper(step): if wrapper is not None: return wrapper if step.app_context is not None: - fallback = step.app_context.components.get(ComponentEnum.AGENT_WRAPPER, {}).get("default") + fallback = step.app_context.shared_components.get(ComponentEnum.AGENT_WRAPPER, {}).get("default") if fallback is not None: configured = step.kwargs.get("agent_wrapper", "default") step.logger.warning(f"[{step.name}] agent_wrapper '{configured}' missing; using default") diff --git a/reme/steps/index/update_changes.py b/reme/steps/index/update_changes.py index 7afa9fed..04c39206 100644 --- a/reme/steps/index/update_changes.py +++ b/reme/steps/index/update_changes.py @@ -12,6 +12,7 @@ from watchfiles import Change from ._change_batch import bucket_changes from ..base_step import BaseStep from ...components import R +from ...components.base_component import _context_workspaces from ...components.file_chunker import BaseFileChunker from ...enumeration import ComponentEnum from ...schema import FileChunk, FileNode @@ -368,8 +369,10 @@ class UpdateIndexStep(ChangeApplyStep): bound = getattr(self.context, "workspace", None) if self.context is not None else None if bound is not None: chunkers: dict[str, BaseFileChunker] = bound.components.get(ComponentEnum.FILE_CHUNKER, {}) - else: + elif _context_workspaces(self.app_context) is None: chunkers = self.app_context.components[ComponentEnum.FILE_CHUNKER] + else: + raise RuntimeError("A workspace binding is required to resolve file chunkers") suffix = path.suffix.lstrip(".").lower() for candidate in chunkers.values(): if suffix and suffix in {ext.lower().lstrip(".") for ext in candidate.supported_extensions}: diff --git a/tests/unit/test_codex_agent_wrapper.py b/tests/unit/test_codex_agent_wrapper.py index b0da6d96..c498fc41 100644 --- a/tests/unit/test_codex_agent_wrapper.py +++ b/tests/unit/test_codex_agent_wrapper.py @@ -61,60 +61,51 @@ def _wrapper(tmp_path, **kwargs): return CodexAgentWrapper(app_context=context, **kwargs), job -@pytest.mark.asyncio -async def test_mcp_config_uses_loopback_bridge_and_selected_jobs(tmp_path): +def test_mcp_config_uses_legacy_child_bridge_and_selected_jobs(tmp_path): wrapper, _job = _wrapper(tmp_path, mcp_config="custom.yaml") - await wrapper._ensure_job_server() # pylint: disable=protected-access - config = wrapper._mcp_server_config( # pylint: disable=protected-access + config = wrapper._mcp_server_config( {"job_tools": ["search", "search"], "tool_context_id": "ctx-1"}, ) - await wrapper.close() assert config["command"] assert config["enabled_tools"] == ["search"] assert config["args"].count("--job") == 1 assert config["args"][config["args"].index("--job") + 1] == "search" assert "reme.components.agent_wrapper.codex_mcp_server" in config["args"] - assert config["args"][config["args"].index("--endpoint") + 1].startswith("http://127.0.0.1:") - assert config["args"][config["args"].index("--token") + 1] + assert config["args"][config["args"].index("--config") + 1] == str(tmp_path / "custom.yaml") + assert config["args"][config["args"].index("--workspace") + 1] == str(tmp_path) assert config["args"][config["args"].index("--tool-context-id") + 1] == "ctx-1" - # mcp_config is accepted for compatibility but no longer builds a child app. - assert "--config" not in config["args"] + assert "--endpoint" not in config["args"] + assert "--token" not in config["args"] -@pytest.mark.asyncio -async def test_mcp_config_serializes_injected_job_kwargs(tmp_path): +def test_mcp_config_serializes_injected_job_kwargs(tmp_path): wrapper, _job = _wrapper(tmp_path, mcp_config="custom.yaml") - await wrapper._ensure_job_server() # pylint: disable=protected-access - config = wrapper._mcp_server_config( # pylint: disable=protected-access + config = wrapper._mcp_server_config( { "job_tools": ["search"], "tool_context_id": "ctx-1", "injected_job_kwargs": {"_allowed_paths": ["daily/2025-06-01/note.md"]}, }, ) - await wrapper.close() raw = config["args"][config["args"].index("--injected-job-kwargs") + 1] assert json.loads(raw) == {"_allowed_paths": ["daily/2025-06-01/note.md"]} - plain = wrapper._mcp_server_config({"job_tools": ["search"]}) # pylint: disable=protected-access + plain = wrapper._mcp_server_config({"job_tools": ["search"]}) assert "--injected-job-kwargs" not in plain["args"] -@pytest.mark.asyncio -async def test_thread_config_preserves_other_mcp_servers(tmp_path): +def test_thread_config_preserves_other_mcp_servers(tmp_path): wrapper, _job = _wrapper(tmp_path) - await wrapper._ensure_job_server() # pylint: disable=protected-access - config = wrapper._thread_config( # pylint: disable=protected-access + config = wrapper._thread_config( { "job_tools": ["search"], "config": {"mcp_servers": {"docs": {"url": "https://example.test/mcp"}}}, }, ) - await wrapper.close() assert "docs" in config["mcp_servers"] assert len(config["mcp_servers"]) == 2 @@ -166,65 +157,15 @@ def test_mcp_config_rejects_background_jobs(tmp_path): wrapper._mcp_server_config({"job_tools": ["watch"]}) -@pytest.mark.asyncio -async def test_loopback_endpoint_serves_parent_jobs(tmp_path): - import httpx - - wrapper, job = _wrapper(tmp_path) - await wrapper.start() - server = await wrapper._ensure_job_server() # pylint: disable=protected-access - base = server.url - headers = {"Authorization": f"Bearer {server.token}"} - try: - async with httpx.AsyncClient(base_url=base, headers=headers, timeout=10) as http: - listed = (await http.get("/jobs")).json()["jobs"] - assert [entry["name"] for entry in listed] == ["search"] - - called = await http.post("/call", json={"job": "search", "kwargs": {"query": "memo"}}) - assert called.json() == {"success": True, "answer": "found:memo"} - # The job executed in the parent process, not in any child. - assert job.calls == [{"query": "memo"}] - - missing = await http.post("/call", json={"job": "missing", "kwargs": {}}) - assert missing.status_code == 404 - - unauthorized = await http.get("/jobs", headers={"Authorization": "Bearer wrong"}) - assert unauthorized.status_code == 401 - finally: - await wrapper.close() - assert server.is_running is False - - -@pytest.mark.asyncio -async def test_loopback_bridge_lists_and_calls_parent_job(tmp_path): - from fastmcp import Client - from fastmcp.client import StdioTransport - - wrapper, job = _wrapper(tmp_path) - await wrapper._ensure_job_server() # pylint: disable=protected-access - server_config = wrapper._mcp_server_config({"job_tools": ["search"]}) # pylint: disable=protected-access - transport = StdioTransport( - command=server_config["command"], - args=server_config["args"], - cwd=server_config["cwd"], - keep_alive=False, - ) - - async with Client(transport, timeout=None, init_timeout=STDIO_INIT_TIMEOUT) as client: - tools = await asyncio.wait_for(client.list_tools(), timeout=10) - assert [tool.name for tool in tools] == ["search"] - result = await asyncio.wait_for(client.call_tool("search", {"query": "via-bridge"}), timeout=10) - await wrapper.close() - - assert "found:via-bridge" in str(result.content[0].text) - assert job.calls == [{"query": "via-bridge"}] - - @pytest.mark.asyncio async def test_stdio_bridge_stdout_is_protocol_clean(tmp_path): - wrapper, _job = _wrapper(tmp_path) - await wrapper._ensure_job_server() # pylint: disable=protected-access - server_config = wrapper._mcp_server_config({"job_tools": ["search"]}) # pylint: disable=protected-access + config_path = tmp_path / "config.json" + config_path.write_text( + json.dumps({"jobs": {"search": {"backend": "base", "steps": []}}}), + encoding="utf-8", + ) + wrapper, _job = _wrapper(tmp_path, mcp_config=str(config_path)) + server_config = wrapper._mcp_server_config({"job_tools": ["search"]}) proc = await asyncio.create_subprocess_exec( server_config["command"], *server_config["args"], diff --git a/tests/unit/test_workspace_binding.py b/tests/unit/test_workspace_binding.py index f7122489..008b78d5 100644 --- a/tests/unit/test_workspace_binding.py +++ b/tests/unit/test_workspace_binding.py @@ -165,7 +165,12 @@ async def test_workspace_scoped_components_instantiate_per_alias(tmp_path): store_beta = app.context.get_component(ComponentEnum.FILE_STORE, "default", "beta") assert store_alpha is not store_beta - assert app.context.components[ComponentEnum.FILE_STORE]["default"] is store_alpha # default alias entry + assert app.context.components[ComponentEnum.FILE_STORE]["default"] is store_alpha + assert ComponentEnum.FILE_STORE not in app.context.shared_components + assert app.context.workspaces["alpha"].get_component(ComponentEnum.FILE_STORE, "default") is store_alpha + assert app.context.workspaces["beta"].get_component(ComponentEnum.FILE_STORE, "default") is store_beta + with pytest.raises(KeyError, match="no instance for workspace 'alpha'"): + app.context.workspaces["alpha"].get_component(ComponentEnum.FILE_STORE, "missing") assert store_alpha.workspace_path == tmp_path / "alpha" assert store_beta.workspace_path == tmp_path / "beta" assert store_alpha.chunks_path.parent == tmp_path / "alpha" / "metadata" / "file_store" @@ -314,7 +319,8 @@ async def test_digest_titles_follow_workspace_binding(tmp_path): assert await step._digest_titles() == ["beta-only"] step.context = RuntimeContext() - assert await step._digest_titles() == ["alpha-only"] + with pytest.raises(RuntimeError, match="workspace binding is required"): + await step._digest_titles() @pytest.mark.asyncio diff --git a/tests/unit/test_workspace_phase3.py b/tests/unit/test_workspace_phase3.py index 81bc8a72..db00f606 100644 --- a/tests/unit/test_workspace_phase3.py +++ b/tests/unit/test_workspace_phase3.py @@ -3,6 +3,8 @@ # pylint: disable=protected-access,missing-function-docstring import asyncio +import json +import os from types import SimpleNamespace import pytest @@ -128,11 +130,8 @@ async def test_tool_context_state_namespaced_per_workspace(tmp_path): @pytest.mark.asyncio async def test_codex_thread_tool_contexts_namespaced_per_workspace(tmp_path): - wrapper = CodexAgentWrapper( - app_context=ApplicationContext( - workspace_dir={"alpha": str(tmp_path / "alpha"), "beta": str(tmp_path / "beta")}, - ), - ) + context = ApplicationContext(workspace_dir={"alpha": str(tmp_path / "alpha"), "beta": str(tmp_path / "beta")}) + wrapper = CodexAgentWrapper(app_context=context) class FakeCodex: """Minimal Codex double echoing the requested thread id.""" @@ -158,7 +157,13 @@ async def test_codex_thread_tool_contexts_namespaced_per_workspace(tmp_path): with pytest.raises(ValueError, match="belongs to workspace 'alpha'"): await wrapper._open_thread(codex, {"session_id": "thread-1", "workspace": "beta"}) - assert sorted(wrapper._thread_tool_contexts) == ["beta/thread-2", "thread-1"] + alpha_state = context.workspaces["alpha"].agent_state(wrapper.name) + beta_state = context.workspaces["beta"].agent_state(wrapper.name) + assert set(alpha_state.thread_tool_contexts) == {"thread-1"} + assert set(beta_state.thread_tool_contexts) == {"thread-2"} + assert not wrapper._thread_tool_contexts + assert alpha_state.owned_thread_ids == {"thread-1"} + assert beta_state.owned_thread_ids == {"thread-2"} @pytest.mark.asyncio @@ -295,27 +300,24 @@ async def test_mcp_tool_boundary_forwards_injected_workspace(tmp_path): @pytest.mark.asyncio -async def test_codex_bridge_uses_loopback_endpoint(tmp_path): - import json +async def test_codex_bridge_uses_resident_mcp_endpoint(tmp_path): + from reme.components.service import HttpService context = _context(tmp_path) + context.service = HttpService(app_context=context) context.jobs = {"search_job": _job(context, "search_job")} wrapper = CodexAgentWrapper(app_context=context) - await wrapper.start() - await wrapper._ensure_job_server() # pylint: disable=protected-access config = wrapper._mcp_server_config({"job_tools": ["search_job"], "workspace": "beta"}) - assert "--workspace-alias" in config["args"] - assert config["args"][config["args"].index("--workspace-alias") + 1] == "beta" + assert config["args"][config["args"].index("--endpoint") + 1] == "http://127.0.0.1:2333/mcp" raw = config["args"][config["args"].index("--injected-job-kwargs") + 1] assert json.loads(raw) == {"workspace": "beta"} assert "--config" not in config["args"] and "--workspace" not in config["args"] + assert "--token" not in config["args"] default_config = wrapper._mcp_server_config({"job_tools": ["search_job"]}) - assert "--workspace-alias" not in default_config["args"] - + assert "--injected-job-kwargs" not in default_config["args"] await wrapper.close() - assert wrapper._job_server is None # ----- Review follow-ups: explicit bindings, session isolation, key collisions @@ -423,18 +425,13 @@ async def test_codex_thread_cwd_follows_bound_workspace(tmp_path): # ----- Codex bridge: real child-process execution ---------------------------- -@pytest.mark.asyncio -async def test_codex_bridge_reads_resident_parent_state(tmp_path): - """Bridge calls execute against the parent's live stores; there is no second owner.""" - from fastmcp import Client - from fastmcp.client import StdioTransport - - app = Application( +def _codex_test_app(tmp_path): + return Application( workspace_dir={"alpha": str(tmp_path / "alpha"), "beta": str(tmp_path / "beta")}, enable_logo=False, log_to_console=False, log_to_file=False, - service={"backend": "cli"}, + service={"backend": "http", "host": "127.0.0.1", "port": 0, "web_enabled": False}, components={ "tokenizer": {"default": {"backend": "regex"}}, "file_store": {"default": {"backend": "local", "embedding_store": "", "tag_index": ""}}, @@ -464,57 +461,104 @@ async def test_codex_bridge_reads_resident_parent_state(tmp_path): }, }, ) + + +async def _start_resident_http(app): + import uvicorn + + service = app.context.service + service.build_service(app) + service.add_jobs(app) + config = uvicorn.Config(service.service, host=service.host, port=0, log_level="error", lifespan="on") + server = uvicorn.Server(config) + task = asyncio.create_task(server.serve()) + for _ in range(100): + if server.started: + break + await asyncio.sleep(0.05) + assert server.started + service.port = server.servers[0].sockets[0].getsockname()[1] + return server, task + + +async def _stop_resident_http(server, task): + server.should_exit = True + await asyncio.wait_for(task, timeout=30) + + +@pytest.mark.asyncio +async def test_codex_bridge_reads_resident_parent_state(tmp_path): + """The stdio bridge reuses the resident MCP service and its live workspace state.""" + from fastmcp import Client + from fastmcp.client import StdioTransport + + app = _codex_test_app(tmp_path) wrapper = CodexAgentWrapper(app_context=app.context) async with app: - await wrapper.start() - # Index a note directly in the parent's beta store. - await app.run_job("write_note", path="daily/resident.md", content="parent-resident-beta", workspace="beta") - await wrapper._ensure_job_server() # pylint: disable=protected-access - server_config = wrapper._mcp_server_config( # pylint: disable=protected-access - {"job_tools": ["read_note", "write_note"], "workspace": "beta"}, - ) - transport = StdioTransport( - command=server_config["command"], - args=server_config["args"], - cwd=server_config["cwd"], - keep_alive=False, - ) - async with Client(transport, timeout=None, init_timeout=30) as client: - read = await asyncio.wait_for(client.call_tool("read_note", {"path": "daily/resident.md"}), timeout=30) - await asyncio.wait_for( - client.call_tool("write_note", {"path": "daily/bridge.md", "content": "via-bridge-beta"}), - timeout=30, + server, task = await _start_resident_http(app) + try: + await app.run_job("write_note", path="daily/resident.md", content="parent-resident-beta", workspace="beta") + from reme.components.client import MCPClient + + resident_client = MCPClient(transport="streamable-http", host="127.0.0.1", port=app.context.service.port) + await resident_client.start() + assert {entry["name"] for entry in await resident_client.list_actions()} == {"read_note", "write_note"} + await resident_client.close() + server_config = wrapper._mcp_server_config( + {"job_tools": ["read_note", "write_note"], "workspace": "beta"}, ) - # The bridge saw the parent's resident beta state and wrote through it. - assert "parent-resident-beta" in str(read.content[0].text) - assert "via-bridge-beta" in (tmp_path / "beta" / "daily" / "bridge.md").read_text(encoding="utf-8") - assert not (tmp_path / "alpha" / "daily" / "bridge.md").exists() - store_alpha = app.context.get_component(ComponentEnum.FILE_STORE, "default", "alpha") - assert all("bridge" not in node.path for node in await store_alpha.get_nodes()) + transport = StdioTransport( + command=server_config["command"], + args=server_config["args"], + cwd=server_config["cwd"], + keep_alive=False, + env=dict(os.environ), + ) + async with Client(transport, timeout=None, init_timeout=30) as client: + tools = await client.list_tools() + assert {tool.name for tool in tools} == {"read_note", "write_note"} + read = await asyncio.wait_for(client.call_tool("read_note", {"path": "daily/resident.md"}), timeout=30) + with pytest.raises(Exception): + await client.call_tool("read_note", {"path": "daily/resident.md", "workspace": "alpha"}) + await asyncio.wait_for( + client.call_tool("write_note", {"path": "daily/bridge.md", "content": "via-bridge-beta"}), + timeout=30, + ) + assert "parent-resident-beta" in str(read.content[0].text) + assert "via-bridge-beta" in (tmp_path / "beta" / "daily" / "bridge.md").read_text(encoding="utf-8") + assert not (tmp_path / "alpha" / "daily" / "bridge.md").exists() + store_alpha = app.context.get_component(ComponentEnum.FILE_STORE, "default", "alpha") + assert all("bridge" not in node.path for node in await store_alpha.get_nodes()) + + invalid_config = wrapper._mcp_server_config({"job_tools": ["read_note"], "workspace": "nope"}) + invalid_transport = StdioTransport( + command=invalid_config["command"], + args=invalid_config["args"], + cwd=invalid_config["cwd"], + keep_alive=False, + env=dict(os.environ), + ) + async with Client(invalid_transport, timeout=None, init_timeout=30) as client: + with pytest.raises(Exception, match="Unknown workspace 'nope'"): + await asyncio.wait_for(client.call_tool("read_note", {"path": "daily/resident.md"}), timeout=30) + finally: + await _stop_resident_http(server, task) await wrapper.close() @pytest.mark.asyncio -async def test_codex_bridge_unknown_alias_fails_at_execution(tmp_path): - """An unknown workspace alias surfaces as a tool error from the parent boundary.""" - from fastmcp import Client - from fastmcp.client import StdioTransport +async def test_codex_multi_workspace_rejects_unsupported_job_transport(tmp_path): + from reme.components.service import CliService context = _context(tmp_path) + context.service = CliService(app_context=context) context.jobs = {"search_job": _job(context, "search_job")} - wrapper = CodexAgentWrapper(app_context=context, project_path=str(tmp_path)) - await wrapper.start() - await wrapper._ensure_job_server() # pylint: disable=protected-access - server_config = wrapper._mcp_server_config( # pylint: disable=protected-access - {"job_tools": ["search_job"], "workspace": "nope"}, - ) - transport = StdioTransport( - command=server_config["command"], - args=server_config["args"], - cwd=server_config["cwd"], - keep_alive=False, - ) - async with Client(transport, timeout=None, init_timeout=30) as client: - with pytest.raises(Exception, match="Unknown workspace 'nope'"): - await asyncio.wait_for(client.call_tool("search_job", {}), timeout=30) + wrapper = CodexAgentWrapper(app_context=context) + + with pytest.raises(RuntimeError, match="resident HTTP MCP service"): + wrapper._mcp_server_config({"job_tools": ["search_job"], "workspace": "beta"}) + with pytest.raises(ValueError, match="only supported for single-workspace"): + wrapper._mcp_server_config( + {"job_tools": ["search_job"], "workspace": "beta", "mcp_config": "custom.yaml"}, + ) await wrapper.close()