refactor(workspace): consolidate ownership and Codex integration

This commit is contained in:
celanwang 2026-10-26 18:56:30 +08:00 • committed by Cursor Agent
parent 440b85b5c4
commit f0afcfd13a
No known key found for this signature in database
17 changed files with 548 additions and 547 deletions

View file

@ -132,8 +132,12 @@ Treat Step instances as invocation-scoped:
- `self.context.response.answer`, `success`, and `metadata` are request-scoped output. Because the same response travels
through the Step chain, later Steps may consume metadata produced earlier, but it is not application-lifetime or
durable storage.
- `self.app_context.metadata` holds in-memory state shared across Job/Step invocations for the life of one
`Application`, such as counters, tool-context state, session maps, or locks.
- `self.app_context.metadata` holds application-lifetime operational state shared across Job/Step invocations,
such as counters. `ApplicationContext.shared_components` owns genuinely shared components; its `components`
compatibility view also exposes default-workspace instances to legacy callers.
- Each `WorkspaceContext` owns its paths, stateful components, tool contexts, search budgets, agent session state,
and locks. `RuntimeContext.workspace` is the explicit binding used by workspace-bound execution; internal Steps
must not resolve a default workspace repeatedly or use the compatibility component view.
- Workspace files or a dedicated Component/store hold durable state that must survive restart.
Use narrow, namespaced keys in `app_context.metadata` and protect shared mutable values against concurrent access. The
@ -144,8 +148,9 @@ dedicated lifecycle, locking, or persistence, promote it to a typed context fiel
Additional Step contracts:
- `Ref` dependencies resolve in this order: Step kwargs, current `RuntimeContext`, then the named application component.
The value is cached only on the current Step instance and cleared before each call.
- `Ref` dependencies resolve in this order: Step kwargs, current `RuntimeContext`, then the bound workspace for scoped
components or the application shared-component registry for shared components. The value is cached only on the
current Step instance and cleared before each call.
- `input_mapping` and `output_mapping` copy keys within `RuntimeContext.data`; missing sources are ignored.
- Dispatched Steps receive the current `RuntimeContext`, so their data and response are shared.
- Base jobs convert uncaught Step errors into `Response(success=False)`; stream jobs emit an error chunk and always a

View file

@ -100,8 +100,13 @@ External stateful backends that cannot isolate workspaces (currently the `neo4j`
startup in multi-workspace configurations.
The binding propagates through nested job calls and agent tool calls (`job_tools`) as a server-owned constraint
that models cannot override. Background and cron jobs bind through their job configuration, so extra workspaces
are watched by declaring explicit per-alias variants:
that models cannot override. For Codex with multiple workspaces, `job_tools` requires the resident HTTP service
with MCP enabled, and the selected jobs must be exposed by that service. Its STDIO bridge forwards to this service
without creating another Application. The multi-workspace one-shot CLI combination is not supported; `mcp_config`
retains its legacy meaning only for single-workspace Codex job tools.
Background and cron jobs bind through their job configuration, so extra workspaces are watched by declaring
explicit per-alias variants:
```yaml
jobs:

View file

@ -102,6 +102,10 @@ Job 调用接受可选的 `workspace` 字段指定绑定的别名,例如
在多 workspace 配置下会在启动时被拒绝。
绑定会经嵌套 Job 调用和 agent 工具调用(`job_tools`)传播,作为模型无法覆盖的服务端约束。
多 workspace 下的 Codex `job_tools` 需要启用 MCP 的常驻 HTTP 服务,且所选 Job 必须由该服务暴露;
其 STDIO bridge 转发到该服务,不会创建另一个 Application。多 workspace 与 one-shot CLI 的组合暂不支持;
`mcp_config` 仅在单 workspace 的 Codex job tools 中保留原有语义。
Background 和 cron Job 通过 Job 配置绑定别名,额外 workspace 的 watch 通过显式配置按别名的变体实现:
```yaml

View file

@ -73,7 +73,6 @@ class Application(BaseComponent):
"""Instantiate declared components; workspace-scoped ones once per alias."""
multi_workspace = len(self.context.workspaces) > 1
for ctype, group in self.config.components.items():
self.context.components[ctype] = {}
for name, cfg in group.items():
label = f"Component '{name}'"
backend_cls = self._backend_class(ctype, cfg, label)
@ -92,12 +91,8 @@ class Application(BaseComponent):
workspace_alias=alias,
)
workspace.components.setdefault(ctype, {})[name] = instance
# The default alias instance doubles as the plain lookup entry.
self.context.components[ctype][name] = next(iter(self.context.workspaces.values())).components[
ctype
][name]
else:
self.context.components[ctype][name] = self._instantiate(
self.context.shared_components.setdefault(ctype, {})[name] = self._instantiate(
ctype,
cfg,
label=label,
@ -166,16 +161,13 @@ class Application(BaseComponent):
) -> dict[_NodeKey, BaseComponent]:
"""One node per component instance; scoped components contribute one node per alias."""
nodes: dict[_NodeKey, BaseComponent] = {}
scoped_keys: set[tuple[str, str]] = set()
for alias, workspace in self.context.workspaces.items():
for ctype, group in workspace.components.items():
for name, instance in group.items():
nodes[(ctype, name, alias)] = instance
scoped_keys.add((ctype, name))
for ctype, group in self.context.components.items():
for ctype, group in self.context.shared_components.items():
for name, comp in group.items():
if (ctype, name) not in scoped_keys:
nodes[(ctype, name, "")] = comp
nodes[(ctype, name, "")] = comp
if replacement is not None:
nodes[replacement[0]] = replacement[1]
return nodes
@ -297,11 +289,10 @@ class Application(BaseComponent):
"""Update an existing component by type/name; never creates missing components."""
async with self._component_mutation_lock:
component_type = component_type_name(component_enum)
group = self.context.components.get(component_type)
if not group or name not in group:
component = self.context.get_component(component_type, name)
if component is None:
raise KeyError(f"Component '{name}' not found in {component_type}")
component = group[name]
# Validate fields without evaluating lazy properties before injection.
for key in kwargs:
if getattr_static(component, key, _UNSET) is _UNSET and not hasattr(component, key):
@ -317,7 +308,7 @@ class Application(BaseComponent):
for group in workspace.components.values():
for instance in group.values():
instances[id(instance)] = instance
for group in self.context.components.values():
for group in self.context.shared_components.values():
for component in group.values():
instances[id(component)] = component
return list(instances.values())
@ -365,19 +356,27 @@ class Application(BaseComponent):
"""
async with self._component_mutation_lock:
component_type = component_type_name(component_enum)
group = self.context.components.get(component_type)
config_group = self.config.components.get(component_type)
if not group or name not in group or config_group is None:
raise KeyError(f"Component '{name}' not found in {component_type}")
old_component = group[name]
scoped_instances = self.context.workspace_components.get(component_type, {}).get(name)
if scoped_instances is not None and len(scoped_instances) > 1:
scoped_instances = {
alias: workspace.components[component_type][name]
for alias, workspace in self.context.workspaces.items()
if name in workspace.components.get(component_type, {})
}
if len(scoped_instances) > 1:
raise NotImplementedError(
f"replace_component does not support workspace-scoped component "
f"'{component_type}:{name}' in multi-workspace configurations",
)
alias = next(iter(scoped_instances)) if scoped_instances is not None else None
alias = next(iter(scoped_instances), None)
group = (
self.context.workspaces[alias].components.get(component_type)
if alias is not None
else self.context.shared_components.get(component_type)
)
if not group or name not in group or config_group is None:
raise KeyError(f"Component '{name}' not found in {component_type}")
old_component = group[name]
replacement_config = (
config.model_copy(deep=True)
if isinstance(config, ComponentConfig)

View file

@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
from collections.abc import AsyncGenerator
from contextlib import suppress
from dataclasses import dataclass
from functools import partial
import hashlib
@ -11,14 +12,16 @@ import json
import os
from pathlib import Path
import sys
import tempfile
from typing import Any, TYPE_CHECKING
from .base_agent_wrapper import BaseAgentWrapper
from .loopback_job_server import LoopbackJobServer
from ..base_component import _context_workspaces
from ..component_registry import R
from ...constants import normalize_connect_host
from ...enumeration import ChunkEnum
from ...schema import StreamChunk, TokenUsage
from ...schema.workspace import WorkspaceAgentState
if TYPE_CHECKING:
from openai_codex import AsyncCodex, AsyncThread, CodexConfig, RunInput
@ -116,9 +119,7 @@ class CodexAgentWrapper(BaseAgentWrapper):
self._release_threads = release_threads
self._turn_lock = asyncio.Lock()
self._thread_tool_contexts: dict[str, str] = {}
self._thread_owners: dict[str, set[str]] = {}
self._job_server: LoopbackJobServer | None = None
self._mcp_config_warned = False
self._mcp_snapshot_path: Path | None = None
@property
def session_path(self) -> Path:
@ -149,15 +150,47 @@ class CodexAgentWrapper(BaseAgentWrapper):
relative_source = os.path.relpath(source, target.parent)
target.symlink_to(relative_source, target_is_directory=True)
def _mcp_config_source_unused(self, kwargs: dict[str, Any]) -> None:
"""Warn once that job_tools now execute in the parent via the loopback endpoint."""
@property
def wrapper_session_path(self) -> Path:
"""ReMe-owned session data, kept separate from a shared OAuth CODEX_HOME."""
if self.app_context is None:
return self.workspace_path / "mem_session" / "codex"
return self.workspace_path / self.app_context.app_config.mem_session_dir / "codex"
def _explicit_mcp_config(self, kwargs: dict[str, Any]) -> str | None:
value = kwargs.get("mcp_config") if "mcp_config" in kwargs else self.mcp_config
if value is not None and not self._mcp_config_warned:
self._mcp_config_warned = True
self.logger.warning(
"mcp_config is ignored for job_tools: Codex job tools execute in the parent "
"process via the wrapper-owned loopback endpoint",
)
if value is None:
return None
source = Path(str(value)).expanduser()
if source.suffix in {".yaml", ".yml", ".json"}:
if not source.is_absolute():
source = self.workspace_path / source
return str(source.absolute())
return str(value)
def _effective_config_snapshot(self) -> Path:
"""Create one private snapshot that remains valid for the client lifetime."""
if self._mcp_snapshot_path is not None:
return self._mcp_snapshot_path
if self.app_context is None:
raise RuntimeError("Cannot snapshot MCP config without an app_context")
snapshot_dir = self.wrapper_session_path / "reme-mcp"
snapshot_dir.mkdir(parents=True, exist_ok=True)
fd, raw_path = tempfile.mkstemp(prefix="config-", suffix=".json", dir=snapshot_dir)
try:
os.fchmod(fd, 0o600)
with os.fdopen(fd, "w", encoding="utf-8") as stream:
json.dump(self.app_context.app_config.model_dump(mode="json"), stream)
except BaseException:
with suppress(OSError):
os.close(fd)
Path(raw_path).unlink(missing_ok=True)
raise
self._mcp_snapshot_path = Path(raw_path)
return self._mcp_snapshot_path
def _mcp_config_source(self, kwargs: dict[str, Any]) -> str:
return self._explicit_mcp_config(kwargs) or str(self._effective_config_snapshot())
@staticmethod
def _resolve_auth_config(auth_mode: str, api_key: str = "", base_url: str = "") -> _CodexAuthConfig:
@ -209,19 +242,32 @@ class CodexAgentWrapper(BaseAgentWrapper):
names = ", ".join(invalid)
raise TypeError(f"Codex client options must be configured on the wrapper: {names}")
async def _ensure_job_server(self) -> LoopbackJobServer:
"""Start the wrapper-owned loopback job endpoint on first bridge use."""
if self._job_server is None:
if self.app_context is None:
raise RuntimeError("Codex job_tools require an app_context")
self._job_server = LoopbackJobServer(self.app_context)
await self._job_server.start()
return self._job_server
def _resident_mcp_endpoint(self) -> str:
from ..service import HttpService, MCPService
service = self.app_context.service if self.app_context is not None else None
if isinstance(service, HttpService) and service.mcp_enabled:
return f"http://{normalize_connect_host(service.host)}:{service.port}{service.mcp_path}"
if isinstance(service, MCPService) and service.transport == "streamable-http":
return f"http://{normalize_connect_host(service.host)}:{service.port}/mcp"
raise RuntimeError(
"Multi-workspace Codex job_tools require a resident HTTP MCP service; "
"one-shot CLI and other service backends are not supported for this combination",
)
@staticmethod
def _validate_resident_jobs(service, jobs) -> None:
unavailable = [
job.name
for job in jobs
if not job.enabled or not job.enable_serve or (service.jobs is not None and job.name not in service.jobs)
]
if unavailable:
raise ValueError(f"Codex job_tools must be exposed by the resident MCP service: {', '.join(unavailable)}")
def _mcp_server_config(self, kwargs: dict[str, Any]) -> dict[str, Any] | None:
from ..job import BackgroundJob, StreamJob
self._mcp_config_source_unused(kwargs)
job_names = list(dict.fromkeys(kwargs.get("job_tools") or []))
if not job_names:
return None
@ -229,26 +275,29 @@ class CodexAgentWrapper(BaseAgentWrapper):
unsupported = [job.name for job in jobs if isinstance(job, (BackgroundJob, StreamJob))]
if unsupported:
raise TypeError(f"Codex job_tools must be non-stream request jobs: {', '.join(unsupported)}")
if self._job_server is None:
raise RuntimeError("Codex job endpoint is not started; call _ensure_job_server first")
alias = str(kwargs.get("workspace") or "")
args = [
"-m",
"reme.components.agent_wrapper.codex_mcp_server",
"--endpoint",
self._job_server.url,
"--token",
self._job_server.token,
]
if alias:
args.extend(["--workspace-alias", alias])
workspaces = _context_workspaces(self.app_context)
multi_workspace = workspaces is not None and len(workspaces) > 1
if multi_workspace:
if self._explicit_mcp_config(kwargs) is not None:
raise ValueError("mcp_config is only supported for single-workspace Codex job_tools")
endpoint = self._resident_mcp_endpoint()
self._validate_resident_jobs(self.app_context.service, jobs)
args = ["-m", "reme.components.agent_wrapper.codex_mcp_server", "--endpoint", endpoint]
else:
args = [
"-m",
"reme.components.agent_wrapper.codex_mcp_server",
"--config",
self._mcp_config_source(kwargs),
"--workspace",
str(self.workspace_path),
]
for name in job_names:
args.extend(["--job", name])
args.extend(["--tool-context-id", str(kwargs.get("tool_context_id") or "")])
injected_job_kwargs = dict(kwargs.get("injected_job_kwargs") or {})
if workspace := kwargs.get("workspace"):
# The bridge injects the binding into nested job calls; models cannot override it.
if multi_workspace and (workspace := kwargs.get("workspace")):
injected_job_kwargs["workspace"] = workspace
if injected_job_kwargs:
args.extend(["--injected-job-kwargs", json.dumps(injected_job_kwargs, sort_keys=True)])
@ -288,6 +337,7 @@ class CodexAgentWrapper(BaseAgentWrapper):
from openai_codex.types import Personality, ThreadSource, ThreadStartSource
alias = str(kwargs.get("workspace") or "")
tool_contexts = self._tool_contexts(alias)
resume = kwargs.get("resume") or ""
session_id = kwargs.get("session_id") or ""
if resume and session_id and resume != session_id:
@ -297,12 +347,9 @@ class CodexAgentWrapper(BaseAgentWrapper):
if fork_session and not thread_id:
raise ValueError("fork_session=True requires resume or session_id")
if thread_id:
# Cross-workspace thread access is rejected before touching the SDK.
await self._check_thread_ownership(thread_id, alias)
if kwargs.get("job_tools"):
await self._ensure_job_server()
if self._release_threads and self._thread_tool_contexts:
await self._prune_thread_tool_contexts(codex)
if self._release_threads and tool_contexts:
await self._prune_thread_tool_contexts(codex, alias)
requested_tool_context = json.dumps(
{
"tool_context_id": str(kwargs.get("tool_context_id") or ""),
@ -311,9 +358,8 @@ class CodexAgentWrapper(BaseAgentWrapper):
sort_keys=True,
default=str,
)
guard_key = f"{alias}/{thread_id}" if alias else thread_id
if not fork_session and guard_key in self._thread_tool_contexts:
if requested_tool_context != self._thread_tool_contexts[guard_key]:
if not fork_session and thread_id in tool_contexts:
if requested_tool_context != tool_contexts[thread_id]:
raise ValueError("tool_context_id and injected_job_kwargs cannot change when resuming a Codex thread")
common = {
@ -348,19 +394,28 @@ class CodexAgentWrapper(BaseAgentWrapper):
**common,
)
if fork_session or not thread_id:
# Threads created under this binding are owned by this workspace.
self._record_thread_ownership(alias, thread.id)
self._thread_tool_contexts[f"{alias}/{thread.id}" if alias else thread.id] = requested_tool_context
tool_contexts[thread.id] = requested_tool_context
return thread
# ----- Thread ownership (per-workspace, persisted) ---------------------
def _normalize_thread_alias(self, alias: str) -> str:
"""Map a call alias to a concrete workspace alias ("" means the default)."""
def _agent_state(self, alias: str) -> WorkspaceAgentState | None:
workspaces = _context_workspaces(self.app_context)
if not workspaces:
return alias
return alias if alias in workspaces else next(iter(workspaces))
return None
return workspaces[self._normalize_thread_alias(alias)].agent_state(self.name)
def _tool_contexts(self, alias: str) -> dict[str, str]:
state = self._agent_state(alias)
return state.thread_tool_contexts if state is not None else self._thread_tool_contexts
def _normalize_thread_alias(self, alias: str) -> str:
"""Map an omitted alias to the default while preserving explicit aliases."""
workspaces = _context_workspaces(self.app_context)
if workspaces and not alias:
return next(iter(workspaces))
return alias
def _thread_ownership_path(self, alias: str) -> Path | None:
workspaces = _context_workspaces(self.app_context)
@ -372,9 +427,10 @@ class CodexAgentWrapper(BaseAgentWrapper):
return workspace.mem_session_path / "codex" / "owned_threads.json"
def _owned_thread_ids(self, alias: str) -> set[str]:
"""Load (once) the persisted ids of threads this workspace created."""
if alias in self._thread_owners:
return self._thread_owners[alias]
"""Load the persisted ids into the owning workspace's agent state."""
state = self._agent_state(alias)
if state is not None and state.owned_thread_ids is not None:
return state.owned_thread_ids
ids: set[str] = set()
path = self._thread_ownership_path(alias)
if path is not None and path.is_file():
@ -383,7 +439,8 @@ class CodexAgentWrapper(BaseAgentWrapper):
ids = {str(thread_id) for thread_id in raw.get("thread_ids", [])}
except (OSError, ValueError, AttributeError) as exc:
self.logger.warning(f"Ignoring unreadable Codex thread ownership file {path}: {exc}")
self._thread_owners[alias] = ids
if state is not None:
state.owned_thread_ids = ids
return ids
def _record_thread_ownership(self, alias: str, thread_id: str) -> None:
@ -453,20 +510,19 @@ class CodexAgentWrapper(BaseAgentWrapper):
return codex
async def _close(self) -> None:
"""Close the persistent app-server and the wrapper-owned job endpoint."""
"""Close the persistent app-server and clear session execution guards."""
async with self._turn_lock:
codex, self._codex = self._codex, None
self._thread_tool_contexts.clear()
try:
if codex is not None:
await codex.close()
finally:
if self._job_server is not None:
job_server, self._job_server = self._job_server, None
await job_server.close()
for workspace in (_context_workspaces(self.app_context) or {}).values():
if state := workspace.agent_states.get(self.name):
state.thread_tool_contexts.clear()
if codex is not None:
await codex.close()
async def _prune_thread_tool_contexts(self, codex: AsyncCodex) -> None:
async def _prune_thread_tool_contexts(self, codex: AsyncCodex, alias: str = "") -> None:
"""Forget tool constraints only after the server confirms a thread is unloaded."""
tool_contexts = self._tool_contexts(alias)
try:
from openai_codex.generated.v2_all import ThreadLoadedListResponse
@ -485,9 +541,9 @@ class CodexAgentWrapper(BaseAgentWrapper):
except Exception as exc: # pylint: disable=broad-exception-caught
self.logger.warning(f"Failed to list loaded Codex threads: {exc}")
return
self._thread_tool_contexts = {
key: context for key, context in self._thread_tool_contexts.items() if key.rsplit("/", 1)[-1] in loaded_ids
}
retained = {thread_id: context for thread_id, context in tool_contexts.items() if thread_id in loaded_ids}
tool_contexts.clear()
tool_contexts.update(retained)
async def _release_thread(self, codex: AsyncCodex, thread_id: str) -> None:
"""Unsubscribe so the app-server can unload the thread once it is idle."""

View file

@ -1,25 +1,27 @@
"""Thin FastMCP STDIO bridge forwarding Codex job tools to the parent process.
"""Expose ReMe jobs to Codex over FastMCP STDIO.
This subprocess never builds a ReMe Application or any workspace store. It
asks the parent-owned loopback endpoint (LoopbackJobServer) for job metadata
and forwards every tool call — together with the server-owned workspace alias,
tool context id, and injected kwargs — back to the parent for execution, so
all workspace state has exactly one owner.
Multi-workspace callers forward to the resident MCP service without creating
an Application. Single-workspace callers retain the legacy child-service path.
"""
import argparse
import asyncio
import json
from pathlib import Path
from typing import Any
import httpx
from .base_agent_wrapper import BaseAgentWrapper
from ..client import MCPClient
from ...config import resolve_app_config
from ...reme import ReMe
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Forward selected ReMe jobs from the parent loopback endpoint")
parser.add_argument("--endpoint", required=True, help="Parent loopback endpoint base URL")
parser.add_argument("--token", required=True, help="Bearer token for the loopback endpoint")
parser = argparse.ArgumentParser(description="Expose selected ReMe jobs over FastMCP STDIO")
parser.add_argument("--config", default="default", help="ReMe config name or file path")
parser.add_argument("--workspace", help="Single-workspace directory for the legacy child service")
parser.add_argument("--endpoint", help="Resident streamable-HTTP MCP endpoint for forwarding")
parser.add_argument("--job", dest="jobs", action="append", required=True, help="ReMe job name; repeat as needed")
parser.add_argument("--workspace-alias", default="", help="Workspace alias injected into every job call")
parser.add_argument("--tool-context-id", default="", help="Context id injected into every job call")
parser.add_argument(
"--injected-job-kwargs",
@ -30,89 +32,145 @@ def _parse_args() -> argparse.Namespace:
return parser.parse_args()
class _EndpointClient:
"""Minimal client for the parent's loopback job server."""
def _prepare_config(
config: dict[str, Any],
job_names: list[str],
tool_context_id: str = "",
injected_job_kwargs: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Configure the legacy single-workspace child Application for MCP STDIO."""
selected = set(job_names)
jobs: dict[str, dict[str, Any]] = {}
for name, raw_job_config in (config.get("jobs") or {}).items():
job_config = dict(raw_job_config)
if job_config.get("backend") in {"background", "cron"}:
continue
if name in selected:
job_config["enable_serve"] = True
jobs[name] = job_config
def __init__(self, endpoint: str, token: str, timeout: float) -> None:
self._client = httpx.Client(
base_url=endpoint,
headers={"Authorization": f"Bearer {token}"},
timeout=timeout,
)
missing = sorted(selected.difference(jobs))
if missing:
raise KeyError(f"Codex job tools not found or not request jobs: {', '.join(missing)}")
def list_jobs(self) -> list[dict[str, Any]]:
"""Fetch the servable job metadata from the parent endpoint."""
response = self._client.get("/jobs")
response.raise_for_status()
return response.json()["jobs"]
service: dict[str, Any] = {
"backend": "mcp",
"transport": "stdio",
"jobs": job_names,
"tool_error_on_failure": True,
}
injected = dict(injected_job_kwargs or {})
if tool_context_id:
injected["tool_context_id"] = tool_context_id
if injected:
service["injected_job_kwargs"] = injected
def call(self, job: str, kwargs: dict[str, Any], injected_job_kwargs: dict[str, Any] | None = None) -> str:
"""Forward one tool call to the parent endpoint and unwrap its answer."""
response = self._client.post(
"/call",
json={"job": job, "kwargs": kwargs, "injected_job_kwargs": injected_job_kwargs or {}},
)
if response.status_code != 200:
try:
detail = response.json().get("error") or response.text
except ValueError:
detail = response.text
raise RuntimeError(str(detail))
payload = response.json()
if not payload.get("success"):
raise RuntimeError(str(payload.get("answer") or payload))
return str(payload.get("answer") or "")
def close(self) -> None:
"""Close the underlying HTTP client."""
self._client.close()
prepared = dict(config)
prepared["jobs"] = jobs
prepared["service"] = service
return prepared
def main() -> None:
"""Fetch job metadata from the parent and serve forwarding tools over STDIO."""
class _ResidentMCPClient:
"""Reuse the standard ReMe MCP client for resident tool discovery and calls."""
def __init__(self, endpoint: str, timeout: float) -> None:
from fastmcp.client import StreamableHttpTransport
self._client = MCPClient(transport=StreamableHttpTransport(url=endpoint), timeout=timeout)
async def start(self) -> None:
"""Connect to the resident service."""
await self._client.start()
async def list_jobs(self) -> list[dict[str, Any]]:
"""Return the resident service's tool descriptors."""
return await self._client.list_actions()
async def call(self, job: str, kwargs: dict[str, Any], injected: dict[str, Any]) -> str:
"""Call one resident tool with server-owned arguments enforced."""
# pylint: disable=protected-access
payload = BaseAgentWrapper._merge_injected_job_kwargs(kwargs, injected)
async for text in self._client(action=job, **payload):
return text
return ""
async def close(self) -> None:
"""Close the resident connection."""
await self._client.close()
def _injected_kwargs(args: argparse.Namespace) -> dict[str, Any]:
injected = json.loads(args.injected_job_kwargs) if args.injected_job_kwargs else {}
if not isinstance(injected, dict):
raise TypeError("--injected-job-kwargs must be a JSON object")
injected = dict(injected)
if args.tool_context_id:
injected["tool_context_id"] = args.tool_context_id
return injected
async def _run_forwarder(args: argparse.Namespace, injected: dict[str, Any]) -> None:
from fastmcp import FastMCP
from fastmcp.exceptions import ToolError
from fastmcp.tools import FunctionTool
args = _parse_args()
injected_job_kwargs = json.loads(args.injected_job_kwargs) if args.injected_job_kwargs else {}
if not isinstance(injected_job_kwargs, dict):
raise TypeError("--injected-job-kwargs must be a JSON object")
injected = dict(injected_job_kwargs)
if args.workspace_alias:
injected["workspace"] = args.workspace_alias
if args.tool_context_id:
injected["tool_context_id"] = args.tool_context_id
client = _EndpointClient(args.endpoint, args.token, args.tool_timeout)
metadata = {entry["name"]: entry for entry in client.list_jobs()}
missing = sorted(set(args.jobs) - set(metadata))
if missing:
raise KeyError(f"Codex job tools not found on the parent endpoint: {', '.join(missing)}")
server = FastMCP("reme-codex-bridge")
for name in args.jobs:
entry = metadata[name]
def forward(_client=client, _name=name, _injected=None, **kwargs):
try:
return _client.call(_name, kwargs, _injected or injected)
except Exception as exc: # noqa: BLE001 - surface every failure as a tool error
raise ToolError(str(exc)) from exc
server.add_tool(
FunctionTool(
name=name,
description=entry.get("description") or "",
parameters=entry.get("parameters") or {"type": "object", "properties": {}},
fn=forward,
),
)
client = _ResidentMCPClient(args.endpoint, args.tool_timeout)
await client.start()
try:
server.run(transport="stdio")
metadata = {entry["name"]: entry for entry in await client.list_jobs()}
missing = sorted(set(args.jobs) - set(metadata))
if missing:
raise KeyError(f"Codex job tools not found on the resident MCP service: {', '.join(missing)}")
server = FastMCP("reme-codex-bridge")
for name in args.jobs:
entry = metadata[name]
def make_forwarder(job_name: str):
async def forward(**kwargs):
try:
return await client.call(job_name, kwargs, injected)
except Exception as exc: # noqa: BLE001 - surface every failure as a tool error
raise ToolError(str(exc)) from exc
return forward
parameters = entry.get("inputSchema") or entry.get("parameters")
# pylint: disable=protected-access
parameters = BaseAgentWrapper._strip_injected_parameters(parameters, injected)
server.add_tool(
FunctionTool(
name=name,
description=entry.get("description") or "",
parameters=parameters or {"type": "object", "properties": {}},
fn=make_forwarder(name),
),
)
await server.run_async(transport="stdio")
finally:
client.close()
await client.close()
def main() -> None:
"""Forward to a resident service or start the legacy single-workspace child service."""
args = _parse_args()
injected = _injected_kwargs(args)
if args.endpoint:
asyncio.run(_run_forwarder(args, injected))
return
if not args.workspace:
raise ValueError("--workspace is required when --endpoint is not supplied")
config = resolve_app_config(
config=args.config,
workspace_dir=str(Path(args.workspace).absolute()),
enable_logo=False,
log_to_console=False,
log_to_file=False,
log_config=False,
)
config = _prepare_config(config, args.jobs, args.tool_context_id, injected)
ReMe(**config).run_app()
if __name__ == "__main__":

View file

@ -1,148 +0,0 @@
"""Wrapper-owned loopback job endpoint for agent-bridge subprocesses.
The Codex job bridge used to spawn a full child Application per thread; every
thread then held its own workspace stores and became a second persistence
writer. This endpoint keeps job execution in the parent process instead: a
minimal HTTP/JSON server on 127.0.0.1 with an ephemeral port and a bearer
token, owned by one CodexAgentWrapper instance for its lifetime. Bridge
subprocesses are thin FastMCP forwarders that call this endpoint.
"""
import asyncio
import json
import secrets
from typing import Any, TYPE_CHECKING
if TYPE_CHECKING:
from ..application_context import ApplicationContext
from ..job import BaseJob
_REASON = {
200: "OK",
400: "Bad Request",
401: "Unauthorized",
404: "Not Found",
500: "Internal Server Error",
}
class LoopbackJobServer:
"""Serve this application's request jobs to trusted local subprocesses."""
def __init__(self, app_context: "ApplicationContext") -> None:
self._app_context = app_context
self.token = secrets.token_urlsafe(24)
self._server: asyncio.Server | None = None
@property
def is_running(self) -> bool:
"""Whether the endpoint is currently bound."""
return self._server is not None
@property
def url(self) -> str:
"""Base URL of the bound endpoint."""
if self._server is None:
raise RuntimeError("Loopback job server is not started")
port = self._server.sockets[0].getsockname()[1]
return f"http://127.0.0.1:{port}"
async def start(self) -> None:
"""Bind the endpoint; idempotent so lazy bridge creation can retry."""
if self._server is None:
self._server = await asyncio.start_server(self._handle, "127.0.0.1", 0)
async def close(self) -> None:
"""Stop the endpoint and release the port."""
server, self._server = self._server, None
if server is not None:
server.close()
await server.wait_closed()
# ----- HTTP handling ---------------------------------------------------
async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
try:
head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), timeout=30)
request_line, *header_lines = head.decode("latin-1").split("\r\n")
method, target, _version = request_line.split(" ", 2)
headers = {}
for line in header_lines:
if ":" in line:
key, value = line.split(":", 1)
headers[key.strip().lower()] = value.strip()
length = int(headers.get("content-length", 0) or 0)
body = await reader.readexactly(length) if length else b""
status, payload = await self._dispatch(method, target, headers, body)
except Exception as exc: # pylint: disable=broad-exception-caught
status, payload = 500, {"error": f"{type(exc).__name__}: {exc}"}
data = json.dumps(payload).encode()
writer.write(
f"HTTP/1.1 {status} {_REASON.get(status, 'OK')}\r\n"
"content-type: application/json\r\n"
f"content-length: {len(data)}\r\n"
"connection: close\r\n\r\n".encode() + data,
)
try:
await writer.drain()
finally:
writer.close()
async def _dispatch(
self,
method: str,
target: str,
headers: dict[str, str],
body: bytes,
) -> tuple[int, dict[str, Any]]:
if headers.get("authorization") != f"Bearer {self.token}":
return 401, {"error": "unauthorized"}
if method == "GET" and target == "/jobs":
return 200, {"jobs": self._job_metadata()}
if method == "POST" and target == "/call":
try:
request = json.loads(body or b"{}")
except json.JSONDecodeError as exc:
return 400, {"error": f"invalid JSON body: {exc}"}
return await self._call(request)
return 404, {"error": "not found"}
# ----- Job surface -------------------------------------------------------
def _request_jobs(self) -> dict[str, "BaseJob"]:
"""Return the application's non-stream, non-background jobs by name."""
from ..job import BackgroundJob, StreamJob
return {
name: job for name, job in self._app_context.jobs.items() if not isinstance(job, (StreamJob, BackgroundJob))
}
def _job_metadata(self) -> list[dict[str, Any]]:
"""Describe every servable job for bridge tool registration."""
return [
{"name": name, "description": job.description, "parameters": job.parameters}
for name, job in sorted(self._request_jobs().items())
]
async def _call(self, request: dict[str, Any]) -> tuple[int, dict[str, Any]]:
name = str(request.get("job") or "")
job = self._request_jobs().get(name)
if job is None:
return 404, {"error": f"job {name!r} not found or not a request job"}
kwargs = dict(request.get("kwargs") or {})
injected = dict(request.get("injected_job_kwargs") or {})
conflicts = sorted(injected.keys() & kwargs.keys())
if conflicts:
return 400, {"error": f"{', '.join(conflicts)} injected by the server and cannot be provided"}
kwargs.update(injected)
if workspace := request.get("workspace"):
kwargs["workspace"] = workspace
if tool_context_id := request.get("tool_context_id"):
kwargs["tool_context_id"] = tool_context_id
try:
response = await job(**kwargs)
except ValueError as exc:
return 400, {"error": str(exc)}
return 200, {"success": response.success, "answer": response.answer}

View file

@ -1,5 +1,6 @@
"""Application context: shared state container for components, jobs, and service."""
from collections.abc import Iterator, MutableMapping
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING, Any
@ -12,65 +13,80 @@ if TYPE_CHECKING:
from .service import BaseService
class ApplicationContext:
"""Passive state container holding parsed config and wired components.
class _LegacyComponentView(MutableMapping):
"""Compatibility mapping combining shared components with default-workspace instances."""
The Application class performs the actual wiring (registry lookups and
component instantiation); this class only stores the results so that
components, jobs, and the service can find each other at runtime.
"""
def __init__(self, context: "ApplicationContext") -> None:
self._context = context
def _default_components(self) -> dict:
return next(iter(self._context.workspaces.values())).components
def __getitem__(self, key: str) -> dict:
if key in self._context.shared_components:
return self._context.shared_components[key]
return self._default_components()[key]
def __setitem__(self, key: str, value: dict) -> None:
if value and all(getattr(component, "workspace_scoped", False) is True for component in value.values()):
self._default_components()[key] = value
self._context.shared_components.pop(key, None)
else:
self._context.shared_components[key] = value
self._default_components().pop(key, None)
def __delitem__(self, key: str) -> None:
if key in self._context.shared_components:
del self._context.shared_components[key]
else:
del self._default_components()[key]
def __iter__(self) -> Iterator[str]:
yield from self._context.shared_components
yield from (key for key in self._default_components() if key not in self._context.shared_components)
def __len__(self) -> int:
return len(set(self._context.shared_components) | set(self._default_components()))
class ApplicationContext:
"""Application-wide configuration, shared resources, and workspace registry."""
def __init__(self, *, registry: ComponentRegistry | None = None, **kwargs):
# Parse raw kwargs into a typed, validated config object.
self.app_config: ApplicationConfig = ApplicationConfig(**kwargs)
self.registry = registry or create_application_registry()
# Ordered per-workspace path views; the first entry is the default workspace.
self.workspaces: dict[str, WorkspaceContext] = {
alias: WorkspaceContext.from_config(alias, path, self.app_config)
for alias, path in self.app_config.workspaces.items()
}
self.workspace: WorkspaceContext = next(iter(self.workspaces.values()))
# Populated by Application during initialization.
self.workspace: WorkspaceContext = next(iter(self.workspaces.values()))
self.service: "BaseService | None" = None
self.components: dict[str, dict[str, "BaseComponent"]] = {}
self.shared_components: dict[str, dict[str, "BaseComponent"]] = {}
self._components = _LegacyComponentView(self)
self.jobs: dict[str, "BaseJob"] = {}
self.thread_pool: ThreadPoolExecutor | None = None
# Application-lifetime shared state. Values remain available across Job and Step
# invocations while this Application is running, so Jobs may keep cross-call state here.
# This is in-memory state, not durable storage; use workspace files or a store when state
# must survive an Application restart.
self.metadata: dict[str, Any] = {}
@property
def workspace_components(self) -> dict[str, dict[str, dict[str, "BaseComponent"]]]:
"""Compatibility view of scoped instances, ctype -> name -> alias.
def components(self) -> MutableMapping:
"""Legacy component lookup; internal execution uses explicit shared or workspace ownership."""
return self._components
The authoritative owner of these instances is each WorkspaceContext;
this view exists for transitional readers.
"""
grouped: dict[str, dict[str, dict[str, "BaseComponent"]]] = {}
for alias, workspace in self.workspaces.items():
for ctype, group in workspace.components.items():
for name, instance in group.items():
grouped.setdefault(ctype, {}).setdefault(name, {})[alias] = instance
return grouped
@components.setter
def components(self, value: dict) -> None:
self.shared_components = {}
for ctype, group in value.items():
self._components[ctype] = group
def get_component(self, ctype: str, name: str, alias: str | None = None) -> "BaseComponent | None":
"""Resolve a component, honoring workspace bindings for scoped components.
Returns None when the component is not configured at all. Raises
KeyError when a workspace-scoped component has no instance for the
requested alias — a missing scoped resource must fail, never fall back.
"""
"""Compatibility resolver for external callers that select a workspace by alias."""
alias = alias or next(iter(self.workspaces))
workspace = self.workspaces.get(alias)
if workspace is None:
raise KeyError(f"Unknown workspace alias {alias!r}")
if name in workspace.components.get(ctype, {}):
return workspace.components[ctype][name]
return workspace.get_component(ctype, name)
if any(name in ws.components.get(ctype, {}) for ws in self.workspaces.values()):
raise KeyError(f"Component '{name}' in {ctype} has no instance for workspace alias {alias!r}")
return self.components.get(ctype, {}).get(name)
return self.shared_components.get(ctype, {}).get(name)

View file

@ -221,20 +221,24 @@ class BaseComponent(ComponentMixin, ABC):
setattr(self, attr, None)
def _resolve_from_context(self, attr: str, dep: Dependency) -> None:
"""Context-bound mode: look up the component from ``app_context.components``.
Workspace-scoped dependencies resolve to the same-alias instance as the
dependent; a missing alias instance is an error, never a fallback.
"""
"""Resolve scoped dependencies from their workspace and shared ones from the application."""
workspaces = getattr(self.app_context, "workspaces", None)
workspace = workspaces.get(self.workspace_alias) if workspaces and self.workspace_alias is not None else None
scoped_group = workspace.components.get(dep.ctype) if workspace is not None else None
if scoped_group is not None:
target = scoped_group.get(dep.name)
if target is None:
raise ValueError(f"{dep.ctype} '{dep.name}' has no instance for workspace '{self.workspace_alias}'.")
if workspace is not None and dep.name in workspace.components.get(dep.ctype, {}):
target = workspace.get_component(dep.ctype, dep.name)
elif workspace is not None and any(
dep.name in candidate.components.get(dep.ctype, {}) for candidate in workspaces.values()
):
raise ValueError(f"{dep.ctype} '{dep.name}' has no instance for workspace '{self.workspace_alias}'.")
else:
target = self.app_context.components.get(dep.ctype, {}).get(dep.name)
shared = getattr(self.app_context, "shared_components", None)
if shared is None:
shared = self.app_context.components
target = shared.get(dep.ctype, {}).get(dep.name)
if target is None and workspaces:
default_workspace = next(iter(workspaces.values()))
if dep.name in default_workspace.components.get(dep.ctype, {}):
target = default_workspace.get_component(dep.ctype, dep.name)
if target is not None:
setattr(self, attr, target)
elif dep.optional:

View file

@ -9,17 +9,17 @@ if TYPE_CHECKING:
from .application_config import ApplicationConfig
@dataclass
class WorkspaceAgentState:
"""Session execution state owned by one agent wrapper within one workspace."""
thread_tool_contexts: dict[str, str] = field(default_factory=dict)
owned_thread_ids: set[str] | None = None
@dataclass(frozen=True)
class WorkspaceContext:
"""Owns everything scoped to one workspace of an application.
Built from ApplicationConfig when the application context is created.
Beyond the resolved paths, the workspace owns its stateful component
instances (stores, graphs, indexes, catalogs), in-memory execution state
(drafts, search dedup, call budgets), session ownership records, and
workspace-scoped locks. Shared, stateless clients stay on the
ApplicationContext; per-call bindings reference this object.
"""
"""Owns the paths, stateful components, execution state, and locks of one workspace."""
alias: str
workspace_path: Path
@ -29,17 +29,16 @@ class WorkspaceContext:
resource_path: Path | None
daily_path: Path | None
digest_path: Path | None
# Workspace-scoped component instances keyed by component type then name.
# Values are BaseComponent instances; typed as Any to keep this module
# free of component imports (schema must not depend on components).
components: dict[str, dict[str, Any]] = field(default_factory=dict)
# Drafts and search dedup state, keyed by tool_context_id.
tool_contexts: dict[str, Any] = field(default_factory=dict)
# Search call budgets keyed by tool_context_id.
search_call_budgets: dict[str, Any] = field(default_factory=dict)
# Workspace-scoped named asyncio locks (e.g. dream integration).
agent_states: dict[str, WorkspaceAgentState] = field(default_factory=dict)
locks: dict[str, asyncio.Lock] = field(default_factory=dict)
def agent_state(self, wrapper_name: str) -> WorkspaceAgentState:
"""Return the named wrapper's state within this workspace."""
return self.agent_states.setdefault(wrapper_name, WorkspaceAgentState())
@classmethod
def from_config(cls, alias: str, workspace: str, config: "ApplicationConfig") -> "WorkspaceContext":
"""Resolve one alias of the config's workspaces mapping into absolute paths."""
@ -59,6 +58,13 @@ class WorkspaceContext:
digest_path=child(config.digest_dir),
)
def get_component(self, ctype: str, name: str) -> Any:
"""Return a component owned by this workspace, never a default or shared substitute."""
try:
return self.components[ctype][name]
except KeyError as exc:
raise KeyError(f"Component '{name}' in {ctype} has no instance for workspace '{self.alias}'") from exc
def storage_dirs(self) -> list[Path]:
"""Subdirectories the application materializes at startup, in configuration order."""
return [

View file

@ -87,14 +87,16 @@ class Ref:
if self.optional:
return None
raise RuntimeError(f"app_context is not set when resolving '{self.key}'")
if _context_workspaces(obj.app_context) is not None:
if (workspaces := _context_workspaces(obj.app_context)) is not None:
context = obj.context
bound = getattr(context, "workspace", None) if context is not None else None
comp = obj.app_context.get_component(
self.comp_enum,
name,
alias=bound.alias if bound is not None else None,
)
is_scoped = any(name in workspace.components.get(self.comp_enum, {}) for workspace in workspaces.values())
if is_scoped:
if bound is None:
raise RuntimeError(f"A workspace binding is required when resolving '{self.key}'")
comp = bound.get_component(self.comp_enum, name)
else:
comp = obj.app_context.shared_components.get(self.comp_enum, {}).get(name)
else:
# Test doubles and standalone-style contexts without the workspace layer.
comp = obj.app_context.components.get(self.comp_enum, {}).get(name)
@ -159,6 +161,8 @@ class BaseStep(ComponentMixin, ABC):
del self.__dict__[key]
self.context = RuntimeContext.from_context(context, **kwargs)
assert self.context is not None
if (workspaces := _context_workspaces(self.app_context)) is not None and self.context.workspace is None:
self.context.workspace = next(iter(workspaces.values()))
if self.input_mapping:
self.context.apply_mapping(self.input_mapping)
result = await self.execute()

View file

@ -343,11 +343,9 @@ class ProactiveTopicsStep(BaseStep):
return []
bound = getattr(self.context, "workspace", None) if self.context is not None else None
if _context_workspaces(self.app_context) is not None:
catalog = self.app_context.get_component(
ComponentEnum.FILE_CATALOG,
"digest",
alias=bound.alias if bound is not None else None,
)
if bound is None:
raise RuntimeError("A workspace binding is required to resolve the digest catalog")
catalog = bound.components.get(ComponentEnum.FILE_CATALOG, {}).get("digest")
else:
# Test doubles and standalone-style contexts without the workspace layer.
catalog = self.app_context.components.get(ComponentEnum.FILE_CATALOG, {}).get("digest")
@ -368,7 +366,7 @@ class ProactiveTopicsStep(BaseStep):
name = self.kwargs.get("as_embedding", "default")
if self.app_context is None:
return None
return self.app_context.components.get(ComponentEnum.AS_EMBEDDING, {}).get(name)
return self.app_context.shared_components.get(ComponentEnum.AS_EMBEDDING, {}).get(name)
async def _semantic_verdict(
self,

View file

@ -681,7 +681,7 @@ def resolve_agent_wrapper(step):
if wrapper is not None:
return wrapper
if step.app_context is not None:
fallback = step.app_context.components.get(ComponentEnum.AGENT_WRAPPER, {}).get("default")
fallback = step.app_context.shared_components.get(ComponentEnum.AGENT_WRAPPER, {}).get("default")
if fallback is not None:
configured = step.kwargs.get("agent_wrapper", "default")
step.logger.warning(f"[{step.name}] agent_wrapper '{configured}' missing; using default")

View file

@ -12,6 +12,7 @@ from watchfiles import Change
from ._change_batch import bucket_changes
from ..base_step import BaseStep
from ...components import R
from ...components.base_component import _context_workspaces
from ...components.file_chunker import BaseFileChunker
from ...enumeration import ComponentEnum
from ...schema import FileChunk, FileNode
@ -368,8 +369,10 @@ class UpdateIndexStep(ChangeApplyStep):
bound = getattr(self.context, "workspace", None) if self.context is not None else None
if bound is not None:
chunkers: dict[str, BaseFileChunker] = bound.components.get(ComponentEnum.FILE_CHUNKER, {})
else:
elif _context_workspaces(self.app_context) is None:
chunkers = self.app_context.components[ComponentEnum.FILE_CHUNKER]
else:
raise RuntimeError("A workspace binding is required to resolve file chunkers")
suffix = path.suffix.lstrip(".").lower()
for candidate in chunkers.values():
if suffix and suffix in {ext.lower().lstrip(".") for ext in candidate.supported_extensions}:

View file

@ -61,60 +61,51 @@ def _wrapper(tmp_path, **kwargs):
return CodexAgentWrapper(app_context=context, **kwargs), job
@pytest.mark.asyncio
async def test_mcp_config_uses_loopback_bridge_and_selected_jobs(tmp_path):
def test_mcp_config_uses_legacy_child_bridge_and_selected_jobs(tmp_path):
wrapper, _job = _wrapper(tmp_path, mcp_config="custom.yaml")
await wrapper._ensure_job_server() # pylint: disable=protected-access
config = wrapper._mcp_server_config( # pylint: disable=protected-access
config = wrapper._mcp_server_config(
{"job_tools": ["search", "search"], "tool_context_id": "ctx-1"},
)
await wrapper.close()
assert config["command"]
assert config["enabled_tools"] == ["search"]
assert config["args"].count("--job") == 1
assert config["args"][config["args"].index("--job") + 1] == "search"
assert "reme.components.agent_wrapper.codex_mcp_server" in config["args"]
assert config["args"][config["args"].index("--endpoint") + 1].startswith("http://127.0.0.1:")
assert config["args"][config["args"].index("--token") + 1]
assert config["args"][config["args"].index("--config") + 1] == str(tmp_path / "custom.yaml")
assert config["args"][config["args"].index("--workspace") + 1] == str(tmp_path)
assert config["args"][config["args"].index("--tool-context-id") + 1] == "ctx-1"
# mcp_config is accepted for compatibility but no longer builds a child app.
assert "--config" not in config["args"]
assert "--endpoint" not in config["args"]
assert "--token" not in config["args"]
@pytest.mark.asyncio
async def test_mcp_config_serializes_injected_job_kwargs(tmp_path):
def test_mcp_config_serializes_injected_job_kwargs(tmp_path):
wrapper, _job = _wrapper(tmp_path, mcp_config="custom.yaml")
await wrapper._ensure_job_server() # pylint: disable=protected-access
config = wrapper._mcp_server_config( # pylint: disable=protected-access
config = wrapper._mcp_server_config(
{
"job_tools": ["search"],
"tool_context_id": "ctx-1",
"injected_job_kwargs": {"_allowed_paths": ["daily/2025-06-01/note.md"]},
},
)
await wrapper.close()
raw = config["args"][config["args"].index("--injected-job-kwargs") + 1]
assert json.loads(raw) == {"_allowed_paths": ["daily/2025-06-01/note.md"]}
plain = wrapper._mcp_server_config({"job_tools": ["search"]}) # pylint: disable=protected-access
plain = wrapper._mcp_server_config({"job_tools": ["search"]})
assert "--injected-job-kwargs" not in plain["args"]
@pytest.mark.asyncio
async def test_thread_config_preserves_other_mcp_servers(tmp_path):
def test_thread_config_preserves_other_mcp_servers(tmp_path):
wrapper, _job = _wrapper(tmp_path)
await wrapper._ensure_job_server() # pylint: disable=protected-access
config = wrapper._thread_config( # pylint: disable=protected-access
config = wrapper._thread_config(
{
"job_tools": ["search"],
"config": {"mcp_servers": {"docs": {"url": "https://example.test/mcp"}}},
},
)
await wrapper.close()
assert "docs" in config["mcp_servers"]
assert len(config["mcp_servers"]) == 2
@ -166,65 +157,15 @@ def test_mcp_config_rejects_background_jobs(tmp_path):
wrapper._mcp_server_config({"job_tools": ["watch"]})
@pytest.mark.asyncio
async def test_loopback_endpoint_serves_parent_jobs(tmp_path):
import httpx
wrapper, job = _wrapper(tmp_path)
await wrapper.start()
server = await wrapper._ensure_job_server() # pylint: disable=protected-access
base = server.url
headers = {"Authorization": f"Bearer {server.token}"}
try:
async with httpx.AsyncClient(base_url=base, headers=headers, timeout=10) as http:
listed = (await http.get("/jobs")).json()["jobs"]
assert [entry["name"] for entry in listed] == ["search"]
called = await http.post("/call", json={"job": "search", "kwargs": {"query": "memo"}})
assert called.json() == {"success": True, "answer": "found:memo"}
# The job executed in the parent process, not in any child.
assert job.calls == [{"query": "memo"}]
missing = await http.post("/call", json={"job": "missing", "kwargs": {}})
assert missing.status_code == 404
unauthorized = await http.get("/jobs", headers={"Authorization": "Bearer wrong"})
assert unauthorized.status_code == 401
finally:
await wrapper.close()
assert server.is_running is False
@pytest.mark.asyncio
async def test_loopback_bridge_lists_and_calls_parent_job(tmp_path):
from fastmcp import Client
from fastmcp.client import StdioTransport
wrapper, job = _wrapper(tmp_path)
await wrapper._ensure_job_server() # pylint: disable=protected-access
server_config = wrapper._mcp_server_config({"job_tools": ["search"]}) # pylint: disable=protected-access
transport = StdioTransport(
command=server_config["command"],
args=server_config["args"],
cwd=server_config["cwd"],
keep_alive=False,
)
async with Client(transport, timeout=None, init_timeout=STDIO_INIT_TIMEOUT) as client:
tools = await asyncio.wait_for(client.list_tools(), timeout=10)
assert [tool.name for tool in tools] == ["search"]
result = await asyncio.wait_for(client.call_tool("search", {"query": "via-bridge"}), timeout=10)
await wrapper.close()
assert "found:via-bridge" in str(result.content[0].text)
assert job.calls == [{"query": "via-bridge"}]
@pytest.mark.asyncio
async def test_stdio_bridge_stdout_is_protocol_clean(tmp_path):
wrapper, _job = _wrapper(tmp_path)
await wrapper._ensure_job_server() # pylint: disable=protected-access
server_config = wrapper._mcp_server_config({"job_tools": ["search"]}) # pylint: disable=protected-access
config_path = tmp_path / "config.json"
config_path.write_text(
json.dumps({"jobs": {"search": {"backend": "base", "steps": []}}}),
encoding="utf-8",
)
wrapper, _job = _wrapper(tmp_path, mcp_config=str(config_path))
server_config = wrapper._mcp_server_config({"job_tools": ["search"]})
proc = await asyncio.create_subprocess_exec(
server_config["command"],
*server_config["args"],

View file

@ -165,7 +165,12 @@ async def test_workspace_scoped_components_instantiate_per_alias(tmp_path):
store_beta = app.context.get_component(ComponentEnum.FILE_STORE, "default", "beta")
assert store_alpha is not store_beta
assert app.context.components[ComponentEnum.FILE_STORE]["default"] is store_alpha # default alias entry
assert app.context.components[ComponentEnum.FILE_STORE]["default"] is store_alpha
assert ComponentEnum.FILE_STORE not in app.context.shared_components
assert app.context.workspaces["alpha"].get_component(ComponentEnum.FILE_STORE, "default") is store_alpha
assert app.context.workspaces["beta"].get_component(ComponentEnum.FILE_STORE, "default") is store_beta
with pytest.raises(KeyError, match="no instance for workspace 'alpha'"):
app.context.workspaces["alpha"].get_component(ComponentEnum.FILE_STORE, "missing")
assert store_alpha.workspace_path == tmp_path / "alpha"
assert store_beta.workspace_path == tmp_path / "beta"
assert store_alpha.chunks_path.parent == tmp_path / "alpha" / "metadata" / "file_store"
@ -314,7 +319,8 @@ async def test_digest_titles_follow_workspace_binding(tmp_path):
assert await step._digest_titles() == ["beta-only"]
step.context = RuntimeContext()
assert await step._digest_titles() == ["alpha-only"]
with pytest.raises(RuntimeError, match="workspace binding is required"):
await step._digest_titles()
@pytest.mark.asyncio

View file

@ -3,6 +3,8 @@
# pylint: disable=protected-access,missing-function-docstring
import asyncio
import json
import os
from types import SimpleNamespace
import pytest
@ -128,11 +130,8 @@ async def test_tool_context_state_namespaced_per_workspace(tmp_path):
@pytest.mark.asyncio
async def test_codex_thread_tool_contexts_namespaced_per_workspace(tmp_path):
wrapper = CodexAgentWrapper(
app_context=ApplicationContext(
workspace_dir={"alpha": str(tmp_path / "alpha"), "beta": str(tmp_path / "beta")},
),
)
context = ApplicationContext(workspace_dir={"alpha": str(tmp_path / "alpha"), "beta": str(tmp_path / "beta")})
wrapper = CodexAgentWrapper(app_context=context)
class FakeCodex:
"""Minimal Codex double echoing the requested thread id."""
@ -158,7 +157,13 @@ async def test_codex_thread_tool_contexts_namespaced_per_workspace(tmp_path):
with pytest.raises(ValueError, match="belongs to workspace 'alpha'"):
await wrapper._open_thread(codex, {"session_id": "thread-1", "workspace": "beta"})
assert sorted(wrapper._thread_tool_contexts) == ["beta/thread-2", "thread-1"]
alpha_state = context.workspaces["alpha"].agent_state(wrapper.name)
beta_state = context.workspaces["beta"].agent_state(wrapper.name)
assert set(alpha_state.thread_tool_contexts) == {"thread-1"}
assert set(beta_state.thread_tool_contexts) == {"thread-2"}
assert not wrapper._thread_tool_contexts
assert alpha_state.owned_thread_ids == {"thread-1"}
assert beta_state.owned_thread_ids == {"thread-2"}
@pytest.mark.asyncio
@ -295,27 +300,24 @@ async def test_mcp_tool_boundary_forwards_injected_workspace(tmp_path):
@pytest.mark.asyncio
async def test_codex_bridge_uses_loopback_endpoint(tmp_path):
import json
async def test_codex_bridge_uses_resident_mcp_endpoint(tmp_path):
from reme.components.service import HttpService
context = _context(tmp_path)
context.service = HttpService(app_context=context)
context.jobs = {"search_job": _job(context, "search_job")}
wrapper = CodexAgentWrapper(app_context=context)
await wrapper.start()
await wrapper._ensure_job_server() # pylint: disable=protected-access
config = wrapper._mcp_server_config({"job_tools": ["search_job"], "workspace": "beta"})
assert "--workspace-alias" in config["args"]
assert config["args"][config["args"].index("--workspace-alias") + 1] == "beta"
assert config["args"][config["args"].index("--endpoint") + 1] == "http://127.0.0.1:2333/mcp"
raw = config["args"][config["args"].index("--injected-job-kwargs") + 1]
assert json.loads(raw) == {"workspace": "beta"}
assert "--config" not in config["args"] and "--workspace" not in config["args"]
assert "--token" not in config["args"]
default_config = wrapper._mcp_server_config({"job_tools": ["search_job"]})
assert "--workspace-alias" not in default_config["args"]
assert "--injected-job-kwargs" not in default_config["args"]
await wrapper.close()
assert wrapper._job_server is None
# ----- Review follow-ups: explicit bindings, session isolation, key collisions
@ -423,18 +425,13 @@ async def test_codex_thread_cwd_follows_bound_workspace(tmp_path):
# ----- Codex bridge: real child-process execution ----------------------------
@pytest.mark.asyncio
async def test_codex_bridge_reads_resident_parent_state(tmp_path):
"""Bridge calls execute against the parent's live stores; there is no second owner."""
from fastmcp import Client
from fastmcp.client import StdioTransport
app = Application(
def _codex_test_app(tmp_path):
return Application(
workspace_dir={"alpha": str(tmp_path / "alpha"), "beta": str(tmp_path / "beta")},
enable_logo=False,
log_to_console=False,
log_to_file=False,
service={"backend": "cli"},
service={"backend": "http", "host": "127.0.0.1", "port": 0, "web_enabled": False},
components={
"tokenizer": {"default": {"backend": "regex"}},
"file_store": {"default": {"backend": "local", "embedding_store": "", "tag_index": ""}},
@ -464,57 +461,104 @@ async def test_codex_bridge_reads_resident_parent_state(tmp_path):
},
},
)
async def _start_resident_http(app):
import uvicorn
service = app.context.service
service.build_service(app)
service.add_jobs(app)
config = uvicorn.Config(service.service, host=service.host, port=0, log_level="error", lifespan="on")
server = uvicorn.Server(config)
task = asyncio.create_task(server.serve())
for _ in range(100):
if server.started:
break
await asyncio.sleep(0.05)
assert server.started
service.port = server.servers[0].sockets[0].getsockname()[1]
return server, task
async def _stop_resident_http(server, task):
server.should_exit = True
await asyncio.wait_for(task, timeout=30)
@pytest.mark.asyncio
async def test_codex_bridge_reads_resident_parent_state(tmp_path):
"""The stdio bridge reuses the resident MCP service and its live workspace state."""
from fastmcp import Client
from fastmcp.client import StdioTransport
app = _codex_test_app(tmp_path)
wrapper = CodexAgentWrapper(app_context=app.context)
async with app:
await wrapper.start()
# Index a note directly in the parent's beta store.
await app.run_job("write_note", path="daily/resident.md", content="parent-resident-beta", workspace="beta")
await wrapper._ensure_job_server() # pylint: disable=protected-access
server_config = wrapper._mcp_server_config( # pylint: disable=protected-access
{"job_tools": ["read_note", "write_note"], "workspace": "beta"},
)
transport = StdioTransport(
command=server_config["command"],
args=server_config["args"],
cwd=server_config["cwd"],
keep_alive=False,
)
async with Client(transport, timeout=None, init_timeout=30) as client:
read = await asyncio.wait_for(client.call_tool("read_note", {"path": "daily/resident.md"}), timeout=30)
await asyncio.wait_for(
client.call_tool("write_note", {"path": "daily/bridge.md", "content": "via-bridge-beta"}),
timeout=30,
server, task = await _start_resident_http(app)
try:
await app.run_job("write_note", path="daily/resident.md", content="parent-resident-beta", workspace="beta")
from reme.components.client import MCPClient
resident_client = MCPClient(transport="streamable-http", host="127.0.0.1", port=app.context.service.port)
await resident_client.start()
assert {entry["name"] for entry in await resident_client.list_actions()} == {"read_note", "write_note"}
await resident_client.close()
server_config = wrapper._mcp_server_config(
{"job_tools": ["read_note", "write_note"], "workspace": "beta"},
)
# The bridge saw the parent's resident beta state and wrote through it.
assert "parent-resident-beta" in str(read.content[0].text)
assert "via-bridge-beta" in (tmp_path / "beta" / "daily" / "bridge.md").read_text(encoding="utf-8")
assert not (tmp_path / "alpha" / "daily" / "bridge.md").exists()
store_alpha = app.context.get_component(ComponentEnum.FILE_STORE, "default", "alpha")
assert all("bridge" not in node.path for node in await store_alpha.get_nodes())
transport = StdioTransport(
command=server_config["command"],
args=server_config["args"],
cwd=server_config["cwd"],
keep_alive=False,
env=dict(os.environ),
)
async with Client(transport, timeout=None, init_timeout=30) as client:
tools = await client.list_tools()
assert {tool.name for tool in tools} == {"read_note", "write_note"}
read = await asyncio.wait_for(client.call_tool("read_note", {"path": "daily/resident.md"}), timeout=30)
with pytest.raises(Exception):
await client.call_tool("read_note", {"path": "daily/resident.md", "workspace": "alpha"})
await asyncio.wait_for(
client.call_tool("write_note", {"path": "daily/bridge.md", "content": "via-bridge-beta"}),
timeout=30,
)
assert "parent-resident-beta" in str(read.content[0].text)
assert "via-bridge-beta" in (tmp_path / "beta" / "daily" / "bridge.md").read_text(encoding="utf-8")
assert not (tmp_path / "alpha" / "daily" / "bridge.md").exists()
store_alpha = app.context.get_component(ComponentEnum.FILE_STORE, "default", "alpha")
assert all("bridge" not in node.path for node in await store_alpha.get_nodes())
invalid_config = wrapper._mcp_server_config({"job_tools": ["read_note"], "workspace": "nope"})
invalid_transport = StdioTransport(
command=invalid_config["command"],
args=invalid_config["args"],
cwd=invalid_config["cwd"],
keep_alive=False,
env=dict(os.environ),
)
async with Client(invalid_transport, timeout=None, init_timeout=30) as client:
with pytest.raises(Exception, match="Unknown workspace 'nope'"):
await asyncio.wait_for(client.call_tool("read_note", {"path": "daily/resident.md"}), timeout=30)
finally:
await _stop_resident_http(server, task)
await wrapper.close()
@pytest.mark.asyncio
async def test_codex_bridge_unknown_alias_fails_at_execution(tmp_path):
"""An unknown workspace alias surfaces as a tool error from the parent boundary."""
from fastmcp import Client
from fastmcp.client import StdioTransport
async def test_codex_multi_workspace_rejects_unsupported_job_transport(tmp_path):
from reme.components.service import CliService
context = _context(tmp_path)
context.service = CliService(app_context=context)
context.jobs = {"search_job": _job(context, "search_job")}
wrapper = CodexAgentWrapper(app_context=context, project_path=str(tmp_path))
await wrapper.start()
await wrapper._ensure_job_server() # pylint: disable=protected-access
server_config = wrapper._mcp_server_config( # pylint: disable=protected-access
{"job_tools": ["search_job"], "workspace": "nope"},
)
transport = StdioTransport(
command=server_config["command"],
args=server_config["args"],
cwd=server_config["cwd"],
keep_alive=False,
)
async with Client(transport, timeout=None, init_timeout=30) as client:
with pytest.raises(Exception, match="Unknown workspace 'nope'"):
await asyncio.wait_for(client.call_tool("search_job", {}), timeout=30)
wrapper = CodexAgentWrapper(app_context=context)
with pytest.raises(RuntimeError, match="resident HTTP MCP service"):
wrapper._mcp_server_config({"job_tools": ["search_job"], "workspace": "beta"})
with pytest.raises(ValueError, match="only supported for single-workspace"):
wrapper._mcp_server_config(
{"job_tools": ["search_job"], "workspace": "beta", "mcp_config": "custom.yaml"},
)
await wrapper.close()