mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-15 09:11:24 +00:00
refactor(workspace): consolidate ownership and Codex integration
This commit is contained in:
parent
440b85b5c4
commit
f0afcfd13a
17 changed files with 548 additions and 547 deletions
13
AGENTS.md
13
AGENTS.md
|
|
@ -132,8 +132,12 @@ Treat Step instances as invocation-scoped:
|
|||
- `self.context.response.answer`, `success`, and `metadata` are request-scoped output. Because the same response travels
|
||||
through the Step chain, later Steps may consume metadata produced earlier, but it is not application-lifetime or
|
||||
durable storage.
|
||||
- `self.app_context.metadata` holds in-memory state shared across Job/Step invocations for the life of one
|
||||
`Application`, such as counters, tool-context state, session maps, or locks.
|
||||
- `self.app_context.metadata` holds application-lifetime operational state shared across Job/Step invocations,
|
||||
such as counters. `ApplicationContext.shared_components` owns genuinely shared components; its `components`
|
||||
compatibility view also exposes default-workspace instances to legacy callers.
|
||||
- Each `WorkspaceContext` owns its paths, stateful components, tool contexts, search budgets, agent session state,
|
||||
and locks. `RuntimeContext.workspace` is the explicit binding used by workspace-bound execution; internal Steps
|
||||
must not resolve a default workspace repeatedly or use the compatibility component view.
|
||||
- Workspace files or a dedicated Component/store hold durable state that must survive restart.
|
||||
|
||||
Use narrow, namespaced keys in `app_context.metadata` and protect shared mutable values against concurrent access. The
|
||||
|
|
@ -144,8 +148,9 @@ dedicated lifecycle, locking, or persistence, promote it to a typed context fiel
|
|||
|
||||
Additional Step contracts:
|
||||
|
||||
- `Ref` dependencies resolve in this order: Step kwargs, current `RuntimeContext`, then the named application component.
|
||||
The value is cached only on the current Step instance and cleared before each call.
|
||||
- `Ref` dependencies resolve in this order: Step kwargs, current `RuntimeContext`, then the bound workspace for scoped
|
||||
components or the application shared-component registry for shared components. The value is cached only on the
|
||||
current Step instance and cleared before each call.
|
||||
- `input_mapping` and `output_mapping` copy keys within `RuntimeContext.data`; missing sources are ignored.
|
||||
- Dispatched Steps receive the current `RuntimeContext`, so their data and response are shared.
|
||||
- Base jobs convert uncaught Step errors into `Response(success=False)`; stream jobs emit an error chunk and always a
|
||||
|
|
|
|||
|
|
@ -100,8 +100,13 @@ External stateful backends that cannot isolate workspaces (currently the `neo4j`
|
|||
startup in multi-workspace configurations.
|
||||
|
||||
The binding propagates through nested job calls and agent tool calls (`job_tools`) as a server-owned constraint
|
||||
that models cannot override. Background and cron jobs bind through their job configuration, so extra workspaces
|
||||
are watched by declaring explicit per-alias variants:
|
||||
that models cannot override. For Codex with multiple workspaces, `job_tools` requires the resident HTTP service
|
||||
with MCP enabled, and the selected jobs must be exposed by that service. Its STDIO bridge forwards to this service
|
||||
without creating another Application. The multi-workspace one-shot CLI combination is not supported; `mcp_config`
|
||||
retains its legacy meaning only for single-workspace Codex job tools.
|
||||
|
||||
Background and cron jobs bind through their job configuration, so extra workspaces are watched by declaring
|
||||
explicit per-alias variants:
|
||||
|
||||
```yaml
|
||||
jobs:
|
||||
|
|
|
|||
|
|
@ -102,6 +102,10 @@ Job 调用接受可选的 `workspace` 字段指定绑定的别名,例如
|
|||
在多 workspace 配置下会在启动时被拒绝。
|
||||
|
||||
绑定会经嵌套 Job 调用和 agent 工具调用(`job_tools`)传播,作为模型无法覆盖的服务端约束。
|
||||
多 workspace 下的 Codex `job_tools` 需要启用 MCP 的常驻 HTTP 服务,且所选 Job 必须由该服务暴露;
|
||||
其 STDIO bridge 转发到该服务,不会创建另一个 Application。多 workspace 与 one-shot CLI 的组合暂不支持;
|
||||
`mcp_config` 仅在单 workspace 的 Codex job tools 中保留原有语义。
|
||||
|
||||
Background 和 cron Job 通过 Job 配置绑定别名,额外 workspace 的 watch 通过显式配置按别名的变体实现:
|
||||
|
||||
```yaml
|
||||
|
|
|
|||
|
|
@ -73,7 +73,6 @@ class Application(BaseComponent):
|
|||
"""Instantiate declared components; workspace-scoped ones once per alias."""
|
||||
multi_workspace = len(self.context.workspaces) > 1
|
||||
for ctype, group in self.config.components.items():
|
||||
self.context.components[ctype] = {}
|
||||
for name, cfg in group.items():
|
||||
label = f"Component '{name}'"
|
||||
backend_cls = self._backend_class(ctype, cfg, label)
|
||||
|
|
@ -92,12 +91,8 @@ class Application(BaseComponent):
|
|||
workspace_alias=alias,
|
||||
)
|
||||
workspace.components.setdefault(ctype, {})[name] = instance
|
||||
# The default alias instance doubles as the plain lookup entry.
|
||||
self.context.components[ctype][name] = next(iter(self.context.workspaces.values())).components[
|
||||
ctype
|
||||
][name]
|
||||
else:
|
||||
self.context.components[ctype][name] = self._instantiate(
|
||||
self.context.shared_components.setdefault(ctype, {})[name] = self._instantiate(
|
||||
ctype,
|
||||
cfg,
|
||||
label=label,
|
||||
|
|
@ -166,16 +161,13 @@ class Application(BaseComponent):
|
|||
) -> dict[_NodeKey, BaseComponent]:
|
||||
"""One node per component instance; scoped components contribute one node per alias."""
|
||||
nodes: dict[_NodeKey, BaseComponent] = {}
|
||||
scoped_keys: set[tuple[str, str]] = set()
|
||||
for alias, workspace in self.context.workspaces.items():
|
||||
for ctype, group in workspace.components.items():
|
||||
for name, instance in group.items():
|
||||
nodes[(ctype, name, alias)] = instance
|
||||
scoped_keys.add((ctype, name))
|
||||
for ctype, group in self.context.components.items():
|
||||
for ctype, group in self.context.shared_components.items():
|
||||
for name, comp in group.items():
|
||||
if (ctype, name) not in scoped_keys:
|
||||
nodes[(ctype, name, "")] = comp
|
||||
nodes[(ctype, name, "")] = comp
|
||||
if replacement is not None:
|
||||
nodes[replacement[0]] = replacement[1]
|
||||
return nodes
|
||||
|
|
@ -297,11 +289,10 @@ class Application(BaseComponent):
|
|||
"""Update an existing component by type/name; never creates missing components."""
|
||||
async with self._component_mutation_lock:
|
||||
component_type = component_type_name(component_enum)
|
||||
group = self.context.components.get(component_type)
|
||||
if not group or name not in group:
|
||||
component = self.context.get_component(component_type, name)
|
||||
if component is None:
|
||||
raise KeyError(f"Component '{name}' not found in {component_type}")
|
||||
|
||||
component = group[name]
|
||||
# Validate fields without evaluating lazy properties before injection.
|
||||
for key in kwargs:
|
||||
if getattr_static(component, key, _UNSET) is _UNSET and not hasattr(component, key):
|
||||
|
|
@ -317,7 +308,7 @@ class Application(BaseComponent):
|
|||
for group in workspace.components.values():
|
||||
for instance in group.values():
|
||||
instances[id(instance)] = instance
|
||||
for group in self.context.components.values():
|
||||
for group in self.context.shared_components.values():
|
||||
for component in group.values():
|
||||
instances[id(component)] = component
|
||||
return list(instances.values())
|
||||
|
|
@ -365,19 +356,27 @@ class Application(BaseComponent):
|
|||
"""
|
||||
async with self._component_mutation_lock:
|
||||
component_type = component_type_name(component_enum)
|
||||
group = self.context.components.get(component_type)
|
||||
config_group = self.config.components.get(component_type)
|
||||
if not group or name not in group or config_group is None:
|
||||
raise KeyError(f"Component '{name}' not found in {component_type}")
|
||||
|
||||
old_component = group[name]
|
||||
scoped_instances = self.context.workspace_components.get(component_type, {}).get(name)
|
||||
if scoped_instances is not None and len(scoped_instances) > 1:
|
||||
scoped_instances = {
|
||||
alias: workspace.components[component_type][name]
|
||||
for alias, workspace in self.context.workspaces.items()
|
||||
if name in workspace.components.get(component_type, {})
|
||||
}
|
||||
if len(scoped_instances) > 1:
|
||||
raise NotImplementedError(
|
||||
f"replace_component does not support workspace-scoped component "
|
||||
f"'{component_type}:{name}' in multi-workspace configurations",
|
||||
)
|
||||
alias = next(iter(scoped_instances)) if scoped_instances is not None else None
|
||||
alias = next(iter(scoped_instances), None)
|
||||
group = (
|
||||
self.context.workspaces[alias].components.get(component_type)
|
||||
if alias is not None
|
||||
else self.context.shared_components.get(component_type)
|
||||
)
|
||||
if not group or name not in group or config_group is None:
|
||||
raise KeyError(f"Component '{name}' not found in {component_type}")
|
||||
|
||||
old_component = group[name]
|
||||
replacement_config = (
|
||||
config.model_copy(deep=True)
|
||||
if isinstance(config, ComponentConfig)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
import hashlib
|
||||
|
|
@ -11,14 +12,16 @@ import json
|
|||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Any, TYPE_CHECKING
|
||||
|
||||
from .base_agent_wrapper import BaseAgentWrapper
|
||||
from .loopback_job_server import LoopbackJobServer
|
||||
from ..base_component import _context_workspaces
|
||||
from ..component_registry import R
|
||||
from ...constants import normalize_connect_host
|
||||
from ...enumeration import ChunkEnum
|
||||
from ...schema import StreamChunk, TokenUsage
|
||||
from ...schema.workspace import WorkspaceAgentState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai_codex import AsyncCodex, AsyncThread, CodexConfig, RunInput
|
||||
|
|
@ -116,9 +119,7 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
self._release_threads = release_threads
|
||||
self._turn_lock = asyncio.Lock()
|
||||
self._thread_tool_contexts: dict[str, str] = {}
|
||||
self._thread_owners: dict[str, set[str]] = {}
|
||||
self._job_server: LoopbackJobServer | None = None
|
||||
self._mcp_config_warned = False
|
||||
self._mcp_snapshot_path: Path | None = None
|
||||
|
||||
@property
|
||||
def session_path(self) -> Path:
|
||||
|
|
@ -149,15 +150,47 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
relative_source = os.path.relpath(source, target.parent)
|
||||
target.symlink_to(relative_source, target_is_directory=True)
|
||||
|
||||
def _mcp_config_source_unused(self, kwargs: dict[str, Any]) -> None:
|
||||
"""Warn once that job_tools now execute in the parent via the loopback endpoint."""
|
||||
@property
|
||||
def wrapper_session_path(self) -> Path:
|
||||
"""ReMe-owned session data, kept separate from a shared OAuth CODEX_HOME."""
|
||||
if self.app_context is None:
|
||||
return self.workspace_path / "mem_session" / "codex"
|
||||
return self.workspace_path / self.app_context.app_config.mem_session_dir / "codex"
|
||||
|
||||
def _explicit_mcp_config(self, kwargs: dict[str, Any]) -> str | None:
|
||||
value = kwargs.get("mcp_config") if "mcp_config" in kwargs else self.mcp_config
|
||||
if value is not None and not self._mcp_config_warned:
|
||||
self._mcp_config_warned = True
|
||||
self.logger.warning(
|
||||
"mcp_config is ignored for job_tools: Codex job tools execute in the parent "
|
||||
"process via the wrapper-owned loopback endpoint",
|
||||
)
|
||||
if value is None:
|
||||
return None
|
||||
source = Path(str(value)).expanduser()
|
||||
if source.suffix in {".yaml", ".yml", ".json"}:
|
||||
if not source.is_absolute():
|
||||
source = self.workspace_path / source
|
||||
return str(source.absolute())
|
||||
return str(value)
|
||||
|
||||
def _effective_config_snapshot(self) -> Path:
|
||||
"""Create one private snapshot that remains valid for the client lifetime."""
|
||||
if self._mcp_snapshot_path is not None:
|
||||
return self._mcp_snapshot_path
|
||||
if self.app_context is None:
|
||||
raise RuntimeError("Cannot snapshot MCP config without an app_context")
|
||||
snapshot_dir = self.wrapper_session_path / "reme-mcp"
|
||||
snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
fd, raw_path = tempfile.mkstemp(prefix="config-", suffix=".json", dir=snapshot_dir)
|
||||
try:
|
||||
os.fchmod(fd, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as stream:
|
||||
json.dump(self.app_context.app_config.model_dump(mode="json"), stream)
|
||||
except BaseException:
|
||||
with suppress(OSError):
|
||||
os.close(fd)
|
||||
Path(raw_path).unlink(missing_ok=True)
|
||||
raise
|
||||
self._mcp_snapshot_path = Path(raw_path)
|
||||
return self._mcp_snapshot_path
|
||||
|
||||
def _mcp_config_source(self, kwargs: dict[str, Any]) -> str:
|
||||
return self._explicit_mcp_config(kwargs) or str(self._effective_config_snapshot())
|
||||
|
||||
@staticmethod
|
||||
def _resolve_auth_config(auth_mode: str, api_key: str = "", base_url: str = "") -> _CodexAuthConfig:
|
||||
|
|
@ -209,19 +242,32 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
names = ", ".join(invalid)
|
||||
raise TypeError(f"Codex client options must be configured on the wrapper: {names}")
|
||||
|
||||
async def _ensure_job_server(self) -> LoopbackJobServer:
|
||||
"""Start the wrapper-owned loopback job endpoint on first bridge use."""
|
||||
if self._job_server is None:
|
||||
if self.app_context is None:
|
||||
raise RuntimeError("Codex job_tools require an app_context")
|
||||
self._job_server = LoopbackJobServer(self.app_context)
|
||||
await self._job_server.start()
|
||||
return self._job_server
|
||||
def _resident_mcp_endpoint(self) -> str:
|
||||
from ..service import HttpService, MCPService
|
||||
|
||||
service = self.app_context.service if self.app_context is not None else None
|
||||
if isinstance(service, HttpService) and service.mcp_enabled:
|
||||
return f"http://{normalize_connect_host(service.host)}:{service.port}{service.mcp_path}"
|
||||
if isinstance(service, MCPService) and service.transport == "streamable-http":
|
||||
return f"http://{normalize_connect_host(service.host)}:{service.port}/mcp"
|
||||
raise RuntimeError(
|
||||
"Multi-workspace Codex job_tools require a resident HTTP MCP service; "
|
||||
"one-shot CLI and other service backends are not supported for this combination",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_resident_jobs(service, jobs) -> None:
|
||||
unavailable = [
|
||||
job.name
|
||||
for job in jobs
|
||||
if not job.enabled or not job.enable_serve or (service.jobs is not None and job.name not in service.jobs)
|
||||
]
|
||||
if unavailable:
|
||||
raise ValueError(f"Codex job_tools must be exposed by the resident MCP service: {', '.join(unavailable)}")
|
||||
|
||||
def _mcp_server_config(self, kwargs: dict[str, Any]) -> dict[str, Any] | None:
|
||||
from ..job import BackgroundJob, StreamJob
|
||||
|
||||
self._mcp_config_source_unused(kwargs)
|
||||
job_names = list(dict.fromkeys(kwargs.get("job_tools") or []))
|
||||
if not job_names:
|
||||
return None
|
||||
|
|
@ -229,26 +275,29 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
unsupported = [job.name for job in jobs if isinstance(job, (BackgroundJob, StreamJob))]
|
||||
if unsupported:
|
||||
raise TypeError(f"Codex job_tools must be non-stream request jobs: {', '.join(unsupported)}")
|
||||
if self._job_server is None:
|
||||
raise RuntimeError("Codex job endpoint is not started; call _ensure_job_server first")
|
||||
|
||||
alias = str(kwargs.get("workspace") or "")
|
||||
args = [
|
||||
"-m",
|
||||
"reme.components.agent_wrapper.codex_mcp_server",
|
||||
"--endpoint",
|
||||
self._job_server.url,
|
||||
"--token",
|
||||
self._job_server.token,
|
||||
]
|
||||
if alias:
|
||||
args.extend(["--workspace-alias", alias])
|
||||
workspaces = _context_workspaces(self.app_context)
|
||||
multi_workspace = workspaces is not None and len(workspaces) > 1
|
||||
if multi_workspace:
|
||||
if self._explicit_mcp_config(kwargs) is not None:
|
||||
raise ValueError("mcp_config is only supported for single-workspace Codex job_tools")
|
||||
endpoint = self._resident_mcp_endpoint()
|
||||
self._validate_resident_jobs(self.app_context.service, jobs)
|
||||
args = ["-m", "reme.components.agent_wrapper.codex_mcp_server", "--endpoint", endpoint]
|
||||
else:
|
||||
args = [
|
||||
"-m",
|
||||
"reme.components.agent_wrapper.codex_mcp_server",
|
||||
"--config",
|
||||
self._mcp_config_source(kwargs),
|
||||
"--workspace",
|
||||
str(self.workspace_path),
|
||||
]
|
||||
for name in job_names:
|
||||
args.extend(["--job", name])
|
||||
args.extend(["--tool-context-id", str(kwargs.get("tool_context_id") or "")])
|
||||
injected_job_kwargs = dict(kwargs.get("injected_job_kwargs") or {})
|
||||
if workspace := kwargs.get("workspace"):
|
||||
# The bridge injects the binding into nested job calls; models cannot override it.
|
||||
if multi_workspace and (workspace := kwargs.get("workspace")):
|
||||
injected_job_kwargs["workspace"] = workspace
|
||||
if injected_job_kwargs:
|
||||
args.extend(["--injected-job-kwargs", json.dumps(injected_job_kwargs, sort_keys=True)])
|
||||
|
|
@ -288,6 +337,7 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
from openai_codex.types import Personality, ThreadSource, ThreadStartSource
|
||||
|
||||
alias = str(kwargs.get("workspace") or "")
|
||||
tool_contexts = self._tool_contexts(alias)
|
||||
resume = kwargs.get("resume") or ""
|
||||
session_id = kwargs.get("session_id") or ""
|
||||
if resume and session_id and resume != session_id:
|
||||
|
|
@ -297,12 +347,9 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
if fork_session and not thread_id:
|
||||
raise ValueError("fork_session=True requires resume or session_id")
|
||||
if thread_id:
|
||||
# Cross-workspace thread access is rejected before touching the SDK.
|
||||
await self._check_thread_ownership(thread_id, alias)
|
||||
if kwargs.get("job_tools"):
|
||||
await self._ensure_job_server()
|
||||
if self._release_threads and self._thread_tool_contexts:
|
||||
await self._prune_thread_tool_contexts(codex)
|
||||
if self._release_threads and tool_contexts:
|
||||
await self._prune_thread_tool_contexts(codex, alias)
|
||||
requested_tool_context = json.dumps(
|
||||
{
|
||||
"tool_context_id": str(kwargs.get("tool_context_id") or ""),
|
||||
|
|
@ -311,9 +358,8 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
sort_keys=True,
|
||||
default=str,
|
||||
)
|
||||
guard_key = f"{alias}/{thread_id}" if alias else thread_id
|
||||
if not fork_session and guard_key in self._thread_tool_contexts:
|
||||
if requested_tool_context != self._thread_tool_contexts[guard_key]:
|
||||
if not fork_session and thread_id in tool_contexts:
|
||||
if requested_tool_context != tool_contexts[thread_id]:
|
||||
raise ValueError("tool_context_id and injected_job_kwargs cannot change when resuming a Codex thread")
|
||||
|
||||
common = {
|
||||
|
|
@ -348,19 +394,28 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
**common,
|
||||
)
|
||||
if fork_session or not thread_id:
|
||||
# Threads created under this binding are owned by this workspace.
|
||||
self._record_thread_ownership(alias, thread.id)
|
||||
self._thread_tool_contexts[f"{alias}/{thread.id}" if alias else thread.id] = requested_tool_context
|
||||
tool_contexts[thread.id] = requested_tool_context
|
||||
return thread
|
||||
|
||||
# ----- Thread ownership (per-workspace, persisted) ---------------------
|
||||
|
||||
def _normalize_thread_alias(self, alias: str) -> str:
|
||||
"""Map a call alias to a concrete workspace alias ("" means the default)."""
|
||||
def _agent_state(self, alias: str) -> WorkspaceAgentState | None:
|
||||
workspaces = _context_workspaces(self.app_context)
|
||||
if not workspaces:
|
||||
return alias
|
||||
return alias if alias in workspaces else next(iter(workspaces))
|
||||
return None
|
||||
return workspaces[self._normalize_thread_alias(alias)].agent_state(self.name)
|
||||
|
||||
def _tool_contexts(self, alias: str) -> dict[str, str]:
|
||||
state = self._agent_state(alias)
|
||||
return state.thread_tool_contexts if state is not None else self._thread_tool_contexts
|
||||
|
||||
def _normalize_thread_alias(self, alias: str) -> str:
|
||||
"""Map an omitted alias to the default while preserving explicit aliases."""
|
||||
workspaces = _context_workspaces(self.app_context)
|
||||
if workspaces and not alias:
|
||||
return next(iter(workspaces))
|
||||
return alias
|
||||
|
||||
def _thread_ownership_path(self, alias: str) -> Path | None:
|
||||
workspaces = _context_workspaces(self.app_context)
|
||||
|
|
@ -372,9 +427,10 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
return workspace.mem_session_path / "codex" / "owned_threads.json"
|
||||
|
||||
def _owned_thread_ids(self, alias: str) -> set[str]:
|
||||
"""Load (once) the persisted ids of threads this workspace created."""
|
||||
if alias in self._thread_owners:
|
||||
return self._thread_owners[alias]
|
||||
"""Load the persisted ids into the owning workspace's agent state."""
|
||||
state = self._agent_state(alias)
|
||||
if state is not None and state.owned_thread_ids is not None:
|
||||
return state.owned_thread_ids
|
||||
ids: set[str] = set()
|
||||
path = self._thread_ownership_path(alias)
|
||||
if path is not None and path.is_file():
|
||||
|
|
@ -383,7 +439,8 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
ids = {str(thread_id) for thread_id in raw.get("thread_ids", [])}
|
||||
except (OSError, ValueError, AttributeError) as exc:
|
||||
self.logger.warning(f"Ignoring unreadable Codex thread ownership file {path}: {exc}")
|
||||
self._thread_owners[alias] = ids
|
||||
if state is not None:
|
||||
state.owned_thread_ids = ids
|
||||
return ids
|
||||
|
||||
def _record_thread_ownership(self, alias: str, thread_id: str) -> None:
|
||||
|
|
@ -453,20 +510,19 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
return codex
|
||||
|
||||
async def _close(self) -> None:
|
||||
"""Close the persistent app-server and the wrapper-owned job endpoint."""
|
||||
"""Close the persistent app-server and clear session execution guards."""
|
||||
async with self._turn_lock:
|
||||
codex, self._codex = self._codex, None
|
||||
self._thread_tool_contexts.clear()
|
||||
try:
|
||||
if codex is not None:
|
||||
await codex.close()
|
||||
finally:
|
||||
if self._job_server is not None:
|
||||
job_server, self._job_server = self._job_server, None
|
||||
await job_server.close()
|
||||
for workspace in (_context_workspaces(self.app_context) or {}).values():
|
||||
if state := workspace.agent_states.get(self.name):
|
||||
state.thread_tool_contexts.clear()
|
||||
if codex is not None:
|
||||
await codex.close()
|
||||
|
||||
async def _prune_thread_tool_contexts(self, codex: AsyncCodex) -> None:
|
||||
async def _prune_thread_tool_contexts(self, codex: AsyncCodex, alias: str = "") -> None:
|
||||
"""Forget tool constraints only after the server confirms a thread is unloaded."""
|
||||
tool_contexts = self._tool_contexts(alias)
|
||||
try:
|
||||
from openai_codex.generated.v2_all import ThreadLoadedListResponse
|
||||
|
||||
|
|
@ -485,9 +541,9 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
self.logger.warning(f"Failed to list loaded Codex threads: {exc}")
|
||||
return
|
||||
self._thread_tool_contexts = {
|
||||
key: context for key, context in self._thread_tool_contexts.items() if key.rsplit("/", 1)[-1] in loaded_ids
|
||||
}
|
||||
retained = {thread_id: context for thread_id, context in tool_contexts.items() if thread_id in loaded_ids}
|
||||
tool_contexts.clear()
|
||||
tool_contexts.update(retained)
|
||||
|
||||
async def _release_thread(self, codex: AsyncCodex, thread_id: str) -> None:
|
||||
"""Unsubscribe so the app-server can unload the thread once it is idle."""
|
||||
|
|
|
|||
|
|
@ -1,25 +1,27 @@
|
|||
"""Thin FastMCP STDIO bridge forwarding Codex job tools to the parent process.
|
||||
"""Expose ReMe jobs to Codex over FastMCP STDIO.
|
||||
|
||||
This subprocess never builds a ReMe Application or any workspace store. It
|
||||
asks the parent-owned loopback endpoint (LoopbackJobServer) for job metadata
|
||||
and forwards every tool call — together with the server-owned workspace alias,
|
||||
tool context id, and injected kwargs — back to the parent for execution, so
|
||||
all workspace state has exactly one owner.
|
||||
Multi-workspace callers forward to the resident MCP service without creating
|
||||
an Application. Single-workspace callers retain the legacy child-service path.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from .base_agent_wrapper import BaseAgentWrapper
|
||||
from ..client import MCPClient
|
||||
from ...config import resolve_app_config
|
||||
from ...reme import ReMe
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Forward selected ReMe jobs from the parent loopback endpoint")
|
||||
parser.add_argument("--endpoint", required=True, help="Parent loopback endpoint base URL")
|
||||
parser.add_argument("--token", required=True, help="Bearer token for the loopback endpoint")
|
||||
parser = argparse.ArgumentParser(description="Expose selected ReMe jobs over FastMCP STDIO")
|
||||
parser.add_argument("--config", default="default", help="ReMe config name or file path")
|
||||
parser.add_argument("--workspace", help="Single-workspace directory for the legacy child service")
|
||||
parser.add_argument("--endpoint", help="Resident streamable-HTTP MCP endpoint for forwarding")
|
||||
parser.add_argument("--job", dest="jobs", action="append", required=True, help="ReMe job name; repeat as needed")
|
||||
parser.add_argument("--workspace-alias", default="", help="Workspace alias injected into every job call")
|
||||
parser.add_argument("--tool-context-id", default="", help="Context id injected into every job call")
|
||||
parser.add_argument(
|
||||
"--injected-job-kwargs",
|
||||
|
|
@ -30,89 +32,145 @@ def _parse_args() -> argparse.Namespace:
|
|||
return parser.parse_args()
|
||||
|
||||
|
||||
class _EndpointClient:
|
||||
"""Minimal client for the parent's loopback job server."""
|
||||
def _prepare_config(
|
||||
config: dict[str, Any],
|
||||
job_names: list[str],
|
||||
tool_context_id: str = "",
|
||||
injected_job_kwargs: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Configure the legacy single-workspace child Application for MCP STDIO."""
|
||||
selected = set(job_names)
|
||||
jobs: dict[str, dict[str, Any]] = {}
|
||||
for name, raw_job_config in (config.get("jobs") or {}).items():
|
||||
job_config = dict(raw_job_config)
|
||||
if job_config.get("backend") in {"background", "cron"}:
|
||||
continue
|
||||
if name in selected:
|
||||
job_config["enable_serve"] = True
|
||||
jobs[name] = job_config
|
||||
|
||||
def __init__(self, endpoint: str, token: str, timeout: float) -> None:
|
||||
self._client = httpx.Client(
|
||||
base_url=endpoint,
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=timeout,
|
||||
)
|
||||
missing = sorted(selected.difference(jobs))
|
||||
if missing:
|
||||
raise KeyError(f"Codex job tools not found or not request jobs: {', '.join(missing)}")
|
||||
|
||||
def list_jobs(self) -> list[dict[str, Any]]:
|
||||
"""Fetch the servable job metadata from the parent endpoint."""
|
||||
response = self._client.get("/jobs")
|
||||
response.raise_for_status()
|
||||
return response.json()["jobs"]
|
||||
service: dict[str, Any] = {
|
||||
"backend": "mcp",
|
||||
"transport": "stdio",
|
||||
"jobs": job_names,
|
||||
"tool_error_on_failure": True,
|
||||
}
|
||||
injected = dict(injected_job_kwargs or {})
|
||||
if tool_context_id:
|
||||
injected["tool_context_id"] = tool_context_id
|
||||
if injected:
|
||||
service["injected_job_kwargs"] = injected
|
||||
|
||||
def call(self, job: str, kwargs: dict[str, Any], injected_job_kwargs: dict[str, Any] | None = None) -> str:
|
||||
"""Forward one tool call to the parent endpoint and unwrap its answer."""
|
||||
response = self._client.post(
|
||||
"/call",
|
||||
json={"job": job, "kwargs": kwargs, "injected_job_kwargs": injected_job_kwargs or {}},
|
||||
)
|
||||
if response.status_code != 200:
|
||||
try:
|
||||
detail = response.json().get("error") or response.text
|
||||
except ValueError:
|
||||
detail = response.text
|
||||
raise RuntimeError(str(detail))
|
||||
payload = response.json()
|
||||
if not payload.get("success"):
|
||||
raise RuntimeError(str(payload.get("answer") or payload))
|
||||
return str(payload.get("answer") or "")
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the underlying HTTP client."""
|
||||
self._client.close()
|
||||
prepared = dict(config)
|
||||
prepared["jobs"] = jobs
|
||||
prepared["service"] = service
|
||||
return prepared
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Fetch job metadata from the parent and serve forwarding tools over STDIO."""
|
||||
class _ResidentMCPClient:
|
||||
"""Reuse the standard ReMe MCP client for resident tool discovery and calls."""
|
||||
|
||||
def __init__(self, endpoint: str, timeout: float) -> None:
|
||||
from fastmcp.client import StreamableHttpTransport
|
||||
|
||||
self._client = MCPClient(transport=StreamableHttpTransport(url=endpoint), timeout=timeout)
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Connect to the resident service."""
|
||||
await self._client.start()
|
||||
|
||||
async def list_jobs(self) -> list[dict[str, Any]]:
|
||||
"""Return the resident service's tool descriptors."""
|
||||
return await self._client.list_actions()
|
||||
|
||||
async def call(self, job: str, kwargs: dict[str, Any], injected: dict[str, Any]) -> str:
|
||||
"""Call one resident tool with server-owned arguments enforced."""
|
||||
# pylint: disable=protected-access
|
||||
payload = BaseAgentWrapper._merge_injected_job_kwargs(kwargs, injected)
|
||||
async for text in self._client(action=job, **payload):
|
||||
return text
|
||||
return ""
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close the resident connection."""
|
||||
await self._client.close()
|
||||
|
||||
|
||||
def _injected_kwargs(args: argparse.Namespace) -> dict[str, Any]:
|
||||
injected = json.loads(args.injected_job_kwargs) if args.injected_job_kwargs else {}
|
||||
if not isinstance(injected, dict):
|
||||
raise TypeError("--injected-job-kwargs must be a JSON object")
|
||||
injected = dict(injected)
|
||||
if args.tool_context_id:
|
||||
injected["tool_context_id"] = args.tool_context_id
|
||||
return injected
|
||||
|
||||
|
||||
async def _run_forwarder(args: argparse.Namespace, injected: dict[str, Any]) -> None:
|
||||
from fastmcp import FastMCP
|
||||
from fastmcp.exceptions import ToolError
|
||||
from fastmcp.tools import FunctionTool
|
||||
|
||||
args = _parse_args()
|
||||
injected_job_kwargs = json.loads(args.injected_job_kwargs) if args.injected_job_kwargs else {}
|
||||
if not isinstance(injected_job_kwargs, dict):
|
||||
raise TypeError("--injected-job-kwargs must be a JSON object")
|
||||
injected = dict(injected_job_kwargs)
|
||||
if args.workspace_alias:
|
||||
injected["workspace"] = args.workspace_alias
|
||||
if args.tool_context_id:
|
||||
injected["tool_context_id"] = args.tool_context_id
|
||||
|
||||
client = _EndpointClient(args.endpoint, args.token, args.tool_timeout)
|
||||
metadata = {entry["name"]: entry for entry in client.list_jobs()}
|
||||
missing = sorted(set(args.jobs) - set(metadata))
|
||||
if missing:
|
||||
raise KeyError(f"Codex job tools not found on the parent endpoint: {', '.join(missing)}")
|
||||
|
||||
server = FastMCP("reme-codex-bridge")
|
||||
for name in args.jobs:
|
||||
entry = metadata[name]
|
||||
|
||||
def forward(_client=client, _name=name, _injected=None, **kwargs):
|
||||
try:
|
||||
return _client.call(_name, kwargs, _injected or injected)
|
||||
except Exception as exc: # noqa: BLE001 - surface every failure as a tool error
|
||||
raise ToolError(str(exc)) from exc
|
||||
|
||||
server.add_tool(
|
||||
FunctionTool(
|
||||
name=name,
|
||||
description=entry.get("description") or "",
|
||||
parameters=entry.get("parameters") or {"type": "object", "properties": {}},
|
||||
fn=forward,
|
||||
),
|
||||
)
|
||||
|
||||
client = _ResidentMCPClient(args.endpoint, args.tool_timeout)
|
||||
await client.start()
|
||||
try:
|
||||
server.run(transport="stdio")
|
||||
metadata = {entry["name"]: entry for entry in await client.list_jobs()}
|
||||
missing = sorted(set(args.jobs) - set(metadata))
|
||||
if missing:
|
||||
raise KeyError(f"Codex job tools not found on the resident MCP service: {', '.join(missing)}")
|
||||
|
||||
server = FastMCP("reme-codex-bridge")
|
||||
for name in args.jobs:
|
||||
entry = metadata[name]
|
||||
|
||||
def make_forwarder(job_name: str):
|
||||
async def forward(**kwargs):
|
||||
try:
|
||||
return await client.call(job_name, kwargs, injected)
|
||||
except Exception as exc: # noqa: BLE001 - surface every failure as a tool error
|
||||
raise ToolError(str(exc)) from exc
|
||||
|
||||
return forward
|
||||
|
||||
parameters = entry.get("inputSchema") or entry.get("parameters")
|
||||
# pylint: disable=protected-access
|
||||
parameters = BaseAgentWrapper._strip_injected_parameters(parameters, injected)
|
||||
server.add_tool(
|
||||
FunctionTool(
|
||||
name=name,
|
||||
description=entry.get("description") or "",
|
||||
parameters=parameters or {"type": "object", "properties": {}},
|
||||
fn=make_forwarder(name),
|
||||
),
|
||||
)
|
||||
await server.run_async(transport="stdio")
|
||||
finally:
|
||||
client.close()
|
||||
await client.close()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Forward to a resident service or start the legacy single-workspace child service."""
|
||||
args = _parse_args()
|
||||
injected = _injected_kwargs(args)
|
||||
if args.endpoint:
|
||||
asyncio.run(_run_forwarder(args, injected))
|
||||
return
|
||||
if not args.workspace:
|
||||
raise ValueError("--workspace is required when --endpoint is not supplied")
|
||||
config = resolve_app_config(
|
||||
config=args.config,
|
||||
workspace_dir=str(Path(args.workspace).absolute()),
|
||||
enable_logo=False,
|
||||
log_to_console=False,
|
||||
log_to_file=False,
|
||||
log_config=False,
|
||||
)
|
||||
config = _prepare_config(config, args.jobs, args.tool_context_id, injected)
|
||||
ReMe(**config).run_app()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,148 +0,0 @@
|
|||
"""Wrapper-owned loopback job endpoint for agent-bridge subprocesses.
|
||||
|
||||
The Codex job bridge used to spawn a full child Application per thread; every
|
||||
thread then held its own workspace stores and became a second persistence
|
||||
writer. This endpoint keeps job execution in the parent process instead: a
|
||||
minimal HTTP/JSON server on 127.0.0.1 with an ephemeral port and a bearer
|
||||
token, owned by one CodexAgentWrapper instance for its lifetime. Bridge
|
||||
subprocesses are thin FastMCP forwarders that call this endpoint.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import secrets
|
||||
from typing import Any, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..application_context import ApplicationContext
|
||||
from ..job import BaseJob
|
||||
|
||||
_REASON = {
|
||||
200: "OK",
|
||||
400: "Bad Request",
|
||||
401: "Unauthorized",
|
||||
404: "Not Found",
|
||||
500: "Internal Server Error",
|
||||
}
|
||||
|
||||
|
||||
class LoopbackJobServer:
|
||||
"""Serve this application's request jobs to trusted local subprocesses."""
|
||||
|
||||
def __init__(self, app_context: "ApplicationContext") -> None:
|
||||
self._app_context = app_context
|
||||
self.token = secrets.token_urlsafe(24)
|
||||
self._server: asyncio.Server | None = None
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
"""Whether the endpoint is currently bound."""
|
||||
return self._server is not None
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
"""Base URL of the bound endpoint."""
|
||||
if self._server is None:
|
||||
raise RuntimeError("Loopback job server is not started")
|
||||
port = self._server.sockets[0].getsockname()[1]
|
||||
return f"http://127.0.0.1:{port}"
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Bind the endpoint; idempotent so lazy bridge creation can retry."""
|
||||
if self._server is None:
|
||||
self._server = await asyncio.start_server(self._handle, "127.0.0.1", 0)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Stop the endpoint and release the port."""
|
||||
server, self._server = self._server, None
|
||||
if server is not None:
|
||||
server.close()
|
||||
await server.wait_closed()
|
||||
|
||||
# ----- HTTP handling ---------------------------------------------------
|
||||
|
||||
async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
||||
try:
|
||||
head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), timeout=30)
|
||||
request_line, *header_lines = head.decode("latin-1").split("\r\n")
|
||||
method, target, _version = request_line.split(" ", 2)
|
||||
headers = {}
|
||||
for line in header_lines:
|
||||
if ":" in line:
|
||||
key, value = line.split(":", 1)
|
||||
headers[key.strip().lower()] = value.strip()
|
||||
length = int(headers.get("content-length", 0) or 0)
|
||||
body = await reader.readexactly(length) if length else b""
|
||||
status, payload = await self._dispatch(method, target, headers, body)
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
status, payload = 500, {"error": f"{type(exc).__name__}: {exc}"}
|
||||
data = json.dumps(payload).encode()
|
||||
writer.write(
|
||||
f"HTTP/1.1 {status} {_REASON.get(status, 'OK')}\r\n"
|
||||
"content-type: application/json\r\n"
|
||||
f"content-length: {len(data)}\r\n"
|
||||
"connection: close\r\n\r\n".encode() + data,
|
||||
)
|
||||
try:
|
||||
await writer.drain()
|
||||
finally:
|
||||
writer.close()
|
||||
|
||||
async def _dispatch(
|
||||
self,
|
||||
method: str,
|
||||
target: str,
|
||||
headers: dict[str, str],
|
||||
body: bytes,
|
||||
) -> tuple[int, dict[str, Any]]:
|
||||
if headers.get("authorization") != f"Bearer {self.token}":
|
||||
return 401, {"error": "unauthorized"}
|
||||
if method == "GET" and target == "/jobs":
|
||||
return 200, {"jobs": self._job_metadata()}
|
||||
if method == "POST" and target == "/call":
|
||||
try:
|
||||
request = json.loads(body or b"{}")
|
||||
except json.JSONDecodeError as exc:
|
||||
return 400, {"error": f"invalid JSON body: {exc}"}
|
||||
return await self._call(request)
|
||||
return 404, {"error": "not found"}
|
||||
|
||||
# ----- Job surface -------------------------------------------------------
|
||||
|
||||
def _request_jobs(self) -> dict[str, "BaseJob"]:
|
||||
"""Return the application's non-stream, non-background jobs by name."""
|
||||
from ..job import BackgroundJob, StreamJob
|
||||
|
||||
return {
|
||||
name: job for name, job in self._app_context.jobs.items() if not isinstance(job, (StreamJob, BackgroundJob))
|
||||
}
|
||||
|
||||
def _job_metadata(self) -> list[dict[str, Any]]:
|
||||
"""Describe every servable job for bridge tool registration."""
|
||||
return [
|
||||
{"name": name, "description": job.description, "parameters": job.parameters}
|
||||
for name, job in sorted(self._request_jobs().items())
|
||||
]
|
||||
|
||||
async def _call(self, request: dict[str, Any]) -> tuple[int, dict[str, Any]]:
|
||||
name = str(request.get("job") or "")
|
||||
job = self._request_jobs().get(name)
|
||||
if job is None:
|
||||
return 404, {"error": f"job {name!r} not found or not a request job"}
|
||||
|
||||
kwargs = dict(request.get("kwargs") or {})
|
||||
injected = dict(request.get("injected_job_kwargs") or {})
|
||||
conflicts = sorted(injected.keys() & kwargs.keys())
|
||||
if conflicts:
|
||||
return 400, {"error": f"{', '.join(conflicts)} injected by the server and cannot be provided"}
|
||||
kwargs.update(injected)
|
||||
if workspace := request.get("workspace"):
|
||||
kwargs["workspace"] = workspace
|
||||
if tool_context_id := request.get("tool_context_id"):
|
||||
kwargs["tool_context_id"] = tool_context_id
|
||||
|
||||
try:
|
||||
response = await job(**kwargs)
|
||||
except ValueError as exc:
|
||||
return 400, {"error": str(exc)}
|
||||
return 200, {"success": response.success, "answer": response.answer}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
"""Application context: shared state container for components, jobs, and service."""
|
||||
|
||||
from collections.abc import Iterator, MutableMapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
|
|
@ -12,65 +13,80 @@ if TYPE_CHECKING:
|
|||
from .service import BaseService
|
||||
|
||||
|
||||
class ApplicationContext:
|
||||
"""Passive state container holding parsed config and wired components.
|
||||
class _LegacyComponentView(MutableMapping):
|
||||
"""Compatibility mapping combining shared components with default-workspace instances."""
|
||||
|
||||
The Application class performs the actual wiring (registry lookups and
|
||||
component instantiation); this class only stores the results so that
|
||||
components, jobs, and the service can find each other at runtime.
|
||||
"""
|
||||
def __init__(self, context: "ApplicationContext") -> None:
|
||||
self._context = context
|
||||
|
||||
def _default_components(self) -> dict:
|
||||
return next(iter(self._context.workspaces.values())).components
|
||||
|
||||
def __getitem__(self, key: str) -> dict:
|
||||
if key in self._context.shared_components:
|
||||
return self._context.shared_components[key]
|
||||
return self._default_components()[key]
|
||||
|
||||
def __setitem__(self, key: str, value: dict) -> None:
|
||||
if value and all(getattr(component, "workspace_scoped", False) is True for component in value.values()):
|
||||
self._default_components()[key] = value
|
||||
self._context.shared_components.pop(key, None)
|
||||
else:
|
||||
self._context.shared_components[key] = value
|
||||
self._default_components().pop(key, None)
|
||||
|
||||
def __delitem__(self, key: str) -> None:
|
||||
if key in self._context.shared_components:
|
||||
del self._context.shared_components[key]
|
||||
else:
|
||||
del self._default_components()[key]
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
yield from self._context.shared_components
|
||||
yield from (key for key in self._default_components() if key not in self._context.shared_components)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(set(self._context.shared_components) | set(self._default_components()))
|
||||
|
||||
|
||||
class ApplicationContext:
|
||||
"""Application-wide configuration, shared resources, and workspace registry."""
|
||||
|
||||
def __init__(self, *, registry: ComponentRegistry | None = None, **kwargs):
|
||||
# Parse raw kwargs into a typed, validated config object.
|
||||
self.app_config: ApplicationConfig = ApplicationConfig(**kwargs)
|
||||
self.registry = registry or create_application_registry()
|
||||
|
||||
# Ordered per-workspace path views; the first entry is the default workspace.
|
||||
self.workspaces: dict[str, WorkspaceContext] = {
|
||||
alias: WorkspaceContext.from_config(alias, path, self.app_config)
|
||||
for alias, path in self.app_config.workspaces.items()
|
||||
}
|
||||
self.workspace: WorkspaceContext = next(iter(self.workspaces.values()))
|
||||
|
||||
# Populated by Application during initialization.
|
||||
self.workspace: WorkspaceContext = next(iter(self.workspaces.values()))
|
||||
self.service: "BaseService | None" = None
|
||||
self.components: dict[str, dict[str, "BaseComponent"]] = {}
|
||||
self.shared_components: dict[str, dict[str, "BaseComponent"]] = {}
|
||||
self._components = _LegacyComponentView(self)
|
||||
self.jobs: dict[str, "BaseJob"] = {}
|
||||
self.thread_pool: ThreadPoolExecutor | None = None
|
||||
|
||||
# Application-lifetime shared state. Values remain available across Job and Step
|
||||
# invocations while this Application is running, so Jobs may keep cross-call state here.
|
||||
# This is in-memory state, not durable storage; use workspace files or a store when state
|
||||
# must survive an Application restart.
|
||||
self.metadata: dict[str, Any] = {}
|
||||
|
||||
@property
|
||||
def workspace_components(self) -> dict[str, dict[str, dict[str, "BaseComponent"]]]:
|
||||
"""Compatibility view of scoped instances, ctype -> name -> alias.
|
||||
def components(self) -> MutableMapping:
|
||||
"""Legacy component lookup; internal execution uses explicit shared or workspace ownership."""
|
||||
return self._components
|
||||
|
||||
The authoritative owner of these instances is each WorkspaceContext;
|
||||
this view exists for transitional readers.
|
||||
"""
|
||||
grouped: dict[str, dict[str, dict[str, "BaseComponent"]]] = {}
|
||||
for alias, workspace in self.workspaces.items():
|
||||
for ctype, group in workspace.components.items():
|
||||
for name, instance in group.items():
|
||||
grouped.setdefault(ctype, {}).setdefault(name, {})[alias] = instance
|
||||
return grouped
|
||||
@components.setter
|
||||
def components(self, value: dict) -> None:
|
||||
self.shared_components = {}
|
||||
for ctype, group in value.items():
|
||||
self._components[ctype] = group
|
||||
|
||||
def get_component(self, ctype: str, name: str, alias: str | None = None) -> "BaseComponent | None":
|
||||
"""Resolve a component, honoring workspace bindings for scoped components.
|
||||
|
||||
Returns None when the component is not configured at all. Raises
|
||||
KeyError when a workspace-scoped component has no instance for the
|
||||
requested alias — a missing scoped resource must fail, never fall back.
|
||||
"""
|
||||
"""Compatibility resolver for external callers that select a workspace by alias."""
|
||||
alias = alias or next(iter(self.workspaces))
|
||||
workspace = self.workspaces.get(alias)
|
||||
if workspace is None:
|
||||
raise KeyError(f"Unknown workspace alias {alias!r}")
|
||||
if name in workspace.components.get(ctype, {}):
|
||||
return workspace.components[ctype][name]
|
||||
return workspace.get_component(ctype, name)
|
||||
if any(name in ws.components.get(ctype, {}) for ws in self.workspaces.values()):
|
||||
raise KeyError(f"Component '{name}' in {ctype} has no instance for workspace alias {alias!r}")
|
||||
return self.components.get(ctype, {}).get(name)
|
||||
return self.shared_components.get(ctype, {}).get(name)
|
||||
|
|
|
|||
|
|
@ -221,20 +221,24 @@ class BaseComponent(ComponentMixin, ABC):
|
|||
setattr(self, attr, None)
|
||||
|
||||
def _resolve_from_context(self, attr: str, dep: Dependency) -> None:
|
||||
"""Context-bound mode: look up the component from ``app_context.components``.
|
||||
|
||||
Workspace-scoped dependencies resolve to the same-alias instance as the
|
||||
dependent; a missing alias instance is an error, never a fallback.
|
||||
"""
|
||||
"""Resolve scoped dependencies from their workspace and shared ones from the application."""
|
||||
workspaces = getattr(self.app_context, "workspaces", None)
|
||||
workspace = workspaces.get(self.workspace_alias) if workspaces and self.workspace_alias is not None else None
|
||||
scoped_group = workspace.components.get(dep.ctype) if workspace is not None else None
|
||||
if scoped_group is not None:
|
||||
target = scoped_group.get(dep.name)
|
||||
if target is None:
|
||||
raise ValueError(f"{dep.ctype} '{dep.name}' has no instance for workspace '{self.workspace_alias}'.")
|
||||
if workspace is not None and dep.name in workspace.components.get(dep.ctype, {}):
|
||||
target = workspace.get_component(dep.ctype, dep.name)
|
||||
elif workspace is not None and any(
|
||||
dep.name in candidate.components.get(dep.ctype, {}) for candidate in workspaces.values()
|
||||
):
|
||||
raise ValueError(f"{dep.ctype} '{dep.name}' has no instance for workspace '{self.workspace_alias}'.")
|
||||
else:
|
||||
target = self.app_context.components.get(dep.ctype, {}).get(dep.name)
|
||||
shared = getattr(self.app_context, "shared_components", None)
|
||||
if shared is None:
|
||||
shared = self.app_context.components
|
||||
target = shared.get(dep.ctype, {}).get(dep.name)
|
||||
if target is None and workspaces:
|
||||
default_workspace = next(iter(workspaces.values()))
|
||||
if dep.name in default_workspace.components.get(dep.ctype, {}):
|
||||
target = default_workspace.get_component(dep.ctype, dep.name)
|
||||
if target is not None:
|
||||
setattr(self, attr, target)
|
||||
elif dep.optional:
|
||||
|
|
|
|||
|
|
@ -9,17 +9,17 @@ if TYPE_CHECKING:
|
|||
from .application_config import ApplicationConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkspaceAgentState:
|
||||
"""Session execution state owned by one agent wrapper within one workspace."""
|
||||
|
||||
thread_tool_contexts: dict[str, str] = field(default_factory=dict)
|
||||
owned_thread_ids: set[str] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkspaceContext:
|
||||
"""Owns everything scoped to one workspace of an application.
|
||||
|
||||
Built from ApplicationConfig when the application context is created.
|
||||
Beyond the resolved paths, the workspace owns its stateful component
|
||||
instances (stores, graphs, indexes, catalogs), in-memory execution state
|
||||
(drafts, search dedup, call budgets), session ownership records, and
|
||||
workspace-scoped locks. Shared, stateless clients stay on the
|
||||
ApplicationContext; per-call bindings reference this object.
|
||||
"""
|
||||
"""Owns the paths, stateful components, execution state, and locks of one workspace."""
|
||||
|
||||
alias: str
|
||||
workspace_path: Path
|
||||
|
|
@ -29,17 +29,16 @@ class WorkspaceContext:
|
|||
resource_path: Path | None
|
||||
daily_path: Path | None
|
||||
digest_path: Path | None
|
||||
# Workspace-scoped component instances keyed by component type then name.
|
||||
# Values are BaseComponent instances; typed as Any to keep this module
|
||||
# free of component imports (schema must not depend on components).
|
||||
components: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
# Drafts and search dedup state, keyed by tool_context_id.
|
||||
tool_contexts: dict[str, Any] = field(default_factory=dict)
|
||||
# Search call budgets keyed by tool_context_id.
|
||||
search_call_budgets: dict[str, Any] = field(default_factory=dict)
|
||||
# Workspace-scoped named asyncio locks (e.g. dream integration).
|
||||
agent_states: dict[str, WorkspaceAgentState] = field(default_factory=dict)
|
||||
locks: dict[str, asyncio.Lock] = field(default_factory=dict)
|
||||
|
||||
def agent_state(self, wrapper_name: str) -> WorkspaceAgentState:
|
||||
"""Return the named wrapper's state within this workspace."""
|
||||
return self.agent_states.setdefault(wrapper_name, WorkspaceAgentState())
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, alias: str, workspace: str, config: "ApplicationConfig") -> "WorkspaceContext":
|
||||
"""Resolve one alias of the config's workspaces mapping into absolute paths."""
|
||||
|
|
@ -59,6 +58,13 @@ class WorkspaceContext:
|
|||
digest_path=child(config.digest_dir),
|
||||
)
|
||||
|
||||
def get_component(self, ctype: str, name: str) -> Any:
|
||||
"""Return a component owned by this workspace, never a default or shared substitute."""
|
||||
try:
|
||||
return self.components[ctype][name]
|
||||
except KeyError as exc:
|
||||
raise KeyError(f"Component '{name}' in {ctype} has no instance for workspace '{self.alias}'") from exc
|
||||
|
||||
def storage_dirs(self) -> list[Path]:
|
||||
"""Subdirectories the application materializes at startup, in configuration order."""
|
||||
return [
|
||||
|
|
|
|||
|
|
@ -87,14 +87,16 @@ class Ref:
|
|||
if self.optional:
|
||||
return None
|
||||
raise RuntimeError(f"app_context is not set when resolving '{self.key}'")
|
||||
if _context_workspaces(obj.app_context) is not None:
|
||||
if (workspaces := _context_workspaces(obj.app_context)) is not None:
|
||||
context = obj.context
|
||||
bound = getattr(context, "workspace", None) if context is not None else None
|
||||
comp = obj.app_context.get_component(
|
||||
self.comp_enum,
|
||||
name,
|
||||
alias=bound.alias if bound is not None else None,
|
||||
)
|
||||
is_scoped = any(name in workspace.components.get(self.comp_enum, {}) for workspace in workspaces.values())
|
||||
if is_scoped:
|
||||
if bound is None:
|
||||
raise RuntimeError(f"A workspace binding is required when resolving '{self.key}'")
|
||||
comp = bound.get_component(self.comp_enum, name)
|
||||
else:
|
||||
comp = obj.app_context.shared_components.get(self.comp_enum, {}).get(name)
|
||||
else:
|
||||
# Test doubles and standalone-style contexts without the workspace layer.
|
||||
comp = obj.app_context.components.get(self.comp_enum, {}).get(name)
|
||||
|
|
@ -159,6 +161,8 @@ class BaseStep(ComponentMixin, ABC):
|
|||
del self.__dict__[key]
|
||||
self.context = RuntimeContext.from_context(context, **kwargs)
|
||||
assert self.context is not None
|
||||
if (workspaces := _context_workspaces(self.app_context)) is not None and self.context.workspace is None:
|
||||
self.context.workspace = next(iter(workspaces.values()))
|
||||
if self.input_mapping:
|
||||
self.context.apply_mapping(self.input_mapping)
|
||||
result = await self.execute()
|
||||
|
|
|
|||
|
|
@ -343,11 +343,9 @@ class ProactiveTopicsStep(BaseStep):
|
|||
return []
|
||||
bound = getattr(self.context, "workspace", None) if self.context is not None else None
|
||||
if _context_workspaces(self.app_context) is not None:
|
||||
catalog = self.app_context.get_component(
|
||||
ComponentEnum.FILE_CATALOG,
|
||||
"digest",
|
||||
alias=bound.alias if bound is not None else None,
|
||||
)
|
||||
if bound is None:
|
||||
raise RuntimeError("A workspace binding is required to resolve the digest catalog")
|
||||
catalog = bound.components.get(ComponentEnum.FILE_CATALOG, {}).get("digest")
|
||||
else:
|
||||
# Test doubles and standalone-style contexts without the workspace layer.
|
||||
catalog = self.app_context.components.get(ComponentEnum.FILE_CATALOG, {}).get("digest")
|
||||
|
|
@ -368,7 +366,7 @@ class ProactiveTopicsStep(BaseStep):
|
|||
name = self.kwargs.get("as_embedding", "default")
|
||||
if self.app_context is None:
|
||||
return None
|
||||
return self.app_context.components.get(ComponentEnum.AS_EMBEDDING, {}).get(name)
|
||||
return self.app_context.shared_components.get(ComponentEnum.AS_EMBEDDING, {}).get(name)
|
||||
|
||||
async def _semantic_verdict(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -681,7 +681,7 @@ def resolve_agent_wrapper(step):
|
|||
if wrapper is not None:
|
||||
return wrapper
|
||||
if step.app_context is not None:
|
||||
fallback = step.app_context.components.get(ComponentEnum.AGENT_WRAPPER, {}).get("default")
|
||||
fallback = step.app_context.shared_components.get(ComponentEnum.AGENT_WRAPPER, {}).get("default")
|
||||
if fallback is not None:
|
||||
configured = step.kwargs.get("agent_wrapper", "default")
|
||||
step.logger.warning(f"[{step.name}] agent_wrapper '{configured}' missing; using default")
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from watchfiles import Change
|
|||
from ._change_batch import bucket_changes
|
||||
from ..base_step import BaseStep
|
||||
from ...components import R
|
||||
from ...components.base_component import _context_workspaces
|
||||
from ...components.file_chunker import BaseFileChunker
|
||||
from ...enumeration import ComponentEnum
|
||||
from ...schema import FileChunk, FileNode
|
||||
|
|
@ -368,8 +369,10 @@ class UpdateIndexStep(ChangeApplyStep):
|
|||
bound = getattr(self.context, "workspace", None) if self.context is not None else None
|
||||
if bound is not None:
|
||||
chunkers: dict[str, BaseFileChunker] = bound.components.get(ComponentEnum.FILE_CHUNKER, {})
|
||||
else:
|
||||
elif _context_workspaces(self.app_context) is None:
|
||||
chunkers = self.app_context.components[ComponentEnum.FILE_CHUNKER]
|
||||
else:
|
||||
raise RuntimeError("A workspace binding is required to resolve file chunkers")
|
||||
suffix = path.suffix.lstrip(".").lower()
|
||||
for candidate in chunkers.values():
|
||||
if suffix and suffix in {ext.lower().lstrip(".") for ext in candidate.supported_extensions}:
|
||||
|
|
|
|||
|
|
@ -61,60 +61,51 @@ def _wrapper(tmp_path, **kwargs):
|
|||
return CodexAgentWrapper(app_context=context, **kwargs), job
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_config_uses_loopback_bridge_and_selected_jobs(tmp_path):
|
||||
def test_mcp_config_uses_legacy_child_bridge_and_selected_jobs(tmp_path):
|
||||
wrapper, _job = _wrapper(tmp_path, mcp_config="custom.yaml")
|
||||
await wrapper._ensure_job_server() # pylint: disable=protected-access
|
||||
|
||||
config = wrapper._mcp_server_config( # pylint: disable=protected-access
|
||||
config = wrapper._mcp_server_config(
|
||||
{"job_tools": ["search", "search"], "tool_context_id": "ctx-1"},
|
||||
)
|
||||
await wrapper.close()
|
||||
|
||||
assert config["command"]
|
||||
assert config["enabled_tools"] == ["search"]
|
||||
assert config["args"].count("--job") == 1
|
||||
assert config["args"][config["args"].index("--job") + 1] == "search"
|
||||
assert "reme.components.agent_wrapper.codex_mcp_server" in config["args"]
|
||||
assert config["args"][config["args"].index("--endpoint") + 1].startswith("http://127.0.0.1:")
|
||||
assert config["args"][config["args"].index("--token") + 1]
|
||||
assert config["args"][config["args"].index("--config") + 1] == str(tmp_path / "custom.yaml")
|
||||
assert config["args"][config["args"].index("--workspace") + 1] == str(tmp_path)
|
||||
assert config["args"][config["args"].index("--tool-context-id") + 1] == "ctx-1"
|
||||
# mcp_config is accepted for compatibility but no longer builds a child app.
|
||||
assert "--config" not in config["args"]
|
||||
assert "--endpoint" not in config["args"]
|
||||
assert "--token" not in config["args"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_config_serializes_injected_job_kwargs(tmp_path):
|
||||
def test_mcp_config_serializes_injected_job_kwargs(tmp_path):
|
||||
wrapper, _job = _wrapper(tmp_path, mcp_config="custom.yaml")
|
||||
await wrapper._ensure_job_server() # pylint: disable=protected-access
|
||||
|
||||
config = wrapper._mcp_server_config( # pylint: disable=protected-access
|
||||
config = wrapper._mcp_server_config(
|
||||
{
|
||||
"job_tools": ["search"],
|
||||
"tool_context_id": "ctx-1",
|
||||
"injected_job_kwargs": {"_allowed_paths": ["daily/2025-06-01/note.md"]},
|
||||
},
|
||||
)
|
||||
await wrapper.close()
|
||||
|
||||
raw = config["args"][config["args"].index("--injected-job-kwargs") + 1]
|
||||
assert json.loads(raw) == {"_allowed_paths": ["daily/2025-06-01/note.md"]}
|
||||
|
||||
plain = wrapper._mcp_server_config({"job_tools": ["search"]}) # pylint: disable=protected-access
|
||||
plain = wrapper._mcp_server_config({"job_tools": ["search"]})
|
||||
assert "--injected-job-kwargs" not in plain["args"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_config_preserves_other_mcp_servers(tmp_path):
|
||||
def test_thread_config_preserves_other_mcp_servers(tmp_path):
|
||||
wrapper, _job = _wrapper(tmp_path)
|
||||
await wrapper._ensure_job_server() # pylint: disable=protected-access
|
||||
config = wrapper._thread_config( # pylint: disable=protected-access
|
||||
config = wrapper._thread_config(
|
||||
{
|
||||
"job_tools": ["search"],
|
||||
"config": {"mcp_servers": {"docs": {"url": "https://example.test/mcp"}}},
|
||||
},
|
||||
)
|
||||
await wrapper.close()
|
||||
|
||||
assert "docs" in config["mcp_servers"]
|
||||
assert len(config["mcp_servers"]) == 2
|
||||
|
|
@ -166,65 +157,15 @@ def test_mcp_config_rejects_background_jobs(tmp_path):
|
|||
wrapper._mcp_server_config({"job_tools": ["watch"]})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_loopback_endpoint_serves_parent_jobs(tmp_path):
|
||||
import httpx
|
||||
|
||||
wrapper, job = _wrapper(tmp_path)
|
||||
await wrapper.start()
|
||||
server = await wrapper._ensure_job_server() # pylint: disable=protected-access
|
||||
base = server.url
|
||||
headers = {"Authorization": f"Bearer {server.token}"}
|
||||
try:
|
||||
async with httpx.AsyncClient(base_url=base, headers=headers, timeout=10) as http:
|
||||
listed = (await http.get("/jobs")).json()["jobs"]
|
||||
assert [entry["name"] for entry in listed] == ["search"]
|
||||
|
||||
called = await http.post("/call", json={"job": "search", "kwargs": {"query": "memo"}})
|
||||
assert called.json() == {"success": True, "answer": "found:memo"}
|
||||
# The job executed in the parent process, not in any child.
|
||||
assert job.calls == [{"query": "memo"}]
|
||||
|
||||
missing = await http.post("/call", json={"job": "missing", "kwargs": {}})
|
||||
assert missing.status_code == 404
|
||||
|
||||
unauthorized = await http.get("/jobs", headers={"Authorization": "Bearer wrong"})
|
||||
assert unauthorized.status_code == 401
|
||||
finally:
|
||||
await wrapper.close()
|
||||
assert server.is_running is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_loopback_bridge_lists_and_calls_parent_job(tmp_path):
|
||||
from fastmcp import Client
|
||||
from fastmcp.client import StdioTransport
|
||||
|
||||
wrapper, job = _wrapper(tmp_path)
|
||||
await wrapper._ensure_job_server() # pylint: disable=protected-access
|
||||
server_config = wrapper._mcp_server_config({"job_tools": ["search"]}) # pylint: disable=protected-access
|
||||
transport = StdioTransport(
|
||||
command=server_config["command"],
|
||||
args=server_config["args"],
|
||||
cwd=server_config["cwd"],
|
||||
keep_alive=False,
|
||||
)
|
||||
|
||||
async with Client(transport, timeout=None, init_timeout=STDIO_INIT_TIMEOUT) as client:
|
||||
tools = await asyncio.wait_for(client.list_tools(), timeout=10)
|
||||
assert [tool.name for tool in tools] == ["search"]
|
||||
result = await asyncio.wait_for(client.call_tool("search", {"query": "via-bridge"}), timeout=10)
|
||||
await wrapper.close()
|
||||
|
||||
assert "found:via-bridge" in str(result.content[0].text)
|
||||
assert job.calls == [{"query": "via-bridge"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stdio_bridge_stdout_is_protocol_clean(tmp_path):
|
||||
wrapper, _job = _wrapper(tmp_path)
|
||||
await wrapper._ensure_job_server() # pylint: disable=protected-access
|
||||
server_config = wrapper._mcp_server_config({"job_tools": ["search"]}) # pylint: disable=protected-access
|
||||
config_path = tmp_path / "config.json"
|
||||
config_path.write_text(
|
||||
json.dumps({"jobs": {"search": {"backend": "base", "steps": []}}}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
wrapper, _job = _wrapper(tmp_path, mcp_config=str(config_path))
|
||||
server_config = wrapper._mcp_server_config({"job_tools": ["search"]})
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
server_config["command"],
|
||||
*server_config["args"],
|
||||
|
|
|
|||
|
|
@ -165,7 +165,12 @@ async def test_workspace_scoped_components_instantiate_per_alias(tmp_path):
|
|||
store_beta = app.context.get_component(ComponentEnum.FILE_STORE, "default", "beta")
|
||||
|
||||
assert store_alpha is not store_beta
|
||||
assert app.context.components[ComponentEnum.FILE_STORE]["default"] is store_alpha # default alias entry
|
||||
assert app.context.components[ComponentEnum.FILE_STORE]["default"] is store_alpha
|
||||
assert ComponentEnum.FILE_STORE not in app.context.shared_components
|
||||
assert app.context.workspaces["alpha"].get_component(ComponentEnum.FILE_STORE, "default") is store_alpha
|
||||
assert app.context.workspaces["beta"].get_component(ComponentEnum.FILE_STORE, "default") is store_beta
|
||||
with pytest.raises(KeyError, match="no instance for workspace 'alpha'"):
|
||||
app.context.workspaces["alpha"].get_component(ComponentEnum.FILE_STORE, "missing")
|
||||
assert store_alpha.workspace_path == tmp_path / "alpha"
|
||||
assert store_beta.workspace_path == tmp_path / "beta"
|
||||
assert store_alpha.chunks_path.parent == tmp_path / "alpha" / "metadata" / "file_store"
|
||||
|
|
@ -314,7 +319,8 @@ async def test_digest_titles_follow_workspace_binding(tmp_path):
|
|||
assert await step._digest_titles() == ["beta-only"]
|
||||
|
||||
step.context = RuntimeContext()
|
||||
assert await step._digest_titles() == ["alpha-only"]
|
||||
with pytest.raises(RuntimeError, match="workspace binding is required"):
|
||||
await step._digest_titles()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
# pylint: disable=protected-access,missing-function-docstring
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
|
@ -128,11 +130,8 @@ async def test_tool_context_state_namespaced_per_workspace(tmp_path):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_thread_tool_contexts_namespaced_per_workspace(tmp_path):
|
||||
wrapper = CodexAgentWrapper(
|
||||
app_context=ApplicationContext(
|
||||
workspace_dir={"alpha": str(tmp_path / "alpha"), "beta": str(tmp_path / "beta")},
|
||||
),
|
||||
)
|
||||
context = ApplicationContext(workspace_dir={"alpha": str(tmp_path / "alpha"), "beta": str(tmp_path / "beta")})
|
||||
wrapper = CodexAgentWrapper(app_context=context)
|
||||
|
||||
class FakeCodex:
|
||||
"""Minimal Codex double echoing the requested thread id."""
|
||||
|
|
@ -158,7 +157,13 @@ async def test_codex_thread_tool_contexts_namespaced_per_workspace(tmp_path):
|
|||
with pytest.raises(ValueError, match="belongs to workspace 'alpha'"):
|
||||
await wrapper._open_thread(codex, {"session_id": "thread-1", "workspace": "beta"})
|
||||
|
||||
assert sorted(wrapper._thread_tool_contexts) == ["beta/thread-2", "thread-1"]
|
||||
alpha_state = context.workspaces["alpha"].agent_state(wrapper.name)
|
||||
beta_state = context.workspaces["beta"].agent_state(wrapper.name)
|
||||
assert set(alpha_state.thread_tool_contexts) == {"thread-1"}
|
||||
assert set(beta_state.thread_tool_contexts) == {"thread-2"}
|
||||
assert not wrapper._thread_tool_contexts
|
||||
assert alpha_state.owned_thread_ids == {"thread-1"}
|
||||
assert beta_state.owned_thread_ids == {"thread-2"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -295,27 +300,24 @@ async def test_mcp_tool_boundary_forwards_injected_workspace(tmp_path):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_bridge_uses_loopback_endpoint(tmp_path):
|
||||
import json
|
||||
async def test_codex_bridge_uses_resident_mcp_endpoint(tmp_path):
|
||||
from reme.components.service import HttpService
|
||||
|
||||
context = _context(tmp_path)
|
||||
context.service = HttpService(app_context=context)
|
||||
context.jobs = {"search_job": _job(context, "search_job")}
|
||||
wrapper = CodexAgentWrapper(app_context=context)
|
||||
await wrapper.start()
|
||||
await wrapper._ensure_job_server() # pylint: disable=protected-access
|
||||
|
||||
config = wrapper._mcp_server_config({"job_tools": ["search_job"], "workspace": "beta"})
|
||||
assert "--workspace-alias" in config["args"]
|
||||
assert config["args"][config["args"].index("--workspace-alias") + 1] == "beta"
|
||||
assert config["args"][config["args"].index("--endpoint") + 1] == "http://127.0.0.1:2333/mcp"
|
||||
raw = config["args"][config["args"].index("--injected-job-kwargs") + 1]
|
||||
assert json.loads(raw) == {"workspace": "beta"}
|
||||
assert "--config" not in config["args"] and "--workspace" not in config["args"]
|
||||
assert "--token" not in config["args"]
|
||||
|
||||
default_config = wrapper._mcp_server_config({"job_tools": ["search_job"]})
|
||||
assert "--workspace-alias" not in default_config["args"]
|
||||
|
||||
assert "--injected-job-kwargs" not in default_config["args"]
|
||||
await wrapper.close()
|
||||
assert wrapper._job_server is None
|
||||
|
||||
|
||||
# ----- Review follow-ups: explicit bindings, session isolation, key collisions
|
||||
|
|
@ -423,18 +425,13 @@ async def test_codex_thread_cwd_follows_bound_workspace(tmp_path):
|
|||
# ----- Codex bridge: real child-process execution ----------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_bridge_reads_resident_parent_state(tmp_path):
|
||||
"""Bridge calls execute against the parent's live stores; there is no second owner."""
|
||||
from fastmcp import Client
|
||||
from fastmcp.client import StdioTransport
|
||||
|
||||
app = Application(
|
||||
def _codex_test_app(tmp_path):
|
||||
return Application(
|
||||
workspace_dir={"alpha": str(tmp_path / "alpha"), "beta": str(tmp_path / "beta")},
|
||||
enable_logo=False,
|
||||
log_to_console=False,
|
||||
log_to_file=False,
|
||||
service={"backend": "cli"},
|
||||
service={"backend": "http", "host": "127.0.0.1", "port": 0, "web_enabled": False},
|
||||
components={
|
||||
"tokenizer": {"default": {"backend": "regex"}},
|
||||
"file_store": {"default": {"backend": "local", "embedding_store": "", "tag_index": ""}},
|
||||
|
|
@ -464,57 +461,104 @@ async def test_codex_bridge_reads_resident_parent_state(tmp_path):
|
|||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _start_resident_http(app):
|
||||
import uvicorn
|
||||
|
||||
service = app.context.service
|
||||
service.build_service(app)
|
||||
service.add_jobs(app)
|
||||
config = uvicorn.Config(service.service, host=service.host, port=0, log_level="error", lifespan="on")
|
||||
server = uvicorn.Server(config)
|
||||
task = asyncio.create_task(server.serve())
|
||||
for _ in range(100):
|
||||
if server.started:
|
||||
break
|
||||
await asyncio.sleep(0.05)
|
||||
assert server.started
|
||||
service.port = server.servers[0].sockets[0].getsockname()[1]
|
||||
return server, task
|
||||
|
||||
|
||||
async def _stop_resident_http(server, task):
|
||||
server.should_exit = True
|
||||
await asyncio.wait_for(task, timeout=30)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_bridge_reads_resident_parent_state(tmp_path):
|
||||
"""The stdio bridge reuses the resident MCP service and its live workspace state."""
|
||||
from fastmcp import Client
|
||||
from fastmcp.client import StdioTransport
|
||||
|
||||
app = _codex_test_app(tmp_path)
|
||||
wrapper = CodexAgentWrapper(app_context=app.context)
|
||||
async with app:
|
||||
await wrapper.start()
|
||||
# Index a note directly in the parent's beta store.
|
||||
await app.run_job("write_note", path="daily/resident.md", content="parent-resident-beta", workspace="beta")
|
||||
await wrapper._ensure_job_server() # pylint: disable=protected-access
|
||||
server_config = wrapper._mcp_server_config( # pylint: disable=protected-access
|
||||
{"job_tools": ["read_note", "write_note"], "workspace": "beta"},
|
||||
)
|
||||
transport = StdioTransport(
|
||||
command=server_config["command"],
|
||||
args=server_config["args"],
|
||||
cwd=server_config["cwd"],
|
||||
keep_alive=False,
|
||||
)
|
||||
async with Client(transport, timeout=None, init_timeout=30) as client:
|
||||
read = await asyncio.wait_for(client.call_tool("read_note", {"path": "daily/resident.md"}), timeout=30)
|
||||
await asyncio.wait_for(
|
||||
client.call_tool("write_note", {"path": "daily/bridge.md", "content": "via-bridge-beta"}),
|
||||
timeout=30,
|
||||
server, task = await _start_resident_http(app)
|
||||
try:
|
||||
await app.run_job("write_note", path="daily/resident.md", content="parent-resident-beta", workspace="beta")
|
||||
from reme.components.client import MCPClient
|
||||
|
||||
resident_client = MCPClient(transport="streamable-http", host="127.0.0.1", port=app.context.service.port)
|
||||
await resident_client.start()
|
||||
assert {entry["name"] for entry in await resident_client.list_actions()} == {"read_note", "write_note"}
|
||||
await resident_client.close()
|
||||
server_config = wrapper._mcp_server_config(
|
||||
{"job_tools": ["read_note", "write_note"], "workspace": "beta"},
|
||||
)
|
||||
# The bridge saw the parent's resident beta state and wrote through it.
|
||||
assert "parent-resident-beta" in str(read.content[0].text)
|
||||
assert "via-bridge-beta" in (tmp_path / "beta" / "daily" / "bridge.md").read_text(encoding="utf-8")
|
||||
assert not (tmp_path / "alpha" / "daily" / "bridge.md").exists()
|
||||
store_alpha = app.context.get_component(ComponentEnum.FILE_STORE, "default", "alpha")
|
||||
assert all("bridge" not in node.path for node in await store_alpha.get_nodes())
|
||||
transport = StdioTransport(
|
||||
command=server_config["command"],
|
||||
args=server_config["args"],
|
||||
cwd=server_config["cwd"],
|
||||
keep_alive=False,
|
||||
env=dict(os.environ),
|
||||
)
|
||||
async with Client(transport, timeout=None, init_timeout=30) as client:
|
||||
tools = await client.list_tools()
|
||||
assert {tool.name for tool in tools} == {"read_note", "write_note"}
|
||||
read = await asyncio.wait_for(client.call_tool("read_note", {"path": "daily/resident.md"}), timeout=30)
|
||||
with pytest.raises(Exception):
|
||||
await client.call_tool("read_note", {"path": "daily/resident.md", "workspace": "alpha"})
|
||||
await asyncio.wait_for(
|
||||
client.call_tool("write_note", {"path": "daily/bridge.md", "content": "via-bridge-beta"}),
|
||||
timeout=30,
|
||||
)
|
||||
assert "parent-resident-beta" in str(read.content[0].text)
|
||||
assert "via-bridge-beta" in (tmp_path / "beta" / "daily" / "bridge.md").read_text(encoding="utf-8")
|
||||
assert not (tmp_path / "alpha" / "daily" / "bridge.md").exists()
|
||||
store_alpha = app.context.get_component(ComponentEnum.FILE_STORE, "default", "alpha")
|
||||
assert all("bridge" not in node.path for node in await store_alpha.get_nodes())
|
||||
|
||||
invalid_config = wrapper._mcp_server_config({"job_tools": ["read_note"], "workspace": "nope"})
|
||||
invalid_transport = StdioTransport(
|
||||
command=invalid_config["command"],
|
||||
args=invalid_config["args"],
|
||||
cwd=invalid_config["cwd"],
|
||||
keep_alive=False,
|
||||
env=dict(os.environ),
|
||||
)
|
||||
async with Client(invalid_transport, timeout=None, init_timeout=30) as client:
|
||||
with pytest.raises(Exception, match="Unknown workspace 'nope'"):
|
||||
await asyncio.wait_for(client.call_tool("read_note", {"path": "daily/resident.md"}), timeout=30)
|
||||
finally:
|
||||
await _stop_resident_http(server, task)
|
||||
await wrapper.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_bridge_unknown_alias_fails_at_execution(tmp_path):
|
||||
"""An unknown workspace alias surfaces as a tool error from the parent boundary."""
|
||||
from fastmcp import Client
|
||||
from fastmcp.client import StdioTransport
|
||||
async def test_codex_multi_workspace_rejects_unsupported_job_transport(tmp_path):
|
||||
from reme.components.service import CliService
|
||||
|
||||
context = _context(tmp_path)
|
||||
context.service = CliService(app_context=context)
|
||||
context.jobs = {"search_job": _job(context, "search_job")}
|
||||
wrapper = CodexAgentWrapper(app_context=context, project_path=str(tmp_path))
|
||||
await wrapper.start()
|
||||
await wrapper._ensure_job_server() # pylint: disable=protected-access
|
||||
server_config = wrapper._mcp_server_config( # pylint: disable=protected-access
|
||||
{"job_tools": ["search_job"], "workspace": "nope"},
|
||||
)
|
||||
transport = StdioTransport(
|
||||
command=server_config["command"],
|
||||
args=server_config["args"],
|
||||
cwd=server_config["cwd"],
|
||||
keep_alive=False,
|
||||
)
|
||||
async with Client(transport, timeout=None, init_timeout=30) as client:
|
||||
with pytest.raises(Exception, match="Unknown workspace 'nope'"):
|
||||
await asyncio.wait_for(client.call_tool("search_job", {}), timeout=30)
|
||||
wrapper = CodexAgentWrapper(app_context=context)
|
||||
|
||||
with pytest.raises(RuntimeError, match="resident HTTP MCP service"):
|
||||
wrapper._mcp_server_config({"job_tools": ["search_job"], "workspace": "beta"})
|
||||
with pytest.raises(ValueError, match="only supported for single-workspace"):
|
||||
wrapper._mcp_server_config(
|
||||
{"job_tools": ["search_job"], "workspace": "beta", "mcp_config": "custom.yaml"},
|
||||
)
|
||||
await wrapper.close()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue