fix(eval): keep candidate code out of agent comparisons (#3715)

* fix(eval): isolate MCP runtime from coding agents

Expose only a per-cell stdio socket to the release comparison agent while
keeping candidate code in a separately supervised read-only sandbox.
Require host-owned runtime exposure receipts for release and PR evidence.

Preserve the 48 historical measurements with their source-exposure caveat
and document the corrected post-merge comparison. Add transport, sandbox,
cleanup, and report rejection coverage.

Related: #3500
Related: #3609

* fix(eval): address CodeQL transport review findings

Remove the redundant endpoint initializer, explain expected process-scan
races, and limit worker error capture to ordinary exceptions while keeping
teardown failure reporting intact.

* fix(eval): synchronize socket transport tests

---------

Co-authored-by: Gergo Magyar <gergomagyar0@gmail.com>

eval/tests/test_pr_evaluation.py

@@ -38,6 +38,7 @@ def evidence():
             "task_base_sha": "e" * 40,
             "oracle_digest": "f" * 64,
             "sandbox_backend": "bwrap",
+            "runtime_exposure": "none" if arm == "baseline_nomcp" else "stdio-only",
             "sandbox_dependency_content_digest": "9" * 64,
             "ok": True,
             "resolved": True,
@@ -66,6 +67,12 @@ def compare(evidence):
     )
 
 
+def test_pr_comparison_rejects_readable_runtime_cells(evidence):
+    evidence[1][0]["runtime_exposure"] = "readable"
+    report = compare(evidence)
+    assert not report["complete"] and not report["improved"]
+
+
 def test_comparison_measures_only_new_mcp_cells_and_preserves_historical_cost(evidence):
     before = deepcopy(evidence)
     report = compare(evidence)

eval/tests/test_proposer_sandbox.py

@@ -5,10 +5,14 @@ from __future__ import annotations
 import json
 import os
 import shutil
+import socket
 import stat
 import subprocess
 import sys
 import threading
+import time
+import uuid
+from contextlib import ExitStack
 from dataclasses import replace
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
@@ -58,6 +62,222 @@ from workflow_bench.review_scoring import REVIEW_OUTPUT, parse_review_output
 from workflow_bench.task_assets import TaskAssetCache, stage_task_assets as stage_immutable_task_assets
 
 
+@pytest.fixture
+def mcp_socket_client(monkeypatch):
+    """Exercise a real connection attempt between bind() and listen()."""
+    bound, attempted = threading.Event(), threading.Event()
+    original_listen = socket.socket.listen
+
+    def delayed_listen(listener, backlog):
+        bound.set()
+        assert attempted.wait(5), "client never attempted to connect"
+        return original_listen(listener, backlog)
+
+    monkeypatch.setattr(socket.socket, "listen", delayed_listen)
+
+    def connect(endpoint):
+        assert bound.wait(5), "server never bound its socket"
+        deadline = time.monotonic() + 5
+        while True:
+            client = socket.socket(socket.AF_UNIX)
+            client.settimeout(5)
+            try:
+                client.connect(str(endpoint))
+                return client
+            except ConnectionRefusedError:
+                client.close()
+                if time.monotonic() >= deadline:
+                    raise
+            except Exception:
+                client.close()
+                raise
+            finally:
+                attempted.set()
+            time.sleep(0.01)
+
+    return connect
+
+
+@pytest.mark.skipif(os.name == "nt", reason="Unix socket transport requires POSIX socket flags")
+def test_mcp_socket_transport_reconnects_and_reports_server_failure(tmp_path: Path, mcp_socket_client) -> None:
+    from workflow_bench.mcp_transport import serve_stdio
+
+    endpoint = tmp_path / "mcp.sock"
+    failure = tmp_path / "failure"
+    command = [sys.executable, "-c", "import sys; line=sys.stdin.readline(); print(line.strip(), flush=True); sys.exit(3 if line.strip() == 'fail' else 0)"]
+    errors: list[Exception] = []
+
+    def serve() -> None:
+        try:
+            serve_stdio(endpoint, failure, command)
+        except Exception as exc:
+            errors.append(exc)
+
+    worker = threading.Thread(target=serve, daemon=True)
+    worker.start()
+    try:
+        for message in (b"first\n", b"second\n", b"fail\n"):
+            with mcp_socket_client(endpoint) as client:
+                client.sendall(message)
+                client.shutdown(socket.SHUT_WR)
+                assert client.recv(100) == message
+        worker.join(timeout=5)
+        assert not worker.is_alive()
+        assert errors and "3" in failure.read_text()
+    finally:
+        worker.join(timeout=1)
+
+
+@pytest.mark.skipif(os.name == "nt", reason="Unix socket transport requires POSIX socket flags")
+def test_mcp_socket_transport_rejects_premature_clean_server_exit(tmp_path, mcp_socket_client):
+    from workflow_bench.mcp_transport import serve_stdio
+
+    endpoint, failure = tmp_path / "mcp.sock", tmp_path / "failure"
+    errors = []
+
+    def serve():
+        try:
+            serve_stdio(endpoint, failure, [sys.executable, "-c", "pass"])
+        except Exception as exc:
+            errors.append(exc)
+
+    worker = threading.Thread(target=serve, daemon=True)
+    worker.start()
+    with mcp_socket_client(endpoint):
+        worker.join(timeout=2)
+        assert errors and "before client EOF" in failure.read_text()
+
+
+def test_mcp_only_sandbox_does_not_supply_a_cli_wrapper(tmp_path):
+    clone = tmp_path / "clone"
+    clone.mkdir()
+    with prepare_sandbox(
+        clone=clone, claude_bin=sys.executable, bwrap_bin=sys.executable,
+        preflight=False, gitnexus_cli_available=False,
+    ) as sandbox:
+        assert not (sandbox.private_root / "gitnexus").exists()
+        assert SANDBOX_GITNEXUS_CLI not in {mount.target for mount in sandbox.read_only_mounts}
+
+
+@pytest.mark.skipif(
+    os.environ.get("GITNEXUS_REQUIRE_BWRAP_CANARY") != "1",
+    reason="real Bubblewrap canary is mandatory in the named Ubuntu CI job",
+)
+def test_real_sibling_mcp_failure_invalidates_cell_after_client_eof(tmp_path):
+    from workflow_bench.mcp_transport import isolated_mcp
+
+    clone = tmp_path / "clone"
+    clone.mkdir()
+    runtime = tmp_path / "runtime"
+    entrypoint = runtime / "dist/cli/index.js"
+    entrypoint.parent.mkdir(parents=True)
+    entrypoint.write_text("process.exit(7)")
+    bwrap = preflight_bubblewrap()
+    with pytest.raises(SandboxError, match="MCP transport failed"):
+        with isolated_mcp(clone=clone, mounts=[ReadOnlyMount(runtime, proposer_sandbox.SANDBOX_GITNEXUS)],
+                          bwrap_bin=bwrap) as endpoint:
+            with socket.socket(socket.AF_UNIX) as client:
+                client.settimeout(10)
+                client.connect(str(endpoint.source))
+                assert client.recv(1) == b""
+    assert not endpoint.source.exists()
+
+
+@pytest.mark.skipif(
+    os.environ.get("GITNEXUS_REQUIRE_BWRAP_CANARY") != "1",
+    reason="real Bubblewrap canary is mandatory in the named Ubuntu CI job",
+)
+def test_real_sibling_mcp_hides_runtime_and_reaps_detached_children(tmp_path):
+    from workflow_bench.mcp_transport import isolated_mcp, connector_config
+
+    marker = "mcp-child-" + uuid.uuid4().hex
+    clone = tmp_path / "clone"
+    clone.mkdir()
+    (clone / "source.txt").write_text("unsolved")
+    runtime = tmp_path / "runtime with spaces"
+    entrypoint = runtime / "dist/cli/index.js"
+    entrypoint.parent.mkdir(parents=True)
+    host_listener = socket.socket()
+    host_listener.bind(("127.0.0.1", 0))
+    host_listener.listen()
+    entrypoint.write_text("""
+const fs = require('fs');
+const cp = require('child_process');
+const net = require('net');
+const readline = require('readline');
+cp.spawn('/usr/bin/python3', ['-c', 'import time; time.sleep(60)', MARKER],
+  {detached:true, stdio:'ignore'}).unref();
+const input = readline.createInterface({input:process.stdin});
+input.on('line', async line => {
+  const request = JSON.parse(line);
+  let writeBlocked = false;
+  try { fs.writeFileSync('/workspace/source.txt', 'solution'); } catch { writeBlocked = true; }
+  const privateState = !fs.existsSync('/home/agent/secret');
+  const runtimeReadable = fs.readFileSync(__filename, 'utf8').length > 0;
+  const credentialAbsent = !process.env.ANTHROPIC_API_KEY;
+  const networkBlocked = await new Promise(resolve => {
+    const client = net.connect({host:'127.0.0.1', port:HOST_PORT});
+    client.once('connect', () => { client.destroy(); resolve(false); });
+    client.once('error', () => resolve(true));
+    client.setTimeout(1000, () => { client.destroy(); resolve(true); });
+  });
+  process.stdout.write(JSON.stringify({id:request.id, writeBlocked, privateState,
+    runtimeReadable, credentialAbsent, networkBlocked}) + '\\n');
+});
+""".replace("MARKER", json.dumps(marker)).replace("HOST_PORT", str(host_listener.getsockname()[1])))
+    bwrap = preflight_bubblewrap()
+    mounts = [ReadOnlyMount(runtime, proposer_sandbox.SANDBOX_GITNEXUS)]
+    with host_listener, ExitStack() as clients, ExitStack() as stack:
+        endpoints = [stack.enter_context(isolated_mcp(clone=clone, mounts=mounts, bwrap_bin=bwrap)) for _ in range(2)]
+        assert endpoints[0].source != endpoints[1].source
+        server = json.loads(connector_config())["mcpServers"]["gitnexus"]
+        connector = [server["command"], *server["args"]]
+        for index, endpoint in enumerate(endpoints):
+            with prepare_sandbox(
+                clone=clone, claude_bin=sys.executable, bwrap_bin=bwrap,
+                read_only_mounts=[endpoint], preflight=False, gitnexus_cli_available=False,
+            ) as sandbox:
+                (sandbox.home / "secret").write_text("agent-secret")
+                probe = f"""
+import json, subprocess
+from pathlib import Path
+for path in ['/opt/gitnexus', '/opt/gitnexus-shared', '/opt/gitnexus-registry',
+             '/opt/claude/gitnexus', {str(endpoint.source.parent)!r}]:
+    assert not Path(path).exists(), path
+for proc in Path('/proc').glob('[0-9]*'):
+    assert not (proc / 'root/opt/gitnexus').exists()
+for request_id in [{index}, {index + 10}]:
+    result = subprocess.run({connector!r}, input=json.dumps({{'id':request_id}})+'\\n',
+        text=True, capture_output=True, timeout=10)
+    assert result.returncode == 0, result.stderr
+    reply = json.loads(result.stdout)
+    assert reply.pop('id') == request_id
+    assert all(reply.values()), reply
+"""
+                result = sandbox.run(["/usr/bin/python3", "-c", probe], timeout=30,
+                                     env=build_sandbox_environment(auth_token="agent-secret"))
+                assert result.ok, result.stderr_tail
+        # Keep requests open while the cell owner cancels its server namespaces.
+        for endpoint in endpoints:
+            client = clients.enter_context(socket.socket(socket.AF_UNIX))
+            client.settimeout(10)
+            client.connect(str(endpoint.source))
+            client.sendall(b'{"id":99}\n')
+            assert json.loads(client.recv(4096))["id"] == 99
+        children = []
+        for cmdline in Path("/proc").glob("[0-9]*/cmdline"):
+            try:
+                if marker.encode() in cmdline.read_bytes():
+                    children.append(cmdline.parent)
+            except (FileNotFoundError, PermissionError):
+                # Processes may exit during the scan or belong to another user.
+                pass
+        assert children, "canary must create detached descendants"
+    assert all(not process.exists() for process in children)
+    assert all(not endpoint.source.exists() for endpoint in endpoints)
+    assert (clone / "source.txt").read_text() == "unsolved"
+
+
 def test_nomcp_sandbox_hides_graph_and_guidance_without_exposing_the_cli(tmp_path):
     clone = tmp_path / "clone"
     (clone / ".gitnexus").mkdir(parents=True)

eval/tests/test_release_evidence.py

@@ -64,6 +64,7 @@ def make_report(*, harness_sha: str = HARNESS, runtime_sha: str = RUNTIME) -> di
             task_base_sha=PINS["one"]["sha"],
             oracle_digest=PINS["one"]["oracle_digest"],
             sandbox_backend="bwrap",
+            runtime_exposure="none" if arm == "baseline_nomcp" else "stdio-only",
             ok=True,
             resolved=True,
             authored_tests_passed=True,

eval/tests/test_release_gate.py

@@ -38,6 +38,7 @@ def reports():
             task_base_sha=pin["sha"],
             oracle_digest=pin["oracle_digest"],
             sandbox_backend="bwrap",
+            runtime_exposure="none" if arm == "baseline_nomcp" else "stdio-only",
             sandbox_dependency_content_digest=DEPENDENCIES,
             ok=True,
             resolved=run != 0,

eval/tests/test_release_report.py

@@ -39,6 +39,7 @@ def rows():
             "task_base_sha": "d" * 40,
             "oracle_digest": "e" * 64,
             "sandbox_backend": "bwrap",
+            "runtime_exposure": "none" if arm == "baseline_nomcp" else "stdio-only",
             "sandbox_dependency_content_digest": "9" * 64,
             "ok": True,
             "resolved": arm == "baseline_nomcp" or run != 1,
@@ -125,6 +126,24 @@ def test_gate_recomputes_measurements_instead_of_trusting_totals():
         validate_report(report, runtime_sha=SHA, task_set_digest=DIGEST, now=NOW)
 
 
+@pytest.mark.parametrize("exposure", [None, "readable", "none"])
+def test_mcp_measurements_require_host_recorded_socket_isolation(exposure):
+    measured = rows()
+    for row in measured:
+        row["runtime_exposure"] = "none" if row["arm"] == "baseline_nomcp" else "stdio-only"
+    measured[1]["runtime_exposure"] = exposure
+    assert build_report(measured, metadata(), now=NOW)["complete"] is False
+
+
+def test_historical_renderer_does_not_claim_socket_isolation():
+    report = build_report(rows(), metadata(), now=NOW)
+    for row in report["per_run"]:
+        row.pop("runtime_exposure", None)
+    assert "readable compiled runtime" in render_markdown(report)
+    with pytest.raises(ValueError, match="summary"):
+        validate_report(report, runtime_sha=SHA, task_set_digest=DIGEST, now=NOW)
+
+
 def test_torn_result_file_publishes_incomplete_evidence(tmp_path, monkeypatch):
     from workflow_bench.release_report import main
 

eval/tests/test_workflow_bench_sessions.py

@@ -463,6 +463,23 @@ def test_graph_enabled_cell_retains_its_graph_registry_and_runtime(monkeypatch,
     assert sandbox_calls[0]["read_only_mounts"] == [dependencies, runtime, registry]
 
 
+def test_baseline_cell_mounts_only_the_endpoint_and_dependencies(monkeypatch, tmp_path):
+    ctx, clone, dependencies, runtime, registry, _, _, sandbox_calls = _probe_cell_provisioning(monkeypatch, tmp_path)
+    endpoint = runner.ReadOnlyMount(tmp_path / "mcp.sock", "/run/gitnexus-mcp.sock")
+    servers = []
+
+    def endpoint_context(**kwargs):
+        servers.append(kwargs)
+        return nullcontext(endpoint)
+
+    monkeypatch.setattr(runner, "isolated_mcp", endpoint_context)
+    runner.run_cell(ctx, 0, "baseline")
+    assert servers[0]["clone"] == clone
+    assert servers[0]["mounts"] == [dependencies, runtime, registry]
+    assert sandbox_calls[0]["read_only_mounts"] == [dependencies, endpoint]
+    assert sandbox_calls[0]["gitnexus_cli_available"] is False
+
+
 def test_nomcp_cell_does_not_require_an_available_graph(monkeypatch, tmp_path):
     ctx, _, dependencies, _, _, _, _, sandbox_calls = _probe_cell_provisioning(monkeypatch, tmp_path)
     ctx = replace(ctx, graph_snapshot=None, graph_snapshot_error=RuntimeError("graph unavailable"))

eval/workflow_bench/README.md

@@ -228,10 +228,19 @@ are disabled in the no-tool arm, and repository/plugin hooks are disabled in
 both arms. It fails closed when Bubblewrap is unavailable. It retains the repository source and
 ordinary test dependencies required by the tasks; rebuilding tools from that
 source is not prevented. The comparison therefore measures access to the
-provided GitNexus tools and their readable compiled runtime on these GitNexus
-development tasks. That runtime can contain later fixes absent from a task's
-historical base, so the comparison cannot isolate tool assistance from access
-to those implementations.
+provided GitNexus tools on these GitNexus development tasks. The MCP baseline
+receives only a per-cell Unix socket: its candidate runtime, shared package,
+registry, and host launcher are outside the agent filesystem. The host records
+`runtime_exposure: stdio-only` for this arm and `none` for the no-tool arm.
+Release and PR comparison validation reject missing or incompatible receipts.
+Older measurements exposed readable compiled runtime code, including later
+task implementations; they remain historical evidence and cannot satisfy this
+isolated comparison contract. Workflow/CLI arms retain their runtime mounts and
+are labeled `readable`.
+
+The [24 October evidence report](measurement-2026-10-24.md) retains the latest
+historical measurements, current tool-accuracy result, and post-merge rerun
+procedure. Those historical agent results do not show an accuracy or cost gain.
 
 This corpus uses a **fresh index at each task base**. A nightly stale Hermes
 index is a separate condition and must not be pooled with it. Import the
@@ -253,7 +262,9 @@ MCP configuration. Bash children do
 not inherit the model credential and their network sandbox denies all
 domains.
 
-The candidate MCP server runs in a separate nested Bubblewrap boundary. Its
+The release baseline's candidate MCP server runs in a sibling Bubblewrap
+boundary, reached by a stdio socket connector. Workflow/CLI arms retain a nested
+server boundary. In both cases its
 workspace (including the graph), registry, compiled runtime and shared package
 mounts are read-only. Its home and temporary directories are private tmpfs,
 and it has a fresh PID namespace and `/proc`, no network or capabilities, and
@@ -262,6 +273,13 @@ cannot write patches that would be credited to the agent; the agent retains
 its writable workspace for implementation tasks. Mutating MCP tools are not
 granted in any phase.
 
+The cell owns the socket listener and its servers through the existing managed
+process supervisor and an outer PID namespace. Each connection starts one
+server; reconnects are sequential. Server failures invalidate the cell, and
+teardown reaps detached descendants before removing the socket. Native canaries
+in `test_proposer_sandbox.py` run in the required Linux containment job; a local
+namespace skip does not prove isolation.
+
 Prebuilt task `.gitnexus` assets are rejected. For each task commit, the
 harness creates the deterministic parentless snapshot first, removes every
 analyzer-visible path or stored source reference to the benchmark harness,

eval/workflow_bench/mcp_transport.py

@@ -0,0 +1,161 @@
+"""Per-cell stdio endpoint; candidate runtime files stay outside the agent."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import threading
+from contextlib import ExitStack, contextmanager
+from pathlib import Path
+from typing import Iterator, Sequence
+
+from .process_control import ManagedProcessResult, run_managed
+from .proposer_sandbox import (
+    SANDBOX_GITNEXUS,
+    SANDBOX_GITNEXUS_REGISTRY,
+    SANDBOX_HOME,
+    SANDBOX_MCP_SOCKET,
+    SANDBOX_NODE,
+    SANDBOX_TMP,
+    SANDBOX_WORKSPACE,
+    ReadOnlyMount,
+    SandboxError,
+    bwrap_base_args,
+    pid_namespace_command,
+)
+
+def serve_stdio(endpoint: Path, failure: Path, command: Sequence[str]) -> None:
+    """Connect each client socket directly to one server's stdin/stdout.
+
+    The trusted owner runs in an owned PID namespace. The command establishes
+    the candidate's separate filesystem, network and process boundaries.
+    No MCP parsing, shell expansion, or concurrent server fan-out happens here.
+    """
+    try:
+        with socket.socket(socket.AF_UNIX) as listener:
+            listener.bind(str(endpoint))
+            endpoint.chmod(0o600)
+            listener.listen(1)
+            print("ready", flush=True)
+            while True:
+                connection, _ = listener.accept()
+                with connection:
+                    try:
+                        with subprocess.Popen(
+                            command, stdin=connection, stdout=connection, close_fds=True,
+                        ) as server:
+                            returncode = server.wait()
+                        if returncode:
+                            raise RuntimeError(f"MCP server exited with status {returncode}")
+                        try:
+                            pending = connection.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT)
+                        except BlockingIOError:
+                            pending = b"client still connected"
+                        except ConnectionResetError:
+                            pending = b""
+                        if pending:
+                            raise RuntimeError("MCP server exited before client EOF")
+                    except Exception as exc:
+                        # Persist failure before EOF lets the client finish its cell.
+                        failure.write_text(f"{type(exc).__name__}: {exc}")
+                        raise
+    except Exception as exc:
+        failure.write_text(f"{type(exc).__name__}: {exc}")
+        raise
+
+
+def server_command(*, clone: Path, mounts: Sequence[ReadOnlyMount], bwrap_bin: Path) -> list[str]:
+    """Build the credential-free sibling server from host-validated inputs."""
+    node = Path(shutil.which("node") or "/usr/bin/node").resolve()
+    command = [str(bwrap_bin), *bwrap_base_args(unshare_network=True, cap_drop_all=True),
+               "--ro-bind", str(clone), SANDBOX_WORKSPACE,
+               "--ro-bind", str(node), SANDBOX_NODE]
+    for mount in mounts:
+        command += ["--ro-bind", str(mount.source), mount.target]
+    command += ["--tmpfs", "/run", "--tmpfs", SANDBOX_TMP,
+                "--tmpfs", "/home", "--dir", SANDBOX_HOME, "--clearenv"]
+    environment = {
+        "HOME": SANDBOX_HOME, "TMPDIR": SANDBOX_TMP,
+        "GITNEXUS_HOME": SANDBOX_GITNEXUS_REGISTRY,
+        "GITNEXUS_MCP_ALLOWED_REPOS": SANDBOX_WORKSPACE,
+        "GITNEXUS_MCP_DEFAULT_REPO": SANDBOX_WORKSPACE,
+        "PATH": "/usr/local/bin:/usr/bin:/bin", "LANG": "C.UTF-8",
+        "GIT_TERMINAL_PROMPT": "0",
+    }
+    for key, value in environment.items():
+        command += ["--setenv", key, value]
+    return [*command, "--chdir", SANDBOX_WORKSPACE, "--", SANDBOX_NODE,
+            f"{SANDBOX_GITNEXUS}/dist/cli/index.js", "mcp"]
+
+
+def connector_config() -> str:
+    return json.dumps({"mcpServers": {"gitnexus": {
+        "type": "stdio", "command": "/usr/bin/socat",
+        "args": ["STDIO", f"UNIX-CONNECT:{SANDBOX_MCP_SOCKET}"],
+    }}}, sort_keys=True, separators=(",", ":"))
+
+
+@contextmanager
+def isolated_mcp(*, clone: Path, mounts: Sequence[ReadOnlyMount], bwrap_bin: Path) -> Iterator[ReadOnlyMount]:
+    """Own the socket and every server until the complete model session ends."""
+    # A short, private host path also stays below Linux's Unix-socket path limit.
+    with ExitStack() as cleanup:
+        root = Path(tempfile.mkdtemp(prefix="wf-mcp-"))
+        endpoint, failure = root / "mcp.sock", root / "failure"
+        ready, stop = threading.Event(), threading.Event()
+        outcomes: list[ManagedProcessResult | Exception] = []
+        command = pid_namespace_command([
+            sys.executable, "-m", "workflow_bench.mcp_transport", str(endpoint), str(failure),
+            *server_command(clone=clone, mounts=mounts, bwrap_bin=bwrap_bin),
+        ], bwrap_bin=bwrap_bin)
+
+        def run() -> None:
+            try:
+                outcomes.append(run_managed(
+                    command, timeout=24 * 60 * 60, require_pid_namespace=True,
+                    env={"PATH": "/usr/local/bin:/usr/bin:/bin", "LANG": "C.UTF-8",
+                         "PYTHONPATH": str(Path(__file__).resolve().parent.parent)},
+                    cancel_event=stop, stdout_observer=lambda _chunk: ready.set(),
+                ))
+            except Exception as exc:
+                outcomes.append(exc)
+            finally:
+                ready.set()
+
+        worker = threading.Thread(target=run, name="mcp-transport", daemon=True)
+        cleanup.callback(lambda: shutil.rmtree(root) if not worker.is_alive() else None)
+        worker.start()
+        primary: BaseException | None = None
+        try:
+            if not ready.wait(10) or not endpoint.exists() or outcomes:
+                raise SandboxError(f"MCP transport failed to start: {outcomes!r}")
+            yield ReadOnlyMount(endpoint, SANDBOX_MCP_SOCKET)
+            if outcomes or failure.exists():
+                raise SandboxError(f"MCP transport failed: {failure.read_text() if failure.exists() else outcomes!r}")
+        except BaseException as exc:
+            primary = exc
+            raise
+        finally:
+            stop.set()
+            worker.join(timeout=15)
+            error = None
+            if worker.is_alive():
+                error = f"MCP transport supervisor did not stop; backing paths retained at {root}"
+            elif failure.exists():
+                error = f"MCP transport failed: {failure.read_text()}"
+            elif not outcomes or isinstance(outcomes[0], Exception):
+                error = f"MCP transport supervision failed: {outcomes!r}"
+            elif outcomes[0].state not in ("cancelled",):
+                error = f"MCP transport exited unexpectedly: {outcomes[0]!r}"
+            if error:
+                if primary is None:
+                    raise SandboxError(error)
+                primary.add_note(error)
+
+
+if __name__ == "__main__":
+    serve_stdio(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3:])

eval/workflow_bench/measurement-2026-10-24.md

@@ -0,0 +1,82 @@
+# Agent evaluation evidence, 24 October 2026
+
+The latest completed comparison does **not** demonstrate an accuracy gain or
+cost saving. Its MCP arm could read compiled code containing later task fixes,
+so it also cannot isolate the effect of tool assistance.
+
+## Historical measurements
+
+[Release evaluation run 40015796596](https://github.com/abhigyanpatwari/GitNexus/actions/runs/40015796596)
+started on 23 October and completed on 24 October. The sanitized artifact is
+`release-agent-evaluation-62a6f67083fcd1f85634a49b118b4faf3fd5198d`
+(artifact ID `12541246950`). Both JSON reports contain all 24 expected cells;
+their totals were independently recomputed from all 48 per-run records.
+The [sanitized source reports](measurement-2026-10-24.json) are retained here
+so these observations remain reproducible after the workflow artifact expires.
+
+- Candidate: `62a6f67083fcd1f85634a49b118b4faf3fd5198d`.
+- Stable runtime and task base: `c4ecf398de637cf779ab2238e69958cfd5d601ce`.
+- Harness: `466d56280a37cb3dae9cdce8788937964824a3d5`.
+- Model: `gpt-5.6-sol`, effort `xhigh`.
+- Task-set digest: `51daf0758e8bafc30f0a9c94b254ee2b1592be1d569f2d64a7b3031ff7488c16`.
+- Four tasks, three repetitions per arm. Both arms use the same task pins and
+  hidden oracles; the no-GitNexus arm receives no supplied graph or tools.
+
+| Runtime | Arm | Solved | Mean reported cost | Mean agent time |
+| --- | --- | --- | --- | --- |
+| Candidate | No GitNexus | 12/12 | $3.0913 | 617.0s |
+| Candidate | MCP, readable runtime | 12/12 | $3.7592 | 585.4s |
+| Stable | No GitNexus | 12/12 | $3.2524 | 756.2s |
+| Stable | MCP, readable runtime | 12/12 | $4.7397 | 729.3s |
+
+Candidate MCP cost increased **21.6%** and agent time decreased **5.1%**.
+Stable MCP cost increased **45.7%** and agent time decreased **3.6%**.
+Accuracy was unchanged in both comparisons. Cost comes from session reports,
+not invoices; indexing, grading, and infrastructure are excluded. Three
+repetitions on four development tasks do not establish significance or a
+general product benefit. The release gate checks quality against stable; a
+passing gate is not evidence of savings against no GitNexus.
+
+## Deterministic tool accuracy
+
+At source revision `d8a256a03a101a767dc4e1118913c679af740336`, the existing
+tool-accuracy corpus passes **32/32**, with no allowances or execution failures.
+This includes the Python receiver, lexical scope, and summary lower-bound
+checks discussed in issues #3497–#3499. It verifies fixed tool answers, not
+coding-agent success or cost.
+
+Reproduce from `gitnexus/`:
+
+```sh
+node --import tsx bench/tool-accuracy/run.ts --check --out /tmp/tool-accuracy
+```
+
+The local run needed a compatible C++ runtime to load the native parser. Its
+first attempt could not load `GLIBCXX_3.4.31`; that attempt was an execution
+failure, not a measured tool result.
+
+## Corrected comparison and remaining work
+
+The harness correction moves the release baseline's MCP runtime into a
+sibling read-only sandbox. The coding agent receives only a stdio socket.
+Host-generated rows record `runtime_exposure`, and release/PR validation
+rejects missing or incompatible receipts. Workflow arms that intentionally
+expose the CLI retain their different boundary.
+
+The protected Release evaluation workflow executes main's trusted harness,
+including PR previews. A fresh isolated comparison therefore requires this
+correction to merge first. After merge, dispatch `release-evaluation.yml` on
+`main` with `candidate_ref` set to the full merged commit SHA, model
+`gpt-5.6-sol`, effort `xhigh`, and `runner_only=false`. The existing workflow
+runs all four tasks three times per arm for candidate and stable, publishes
+sanitized reports, and stops its runner.
+
+Verify all 48 unique cells, the exact runtime/harness/task bindings, both arm
+receipts, unsuccessful outcomes, and cost/time totals. Publish the result even
+if it shows no gain. Historical artifacts cannot substitute for this run.
+
+Keep [issue #3500](https://github.com/abhigyanpatwari/GitNexus/issues/3500)
+open until the clean measurement is published. A broader SWE-bench or reporter
+corpus, including separate fresh/stale-index conditions, remains necessary
+before making claims beyond this small task set. The reporter paused their
+rerun offer in [discussion #3493](https://github.com/abhigyanpatwari/GitNexus/discussions/3493#discussioncomment-18772530).

eval/workflow_bench/pr_evaluation.py

@@ -321,6 +321,7 @@ def compare_candidate(
                 for field in ("model", "effort", "task_base_sha", "oracle_digest", "sandbox_dependency_content_digest")
             )
             and row.get("sandbox_backend") == "bwrap"
+            and row.get("runtime_exposure") == "stdio-only"
             and row.get("ok") is True
             and all(type(row.get(field)) is bool for field in ("resolved", "authored_tests_passed", "oracle_passed"))
             and row["resolved"] == (row["authored_tests_passed"] and row["oracle_passed"])

eval/workflow_bench/proposer_sandbox.py

@@ -38,6 +38,7 @@ SANDBOX_CLAUDE = "/opt/claude/claude"
 SANDBOX_SHELL_PREFIX = "/opt/claude/shell-prefix"
 SANDBOX_PYTHON3 = "/opt/claude/python3"
 SANDBOX_GITNEXUS_CLI = "/opt/claude/gitnexus"
+SANDBOX_MCP_SOCKET = "/run/gitnexus-mcp.sock"
 SANDBOX_GIT_EXCLUDES = "/opt/claude/git-excludes"
 SANDBOX_NODE = "/opt/claude/node"
 SANDBOX_BWRAP = "/opt/claude/bwrap"
@@ -1219,6 +1220,7 @@ def prepare_sandbox(
     preflight: bool = True,
     backend: str = "bwrap",
     gitnexus_available: bool = True,
+    gitnexus_cli_available: bool = True,
 ) -> Iterator[SandboxSession]:
     """Create private host backing dirs and one virtualized command."""
 
@@ -1235,6 +1237,7 @@ def prepare_sandbox(
             SANDBOX_GITNEXUS_SHARED,
             SANDBOX_GITNEXUS_REGISTRY,
             SANDBOX_GITNEXUS_CLI,
+            SANDBOX_MCP_SOCKET,
             f"{SANDBOX_WORKSPACE}/.gitnexus",
         )
         for mount in read_only_mounts:
@@ -1260,7 +1263,8 @@ def prepare_sandbox(
     for directory in (home, temp):
         directory.mkdir(mode=0o700)
         directory.chmod(0o700)
-    shell_prefix = _create_shell_prefix_wrapper(private_root, gitnexus_available=gitnexus_available)
+    cli_available = gitnexus_available and gitnexus_cli_available
+    shell_prefix = _create_shell_prefix_wrapper(private_root, gitnexus_available=cli_available)
     python3_wrapper = _create_python3_wrapper(private_root)
     git_excludes = _create_git_excludes(private_root)
     # Claude may discover user-level skills below HOME.  Keep the rest of HOME
@@ -1280,7 +1284,7 @@ def prepare_sandbox(
             *read_only_mounts,
             *(
                 (ReadOnlyMount(source=bwrap, target=SANDBOX_BWRAP),)
-                if backend == "bwrap" and gitnexus_available
+                if backend == "bwrap" and cli_available
                 else ()
             ),
             ReadOnlyMount(source=user_skills, target=SANDBOX_USER_SKILLS),
@@ -1288,7 +1292,7 @@ def prepare_sandbox(
             ReadOnlyMount(source=python3_wrapper, target=SANDBOX_PYTHON3),
             *(
                 (ReadOnlyMount(source=_create_gitnexus_wrapper(private_root), target=SANDBOX_GITNEXUS_CLI),)
-                if gitnexus_available
+                if cli_available
                 else baseline_mounts
             ),
             ReadOnlyMount(source=git_excludes, target=SANDBOX_GIT_EXCLUDES),

eval/workflow_bench/release_report.py

@@ -32,6 +32,7 @@ ROW_FIELDS = (
     "task_base_sha",
     "oracle_digest",
     "sandbox_backend",
+    "runtime_exposure",
     DEPENDENCY_CONTENT_BINDING_FIELD,
     "ok",
     "resolved",
@@ -126,6 +127,7 @@ def build_report(rows: list[Any], metadata: dict[str, Any], *, now: datetime | N
             and row.get("model") == report["model"]
             and row.get("effort") == report["effort"]
             and row.get("sandbox_backend") == "bwrap"
+            and row.get("runtime_exposure") == ("none" if arm == "baseline_nomcp" else "stdio-only")
             and row.get("ok") is True
             and type(row.get("resolved")) is bool
             and type(row.get("authored_tests_passed")) is bool
@@ -192,7 +194,14 @@ def render_markdown(report: dict[str, Any]) -> str:
     lines += [
         "",
         "Negative agent outcomes remain in the denominator. Indexing and grading are excluded from agent cost/time.",
-        "The MCP arm also has readable compiled runtime code; this comparison includes that access and cannot isolate tool assistance from access to later fixes.",
+        (
+            "The MCP arm receives a stdio connection; the candidate runtime and registry are absent from the agent filesystem."
+            if report["complete"] and report.get("per_run") and all(
+                row.get("runtime_exposure") == ("none" if row.get("arm") == "baseline_nomcp" else "stdio-only")
+                for row in report["per_run"]
+            )
+            else "Runtime isolation is unproven. Historical MCP cells had readable compiled runtime code and cannot isolate tool assistance from access to later fixes."
+        ),
         "This small pinned task set is release evidence; it does not establish general gains or statistical significance.",
         "",
     ]

eval/workflow_bench/runner.py

@@ -36,6 +36,7 @@ from __future__ import annotations
 
 import argparse
 from concurrent.futures import ThreadPoolExecutor, wait
+from contextlib import ExitStack
 import hashlib
 import json
 import os
@@ -57,6 +58,8 @@ from typing import Any
 
 import yaml
 
+from .mcp_transport import connector_config, isolated_mcp
+
 from .comparator_reuse import (
     REUSE_EXCLUDED_ERROR_KINDS,
     CellKey,
@@ -458,6 +461,7 @@ def run_arm(
     enforce_phase_boundary: bool = False,
     ce_plugin_dir: str | None = None,
     oracle_snapshot: TaskOracleSnapshot | None = None,
+    mcp_config_json: str | None = None,
 ) -> dict[str, Any]:
     sessions: list[dict[str, Any]] = []
     environment_builder = getattr(sandbox, "environment", build_sandbox_environment)
@@ -491,7 +495,7 @@ def run_arm(
         "require_pid_namespace": getattr(sandbox, "require_pid_namespace", True),
         "settings_json": sandbox.settings_json,
         "strict_mcp_config": True,
-        "mcp_config_json": host_text(sandbox_mcp_config(unsafe_host=backend == "host-unsafe")),
+        "mcp_config_json": mcp_config_json or host_text(sandbox_mcp_config(unsafe_host=backend == "host-unsafe")),
         "transcript_projects": sandbox.transcript_projects,
         "transcript_cwd": Path(SANDBOX_WORKSPACE),
         "transcript_wait_seconds": 5,
@@ -1189,6 +1193,9 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
     cleanup_error: OSError | None = None
     execution_arm = CANDIDATE_ARMS.get(arm, arm)
     gitnexus_available = execution_arm != "baseline_nomcp"
+    isolated_baseline = execution_arm == "baseline" and ctx.sandbox_backend == "bwrap"
+    runtime_exposure = "none" if not gitnexus_available else "readable"
+    resources = ExitStack()
     try:
         if ctx.asset_snapshot_error is not None:
             raise RuntimeError(f"task asset snapshot preparation failed: {ctx.asset_snapshot_error}")
@@ -1222,6 +1229,12 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
             else []
         )
         ce_mounts = ce_plugin_mounts_for_arm(execution_arm, ctx.ce_plugin_snapshot)
+        if isolated_baseline:
+            endpoint = resources.enter_context(isolated_mcp(
+                clone=worktree, mounts=[*dependency_mounts, *gitnexus_mounts], bwrap_bin=ctx.bwrap_bin,
+            ))
+            gitnexus_mounts = [endpoint]
+            runtime_exposure = "stdio-only"
         with prepare_sandbox(
             clone=worktree,
             claude_bin=args.claude_bin,
@@ -1234,6 +1247,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
             preflight=False,
             backend=ctx.sandbox_backend,
             gitnexus_available=gitnexus_available,
+            gitnexus_cli_available=not isolated_baseline,
         ) as sandbox:
             # Capture the BASE (pre-overlay) skill digest — identical
             # for the incumbent and candidate arms — then run the
@@ -1314,6 +1328,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
                 enforce_phase_boundary=True,
                 ce_plugin_dir=ce_plugin_dir_for_arm(execution_arm, ctx.ce_plugin_snapshot),
                 oracle_snapshot=ctx.oracle_snapshot,
+                mcp_config_json=connector_config() if isolated_baseline else None,
             )
             if execution_arm in ("review", "ce_review"):
                 review_source = review_output_path(sandbox, REVIEW_OUTPUT)
@@ -1345,6 +1360,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
                 after_digest=after_work_digest,
             )
             patch_bytes = capture_patch(sandbox, worktree, orig_sha)
+        resources.close()
         record["arm"] = arm
         record.update(
             {
@@ -1387,6 +1403,11 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
         detail = redact_text(str(exc), credential_secrets(args))
         print(f"[{task['id']}][{arm}][run {run_idx}] infra-error: {detail}")
     finally:
+        try:
+            resources.close()
+        except (SandboxError, OSError, RuntimeError) as exc:
+            record = infra_error_record(exc)
+            record["arm"] = arm
         if worktree is not None and worktree.exists():
             try:
                 remove_clone(worktree)
@@ -1408,6 +1429,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
             "class": task.get("class", ""),
             "run": run_idx,
             "sandbox_backend": ctx.sandbox_backend,
+            "runtime_exposure": runtime_exposure,
             "task_asset_snapshot_digest": (ctx.asset_snapshot.digest if ctx.asset_snapshot is not None else None),
             "task_asset_manifest_digest": (
                 ctx.asset_snapshot.manifest_digest if ctx.asset_snapshot is not None else None