diff --git a/eval/tests/test_pr_evaluation.py b/eval/tests/test_pr_evaluation.py index 9b5ccedce..0a7f4589c 100644 --- a/eval/tests/test_pr_evaluation.py +++ b/eval/tests/test_pr_evaluation.py @@ -38,6 +38,7 @@ def evidence(): "task_base_sha": "e" * 40, "oracle_digest": "f" * 64, "sandbox_backend": "bwrap", + "runtime_exposure": "none" if arm == "baseline_nomcp" else "stdio-only", "sandbox_dependency_content_digest": "9" * 64, "ok": True, "resolved": True, @@ -66,6 +67,12 @@ def compare(evidence): ) +def test_pr_comparison_rejects_readable_runtime_cells(evidence): + evidence[1][0]["runtime_exposure"] = "readable" + report = compare(evidence) + assert not report["complete"] and not report["improved"] + + def test_comparison_measures_only_new_mcp_cells_and_preserves_historical_cost(evidence): before = deepcopy(evidence) report = compare(evidence) diff --git a/eval/tests/test_proposer_sandbox.py b/eval/tests/test_proposer_sandbox.py index a91166b27..a439b30b9 100644 --- a/eval/tests/test_proposer_sandbox.py +++ b/eval/tests/test_proposer_sandbox.py @@ -5,10 +5,14 @@ from __future__ import annotations import json import os import shutil +import socket import stat import subprocess import sys import threading +import time +import uuid +from contextlib import ExitStack from dataclasses import replace from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -58,6 +62,222 @@ from workflow_bench.review_scoring import REVIEW_OUTPUT, parse_review_output from workflow_bench.task_assets import TaskAssetCache, stage_task_assets as stage_immutable_task_assets +@pytest.fixture +def mcp_socket_client(monkeypatch): + """Exercise a real connection attempt between bind() and listen().""" + bound, attempted = threading.Event(), threading.Event() + original_listen = socket.socket.listen + + def delayed_listen(listener, backlog): + bound.set() + assert attempted.wait(5), "client never attempted to connect" + return original_listen(listener, backlog) + + monkeypatch.setattr(socket.socket, "listen", delayed_listen) + + def connect(endpoint): + assert bound.wait(5), "server never bound its socket" + deadline = time.monotonic() + 5 + while True: + client = socket.socket(socket.AF_UNIX) + client.settimeout(5) + try: + client.connect(str(endpoint)) + return client + except ConnectionRefusedError: + client.close() + if time.monotonic() >= deadline: + raise + except Exception: + client.close() + raise + finally: + attempted.set() + time.sleep(0.01) + + return connect + + +@pytest.mark.skipif(os.name == "nt", reason="Unix socket transport requires POSIX socket flags") +def test_mcp_socket_transport_reconnects_and_reports_server_failure(tmp_path: Path, mcp_socket_client) -> None: + from workflow_bench.mcp_transport import serve_stdio + + endpoint = tmp_path / "mcp.sock" + failure = tmp_path / "failure" + command = [sys.executable, "-c", "import sys; line=sys.stdin.readline(); print(line.strip(), flush=True); sys.exit(3 if line.strip() == 'fail' else 0)"] + errors: list[Exception] = [] + + def serve() -> None: + try: + serve_stdio(endpoint, failure, command) + except Exception as exc: + errors.append(exc) + + worker = threading.Thread(target=serve, daemon=True) + worker.start() + try: + for message in (b"first\n", b"second\n", b"fail\n"): + with mcp_socket_client(endpoint) as client: + client.sendall(message) + client.shutdown(socket.SHUT_WR) + assert client.recv(100) == message + worker.join(timeout=5) + assert not worker.is_alive() + assert errors and "3" in failure.read_text() + finally: + worker.join(timeout=1) + + +@pytest.mark.skipif(os.name == "nt", reason="Unix socket transport requires POSIX socket flags") +def test_mcp_socket_transport_rejects_premature_clean_server_exit(tmp_path, mcp_socket_client): + from workflow_bench.mcp_transport import serve_stdio + + endpoint, failure = tmp_path / "mcp.sock", tmp_path / "failure" + errors = [] + + def serve(): + try: + serve_stdio(endpoint, failure, [sys.executable, "-c", "pass"]) + except Exception as exc: + errors.append(exc) + + worker = threading.Thread(target=serve, daemon=True) + worker.start() + with mcp_socket_client(endpoint): + worker.join(timeout=2) + assert errors and "before client EOF" in failure.read_text() + + +def test_mcp_only_sandbox_does_not_supply_a_cli_wrapper(tmp_path): + clone = tmp_path / "clone" + clone.mkdir() + with prepare_sandbox( + clone=clone, claude_bin=sys.executable, bwrap_bin=sys.executable, + preflight=False, gitnexus_cli_available=False, + ) as sandbox: + assert not (sandbox.private_root / "gitnexus").exists() + assert SANDBOX_GITNEXUS_CLI not in {mount.target for mount in sandbox.read_only_mounts} + + +@pytest.mark.skipif( + os.environ.get("GITNEXUS_REQUIRE_BWRAP_CANARY") != "1", + reason="real Bubblewrap canary is mandatory in the named Ubuntu CI job", +) +def test_real_sibling_mcp_failure_invalidates_cell_after_client_eof(tmp_path): + from workflow_bench.mcp_transport import isolated_mcp + + clone = tmp_path / "clone" + clone.mkdir() + runtime = tmp_path / "runtime" + entrypoint = runtime / "dist/cli/index.js" + entrypoint.parent.mkdir(parents=True) + entrypoint.write_text("process.exit(7)") + bwrap = preflight_bubblewrap() + with pytest.raises(SandboxError, match="MCP transport failed"): + with isolated_mcp(clone=clone, mounts=[ReadOnlyMount(runtime, proposer_sandbox.SANDBOX_GITNEXUS)], + bwrap_bin=bwrap) as endpoint: + with socket.socket(socket.AF_UNIX) as client: + client.settimeout(10) + client.connect(str(endpoint.source)) + assert client.recv(1) == b"" + assert not endpoint.source.exists() + + +@pytest.mark.skipif( + os.environ.get("GITNEXUS_REQUIRE_BWRAP_CANARY") != "1", + reason="real Bubblewrap canary is mandatory in the named Ubuntu CI job", +) +def test_real_sibling_mcp_hides_runtime_and_reaps_detached_children(tmp_path): + from workflow_bench.mcp_transport import isolated_mcp, connector_config + + marker = "mcp-child-" + uuid.uuid4().hex + clone = tmp_path / "clone" + clone.mkdir() + (clone / "source.txt").write_text("unsolved") + runtime = tmp_path / "runtime with spaces" + entrypoint = runtime / "dist/cli/index.js" + entrypoint.parent.mkdir(parents=True) + host_listener = socket.socket() + host_listener.bind(("127.0.0.1", 0)) + host_listener.listen() + entrypoint.write_text(""" +const fs = require('fs'); +const cp = require('child_process'); +const net = require('net'); +const readline = require('readline'); +cp.spawn('/usr/bin/python3', ['-c', 'import time; time.sleep(60)', MARKER], + {detached:true, stdio:'ignore'}).unref(); +const input = readline.createInterface({input:process.stdin}); +input.on('line', async line => { + const request = JSON.parse(line); + let writeBlocked = false; + try { fs.writeFileSync('/workspace/source.txt', 'solution'); } catch { writeBlocked = true; } + const privateState = !fs.existsSync('/home/agent/secret'); + const runtimeReadable = fs.readFileSync(__filename, 'utf8').length > 0; + const credentialAbsent = !process.env.ANTHROPIC_API_KEY; + const networkBlocked = await new Promise(resolve => { + const client = net.connect({host:'127.0.0.1', port:HOST_PORT}); + client.once('connect', () => { client.destroy(); resolve(false); }); + client.once('error', () => resolve(true)); + client.setTimeout(1000, () => { client.destroy(); resolve(true); }); + }); + process.stdout.write(JSON.stringify({id:request.id, writeBlocked, privateState, + runtimeReadable, credentialAbsent, networkBlocked}) + '\\n'); +}); +""".replace("MARKER", json.dumps(marker)).replace("HOST_PORT", str(host_listener.getsockname()[1]))) + bwrap = preflight_bubblewrap() + mounts = [ReadOnlyMount(runtime, proposer_sandbox.SANDBOX_GITNEXUS)] + with host_listener, ExitStack() as clients, ExitStack() as stack: + endpoints = [stack.enter_context(isolated_mcp(clone=clone, mounts=mounts, bwrap_bin=bwrap)) for _ in range(2)] + assert endpoints[0].source != endpoints[1].source + server = json.loads(connector_config())["mcpServers"]["gitnexus"] + connector = [server["command"], *server["args"]] + for index, endpoint in enumerate(endpoints): + with prepare_sandbox( + clone=clone, claude_bin=sys.executable, bwrap_bin=bwrap, + read_only_mounts=[endpoint], preflight=False, gitnexus_cli_available=False, + ) as sandbox: + (sandbox.home / "secret").write_text("agent-secret") + probe = f""" +import json, subprocess +from pathlib import Path +for path in ['/opt/gitnexus', '/opt/gitnexus-shared', '/opt/gitnexus-registry', + '/opt/claude/gitnexus', {str(endpoint.source.parent)!r}]: + assert not Path(path).exists(), path +for proc in Path('/proc').glob('[0-9]*'): + assert not (proc / 'root/opt/gitnexus').exists() +for request_id in [{index}, {index + 10}]: + result = subprocess.run({connector!r}, input=json.dumps({{'id':request_id}})+'\\n', + text=True, capture_output=True, timeout=10) + assert result.returncode == 0, result.stderr + reply = json.loads(result.stdout) + assert reply.pop('id') == request_id + assert all(reply.values()), reply +""" + result = sandbox.run(["/usr/bin/python3", "-c", probe], timeout=30, + env=build_sandbox_environment(auth_token="agent-secret")) + assert result.ok, result.stderr_tail + # Keep requests open while the cell owner cancels its server namespaces. + for endpoint in endpoints: + client = clients.enter_context(socket.socket(socket.AF_UNIX)) + client.settimeout(10) + client.connect(str(endpoint.source)) + client.sendall(b'{"id":99}\n') + assert json.loads(client.recv(4096))["id"] == 99 + children = [] + for cmdline in Path("/proc").glob("[0-9]*/cmdline"): + try: + if marker.encode() in cmdline.read_bytes(): + children.append(cmdline.parent) + except (FileNotFoundError, PermissionError): + # Processes may exit during the scan or belong to another user. + pass + assert children, "canary must create detached descendants" + assert all(not process.exists() for process in children) + assert all(not endpoint.source.exists() for endpoint in endpoints) + assert (clone / "source.txt").read_text() == "unsolved" + + def test_nomcp_sandbox_hides_graph_and_guidance_without_exposing_the_cli(tmp_path): clone = tmp_path / "clone" (clone / ".gitnexus").mkdir(parents=True) diff --git a/eval/tests/test_release_evidence.py b/eval/tests/test_release_evidence.py index 3dbdaced4..97adc1256 100644 --- a/eval/tests/test_release_evidence.py +++ b/eval/tests/test_release_evidence.py @@ -64,6 +64,7 @@ def make_report(*, harness_sha: str = HARNESS, runtime_sha: str = RUNTIME) -> di task_base_sha=PINS["one"]["sha"], oracle_digest=PINS["one"]["oracle_digest"], sandbox_backend="bwrap", + runtime_exposure="none" if arm == "baseline_nomcp" else "stdio-only", ok=True, resolved=True, authored_tests_passed=True, diff --git a/eval/tests/test_release_gate.py b/eval/tests/test_release_gate.py index 53812a37d..cadedda41 100644 --- a/eval/tests/test_release_gate.py +++ b/eval/tests/test_release_gate.py @@ -38,6 +38,7 @@ def reports(): task_base_sha=pin["sha"], oracle_digest=pin["oracle_digest"], sandbox_backend="bwrap", + runtime_exposure="none" if arm == "baseline_nomcp" else "stdio-only", sandbox_dependency_content_digest=DEPENDENCIES, ok=True, resolved=run != 0, diff --git a/eval/tests/test_release_report.py b/eval/tests/test_release_report.py index 6abac8226..dd483fbc0 100644 --- a/eval/tests/test_release_report.py +++ b/eval/tests/test_release_report.py @@ -39,6 +39,7 @@ def rows(): "task_base_sha": "d" * 40, "oracle_digest": "e" * 64, "sandbox_backend": "bwrap", + "runtime_exposure": "none" if arm == "baseline_nomcp" else "stdio-only", "sandbox_dependency_content_digest": "9" * 64, "ok": True, "resolved": arm == "baseline_nomcp" or run != 1, @@ -125,6 +126,24 @@ def test_gate_recomputes_measurements_instead_of_trusting_totals(): validate_report(report, runtime_sha=SHA, task_set_digest=DIGEST, now=NOW) +@pytest.mark.parametrize("exposure", [None, "readable", "none"]) +def test_mcp_measurements_require_host_recorded_socket_isolation(exposure): + measured = rows() + for row in measured: + row["runtime_exposure"] = "none" if row["arm"] == "baseline_nomcp" else "stdio-only" + measured[1]["runtime_exposure"] = exposure + assert build_report(measured, metadata(), now=NOW)["complete"] is False + + +def test_historical_renderer_does_not_claim_socket_isolation(): + report = build_report(rows(), metadata(), now=NOW) + for row in report["per_run"]: + row.pop("runtime_exposure", None) + assert "readable compiled runtime" in render_markdown(report) + with pytest.raises(ValueError, match="summary"): + validate_report(report, runtime_sha=SHA, task_set_digest=DIGEST, now=NOW) + + def test_torn_result_file_publishes_incomplete_evidence(tmp_path, monkeypatch): from workflow_bench.release_report import main diff --git a/eval/tests/test_workflow_bench_sessions.py b/eval/tests/test_workflow_bench_sessions.py index f203f0d43..b599f916b 100644 --- a/eval/tests/test_workflow_bench_sessions.py +++ b/eval/tests/test_workflow_bench_sessions.py @@ -463,6 +463,23 @@ def test_graph_enabled_cell_retains_its_graph_registry_and_runtime(monkeypatch, assert sandbox_calls[0]["read_only_mounts"] == [dependencies, runtime, registry] +def test_baseline_cell_mounts_only_the_endpoint_and_dependencies(monkeypatch, tmp_path): + ctx, clone, dependencies, runtime, registry, _, _, sandbox_calls = _probe_cell_provisioning(monkeypatch, tmp_path) + endpoint = runner.ReadOnlyMount(tmp_path / "mcp.sock", "/run/gitnexus-mcp.sock") + servers = [] + + def endpoint_context(**kwargs): + servers.append(kwargs) + return nullcontext(endpoint) + + monkeypatch.setattr(runner, "isolated_mcp", endpoint_context) + runner.run_cell(ctx, 0, "baseline") + assert servers[0]["clone"] == clone + assert servers[0]["mounts"] == [dependencies, runtime, registry] + assert sandbox_calls[0]["read_only_mounts"] == [dependencies, endpoint] + assert sandbox_calls[0]["gitnexus_cli_available"] is False + + def test_nomcp_cell_does_not_require_an_available_graph(monkeypatch, tmp_path): ctx, _, dependencies, _, _, _, _, sandbox_calls = _probe_cell_provisioning(monkeypatch, tmp_path) ctx = replace(ctx, graph_snapshot=None, graph_snapshot_error=RuntimeError("graph unavailable")) diff --git a/eval/workflow_bench/README.md b/eval/workflow_bench/README.md index d8132d118..2610816ef 100644 --- a/eval/workflow_bench/README.md +++ b/eval/workflow_bench/README.md @@ -228,10 +228,19 @@ are disabled in the no-tool arm, and repository/plugin hooks are disabled in both arms. It fails closed when Bubblewrap is unavailable. It retains the repository source and ordinary test dependencies required by the tasks; rebuilding tools from that source is not prevented. The comparison therefore measures access to the -provided GitNexus tools and their readable compiled runtime on these GitNexus -development tasks. That runtime can contain later fixes absent from a task's -historical base, so the comparison cannot isolate tool assistance from access -to those implementations. +provided GitNexus tools on these GitNexus development tasks. The MCP baseline +receives only a per-cell Unix socket: its candidate runtime, shared package, +registry, and host launcher are outside the agent filesystem. The host records +`runtime_exposure: stdio-only` for this arm and `none` for the no-tool arm. +Release and PR comparison validation reject missing or incompatible receipts. +Older measurements exposed readable compiled runtime code, including later +task implementations; they remain historical evidence and cannot satisfy this +isolated comparison contract. Workflow/CLI arms retain their runtime mounts and +are labeled `readable`. + +The [24 October evidence report](measurement-2026-10-24.md) retains the latest +historical measurements, current tool-accuracy result, and post-merge rerun +procedure. Those historical agent results do not show an accuracy or cost gain. This corpus uses a **fresh index at each task base**. A nightly stale Hermes index is a separate condition and must not be pooled with it. Import the @@ -253,7 +262,9 @@ MCP configuration. Bash children do not inherit the model credential and their network sandbox denies all domains. -The candidate MCP server runs in a separate nested Bubblewrap boundary. Its +The release baseline's candidate MCP server runs in a sibling Bubblewrap +boundary, reached by a stdio socket connector. Workflow/CLI arms retain a nested +server boundary. In both cases its workspace (including the graph), registry, compiled runtime and shared package mounts are read-only. Its home and temporary directories are private tmpfs, and it has a fresh PID namespace and `/proc`, no network or capabilities, and @@ -262,6 +273,13 @@ cannot write patches that would be credited to the agent; the agent retains its writable workspace for implementation tasks. Mutating MCP tools are not granted in any phase. +The cell owns the socket listener and its servers through the existing managed +process supervisor and an outer PID namespace. Each connection starts one +server; reconnects are sequential. Server failures invalidate the cell, and +teardown reaps detached descendants before removing the socket. Native canaries +in `test_proposer_sandbox.py` run in the required Linux containment job; a local +namespace skip does not prove isolation. + Prebuilt task `.gitnexus` assets are rejected. For each task commit, the harness creates the deterministic parentless snapshot first, removes every analyzer-visible path or stored source reference to the benchmark harness, diff --git a/eval/workflow_bench/mcp_transport.py b/eval/workflow_bench/mcp_transport.py new file mode 100644 index 000000000..3958fe260 --- /dev/null +++ b/eval/workflow_bench/mcp_transport.py @@ -0,0 +1,161 @@ +"""Per-cell stdio endpoint; candidate runtime files stay outside the agent.""" + +from __future__ import annotations + +import json +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +from contextlib import ExitStack, contextmanager +from pathlib import Path +from typing import Iterator, Sequence + +from .process_control import ManagedProcessResult, run_managed +from .proposer_sandbox import ( + SANDBOX_GITNEXUS, + SANDBOX_GITNEXUS_REGISTRY, + SANDBOX_HOME, + SANDBOX_MCP_SOCKET, + SANDBOX_NODE, + SANDBOX_TMP, + SANDBOX_WORKSPACE, + ReadOnlyMount, + SandboxError, + bwrap_base_args, + pid_namespace_command, +) + +def serve_stdio(endpoint: Path, failure: Path, command: Sequence[str]) -> None: + """Connect each client socket directly to one server's stdin/stdout. + + The trusted owner runs in an owned PID namespace. The command establishes + the candidate's separate filesystem, network and process boundaries. + No MCP parsing, shell expansion, or concurrent server fan-out happens here. + """ + try: + with socket.socket(socket.AF_UNIX) as listener: + listener.bind(str(endpoint)) + endpoint.chmod(0o600) + listener.listen(1) + print("ready", flush=True) + while True: + connection, _ = listener.accept() + with connection: + try: + with subprocess.Popen( + command, stdin=connection, stdout=connection, close_fds=True, + ) as server: + returncode = server.wait() + if returncode: + raise RuntimeError(f"MCP server exited with status {returncode}") + try: + pending = connection.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT) + except BlockingIOError: + pending = b"client still connected" + except ConnectionResetError: + pending = b"" + if pending: + raise RuntimeError("MCP server exited before client EOF") + except Exception as exc: + # Persist failure before EOF lets the client finish its cell. + failure.write_text(f"{type(exc).__name__}: {exc}") + raise + except Exception as exc: + failure.write_text(f"{type(exc).__name__}: {exc}") + raise + + +def server_command(*, clone: Path, mounts: Sequence[ReadOnlyMount], bwrap_bin: Path) -> list[str]: + """Build the credential-free sibling server from host-validated inputs.""" + node = Path(shutil.which("node") or "/usr/bin/node").resolve() + command = [str(bwrap_bin), *bwrap_base_args(unshare_network=True, cap_drop_all=True), + "--ro-bind", str(clone), SANDBOX_WORKSPACE, + "--ro-bind", str(node), SANDBOX_NODE] + for mount in mounts: + command += ["--ro-bind", str(mount.source), mount.target] + command += ["--tmpfs", "/run", "--tmpfs", SANDBOX_TMP, + "--tmpfs", "/home", "--dir", SANDBOX_HOME, "--clearenv"] + environment = { + "HOME": SANDBOX_HOME, "TMPDIR": SANDBOX_TMP, + "GITNEXUS_HOME": SANDBOX_GITNEXUS_REGISTRY, + "GITNEXUS_MCP_ALLOWED_REPOS": SANDBOX_WORKSPACE, + "GITNEXUS_MCP_DEFAULT_REPO": SANDBOX_WORKSPACE, + "PATH": "/usr/local/bin:/usr/bin:/bin", "LANG": "C.UTF-8", + "GIT_TERMINAL_PROMPT": "0", + } + for key, value in environment.items(): + command += ["--setenv", key, value] + return [*command, "--chdir", SANDBOX_WORKSPACE, "--", SANDBOX_NODE, + f"{SANDBOX_GITNEXUS}/dist/cli/index.js", "mcp"] + + +def connector_config() -> str: + return json.dumps({"mcpServers": {"gitnexus": { + "type": "stdio", "command": "/usr/bin/socat", + "args": ["STDIO", f"UNIX-CONNECT:{SANDBOX_MCP_SOCKET}"], + }}}, sort_keys=True, separators=(",", ":")) + + +@contextmanager +def isolated_mcp(*, clone: Path, mounts: Sequence[ReadOnlyMount], bwrap_bin: Path) -> Iterator[ReadOnlyMount]: + """Own the socket and every server until the complete model session ends.""" + # A short, private host path also stays below Linux's Unix-socket path limit. + with ExitStack() as cleanup: + root = Path(tempfile.mkdtemp(prefix="wf-mcp-")) + endpoint, failure = root / "mcp.sock", root / "failure" + ready, stop = threading.Event(), threading.Event() + outcomes: list[ManagedProcessResult | Exception] = [] + command = pid_namespace_command([ + sys.executable, "-m", "workflow_bench.mcp_transport", str(endpoint), str(failure), + *server_command(clone=clone, mounts=mounts, bwrap_bin=bwrap_bin), + ], bwrap_bin=bwrap_bin) + + def run() -> None: + try: + outcomes.append(run_managed( + command, timeout=24 * 60 * 60, require_pid_namespace=True, + env={"PATH": "/usr/local/bin:/usr/bin:/bin", "LANG": "C.UTF-8", + "PYTHONPATH": str(Path(__file__).resolve().parent.parent)}, + cancel_event=stop, stdout_observer=lambda _chunk: ready.set(), + )) + except Exception as exc: + outcomes.append(exc) + finally: + ready.set() + + worker = threading.Thread(target=run, name="mcp-transport", daemon=True) + cleanup.callback(lambda: shutil.rmtree(root) if not worker.is_alive() else None) + worker.start() + primary: BaseException | None = None + try: + if not ready.wait(10) or not endpoint.exists() or outcomes: + raise SandboxError(f"MCP transport failed to start: {outcomes!r}") + yield ReadOnlyMount(endpoint, SANDBOX_MCP_SOCKET) + if outcomes or failure.exists(): + raise SandboxError(f"MCP transport failed: {failure.read_text() if failure.exists() else outcomes!r}") + except BaseException as exc: + primary = exc + raise + finally: + stop.set() + worker.join(timeout=15) + error = None + if worker.is_alive(): + error = f"MCP transport supervisor did not stop; backing paths retained at {root}" + elif failure.exists(): + error = f"MCP transport failed: {failure.read_text()}" + elif not outcomes or isinstance(outcomes[0], Exception): + error = f"MCP transport supervision failed: {outcomes!r}" + elif outcomes[0].state not in ("cancelled",): + error = f"MCP transport exited unexpectedly: {outcomes[0]!r}" + if error: + if primary is None: + raise SandboxError(error) + primary.add_note(error) + + +if __name__ == "__main__": + serve_stdio(Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3:]) diff --git a/eval/workflow_bench/measurement-2026-10-24.json b/eval/workflow_bench/measurement-2026-10-24.json new file mode 100644 index 000000000..7c5294862 --- /dev/null +++ b/eval/workflow_bench/measurement-2026-10-24.json @@ -0,0 +1,1212 @@ +{ + "historical": true, + "source_run": "https://github.com/abhigyanpatwari/GitNexus/actions/runs/40015796596", + "reports": { + "candidate": { + "runtime_sha": "62a6f67083fcd1f85634a49b118b4faf3fd5198d", + "harness_sha": "466d56280a37cb3dae9cdce8788937964824a3d5", + "task_set_digest": "51daf0758e8bafc30f0a9c94b254ee2b1592be1d569f2d64a7b3031ff7488c16", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "runs": 3, + "tasks": { + "trivial-status-json-alias": { + "sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7" + }, + "inv-feature-c-include-provenance": { + "sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea" + }, + "inv-feature-list-repos-filter": { + "sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14" + }, + "cross-module-parse-retry": { + "sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02" + } + }, + "schema": "gitnexus.release-evaluation/v1", + "generated_at": "2026-10-24T00:34:15.542969+00:00", + "complete": true, + "problems": [], + "per_run": [ + { + "task": "trivial-status-json-alias", + "run": 0, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 0.8601, + "duration_s": 319.4, + "input_tokens": 54, + "cache_creation_input_tokens": 54251, + "cache_read_input_tokens": 717501, + "output_tokens": 6480 + }, + { + "task": "trivial-status-json-alias", + "run": 0, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 1.2159, + "duration_s": 407.8, + "input_tokens": 54, + "cache_creation_input_tokens": 81848, + "cache_read_input_tokens": 1050301, + "output_tokens": 7156 + }, + { + "task": "trivial-status-json-alias", + "run": 1, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 0.7967, + "duration_s": 321.8, + "input_tokens": 48, + "cache_creation_input_tokens": 50844, + "cache_read_input_tokens": 564499, + "output_tokens": 7859 + }, + { + "task": "trivial-status-json-alias", + "run": 1, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 1.7795, + "duration_s": 405.9, + "input_tokens": 96, + "cache_creation_input_tokens": 85654, + "cache_read_input_tokens": 1962444, + "output_tokens": 10500 + }, + { + "task": "trivial-status-json-alias", + "run": 2, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 0.9767, + "duration_s": 261.1, + "input_tokens": 54, + "cache_creation_input_tokens": 62891, + "cache_read_input_tokens": 819460, + "output_tokens": 6946 + }, + { + "task": "trivial-status-json-alias", + "run": 2, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 1.1557, + "duration_s": 385.0, + "input_tokens": 63, + "cache_creation_input_tokens": 70647, + "cache_read_input_tokens": 1000642, + "output_tokens": 8542 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 0, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 2.2548, + "duration_s": 342.9, + "input_tokens": 99, + "cache_creation_input_tokens": 102077, + "cache_read_input_tokens": 2449500, + "output_tokens": 15662 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 0, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 5.3164, + "duration_s": 601.7, + "input_tokens": 135, + "cache_creation_input_tokens": 202129, + "cache_read_input_tokens": 6693294, + "output_tokens": 28232 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 1, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 2.9411, + "duration_s": 479.5, + "input_tokens": 559, + "cache_creation_input_tokens": 129574, + "cache_read_input_tokens": 3121264, + "output_tokens": 22713 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 1, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 2.77, + "duration_s": 352.8, + "input_tokens": 69, + "cache_creation_input_tokens": 164365, + "cache_read_input_tokens": 2536346, + "output_tokens": 18967 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 2, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 4.5127, + "duration_s": 664.7, + "input_tokens": 1718, + "cache_creation_input_tokens": 141945, + "cache_read_input_tokens": 5821552, + "output_tokens": 28246 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 2, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 2.6367, + "duration_s": 465.7, + "input_tokens": 279, + "cache_creation_input_tokens": 135405, + "cache_read_input_tokens": 2486802, + "output_tokens": 21826 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 0, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 2.2716, + "duration_s": 511.6, + "input_tokens": 2621, + "cache_creation_input_tokens": 77766, + "cache_read_input_tokens": 2262735, + "output_tokens": 25645 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 0, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 2.953, + "duration_s": 364.7, + "input_tokens": 324, + "cache_creation_input_tokens": 125985, + "cache_read_input_tokens": 3339609, + "output_tokens": 19766 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 1, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 3.3959, + "duration_s": 626.7, + "input_tokens": 2493, + "cache_creation_input_tokens": 112659, + "cache_read_input_tokens": 3780944, + "output_tokens": 31555 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 1, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 3.159, + "duration_s": 463.4, + "input_tokens": 171, + "cache_creation_input_tokens": 103164, + "cache_read_input_tokens": 3986392, + "output_tokens": 20805 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 2, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 3.6101, + "duration_s": 1085.6, + "input_tokens": 323, + "cache_creation_input_tokens": 148649, + "cache_read_input_tokens": 4111000, + "output_tokens": 24959 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 2, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 2.6612, + "duration_s": 377.6, + "input_tokens": 3536, + "cache_creation_input_tokens": 126703, + "cache_read_input_tokens": 2731099, + "output_tokens": 19445 + }, + { + "task": "cross-module-parse-retry", + "run": 0, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 4.7241, + "duration_s": 604.2, + "input_tokens": 563, + "cache_creation_input_tokens": 140418, + "cache_read_input_tokens": 5947974, + "output_tokens": 34787 + }, + { + "task": "cross-module-parse-retry", + "run": 0, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 8.9079, + "duration_s": 829.8, + "input_tokens": 353, + "cache_creation_input_tokens": 226563, + "cache_read_input_tokens": 13170789, + "output_tokens": 36187 + }, + { + "task": "cross-module-parse-retry", + "run": 1, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 5.7595, + "duration_s": 752.2, + "input_tokens": 311, + "cache_creation_input_tokens": 163110, + "cache_read_input_tokens": 7786402, + "output_tokens": 33813 + }, + { + "task": "cross-module-parse-retry", + "run": 1, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 4.4676, + "duration_s": 813.7, + "input_tokens": 746, + "cache_creation_input_tokens": 144869, + "cache_read_input_tokens": 5081193, + "output_tokens": 40714 + }, + { + "task": "cross-module-parse-retry", + "run": 2, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 4.992, + "duration_s": 1434.0, + "input_tokens": 2186, + "cache_creation_input_tokens": 150371, + "cache_read_input_tokens": 6492405, + "output_tokens": 31803 + }, + { + "task": "cross-module-parse-retry", + "run": 2, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 8.0877, + "duration_s": 1556.2, + "input_tokens": 315, + "cache_creation_input_tokens": 236142, + "cache_read_input_tokens": 11492978, + "output_tokens": 34551 + } + ], + "arms": { + "baseline_nomcp": { + "cells": 12, + "solved": 12, + "mean_cost_usd": 3.091275, + "mean_wall_s": 616.975 + }, + "baseline": { + "cells": 12, + "solved": 12, + "mean_cost_usd": 3.7592166666666667, + "mean_wall_s": 585.3583333333333 + } + }, + "paired": { + "solve_delta": 0, + "mean_cost_change_pct": 21.607319525654194, + "mean_wall_change_pct": -5.124464794629713 + } + }, + "stable": { + "runtime_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "harness_sha": "466d56280a37cb3dae9cdce8788937964824a3d5", + "task_set_digest": "51daf0758e8bafc30f0a9c94b254ee2b1592be1d569f2d64a7b3031ff7488c16", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "runs": 3, + "tasks": { + "trivial-status-json-alias": { + "sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7" + }, + "inv-feature-c-include-provenance": { + "sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea" + }, + "inv-feature-list-repos-filter": { + "sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14" + }, + "cross-module-parse-retry": { + "sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02" + } + }, + "schema": "gitnexus.release-evaluation/v1", + "generated_at": "2026-10-24T00:34:15.377904+00:00", + "complete": true, + "problems": [], + "per_run": [ + { + "task": "trivial-status-json-alias", + "run": 0, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 0.7794, + "duration_s": 209.8, + "input_tokens": 42, + "cache_creation_input_tokens": 56525, + "cache_read_input_tokens": 592095, + "output_tokens": 5196 + }, + { + "task": "trivial-status-json-alias", + "run": 0, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 0.945, + "duration_s": 312.1, + "input_tokens": 42, + "cache_creation_input_tokens": 69816, + "cache_read_input_tokens": 705418, + "output_tokens": 6229 + }, + { + "task": "trivial-status-json-alias", + "run": 1, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 0.966, + "duration_s": 760.7, + "input_tokens": 78, + "cache_creation_input_tokens": 48367, + "cache_read_input_tokens": 985430, + "output_tokens": 6826 + }, + { + "task": "trivial-status-json-alias", + "run": 1, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 1.3409, + "duration_s": 406.0, + "input_tokens": 90, + "cache_creation_input_tokens": 62393, + "cache_read_input_tokens": 1485484, + "output_tokens": 8311 + }, + { + "task": "trivial-status-json-alias", + "run": 2, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 0.9274, + "duration_s": 462.1, + "input_tokens": 54, + "cache_creation_input_tokens": 59154, + "cache_read_input_tokens": 694309, + "output_tokens": 8410 + }, + { + "task": "trivial-status-json-alias", + "run": 2, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "90105c7840b1ff13a5160da2358c0b29877dc6e74b63bdadf5a5a9b08f3315c7", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 1.0308, + "duration_s": 265.0, + "input_tokens": 48, + "cache_creation_input_tokens": 73846, + "cache_read_input_tokens": 870782, + "output_tokens": 5347 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 0, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 2.8105, + "duration_s": 1108.7, + "input_tokens": 123, + "cache_creation_input_tokens": 108080, + "cache_read_input_tokens": 3227177, + "output_tokens": 20830 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 0, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 5.2765, + "duration_s": 736.7, + "input_tokens": 2880, + "cache_creation_input_tokens": 181957, + "cache_read_input_tokens": 6750719, + "output_tokens": 29981 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 1, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 3.8868, + "duration_s": 1211.0, + "input_tokens": 171, + "cache_creation_input_tokens": 112091, + "cache_read_input_tokens": 4902649, + "output_tokens": 29363 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 1, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 3.9381, + "duration_s": 588.2, + "input_tokens": 393, + "cache_creation_input_tokens": 153289, + "cache_read_input_tokens": 4674868, + "output_tokens": 25625 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 2, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 2.1343, + "duration_s": 368.1, + "input_tokens": 389, + "cache_creation_input_tokens": 92506, + "cache_read_input_tokens": 2215318, + "output_tokens": 17863 + }, + { + "task": "inv-feature-c-include-provenance", + "run": 2, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "0fdf26865f1ac41876b6af1fcad729124e293ef9b55c97c9b99fcabce4fd95ea", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 5.5732, + "duration_s": 717.5, + "input_tokens": 180, + "cache_creation_input_tokens": 168774, + "cache_read_input_tokens": 7475148, + "output_tokens": 31194 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 0, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 1.9724, + "duration_s": 455.4, + "input_tokens": 105, + "cache_creation_input_tokens": 67341, + "cache_read_input_tokens": 1818058, + "output_tokens": 25680 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 0, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 2.9823, + "duration_s": 524.1, + "input_tokens": 413, + "cache_creation_input_tokens": 120788, + "cache_read_input_tokens": 3197003, + "output_tokens": 25074 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 1, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 1.2148, + "duration_s": 265.8, + "input_tokens": 2533, + "cache_creation_input_tokens": 56933, + "cache_read_input_tokens": 931884, + "output_tokens": 15214 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 1, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 3.0637, + "duration_s": 406.6, + "input_tokens": 437, + "cache_creation_input_tokens": 125557, + "cache_read_input_tokens": 3662273, + "output_tokens": 17825 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 2, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 1.7031, + "duration_s": 380.2, + "input_tokens": 601, + "cache_creation_input_tokens": 70224, + "cache_read_input_tokens": 1627510, + "output_tokens": 17897 + }, + { + "task": "inv-feature-list-repos-filter", + "run": 2, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "4a9ef06e50560bee3bfccc4f6bf30ad3e8f72ae375eddf47a0bfef19df966d14", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 1.7256, + "duration_s": 364.3, + "input_tokens": 2425, + "cache_creation_input_tokens": 84220, + "cache_read_input_tokens": 1556455, + "output_tokens": 16354 + }, + { + "task": "cross-module-parse-retry", + "run": 0, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 8.1963, + "duration_s": 1556.2, + "input_tokens": 1806, + "cache_creation_input_tokens": 188015, + "cache_read_input_tokens": 12149859, + "output_tokens": 37491 + }, + { + "task": "cross-module-parse-retry", + "run": 0, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 8.8275, + "duration_s": 1775.9, + "input_tokens": 4943, + "cache_creation_input_tokens": 209697, + "cache_read_input_tokens": 13003539, + "output_tokens": 39618 + }, + { + "task": "cross-module-parse-retry", + "run": 1, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 8.9686, + "duration_s": 1408.0, + "input_tokens": 2635, + "cache_creation_input_tokens": 201014, + "cache_read_input_tokens": 13604688, + "output_tokens": 35870 + }, + { + "task": "cross-module-parse-retry", + "run": 1, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 8.9339, + "duration_s": 1359.5, + "input_tokens": 330, + "cache_creation_input_tokens": 216286, + "cache_read_input_tokens": 13145230, + "output_tokens": 40315 + }, + { + "task": "cross-module-parse-retry", + "run": 2, + "arm": "baseline_nomcp", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 5.4686, + "duration_s": 888.1, + "input_tokens": 535, + "cache_creation_input_tokens": 160231, + "cache_read_input_tokens": 7102745, + "output_tokens": 36526 + }, + { + "task": "cross-module-parse-retry", + "run": 2, + "arm": "baseline", + "model": "gpt-5.6-sol", + "effort": "xhigh", + "task_base_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "oracle_digest": "39396d4593cf6bfa6cf838d55a90bbb21f45b20ba969c33feb8d05f8cce92d02", + "sandbox_backend": "bwrap", + "sandbox_dependency_content_digest": "7a3e0c761591e79068655997f4f2d6d5b75dd1126781c6387f4aa1be5bea141e", + "ok": true, + "resolved": true, + "authored_tests_passed": true, + "oracle_passed": true, + "error_kind": null, + "transcript_missing": false, + "cost_usd": 13.2384, + "duration_s": 1295.2, + "input_tokens": 2897, + "cache_creation_input_tokens": 222817, + "cache_read_input_tokens": 20977371, + "output_tokens": 53704 + } + ], + "arms": { + "baseline_nomcp": { + "cells": 12, + "solved": 12, + "mean_cost_usd": 3.2523500000000003, + "mean_wall_s": 756.1750000000001 + }, + "baseline": { + "cells": 12, + "solved": 12, + "mean_cost_usd": 4.739658333333334, + "mean_wall_s": 729.2583333333333 + } + }, + "paired": { + "solve_delta": 0, + "mean_cost_change_pct": 45.73026683270046, + "mean_wall_change_pct": -3.5595816665013706 + } + } + } +} diff --git a/eval/workflow_bench/measurement-2026-10-24.md b/eval/workflow_bench/measurement-2026-10-24.md new file mode 100644 index 000000000..aa59a52fe --- /dev/null +++ b/eval/workflow_bench/measurement-2026-10-24.md @@ -0,0 +1,82 @@ +# Agent evaluation evidence, 24 October 2026 + +The latest completed comparison does **not** demonstrate an accuracy gain or +cost saving. Its MCP arm could read compiled code containing later task fixes, +so it also cannot isolate the effect of tool assistance. + +## Historical measurements + +[Release evaluation run 40015796596](https://github.com/abhigyanpatwari/GitNexus/actions/runs/40015796596) +started on 23 October and completed on 24 October. The sanitized artifact is +`release-agent-evaluation-62a6f67083fcd1f85634a49b118b4faf3fd5198d` +(artifact ID `12541246950`). Both JSON reports contain all 24 expected cells; +their totals were independently recomputed from all 48 per-run records. +The [sanitized source reports](measurement-2026-10-24.json) are retained here +so these observations remain reproducible after the workflow artifact expires. + +- Candidate: `62a6f67083fcd1f85634a49b118b4faf3fd5198d`. +- Stable runtime and task base: `c4ecf398de637cf779ab2238e69958cfd5d601ce`. +- Harness: `466d56280a37cb3dae9cdce8788937964824a3d5`. +- Model: `gpt-5.6-sol`, effort `xhigh`. +- Task-set digest: `51daf0758e8bafc30f0a9c94b254ee2b1592be1d569f2d64a7b3031ff7488c16`. +- Four tasks, three repetitions per arm. Both arms use the same task pins and + hidden oracles; the no-GitNexus arm receives no supplied graph or tools. + +| Runtime | Arm | Solved | Mean reported cost | Mean agent time | +| --- | --- | --- | --- | --- | +| Candidate | No GitNexus | 12/12 | $3.0913 | 617.0s | +| Candidate | MCP, readable runtime | 12/12 | $3.7592 | 585.4s | +| Stable | No GitNexus | 12/12 | $3.2524 | 756.2s | +| Stable | MCP, readable runtime | 12/12 | $4.7397 | 729.3s | + +Candidate MCP cost increased **21.6%** and agent time decreased **5.1%**. +Stable MCP cost increased **45.7%** and agent time decreased **3.6%**. +Accuracy was unchanged in both comparisons. Cost comes from session reports, +not invoices; indexing, grading, and infrastructure are excluded. Three +repetitions on four development tasks do not establish significance or a +general product benefit. The release gate checks quality against stable; a +passing gate is not evidence of savings against no GitNexus. + +## Deterministic tool accuracy + +At source revision `d8a256a03a101a767dc4e1118913c679af740336`, the existing +tool-accuracy corpus passes **32/32**, with no allowances or execution failures. +This includes the Python receiver, lexical scope, and summary lower-bound +checks discussed in issues #3497–#3499. It verifies fixed tool answers, not +coding-agent success or cost. + +Reproduce from `gitnexus/`: + +```sh +node --import tsx bench/tool-accuracy/run.ts --check --out /tmp/tool-accuracy +``` + +The local run needed a compatible C++ runtime to load the native parser. Its +first attempt could not load `GLIBCXX_3.4.31`; that attempt was an execution +failure, not a measured tool result. + +## Corrected comparison and remaining work + +The harness correction moves the release baseline's MCP runtime into a +sibling read-only sandbox. The coding agent receives only a stdio socket. +Host-generated rows record `runtime_exposure`, and release/PR validation +rejects missing or incompatible receipts. Workflow arms that intentionally +expose the CLI retain their different boundary. + +The protected Release evaluation workflow executes main's trusted harness, +including PR previews. A fresh isolated comparison therefore requires this +correction to merge first. After merge, dispatch `release-evaluation.yml` on +`main` with `candidate_ref` set to the full merged commit SHA, model +`gpt-5.6-sol`, effort `xhigh`, and `runner_only=false`. The existing workflow +runs all four tasks three times per arm for candidate and stable, publishes +sanitized reports, and stops its runner. + +Verify all 48 unique cells, the exact runtime/harness/task bindings, both arm +receipts, unsuccessful outcomes, and cost/time totals. Publish the result even +if it shows no gain. Historical artifacts cannot substitute for this run. + +Keep [issue #3500](https://github.com/abhigyanpatwari/GitNexus/issues/3500) +open until the clean measurement is published. A broader SWE-bench or reporter +corpus, including separate fresh/stale-index conditions, remains necessary +before making claims beyond this small task set. The reporter paused their +rerun offer in [discussion #3493](https://github.com/abhigyanpatwari/GitNexus/discussions/3493#discussioncomment-18772530). diff --git a/eval/workflow_bench/pr_evaluation.py b/eval/workflow_bench/pr_evaluation.py index c09405ba2..401aa482c 100644 --- a/eval/workflow_bench/pr_evaluation.py +++ b/eval/workflow_bench/pr_evaluation.py @@ -321,6 +321,7 @@ def compare_candidate( for field in ("model", "effort", "task_base_sha", "oracle_digest", "sandbox_dependency_content_digest") ) and row.get("sandbox_backend") == "bwrap" + and row.get("runtime_exposure") == "stdio-only" and row.get("ok") is True and all(type(row.get(field)) is bool for field in ("resolved", "authored_tests_passed", "oracle_passed")) and row["resolved"] == (row["authored_tests_passed"] and row["oracle_passed"]) diff --git a/eval/workflow_bench/proposer_sandbox.py b/eval/workflow_bench/proposer_sandbox.py index c17afd19a..dd2608df0 100644 --- a/eval/workflow_bench/proposer_sandbox.py +++ b/eval/workflow_bench/proposer_sandbox.py @@ -38,6 +38,7 @@ SANDBOX_CLAUDE = "/opt/claude/claude" SANDBOX_SHELL_PREFIX = "/opt/claude/shell-prefix" SANDBOX_PYTHON3 = "/opt/claude/python3" SANDBOX_GITNEXUS_CLI = "/opt/claude/gitnexus" +SANDBOX_MCP_SOCKET = "/run/gitnexus-mcp.sock" SANDBOX_GIT_EXCLUDES = "/opt/claude/git-excludes" SANDBOX_NODE = "/opt/claude/node" SANDBOX_BWRAP = "/opt/claude/bwrap" @@ -1219,6 +1220,7 @@ def prepare_sandbox( preflight: bool = True, backend: str = "bwrap", gitnexus_available: bool = True, + gitnexus_cli_available: bool = True, ) -> Iterator[SandboxSession]: """Create private host backing dirs and one virtualized command.""" @@ -1235,6 +1237,7 @@ def prepare_sandbox( SANDBOX_GITNEXUS_SHARED, SANDBOX_GITNEXUS_REGISTRY, SANDBOX_GITNEXUS_CLI, + SANDBOX_MCP_SOCKET, f"{SANDBOX_WORKSPACE}/.gitnexus", ) for mount in read_only_mounts: @@ -1260,7 +1263,8 @@ def prepare_sandbox( for directory in (home, temp): directory.mkdir(mode=0o700) directory.chmod(0o700) - shell_prefix = _create_shell_prefix_wrapper(private_root, gitnexus_available=gitnexus_available) + cli_available = gitnexus_available and gitnexus_cli_available + shell_prefix = _create_shell_prefix_wrapper(private_root, gitnexus_available=cli_available) python3_wrapper = _create_python3_wrapper(private_root) git_excludes = _create_git_excludes(private_root) # Claude may discover user-level skills below HOME. Keep the rest of HOME @@ -1280,7 +1284,7 @@ def prepare_sandbox( *read_only_mounts, *( (ReadOnlyMount(source=bwrap, target=SANDBOX_BWRAP),) - if backend == "bwrap" and gitnexus_available + if backend == "bwrap" and cli_available else () ), ReadOnlyMount(source=user_skills, target=SANDBOX_USER_SKILLS), @@ -1288,7 +1292,7 @@ def prepare_sandbox( ReadOnlyMount(source=python3_wrapper, target=SANDBOX_PYTHON3), *( (ReadOnlyMount(source=_create_gitnexus_wrapper(private_root), target=SANDBOX_GITNEXUS_CLI),) - if gitnexus_available + if cli_available else baseline_mounts ), ReadOnlyMount(source=git_excludes, target=SANDBOX_GIT_EXCLUDES), diff --git a/eval/workflow_bench/release_report.py b/eval/workflow_bench/release_report.py index 030d04c34..3ceeb089a 100644 --- a/eval/workflow_bench/release_report.py +++ b/eval/workflow_bench/release_report.py @@ -32,6 +32,7 @@ ROW_FIELDS = ( "task_base_sha", "oracle_digest", "sandbox_backend", + "runtime_exposure", DEPENDENCY_CONTENT_BINDING_FIELD, "ok", "resolved", @@ -126,6 +127,7 @@ def build_report(rows: list[Any], metadata: dict[str, Any], *, now: datetime | N and row.get("model") == report["model"] and row.get("effort") == report["effort"] and row.get("sandbox_backend") == "bwrap" + and row.get("runtime_exposure") == ("none" if arm == "baseline_nomcp" else "stdio-only") and row.get("ok") is True and type(row.get("resolved")) is bool and type(row.get("authored_tests_passed")) is bool @@ -192,7 +194,14 @@ def render_markdown(report: dict[str, Any]) -> str: lines += [ "", "Negative agent outcomes remain in the denominator. Indexing and grading are excluded from agent cost/time.", - "The MCP arm also has readable compiled runtime code; this comparison includes that access and cannot isolate tool assistance from access to later fixes.", + ( + "The MCP arm receives a stdio connection; the candidate runtime and registry are absent from the agent filesystem." + if report["complete"] and report.get("per_run") and all( + row.get("runtime_exposure") == ("none" if row.get("arm") == "baseline_nomcp" else "stdio-only") + for row in report["per_run"] + ) + else "Runtime isolation is unproven. Historical MCP cells had readable compiled runtime code and cannot isolate tool assistance from access to later fixes." + ), "This small pinned task set is release evidence; it does not establish general gains or statistical significance.", "", ] diff --git a/eval/workflow_bench/runner.py b/eval/workflow_bench/runner.py index 90994de34..011d63938 100644 --- a/eval/workflow_bench/runner.py +++ b/eval/workflow_bench/runner.py @@ -36,6 +36,7 @@ from __future__ import annotations import argparse from concurrent.futures import ThreadPoolExecutor, wait +from contextlib import ExitStack import hashlib import json import os @@ -57,6 +58,8 @@ from typing import Any import yaml +from .mcp_transport import connector_config, isolated_mcp + from .comparator_reuse import ( REUSE_EXCLUDED_ERROR_KINDS, CellKey, @@ -458,6 +461,7 @@ def run_arm( enforce_phase_boundary: bool = False, ce_plugin_dir: str | None = None, oracle_snapshot: TaskOracleSnapshot | None = None, + mcp_config_json: str | None = None, ) -> dict[str, Any]: sessions: list[dict[str, Any]] = [] environment_builder = getattr(sandbox, "environment", build_sandbox_environment) @@ -491,7 +495,7 @@ def run_arm( "require_pid_namespace": getattr(sandbox, "require_pid_namespace", True), "settings_json": sandbox.settings_json, "strict_mcp_config": True, - "mcp_config_json": host_text(sandbox_mcp_config(unsafe_host=backend == "host-unsafe")), + "mcp_config_json": mcp_config_json or host_text(sandbox_mcp_config(unsafe_host=backend == "host-unsafe")), "transcript_projects": sandbox.transcript_projects, "transcript_cwd": Path(SANDBOX_WORKSPACE), "transcript_wait_seconds": 5, @@ -1189,6 +1193,9 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: cleanup_error: OSError | None = None execution_arm = CANDIDATE_ARMS.get(arm, arm) gitnexus_available = execution_arm != "baseline_nomcp" + isolated_baseline = execution_arm == "baseline" and ctx.sandbox_backend == "bwrap" + runtime_exposure = "none" if not gitnexus_available else "readable" + resources = ExitStack() try: if ctx.asset_snapshot_error is not None: raise RuntimeError(f"task asset snapshot preparation failed: {ctx.asset_snapshot_error}") @@ -1222,6 +1229,12 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: else [] ) ce_mounts = ce_plugin_mounts_for_arm(execution_arm, ctx.ce_plugin_snapshot) + if isolated_baseline: + endpoint = resources.enter_context(isolated_mcp( + clone=worktree, mounts=[*dependency_mounts, *gitnexus_mounts], bwrap_bin=ctx.bwrap_bin, + )) + gitnexus_mounts = [endpoint] + runtime_exposure = "stdio-only" with prepare_sandbox( clone=worktree, claude_bin=args.claude_bin, @@ -1234,6 +1247,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: preflight=False, backend=ctx.sandbox_backend, gitnexus_available=gitnexus_available, + gitnexus_cli_available=not isolated_baseline, ) as sandbox: # Capture the BASE (pre-overlay) skill digest — identical # for the incumbent and candidate arms — then run the @@ -1314,6 +1328,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: enforce_phase_boundary=True, ce_plugin_dir=ce_plugin_dir_for_arm(execution_arm, ctx.ce_plugin_snapshot), oracle_snapshot=ctx.oracle_snapshot, + mcp_config_json=connector_config() if isolated_baseline else None, ) if execution_arm in ("review", "ce_review"): review_source = review_output_path(sandbox, REVIEW_OUTPUT) @@ -1345,6 +1360,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: after_digest=after_work_digest, ) patch_bytes = capture_patch(sandbox, worktree, orig_sha) + resources.close() record["arm"] = arm record.update( { @@ -1387,6 +1403,11 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: detail = redact_text(str(exc), credential_secrets(args)) print(f"[{task['id']}][{arm}][run {run_idx}] infra-error: {detail}") finally: + try: + resources.close() + except (SandboxError, OSError, RuntimeError) as exc: + record = infra_error_record(exc) + record["arm"] = arm if worktree is not None and worktree.exists(): try: remove_clone(worktree) @@ -1408,6 +1429,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: "class": task.get("class", ""), "run": run_idx, "sandbox_backend": ctx.sandbox_backend, + "runtime_exposure": runtime_exposure, "task_asset_snapshot_digest": (ctx.asset_snapshot.digest if ctx.asset_snapshot is not None else None), "task_asset_manifest_digest": ( ctx.asset_snapshot.manifest_digest if ctx.asset_snapshot is not None else None