diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index d4a98bbce..3fcd451a3 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -1532,6 +1532,7 @@ jobs: uv run --locked --extra dev python -m pytest tests/test_process_control.py tests/test_proposer_sandbox.py + tests/test_cli_shim.py tests/test_workflow_bench_sessions.py tests/test_ce_plugin_runtime.py tests/test_offline_sweep_integration.py diff --git a/.github/workflows/release-evaluation.yml b/.github/workflows/release-evaluation.yml index d4376ef15..94f1e01aa 100644 --- a/.github/workflows/release-evaluation.yml +++ b/.github/workflows/release-evaluation.yml @@ -30,6 +30,10 @@ on: type: choice default: xhigh options: [low, medium, high, xhigh, max] + cli_arm: + description: 'Also run baseline_cli (same tools via a gitnexus shell command, no MCP catalog) on the candidate; adds 12 paid cells, descriptive only, never gates' + type: boolean + default: false # PR preview inputs are available only to the maintainer-authorized label # workflow. Manual dispatch cannot select an unmerged PR runtime. workflow_call: @@ -207,6 +211,8 @@ jobs: COMPARISON_RUN_ID: ${{ inputs.comparison_run_id }} MODEL: ${{ inputs.model || 'gpt-5.6-sol' }} EFFORT: ${{ inputs.effort || 'xhigh' }} + # Manual dispatch only; schedules and PR previews never pay for the extra arm. + CLI_ARM: ${{ inputs.cli_arm == true && inputs.candidate_pr == '' && 'true' || 'false' }} steps: - name: Require startup in this run attempt run: | @@ -360,7 +366,7 @@ jobs: GITNEXUS_REQUIRE_CLAUDE_CANARY: '1' CLAUDE_CANARY_BIN: ${{ runner.temp }}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude run: >- - uv run --locked --extra dev python -m pytest tests/test_proposer_sandbox.py + uv run --locked --extra dev python -m pytest tests/test_proposer_sandbox.py tests/test_cli_shim.py tests/test_workflow_bench_sessions.py::test_real_cli_background_completion_is_one_complete_measurement -q - name: Materialize both release graphs before paid sessions timeout-minutes: 120 @@ -405,6 +411,28 @@ jobs: --claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \ --out "$RUNNER_TEMP/release-evaluation/$runtime/raw" done + # Descriptive only: its own output directory leaves the paired evidence + # that release_report and release_gate read exactly as without the opt-in. + - name: Run the opt-in CLI-only arm on the candidate + if: env.CLI_ARM == 'true' + continue-on-error: true + timeout-minutes: 600 + working-directory: eval + env: + GITNEXUS_BENCH_OPENAI_API_KEY: ${{ secrets.GITNEXUS_BENCH_OPENAI_API_KEY }} + GITNEXUS_BENCH_ANTHROPIC_API_KEY: ${{ secrets.GITNEXUS_BENCH_ANTHROPIC_API_KEY }} + run: | + set -euo pipefail + remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 )) + (( remaining > 0 )) || { echo '::warning::No time remains for the CLI-only arm.'; exit 1; } + timeout --signal=TERM --kill-after=60 "${remaining}s" \ + uv run --locked --extra dev python -m workflow_bench.runner \ + --tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \ + --gitnexus-root "$GITHUB_WORKSPACE/candidate" \ + --arms baseline_cli --runs 3 --include-expensive \ + --workers 3 --timeout 3600 --model "$MODEL" --effort "$EFFORT" \ + --claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \ + --out "$RUNNER_TEMP/release-evaluation/candidate/cli-raw" - name: Summarize complete or incomplete measurements if: ${{ always() && steps.resolve.outputs.sha != '' }} working-directory: eval @@ -460,6 +488,17 @@ jobs: for report in release-quality-gate agent-evaluation; do if [[ -f "$public/$report.md" ]]; then cat "$public/$report.md" >> "$GITHUB_STEP_SUMMARY"; fi done + - name: Summarize the opt-in CLI-only arm + if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' }} + continue-on-error: true + working-directory: eval + run: | + set -euo pipefail + candidate="$RUNNER_TEMP/release-evaluation/candidate" + uv run --locked --extra dev python -m workflow_bench.cli_arm_report \ + --paired "$candidate/raw/results.jsonl" --cli "$candidate/cli-raw/results.jsonl" \ + --out "$RUNNER_TEMP/release-evaluation/public" + cat "$RUNNER_TEMP/release-evaluation/public/cli-arm-comparison.md" >> "$GITHUB_STEP_SUMMARY" - name: Upload sanitized release evidence if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -474,6 +513,16 @@ jobs: ${{ runner.temp }}/release-evaluation/public/release-quality-gate.md if-no-files-found: error retention-days: 30 + - name: Upload the opt-in CLI-only arm comparison + if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: release-cli-arm-${{ steps.resolve.outputs.sha }} + path: | + ${{ runner.temp }}/release-evaluation/public/cli-arm-comparison.json + ${{ runner.temp }}/release-evaluation/public/cli-arm-comparison.md + if-no-files-found: warn + retention-days: 30 - name: Upload sanitized PR cost comparison if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr != '' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/eval/tests/test_cli_shim.py b/eval/tests/test_cli_shim.py new file mode 100644 index 000000000..8686d65b2 --- /dev/null +++ b/eval/tests/test_cli_shim.py @@ -0,0 +1,330 @@ +"""The baseline_cli arm's `gitnexus` shim: argument mapping, MCP round trip, containment.""" + +from __future__ import annotations + +import json +import os +import socket +import subprocess +import sys +import threading +from pathlib import Path + +import pytest + +from workflow_bench import proposer_sandbox +from workflow_bench.cli_shim import CLI_TOOLS, GITNEXUS_CLI_NOTE, shim_source +from workflow_bench.proposer_sandbox import ( + SANDBOX_GITNEXUS_CLI, + SANDBOX_MCP_SOCKET, + ReadOnlyMount, + build_sandbox_environment, + preflight_bubblewrap, + prepare_sandbox, +) +from workflow_bench.runner_sessions import GITNEXUS_MUTATING_TOOLS, GITNEXUS_READ_ONLY_TOOLS + +IMPACT = { + "name": "impact", + "description": "Analyze the blast radius of changing a code symbol. Long details follow.", + "inputSchema": { + "type": "object", + "properties": { + "target": {"type": "string", "description": "Symbol name."}, + "direction": {"type": "string", "description": "upstream or downstream."}, + "maxDepth": {"type": "integer", "description": "Traversal depth."}, + "includeTests": {"type": "boolean", "description": "Include tests."}, + "relationTypes": {"type": "array", "items": {"type": "string"}}, + "minConfidence": {"anyOf": [{"type": "number"}, {"type": "null"}]}, + }, + "required": ["target"], + }, +} +# The real impact contract (gitnexus/src/mcp/tools.ts): direction required, target optional beside target_uid. +REAL_IMPACT = {**IMPACT, "inputSchema": {**IMPACT["inputSchema"], "required": ["direction"]}} +RENAME = {"name": "rename", "description": "Mutating.", "inputSchema": {"type": "object", "properties": {}}} + + +def _shim_namespace() -> dict: + namespace: dict = {"__name__": "gitnexus_shim"} + exec(compile(shim_source(), "gitnexus", "exec"), namespace) + return namespace + + +def test_shim_binds_the_cell_socket_and_only_read_only_tools(): + source = shim_source() + assert json.dumps(SANDBOX_MCP_SOCKET) in source + assert list(CLI_TOOLS) == [tool.removeprefix("mcp__gitnexus__") for tool in GITNEXUS_READ_ONLY_TOOLS] + assert not {tool.removeprefix("mcp__gitnexus__") for tool in GITNEXUS_MUTATING_TOOLS} & set(CLI_TOOLS) + # The note is the arm's only fixed prompt cost; keep it far below a tool catalog. + assert len(GITNEXUS_CLI_NOTE.encode()) < 1024 + + +def test_shim_maps_positionals_and_flags_to_the_tool_schema(): + parse = _shim_namespace()["parse"] + assert parse(IMPACT, ["foo", "--direction", "upstream", "--max-depth", "2", "--include-tests"]) == { + "target": "foo", + "direction": "upstream", + "maxDepth": 2, + "includeTests": True, + } + assert parse(IMPACT, ["--target=bar", "--relation-types", "CALLS,IMPORTS", "--min-confidence", "0.5"]) == { + "target": "bar", + "relationTypes": ["CALLS", "IMPORTS"], + "minConfidence": 0.5, + } + assert parse(IMPACT, ["baz", "--json", '{"maxDepth": 1}']) == {"target": "baz", "maxDepth": 1} + # An explicit empty value is forwarded, not mistaken for a missing one. + assert parse(IMPACT, ["--direction=", "qux"]) == {"direction": "", "target": "qux"} + # The real impact schema requires only direction; the symbol is still positional. + assert parse(REAL_IMPACT, ["statusCommand", "--direction", "upstream"]) == { + "target": "statusCommand", "direction": "upstream"} + assert parse(REAL_IMPACT, ["statusCommand", "upstream"]) == {"target": "statusCommand", "direction": "upstream"} + # A boolean switch before the positional leaves the positional alone. + assert parse(IMPACT, ["--include-tests", "foo"]) == {"includeTests": True, "target": "foo"} + assert parse(IMPACT, ["--include-tests", "false", "foo"]) == {"includeTests": False, "target": "foo"} + + +def test_shim_rejects_unknown_options_and_extra_values(): + parse = _shim_namespace()["parse"] + with pytest.raises(SystemExit, match="unknown option --bogus"): + parse(IMPACT, ["foo", "--bogus", "1"]) + with pytest.raises(SystemExit, match="too many values"): + parse(IMPACT, ["foo", "bar"]) + + +def test_shim_help_is_on_demand_and_short_per_tool(): + describe = _shim_namespace()["describe"] + text = describe(IMPACT) + assert text.startswith("gitnexus impact") + assert "--maxDepth " in text and "--target (required)" in text + assert "Positional values fill: target" in text + + +def test_sandbox_mounts_the_shim_instead_of_the_runtime_cli(tmp_path): + clone = tmp_path / "clone" + clone.mkdir() + with prepare_sandbox( + clone=clone, claude_bin=sys.executable, bwrap_bin=sys.executable, + preflight=False, gitnexus_cli_available=False, gitnexus_cli_shim=True, + ) as sandbox: + mounts = {mount.target: mount.source for mount in sandbox.read_only_mounts} + shim = Path(mounts[SANDBOX_GITNEXUS_CLI]) + assert shim.read_text() == shim_source() + assert proposer_sandbox.SANDBOX_GITNEXUS + "/dist" not in shim.read_text() + # Claude's Bash sandbox blocks AF_UNIX unless told otherwise; the shim needs it. + assert json.loads(sandbox.settings_json)["sandbox"]["network"]["allowAllUnixSockets"] is True + with prepare_sandbox( + clone=clone, claude_bin=sys.executable, bwrap_bin=sys.executable, + preflight=False, gitnexus_cli_available=False, + ) as sandbox: + assert json.loads(sandbox.settings_json)["sandbox"]["network"]["allowAllUnixSockets"] is False + + +class _FakeMcpServer: + """Serve one MCP session per connection, like mcp_transport.serve_stdio.""" + + def __init__(self, path: Path, tools: list[dict]) -> None: + self.path, self.tools, self.calls = path, tools, [] + self.listener = socket.socket(socket.AF_UNIX) + self.listener.bind(str(path)) + self.listener.listen(1) + self.thread = threading.Thread(target=self._serve, daemon=True) + self.thread.start() + + def _serve(self) -> None: + while True: + try: + connection, _ = self.listener.accept() + except OSError: + return + with connection, connection.makefile("rwb") as stream: + for line in stream: + message = json.loads(line) + if "id" not in message: + continue + method = message["method"] + if method == "initialize": + result = {"protocolVersion": "2025-06-18", "capabilities": {}, "serverInfo": {"name": "fake"}} + elif method == "tools/list": + result = {"tools": self.tools} + else: + self.calls.append(message["params"]) + arguments = message["params"]["arguments"] + result = {"content": [{"type": "text", "text": json.dumps(arguments, sort_keys=True)}], + "isError": arguments.get("target") == "broken"} + stream.write(json.dumps({"jsonrpc": "2.0", "id": message["id"], "result": result}).encode() + b"\n") + stream.flush() + + def close(self) -> None: + self.listener.close() + + +@pytest.mark.skipif(not hasattr(socket, "AF_UNIX"), reason="the shim speaks MCP over a Unix socket") +def test_shim_round_trips_through_a_fake_mcp_endpoint(tmp_path): + endpoint = tmp_path / "mcp.sock" + server = _FakeMcpServer(endpoint, [IMPACT, RENAME]) + shim = tmp_path / "gitnexus" + shim.write_text(shim_source().replace(json.dumps(SANDBOX_MCP_SOCKET), json.dumps(str(endpoint)))) + + def run(*argv: str) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, str(shim), *argv], capture_output=True, text=True, timeout=30) + + try: + listing = run("help") + assert listing.returncode == 0, listing.stderr + assert "impact" in listing.stdout and "Analyze the blast radius" in listing.stdout + assert "rename" not in listing.stdout and "Long details" not in listing.stdout + + call = run("impact", "foo", "--direction", "upstream") + assert call.returncode == 0, call.stderr + assert json.loads(call.stdout) == {"direction": "upstream", "target": "foo"} + + failed = run("impact", "broken") + assert failed.returncode == 1 and "broken" in failed.stderr + + assert run("rename", "x").returncode != 0 + assert len(server.calls) == 2 + finally: + server.close() + + +def _report_row(arm: str, run: int, cost: float | None, task: str = "t", **extra) -> dict: + exposure = "none" if arm == "baseline_nomcp" else "stdio-only" + return {"task": task, "run": run, "arm": arm, "ok": True, "sandbox_backend": "bwrap", + "runtime_exposure": exposure, "error_kind": None, "resolved": True, + "transcript_missing": False, "cost_usd": cost, "duration_s": 10.0, "num_turns": 4, + "cache_read_input_tokens": 1000, "output_tokens": 50, **extra} + + +def _write_results(tmp_path: Path, paired_rows: list[dict], cli_rows: list[dict]) -> tuple[Path, Path]: + paired, cli = tmp_path / "raw/results.jsonl", tmp_path / "cli-raw/results.jsonl" + for path, rows in ((paired, paired_rows), (cli, cli_rows)): + path.parent.mkdir(exist_ok=True) + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + return paired, cli + + +def test_cli_arm_report_compares_arms_and_counts_shell_calls(tmp_path): + from workflow_bench.cli_arm_report import build_report, render_markdown + + paired, cli = _write_results( + tmp_path, + [_report_row("baseline_nomcp", 0, 1.0), _report_row("baseline", 0, 1.3, gitnexus_tool_uses=3)], + # The record shape runner_sessions._persist_parent_event_stream writes into result rows. + [_report_row("baseline_cli", 0, 1.1, transcript_artifacts=[ + {"path": "transcripts/t-cli.jsonl", "sha256": "0" * 64, "bytes": 1, + "source": "parent-captured-stream-json"}, + {"path": "../outside.jsonl", "sha256": "0" * 64, "bytes": 1, "source": "parent-captured-stream-json"}])], + ) + (tmp_path / "outside.jsonl").write_text(json.dumps( + {"type": "assistant", "message": {"content": [ + {"type": "tool_use", "name": "Bash", "input": {"command": "gitnexus query x"}}]}}) + "\n") + (cli.parent / "transcripts").mkdir() + event = {"type": "assistant", "message": {"content": [ + {"type": "tool_use", "name": "Bash", "input": {"command": "gitnexus impact foo && gitnexus help"}}, + {"type": "tool_use", "name": "Bash", "input": {"command": "npm --prefix gitnexus test -- --run x"}}]}} + (cli.parent / "transcripts/t-cli.jsonl").write_text(json.dumps(event) + "\n") + + report = build_report(paired, cli) + assert not report["problems"] + assert report["arms"]["baseline_cli"]["gitnexus_commands"] == {"impact": 1, "help": 1} + assert round(report["changes"]["baseline_cli vs baseline_nomcp: cost_usd"]["percent"], 1) == 10.0 + assert round(report["changes"]["baseline_cli vs baseline: cost_usd"]["percent"], 1) == -15.4 + markdown = render_markdown(report) + assert "| baseline_cli | 1/1 | $1.1000" in markdown + assert "baseline_cli vs baseline: cost_usd: -15.4% (1/1 pairs)" in markdown + + +def _two_task_paired() -> list[dict]: + return [_report_row(arm, 0, cost, task=task) + for arm in ("baseline_nomcp", "baseline") for task, cost in (("cheap", 1.0), ("dear", 9.0))] + + +def test_cli_arm_report_compares_an_interrupted_sweep_on_matched_pairs_only(tmp_path): + from workflow_bench.cli_arm_report import build_report, render_markdown + + # The CLI sweep stopped after the $1 task; per-arm means alone would read as an 80% saving. + paired, cli = _write_results(tmp_path, _two_task_paired(), [_report_row("baseline_cli", 0, 1.0, task="cheap")]) + + report = build_report(paired, cli) + change = report["changes"]["baseline_cli vs baseline: cost_usd"] + assert change == {"percent": 0.0, "pairs": 1, "expected_pairs": 2} + assert "baseline_cli vs baseline: 1 of 2 task/run pair(s) matched; deltas use matched pairs only" in report["problems"] + assert "baseline_cli vs baseline: cost_usd: +0.0% (1/2 pairs, partial)" in render_markdown(report) + + +def test_cli_arm_report_treats_missing_billing_as_incomplete(tmp_path): + from workflow_bench.cli_arm_report import build_report, render_markdown + + # run_claude can return a successful session with cost_usd=None. + paired, cli = _write_results(tmp_path, _two_task_paired(), [ + _report_row("baseline_cli", 0, None, task="cheap"), _report_row("baseline_cli", 0, 9.0, task="dear")]) + + report = build_report(paired, cli) + arm = report["arms"]["baseline_cli"] + assert arm["mean_cost_usd"] is None and arm["incomplete_metrics"] == ["cost_usd"] + assert "baseline_cli: cost_usd missing on 1 of 2 cell(s)" in report["problems"] + assert report["changes"]["baseline_cli vs baseline: cost_usd"] == {"percent": 0.0, "pairs": 1, "expected_pairs": 2} + assert report["changes"]["baseline_cli vs baseline: num_turns"]["pairs"] == 2 + markdown = render_markdown(report) + assert "| baseline_cli | 2/2 | incomplete |" in markdown + assert "cost_usd: +0.0% (1/2 pairs, partial)" in markdown + + +def test_cli_arm_report_excludes_duplicate_pairs_from_deltas(tmp_path): + from workflow_bench.cli_arm_report import build_report + + paired, cli = _write_results(tmp_path, _two_task_paired(), [ + _report_row("baseline_cli", 0, 1.0, task="cheap"), + _report_row("baseline_cli", 0, 5.0, task="cheap"), + _report_row("baseline_cli", 0, 9.0, task="dear")]) + + report = build_report(paired, cli) + assert any("duplicate task/run pair(s)" in problem and "cheap" in problem for problem in report["problems"]) + assert report["changes"]["baseline_cli vs baseline: cost_usd"] == {"percent": 0.0, "pairs": 1, "expected_pairs": 2} + + +@pytest.mark.skipif( + os.environ.get("GITNEXUS_REQUIRE_BWRAP_CANARY") != "1", + reason="real Bubblewrap canary is mandatory in the named Ubuntu CI job", +) +def test_real_cli_shim_reaches_isolated_mcp_without_exposing_the_runtime(tmp_path): + from workflow_bench.mcp_transport import isolated_mcp + + clone = tmp_path / "clone" + clone.mkdir() + runtime = tmp_path / "runtime" + entrypoint = runtime / "dist/cli/index.js" + entrypoint.parent.mkdir(parents=True) + entrypoint.write_text(""" +const readline = require('readline'); +const tool = TOOL; +readline.createInterface({input: process.stdin}).on('line', line => { + const message = JSON.parse(line); + if (message.id === undefined) return; + const args = (message.params || {}).arguments || {}; + const missing = (tool.inputSchema.required || []).filter(name => !(name in args)); + const result = message.method === 'initialize' ? {protocolVersion: '2025-06-18', capabilities: {}} + : message.method === 'tools/list' ? {tools: [tool]} + : missing.length ? {isError: true, content: [{type: 'text', text: 'missing ' + missing.join(',')}]} + : {content: [{type: 'text', text: 'impact of ' + args.target + ' ' + args.direction}]}; + process.stdout.write(JSON.stringify({jsonrpc: '2.0', id: message.id, result}) + '\\n'); +}); +""".replace("TOOL", json.dumps(REAL_IMPACT))) + bwrap = preflight_bubblewrap() + mounts = [ReadOnlyMount(runtime, proposer_sandbox.SANDBOX_GITNEXUS)] + with isolated_mcp(clone=clone, mounts=mounts, bwrap_bin=bwrap) as endpoint: + with prepare_sandbox( + clone=clone, claude_bin=sys.executable, bwrap_bin=bwrap, read_only_mounts=[endpoint], + preflight=False, gitnexus_cli_available=False, gitnexus_cli_shim=True, + ) as sandbox: + script = ( + "test ! -e /opt/gitnexus && test ! -e /opt/gitnexus-registry && " + "gitnexus help | grep -q impact && gitnexus impact parseFile --direction upstream && " + "gitnexus impact second downstream && ! gitnexus impact missingDirection" + ) + result = sandbox.run(["/bin/bash", "-c", script], timeout=60, env=build_sandbox_environment()) + assert result.ok, result.stderr_tail + assert "impact of parseFile upstream" in result.stdout_tail + assert "impact of second downstream" in result.stdout_tail diff --git a/eval/tests/test_pr_evaluation.py b/eval/tests/test_pr_evaluation.py index 5f3af3b3c..0a4c3da15 100644 --- a/eval/tests/test_pr_evaluation.py +++ b/eval/tests/test_pr_evaluation.py @@ -448,6 +448,39 @@ def test_workflow_protocol_rejects_extra_paid_invocations(workflow_protocol_pair pr_evaluation.measurement_protocol(yaml.safe_dump(new)) +def test_workflow_protocol_ignores_opt_in_arms_after_the_paid_step(workflow_protocol_pair): + _, new = workflow_protocol_pair + expected = pr_evaluation.measurement_protocol(yaml.safe_dump(new)) + for gate in sorted(pr_evaluation.OPT_IN_ARM_CONDITIONS): + new["jobs"]["evaluate"]["steps"].append({"if": gate, "run": "uv run python -m workflow_bench.runner --arms x\n"}) + assert pr_evaluation.measurement_protocol(yaml.safe_dump(new)) == expected + + +@pytest.mark.parametrize( + "position, step, message", + [ + (0, {"if": "env.CLI_ARM == 'true'", "run": "python -m workflow_bench.runner"}, "must follow"), + (None, {"if": "env.CLI_ARM == 'true'", "run": "python -m workflow_bench.runner\npython -m workflow_bench.runner"}, + "must follow"), + (None, {"if": "env.OTHER == 'true'", "run": "python -m workflow_bench.runner"}, "one paid"), + ], +) +def test_workflow_protocol_rejects_misplaced_or_unlisted_extra_arms(workflow_protocol_pair, position, step, message): + _, new = workflow_protocol_pair + steps = new["jobs"]["evaluate"]["steps"] + steps.insert(len(steps) if position is None else position, step) + with pytest.raises(ValueError, match=message): + pr_evaluation.measurement_protocol(yaml.safe_dump(new)) + + +def test_checked_in_release_workflow_has_a_valid_measurement_protocol(): + from pathlib import Path + + workflow = Path(__file__).resolve().parents[2] / pr_evaluation.EVALUATION_WORKFLOW + protocol = pr_evaluation.measurement_protocol(workflow.read_text()) + assert "-m workflow_bench.runner" in protocol["steps"][-1]["run"] + + @pytest.mark.parametrize( "comparison", [ diff --git a/eval/tests/test_workflow_bench.py b/eval/tests/test_workflow_bench.py index 8e45f079c..9e5714296 100644 --- a/eval/tests/test_workflow_bench.py +++ b/eval/tests/test_workflow_bench.py @@ -267,6 +267,8 @@ def test_eval_ci_uses_locked_uv_and_blocking_native_containment_jobs(): "pytest", "tests/test_process_control.py", "tests/test_proposer_sandbox.py", + # Real-Bubblewrap baseline_cli shim canary; skipped everywhere else. + "tests/test_cli_shim.py", "tests/test_workflow_bench_sessions.py", "tests/test_ce_plugin_runtime.py", # The offline sweep, run here with nothing stubbed: this job is the only @@ -295,7 +297,7 @@ def test_eval_ci_uses_locked_uv_and_blocking_native_containment_jobs(): for path in (repo_root / "eval" / "tests").glob("test_*.py") if bwrap_canary_marker.search(path.read_text()) ) - assert bwrap_canary_files == ["test_proposer_sandbox.py", "test_workflow_bench_sessions.py"] + assert bwrap_canary_files == ["test_cli_shim.py", "test_proposer_sandbox.py", "test_workflow_bench_sessions.py"] assert all(f"tests/{name}" in selected_containment_tests for name in bwrap_canary_files) assert "eval-containment-windows:" in workflow diff --git a/eval/workflow_bench/README.md b/eval/workflow_bench/README.md index 37a57f6c8..ede2a0b13 100644 --- a/eval/workflow_bench/README.md +++ b/eval/workflow_bench/README.md @@ -20,6 +20,7 @@ of indexing, infrastructure, or subscription costs. | `candidate_review` | the same review with a `gitnexus-review` candidate overlay | Paired with `review` on the same case/ref/model/runtime | | `ce_review` | one pinned `ce-code-review` session over the same changes | External comparator paired with both review arms | | `baseline` | one session with the identical task text | `--disallowedTools Skill` so it cannot borrow the workflow; same repo, same MCP tools | +| `baseline_cli` | one session with the same read-only tools behind a `gitnexus` command | No MCP tools, so no tool catalog in context; opt-in, Bubblewrap only (see below) | | `baseline_nomcp` | one session without installed GitNexus tools, graph, or workflow skills | Measures the no-GitNexus comparator; included by default | The default sweep runs `workflow`, `workflow_direct`, `baseline`, and @@ -48,6 +49,24 @@ the model, source, runtime, and grader. Three repetitions are a starting point, not statistical proof. Smaller tool responses measure one input-cost component; they do not establish lower total session cost or better task accuracy. +### CLI-only arm + +`baseline_cli` asks how much of `baseline`'s extra cost is the MCP tool +catalog, which the client re-sends on every turn whether or not a tool is +called. The agent has no MCP tools; `gitnexus` on its `PATH` is a +standard-library shim that forwards one call per invocation to the same +per-cell isolated MCP endpoint `baseline` uses, restricted to the same +read-only tools. Tool output is the server's own text, so the arms differ only +in how tools are described: always (MCP catalog) or on demand (`gitnexus help`, +`gitnexus help `). The prompt adds one short note naming the command. +The candidate runtime stays in the sibling server sandbox, as for `baseline`. + +Compare `baseline_cli` with `baseline` for the catalog's share of the cost and +with `baseline_nomcp` for what remains. It is not a release gate input: the +Release evaluation workflow runs it only when dispatched with `cli_arm: true`, +into its own output directory, and publishes a descriptive +`cli-arm-comparison` summary (`python -m workflow_bench.cli_arm_report`). + ## Quick start ```bash diff --git a/eval/workflow_bench/cli_arm_report.py b/eval/workflow_bench/cli_arm_report.py new file mode 100644 index 000000000..dd33ea6a8 --- /dev/null +++ b/eval/workflow_bench/cli_arm_report.py @@ -0,0 +1,200 @@ +"""Descriptive comparison of the opt-in baseline_cli arm with the paired release arms. + +The release gate stays bound to `baseline_nomcp` and `baseline`; this report never +feeds it. It answers one question: how much of the MCP arm's cost remains when the +same tools are reached through a shell command instead of a tool catalog. +""" + +from __future__ import annotations + +import argparse +import json +import re +import statistics +from collections import Counter +from pathlib import Path +from typing import Any + +ARMS = ("baseline_nomcp", "baseline", "baseline_cli") +EXPOSURE = {"baseline_nomcp": "none", "baseline": "stdio-only", "baseline_cli": "stdio-only"} +METRICS = ("cost_usd", "duration_s", "num_turns", "cache_read_input_tokens", "output_tokens") +# Command position only: `npm --prefix gitnexus test` names this repo's package, not the CLI. +_CLI_COMMAND = re.compile(r"(?:^|[;&|(`\n]\s*|\$\(\s*)gitnexus\s+([a-z_-]+)") + + +def _rows(path: Path) -> list[dict[str, Any]]: + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + +def _trustworthy(row: dict[str, Any]) -> bool: + return ( + row.get("ok") is True + and row.get("sandbox_backend") == "bwrap" + and row.get("runtime_exposure") == EXPOSURE.get(row.get("arm")) + and row.get("error_kind") in (None, "verify-failed", "oracle-failed") + and type(row.get("resolved")) is bool + and row.get("transcript_missing") is False + ) + + +def cli_commands(row: dict[str, Any], results_dir: Path) -> Counter: + """Count `gitnexus ` invocations in a cell's Bash tool calls.""" + + counts: Counter = Counter() + root = results_dir.resolve() + # Each entry is the record runner_sessions persists: {"path", "sha256", "bytes", "source"}. + for artifact in row.get("transcript_artifacts", []): + relative = artifact.get("path") if isinstance(artifact, dict) else None + if not isinstance(relative, str): + continue + path = (results_dir / relative).resolve() + if not path.is_relative_to(root) or not path.is_file(): + continue + for line in path.read_text(errors="replace").splitlines(): + try: + event = json.loads(line) + except ValueError: + continue + content = (event.get("message") or {}).get("content") + if event.get("type") != "assistant" or not isinstance(content, list): + continue + for part in content: + if isinstance(part, dict) and part.get("type") == "tool_use" and part.get("name") == "Bash": + command = str((part.get("input") or {}).get("command", "")) + counts.update(match.group(1) for match in _CLI_COMMAND.finditer(command)) + return counts + + +def _measured(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def build_report(paired: Path, cli: Path) -> dict[str, Any]: + rows = [(row, paired.parent) for row in _rows(paired) if row.get("arm") in ("baseline_nomcp", "baseline")] + rows += [(row, cli.parent) for row in _rows(cli) if row.get("arm") == "baseline_cli"] + report: dict[str, Any] = {"schema": "gitnexus.cli-arm-comparison/v1", "arms": {}, "problems": []} + # Deltas compare the same (task, run) pairs; per-arm means alone hide interrupted sweeps. + expected: dict[str, set[tuple[Any, Any]]] = {} + trusted: dict[str, dict[tuple[Any, Any], dict[str, Any]]] = {} + for arm in ARMS: + cells = [(row, where) for row, where in rows if row["arm"] == arm] + valid = [(row, where) for row, where in cells if _trustworthy(row)] + pairs = Counter((row.get("task"), row.get("run")) for row, _ in cells) + duplicates = sorted((pair for pair, count in pairs.items() if count > 1), key=str) + if duplicates: + report["problems"].append(f"{arm}: duplicate task/run pair(s) {duplicates}; excluded from deltas") + expected[arm] = set(pairs) + trusted[arm] = { + (row.get("task"), row.get("run")): row + for row, _ in valid + if pairs[(row.get("task"), row.get("run"))] == 1 + } + if len(valid) != len(cells): + report["problems"].append(f"{arm}: {len(cells) - len(valid)} of {len(cells)} cell(s) untrustworthy") + if not valid: + report["problems"].append(f"{arm}: no trustworthy cells") + continue + summary: dict[str, Any] = {"cells": len(valid), "solved": sum(row["resolved"] for row, _ in valid)} + summary["incomplete_metrics"] = [] + for metric in METRICS: + values = [row[metric] for row, _ in valid if _measured(row.get(metric))] + if len(values) == len(valid): + summary[f"mean_{metric}"] = statistics.mean(values) + continue + # A mean over the measured subset would read as the arm's mean; leave it unknown. + summary[f"mean_{metric}"] = None + summary["incomplete_metrics"].append(metric) + report["problems"].append(f"{arm}: {metric} missing on {len(valid) - len(values)} of {len(valid)} cell(s)") + if arm == "baseline_cli": + per_cell = [cli_commands(row, where) for row, where in valid] + summary["mean_gitnexus_calls"] = statistics.mean(sum(c.values()) for c in per_cell) + summary["cells_using_gitnexus"] = sum(1 for c in per_cell if c) + summary["gitnexus_commands"] = dict(sum(per_cell, Counter()).most_common()) + else: + summary["mean_gitnexus_calls"] = statistics.mean(row.get("gitnexus_tool_uses", 0) for row, _ in valid) + report["arms"][arm] = summary + report["changes"] = {} + for reference in ("baseline_nomcp", "baseline"): + for arm in ARMS: + if arm == reference or arm not in report["arms"] or reference not in report["arms"]: + continue + pairs = expected[arm] | expected[reference] + shared = [pair for pair in pairs if pair in trusted[arm] and pair in trusted[reference]] + if len(shared) < len(pairs): + report["problems"].append( + f"{arm} vs {reference}: {len(shared)} of {len(pairs)} task/run pair(s) matched; " + + "deltas use matched pairs only" + ) + for metric in ("cost_usd", "duration_s", "cache_read_input_tokens", "num_turns"): + matched = [ + pair + for pair in shared + if _measured(trusted[arm][pair].get(metric)) and _measured(trusted[reference][pair].get(metric)) + ] + if not matched: + continue + b = statistics.mean(trusted[reference][pair][metric] for pair in matched) + v = statistics.mean(trusted[arm][pair][metric] for pair in matched) + if b: + report["changes"][f"{arm} vs {reference}: {metric}"] = { + "percent": 100 * (v - b) / b, + "pairs": len(matched), + "expected_pairs": len(pairs), + } + return report + + +def render_markdown(report: dict[str, Any]) -> str: + lines = [ + "## CLI-only arm (descriptive)", + "", + "`baseline_cli` reaches the same read-only GitNexus tools as `baseline` through a `gitnexus` shell " + + "command, so no tool catalog enters the model context. It is not part of the release gate.", + "", + "| Arm | Solved | Mean cost | Mean wall | Mean turns | Mean cache-read tokens | GitNexus calls/cell |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + for arm, values in report["arms"].items(): + + def show(metric: str, pattern: str) -> str: + value = values.get(metric) + if metric.removeprefix("mean_") in values.get("incomplete_metrics", ()): + return "incomplete" + return "-" if value is None else pattern.format(value) + + lines.append( + f"| {arm} | {values['solved']}/{values['cells']} | {show('mean_cost_usd', '${:.4f}')} " + f"| {show('mean_duration_s', '{:.1f}s')} | {show('mean_num_turns', '{:.1f}')} " + f"| {show('mean_cache_read_input_tokens', '{:,.0f}')} | {show('mean_gitnexus_calls', '{:.1f}')} |" + ) + if report["changes"]: + lines.append("") + for name, change in report["changes"].items(): + sample = f"{change['pairs']}/{change['expected_pairs']} pairs" + if change["pairs"] < change["expected_pairs"]: + sample += ", partial" + lines.append(f"- {name}: {change['percent']:+.1f}% ({sample})") + cli = report["arms"].get("baseline_cli") + if cli: + lines += ["", f"CLI usage: {cli['cells_using_gitnexus']}/{cli['cells']} cells; commands {cli['gitnexus_commands']}."] + if report["problems"]: + lines += ["", "**Problems:**", *[f"- {problem}" for problem in report["problems"]]] + lines += ["", "Small pinned task set; descriptive evidence only, not statistical significance.", ""] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--paired", required=True, type=Path, help="results.jsonl with baseline_nomcp and baseline") + parser.add_argument("--cli", required=True, type=Path, help="results.jsonl with baseline_cli") + parser.add_argument("--out", required=True, type=Path) + args = parser.parse_args(argv) + report = build_report(args.paired, args.cli) + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "cli-arm-comparison.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + (args.out / "cli-arm-comparison.md").write_text(render_markdown(report)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/workflow_bench/cli_shim.py b/eval/workflow_bench/cli_shim.py new file mode 100644 index 000000000..608e50343 --- /dev/null +++ b/eval/workflow_bench/cli_shim.py @@ -0,0 +1,263 @@ +"""CLI-only arm: a `gitnexus` command that forwards to the isolated MCP endpoint. + +The `baseline_cli` arm measures how much of the MCP arm's cost is the tool +catalog itself. Its agent has no MCP tools, so no tool schema enters the model +context; it reaches the same read-only tools through a shell command that +speaks MCP over the per-cell socket. Results are the server's own text, so the +two arms receive identical tool output and differ only in how tools are +described: always (MCP) or on demand (`gitnexus help`). + +The candidate runtime stays in the sibling server sandbox, exactly as for the +MCP arm; the agent's filesystem holds only this standard-library script. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from .proposer_sandbox import SANDBOX_MCP_SOCKET +from .runner_sessions import GITNEXUS_READ_ONLY_TOOLS + +_MCP_PREFIX = "mcp__gitnexus__" +CLI_TOOLS = tuple(tool.removeprefix(_MCP_PREFIX) for tool in GITNEXUS_READ_ONLY_TOOLS) + +GITNEXUS_CLI_NOTE = ( + "GitNexus is available in this arm only as the `gitnexus` shell command (there are no " + "GitNexus MCP tools). It queries a prebuilt code knowledge graph of this repository. " + "Commands include query, context, impact, detect_changes and trace; run `gitnexus help` " + "for the full list and `gitnexus help ` for its options. Example: " + "`gitnexus impact parseFile --direction upstream`. GitNexus MCP tool names in repository " + "docs map to these commands; `npx gitnexus` and `.gitnexus/run.cjs` are not available.\n" +) + +_SHIM_TEMPLATE = r'''#!/usr/bin/python3 +"""gitnexus: forward one read-only GitNexus tool call to the MCP endpoint.""" +import json, socket, sys + +SOCKET = __SOCKET__ +TOOLS = __TOOLS__ +USAGE = "usage: gitnexus [value ...] [--option value] [--json '{...}']" + + +class Session: + def __init__(self): + self.sock = socket.socket(socket.AF_UNIX) + self.sock.settimeout(600) + self.sock.connect(SOCKET) + self.buffer = b"" + self.next_id = 0 + self.request("initialize", { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "gitnexus-cli-shim", "version": "1"}, + }) + self.send({"jsonrpc": "2.0", "method": "notifications/initialized"}) + + def send(self, message): + self.sock.sendall(json.dumps(message, separators=(",", ":")).encode() + b"\n") + + def read(self): + while b"\n" not in self.buffer: + chunk = self.sock.recv(65536) + if not chunk: + raise SystemExit("gitnexus: the GitNexus server closed the connection") + self.buffer += chunk + line, self.buffer = self.buffer.split(b"\n", 1) + return json.loads(line) if line.strip() else None + + def request(self, method, params): + self.next_id += 1 + wanted = self.next_id + self.send({"jsonrpc": "2.0", "id": wanted, "method": method, "params": params}) + while True: + message = self.read() + if not message: + continue + if "method" in message: + if "id" in message: + self.send({"jsonrpc": "2.0", "id": message["id"], + "error": {"code": -32601, "message": "unsupported by gitnexus CLI"}}) + continue + if message.get("id") != wanted: + continue + if "error" in message: + raise SystemExit("gitnexus: " + str(message["error"].get("message", message["error"]))) + return message.get("result", {}) + + def close(self): + try: + self.sock.shutdown(socket.SHUT_WR) + while self.sock.recv(65536): + pass + except OSError: + pass + self.sock.close() + + +def key(name): + return name.replace("-", "").replace("_", "").lower() + + +def types_of(spec): + found = spec.get("type") + if isinstance(found, list): + return found + if found: + return [found] + return [t for option in spec.get("anyOf", []) + spec.get("oneOf", []) for t in types_of(option)] + + +def coerce(name, raw, spec): + kinds = types_of(spec) + if raw is True: + if "boolean" in kinds or not kinds: + return True + raise SystemExit(f"gitnexus: --{name} needs a value") + if "boolean" in kinds and raw.lower() in ("true", "false"): + return raw.lower() == "true" + if "integer" in kinds: + try: + return int(raw) + except ValueError: + pass + if "number" in kinds: + try: + return float(raw) + except ValueError: + pass + if "array" in kinds: + if raw.startswith("["): + return json.loads(raw) + items = spec.get("items", {}) + return [coerce(name, part.strip(), items) for part in raw.split(",") if part.strip()] + if "object" in kinds: + return json.loads(raw) + return raw + + +def first_sentence(text): + text = " ".join((text or "").split()) + end = text.find(". ") + return text if end < 0 else text[: end + 1] + + +def describe(tool): + lines = [f"gitnexus {tool['name']}", "", (tool.get("description") or "").strip(), "", "Options:"] + schema = tool.get("inputSchema") or {} + required = set(schema.get("required", [])) + for name, spec in (schema.get("properties") or {}).items(): + kinds = "|".join(types_of(spec)) or "value" + mark = " (required)" if name in required else "" + lines.append(f" --{name} <{kinds}>{mark} {first_sentence(spec.get('description', ''))}") + positional = positional_names(schema) + if positional: + lines += ["", "Positional values fill: " + ", ".join(positional)] + return "\n".join(lines) + + +def positional_names(schema): + # The first property names the subject (target, name, search_query, from); + # it comes first even when optional, as impact's target is, then required ones. + properties = schema.get("properties") or {} + subject = [name for name in list(properties)[:1] if "string" in types_of(properties[name])] + required = [name for name in schema.get("required", []) if name in properties and name not in subject] + return subject + required + + +def parse(tool, argv): + schema = tool.get("inputSchema") or {} + properties = schema.get("properties") or {} + by_key = {key(name): name for name in properties} + arguments, positional = {}, [] + index = 0 + while index < len(argv): + token = argv[index] + if token == "--json": + index += 1 + if index >= len(argv): + raise SystemExit("gitnexus: --json needs a JSON object") + arguments.update(json.loads(argv[index])) + elif token.startswith("--") and len(token) > 2: + flag, separator, inline = token[2:].partition("=") + name = by_key.get(key(flag)) + if name is None: + raise SystemExit(f"gitnexus: unknown option --{flag} for {tool['name']}; " + f"options: {', '.join('--' + n for n in properties)}") + kinds = set(types_of(properties[name])) + if separator: + raw = inline + elif "boolean" in kinds and kinds <= {"boolean", "null"}: + # A switch takes only an explicit true/false, never the next positional. + if index + 1 < len(argv) and argv[index + 1].lower() in ("true", "false"): + index += 1 + raw = argv[index] + else: + raw = True + elif index + 1 < len(argv) and not argv[index + 1].startswith("--"): + index += 1 + raw = argv[index] + else: + raw = True + arguments[name] = coerce(flag, raw, properties[name]) + else: + positional.append(token) + index += 1 + slots = [name for name in positional_names(schema) if name not in arguments] + if len(positional) > len(slots): + raise SystemExit(f"gitnexus: too many values for {tool['name']}; use --option value") + for name, raw in zip(slots, positional): + arguments[name] = coerce(name, raw, properties[name]) + return arguments + + +def main(argv): + session = Session() + try: + listed = session.request("tools/list", {}).get("tools", []) + tools = {tool["name"]: tool for tool in listed if tool["name"] in TOOLS} + if not argv or argv[0] in ("help", "--help", "-h") and len(argv) == 1: + print(USAGE + "\n\nCommands:") + for name in TOOLS: + if name in tools: + print(f" {name:<16} {first_sentence(tools[name].get('description'))}") + print("\nRun `gitnexus help ` for options.") + return 0 + if argv[0] in ("help", "--help", "-h"): + argv = [argv[1], "--help"] + name = argv[0].replace("-", "_") + if name not in tools: + raise SystemExit(f"gitnexus: unknown command {argv[0]}; run `gitnexus help`") + if any(token in ("--help", "-h") for token in argv[1:]): + print(describe(tools[name])) + return 0 + result = session.request("tools/call", {"name": name, "arguments": parse(tools[name], argv[1:])}) + finally: + session.close() + texts = [part.get("text", "") for part in result.get("content", []) if part.get("type") == "text"] + output = "\n".join(texts) if texts else json.dumps(result.get("structuredContent", result), indent=2) + stream = sys.stderr if result.get("isError") else sys.stdout + stream.write(output + ("" if output.endswith("\n") else "\n")) + return 1 if result.get("isError") else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) +''' + + +def shim_source() -> str: + """Return the shim with its socket path and tool allowlist bound in.""" + + return _SHIM_TEMPLATE.replace("__SOCKET__", json.dumps(SANDBOX_MCP_SOCKET)).replace( + "__TOOLS__", json.dumps(list(CLI_TOOLS)) + ) + + +def create_cli_shim(private_root: Path) -> Path: + """Write the immutable shim the agent sandbox mounts as `gitnexus`.""" + + shim = private_root / "gitnexus" + shim.write_text(shim_source()) + shim.chmod(0o500) + return shim diff --git a/eval/workflow_bench/pr_evaluation.py b/eval/workflow_bench/pr_evaluation.py index 358ea1931..3ea3d2dc3 100644 --- a/eval/workflow_bench/pr_evaluation.py +++ b/eval/workflow_bench/pr_evaluation.py @@ -30,6 +30,8 @@ PREVIEW_PATHS = { "eval/workflow_bench/README.md", } EVALUATION_WORKFLOW = ".github/workflows/release-evaluation.yml" +# Exact `if:` gates of the dispatch-only descriptive arms (never PR previews). +OPT_IN_ARM_CONDITIONS = frozenset({"env.CLI_ARM == 'true'", "env.CODE_ARM == 'true'"}) MEASUREMENT_PATHS = ("eval/", ".github/claude-canary-runtime/", EVALUATION_WORKFLOW) @@ -121,8 +123,18 @@ def measurement_protocol(source: str) -> dict[str, Any]: key: value for key, value in job["env"].items() if key not in {"INPUT_PR", "INPUT_PR_SHA", "COMPARISON_RUN_ID"} } steps = [] - if sum(step.get("run", "").count("-m workflow_bench.runner") for step in job["steps"]) != 1: + invocations = [ + (index, step.get("run", "").count("-m workflow_bench.runner"), step.get("if")) + for index, step in enumerate(job["steps"]) + if "-m workflow_bench.runner" in step.get("run", "") + ] + paid = [(index, count) for index, count, condition in invocations if condition not in OPT_IN_ARM_CONDITIONS] + if len(paid) != 1 or paid[0][1] != 1: raise ValueError("expected one paid measurement step") + # Descriptive opt-in arms run after the paired measurement, which the + # fingerprint below ends at; one placed earlier could alter what it measures. + if any(count != 1 or index < paid[0][0] for index, count, condition in invocations if condition in OPT_IN_ARM_CONDITIONS): + raise ValueError("opt-in arm runs must follow the paid measurement step") for step in job["steps"]: if step.get("id") == "resolve": continue # Runtime selection is intentionally different for an unmerged PR. diff --git a/eval/workflow_bench/proposer_sandbox.py b/eval/workflow_bench/proposer_sandbox.py index dd2608df0..04296b2d5 100644 --- a/eval/workflow_bench/proposer_sandbox.py +++ b/eval/workflow_bench/proposer_sandbox.py @@ -213,6 +213,8 @@ class SandboxSession: read_only_mounts: tuple[ReadOnlyMount, ...] # Harness-authored, immutable overlays must not appear as agent work. synthetic_guidance_paths: tuple[str, ...] = () + # The baseline_cli shim connects to the cell's MCP socket from Bash. + allow_unix_sockets: bool = False @property def require_pid_namespace(self) -> bool: @@ -279,7 +281,9 @@ class SandboxSession: @property def settings_json(self) -> str: - return build_claude_settings(sandbox_enabled=self.backend == "bwrap") + return build_claude_settings( + sandbox_enabled=self.backend == "bwrap", allow_unix_sockets=self.allow_unix_sockets + ) def environment( self, @@ -549,12 +553,16 @@ def build_sandbox_environment( return env -def build_claude_settings(*, sandbox_enabled: bool = True) -> str: +def build_claude_settings(*, sandbox_enabled: bool = True, allow_unix_sockets: bool = False) -> str: """Inline settings that keep every Bash sandboxed and pre-approve the tools. Inline settings disable repository/plugin hooks even when ordinary CLAUDE.md startup context is loaded. Confinement rests on the sandbox policy below, ``--tools``/``--allowedTools``, and the bwrap mounts. + + ``allow_unix_sockets`` lifts the Bash sandbox's AF_UNIX block for the + baseline_cli shim. The bwrap mounts still bound what it can reach: the + only harness socket in the agent namespace is the cell's MCP endpoint. """ permissions = { @@ -573,7 +581,7 @@ def build_claude_settings(*, sandbox_enabled: bool = True) -> str: "network": { "allowedDomains": [], "deniedDomains": ["*"], - "allowAllUnixSockets": False, + "allowAllUnixSockets": allow_unix_sockets, "allowLocalBinding": False, }, "filesystem": { @@ -1221,8 +1229,14 @@ def prepare_sandbox( backend: str = "bwrap", gitnexus_available: bool = True, gitnexus_cli_available: bool = True, + gitnexus_cli_shim: bool = False, ) -> Iterator[SandboxSession]: - """Create private host backing dirs and one virtualized command.""" + """Create private host backing dirs and one virtualized command. + + ``gitnexus_cli_shim`` mounts the baseline_cli arm's MCP-forwarding + `gitnexus` command in place of the pinned CLI; it needs the per-cell MCP + socket among ``read_only_mounts`` and never exposes the runtime itself. + """ # Validate the lexical path before resolving it. Resolving first would # erase the evidence that the caller supplied a symlinked clone root. @@ -1264,7 +1278,8 @@ def prepare_sandbox( directory.mkdir(mode=0o700) directory.chmod(0o700) cli_available = gitnexus_available and gitnexus_cli_available - shell_prefix = _create_shell_prefix_wrapper(private_root, gitnexus_available=cli_available) + cli_shim = gitnexus_available and gitnexus_cli_shim and not cli_available + shell_prefix = _create_shell_prefix_wrapper(private_root, gitnexus_available=cli_available or cli_shim) python3_wrapper = _create_python3_wrapper(private_root) git_excludes = _create_git_excludes(private_root) # Claude may discover user-level skills below HOME. Keep the rest of HOME @@ -1280,6 +1295,17 @@ def prepare_sandbox( else: baseline_mounts = baseline_gitnexus_mounts(clone, private_root) + if cli_available: + cli_mounts: tuple[ReadOnlyMount, ...] = ( + ReadOnlyMount(source=_create_gitnexus_wrapper(private_root), target=SANDBOX_GITNEXUS_CLI), + ) + elif cli_shim: + # Imported here: cli_shim reads this module's sandbox paths. + from .cli_shim import create_cli_shim + + cli_mounts = (ReadOnlyMount(source=create_cli_shim(private_root), target=SANDBOX_GITNEXUS_CLI),) + else: + cli_mounts = baseline_mounts protected_mounts = ( *read_only_mounts, *( @@ -1290,11 +1316,7 @@ def prepare_sandbox( ReadOnlyMount(source=user_skills, target=SANDBOX_USER_SKILLS), ReadOnlyMount(source=shell_prefix, target=SANDBOX_SHELL_PREFIX), ReadOnlyMount(source=python3_wrapper, target=SANDBOX_PYTHON3), - *( - (ReadOnlyMount(source=_create_gitnexus_wrapper(private_root), target=SANDBOX_GITNEXUS_CLI),) - if cli_available - else baseline_mounts - ), + *cli_mounts, ReadOnlyMount(source=git_excludes, target=SANDBOX_GIT_EXCLUDES), ) command_prefix = ( @@ -1324,6 +1346,7 @@ def prepare_sandbox( for mount in baseline_mounts if mount.target != f"{SANDBOX_WORKSPACE}/.gitnexus" ), + allow_unix_sockets=cli_shim, ) except BaseException as exc: primary = exc diff --git a/eval/workflow_bench/runner.py b/eval/workflow_bench/runner.py index 011d63938..0368f2f7d 100644 --- a/eval/workflow_bench/runner.py +++ b/eval/workflow_bench/runner.py @@ -110,6 +110,7 @@ from .review_scoring import ( score_review, ) from .baseline_guidance import GITNEXUS_UNAVAILABLE_NOTE +from .cli_shim import GITNEXUS_CLI_NOTE from .proposer_sandbox import ( SANDBOX_GITNEXUS as SANDBOX_GITNEXUS, SANDBOX_GITNEXUS_REGISTRY, @@ -656,6 +657,23 @@ def run_arm( }, ) ) + elif arm == "baseline_cli": + # Same graph and read-only tools as baseline, reached through the + # `gitnexus` shell command instead of MCP: no tool schema in context. + sessions.append( + run_claude( + BASELINE_PROMPT.format(task=task["prompt"]) + "\n\n" + GITNEXUS_CLI_NOTE, + worktree, + disallowed_tools=["Skill", "mcp__gitnexus"], + **{ + **common, + "mcp_config_json": '{"mcpServers":{}}', + "allowed_tools": [ + tool for tool in allowed_agent_tools(implementation=True, include_mcp=False) if tool != "Skill" + ], + }, + ) + ) else: sessions.append( run_claude( @@ -1193,10 +1211,13 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: cleanup_error: OSError | None = None execution_arm = CANDIDATE_ARMS.get(arm, arm) gitnexus_available = execution_arm != "baseline_nomcp" - isolated_baseline = execution_arm == "baseline" and ctx.sandbox_backend == "bwrap" + # baseline_cli reaches the same isolated endpoint through its shim. + isolated_baseline = execution_arm in ("baseline", "baseline_cli") and ctx.sandbox_backend == "bwrap" runtime_exposure = "none" if not gitnexus_available else "readable" resources = ExitStack() try: + if execution_arm == "baseline_cli" and not isolated_baseline: + raise RuntimeError("baseline_cli requires Bubblewrap: its CLI forwards to the isolated MCP endpoint") if ctx.asset_snapshot_error is not None: raise RuntimeError(f"task asset snapshot preparation failed: {ctx.asset_snapshot_error}") if gitnexus_available: @@ -1248,6 +1269,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: backend=ctx.sandbox_backend, gitnexus_available=gitnexus_available, gitnexus_cli_available=not isolated_baseline, + gitnexus_cli_shim=execution_arm == "baseline_cli", ) as sandbox: # Capture the BASE (pre-overlay) skill digest — identical # for the incumbent and candidate arms — then run the @@ -1977,6 +1999,7 @@ def build_parser() -> argparse.ArgumentParser: "review", "ce_review", "baseline", + "baseline_cli", "baseline_nomcp", ], help="arms to compare (default: workflow, workflow_direct, baseline, baseline_nomcp)", diff --git a/eval/workflow_bench/runner_artifacts.py b/eval/workflow_bench/runner_artifacts.py index 72e25a543..ef79f5259 100644 --- a/eval/workflow_bench/runner_artifacts.py +++ b/eval/workflow_bench/runner_artifacts.py @@ -100,6 +100,7 @@ IMPLEMENTATION_ARMS = frozenset( "ce_workflow", "ce_workflow_direct", "baseline", + "baseline_cli", "baseline_nomcp", } ) diff --git a/gitnexus/test/unit/skill-evolution-workflow.test.ts b/gitnexus/test/unit/skill-evolution-workflow.test.ts index fbe1fdf22..bc7556998 100644 --- a/gitnexus/test/unit/skill-evolution-workflow.test.ts +++ b/gitnexus/test/unit/skill-evolution-workflow.test.ts @@ -127,6 +127,7 @@ describe('dedicated runner readiness', () => { } const releaseDocument = load(release) as typeof workflowDocument; expect(Object.keys(releaseDocument.jobs?.evaluate?.env ?? {}).sort()).toEqual([ + 'CLI_ARM', 'COMPARISON_RUN_ID', 'EFFORT', 'INPUT_PR',