diff --git a/eval/tests/test_workflow_bench_sessions.py b/eval/tests/test_workflow_bench_sessions.py index c48ea10a1..e03104ed3 100644 --- a/eval/tests/test_workflow_bench_sessions.py +++ b/eval/tests/test_workflow_bench_sessions.py @@ -167,6 +167,56 @@ def test_run_claude_forwards_the_named_model_to_every_session(monkeypatch, tmp_p assert captured[captured.index("--model") + 1] == "claude-sonnet-4-20250514" +def test_run_claude_restricts_tools_via_tools_flag_outside_bare(monkeypatch, tmp_path): + # Outside --bare, the built-in toolset defaults to everything (subagents, + # WebFetch, Task, ...) and --allowedTools only pre-approves within that — + # it does not narrow it. --tools is what actually restricts the set, so a + # non-bare arm session must pass it or it silently gets a far wider + # toolset than intended. + captured: list[str] = [] + + def fake_run(command, **kwargs): + captured.extend(command) + return fake_cli_result(VALID_REPORT) + + monkeypatch.setattr(runner_sessions, "run_managed", fake_run) + runner.run_claude( + "task", + tmp_path, + claude_bin="claude", + timeout=5, + bare=False, + allowed_tools=["Read", "Edit", "Bash", "Skill"], + ) + tools_idx = captured.index("--tools") + assert captured[tools_idx + 1 : tools_idx + 5] == ["Read", "Edit", "Bash", "Skill"] + allowed_idx = captured.index("--allowedTools") + assert captured[allowed_idx + 1 : allowed_idx + 5] == ["Read", "Edit", "Bash", "Skill"] + + +def test_run_claude_omits_tools_flag_under_bare(monkeypatch, tmp_path): + # --bare already hard-restricts to Bash/Edit/Read on its own (a Claude + # Code design choice, not something --tools/--allowedTools can widen or + # narrow further), so bare sessions must not also pass --tools. + captured: list[str] = [] + + def fake_run(command, **kwargs): + captured.extend(command) + return fake_cli_result(VALID_REPORT) + + monkeypatch.setattr(runner_sessions, "run_managed", fake_run) + runner.run_claude( + "task", + tmp_path, + claude_bin="claude", + timeout=5, + bare=True, + allowed_tools=["Read", "Edit", "Bash", "Skill"], + ) + assert "--tools" not in captured + assert "--allowedTools" in captured + + @pytest.mark.parametrize( ("proc", "expected_kind"), [ @@ -282,6 +332,15 @@ def test_agent_tool_grants_are_exact_and_nomcp_has_no_graph_tools(monkeypatch, t assert captured[3]["mcp_config_json"] == '{"mcpServers":{}}' assert captured[3]["disallowed_tools"] == ["Skill", "mcp__gitnexus"] + # --bare hard-disables the Skill tool and every mcp__* tool regardless of + # --allowedTools (a Claude Code design choice, not something the harness + # can override) -- every arm here except baseline_nomcp needs Skill + # and/or MCP tools, so only baseline_nomcp may still run under --bare. + assert captured[0]["bare"] is False # workflow: planning session + assert captured[1]["bare"] is False # review + assert captured[2]["bare"] is False # workflow_direct + assert captured[3]["bare"] is True # baseline_nomcp + def test_mcp_config_uses_only_the_minimal_pinned_harness_runtime(monkeypatch, tmp_path): runtime = tmp_path / "gitnexus" diff --git a/eval/workflow_bench/runner.py b/eval/workflow_bench/runner.py index cd760eb98..2a180f43f 100644 --- a/eval/workflow_bench/runner.py +++ b/eval/workflow_bench/runner.py @@ -402,6 +402,13 @@ def run_arm( auth_token=args.auth_token, base_url=args.base_url, ) + # --bare hard-disables the Skill tool and every mcp__* tool — by Claude + # Code design, not a bug (--allowedTools can't restore what --bare + # removes). Every arm except baseline_nomcp needs Skill and/or MCP tools, + # so only baseline_nomcp can keep --bare's tighter isolation; the rest + # rely on ANTHROPIC_API_KEY alone (the sandboxed HOME has no OAuth/ + # keychain state to conflict with it). + bare = arm == "baseline_nomcp" common = { "claude_bin": sandbox.claude_bin, "timeout": args.timeout, @@ -412,7 +419,7 @@ def run_arm( read_only_paths=_evaluated_skill_roots(worktree, arm), ), "require_pid_namespace": True, - "bare": True, + "bare": bare, "settings_json": sandbox.settings_json, "strict_mcp_config": True, "mcp_config_json": sandbox_mcp_config(), diff --git a/eval/workflow_bench/runner_sessions.py b/eval/workflow_bench/runner_sessions.py index 60e92b6bc..53e7335ee 100644 --- a/eval/workflow_bench/runner_sessions.py +++ b/eval/workflow_bench/runner_sessions.py @@ -377,6 +377,13 @@ def run_claude( if strict_mcp_config: cmd += ["--strict-mcp-config", "--mcp-config", mcp_config_json or '{"mcpServers":{}}'] if allowed_tools: + # --bare's own hard-coded Bash/Edit/Read ceiling already scopes bare + # sessions; outside --bare the built-in toolset defaults to + # everything (subagents, WebFetch, Task, ...), so --tools is needed + # to actually restrict it — --allowedTools only pre-approves within + # whatever set is available, it does not narrow that set. + if not bare: + cmd += ["--tools", *allowed_tools] cmd += ["--allowedTools", *allowed_tools] if disable_slash_commands: cmd.append("--disable-slash-commands") diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index fe1643b96..b322f0814 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -24,7 +24,7 @@ "graphology-indices": "^0.17.0", "graphology-utils": "^2.3.0", "ignore": "^7.0.5", - "js-yaml": "^4.1.1", + "js-yaml": "^5.0.0", "jsonc-parser": "^3.3.1", "mnemonist": "^0.40.3", "node-addon-api": "^8.0.0", @@ -59,7 +59,7 @@ "@types/cors": "^2.8.17", "@types/express": "^5.0.6", "@types/js-yaml": "^4.0.9", - "@types/node": "^25.6.0", + "@types/node": "^26.0.0", "@types/uuid": "^11.0.0", "@vitest/coverage-v8": "^4.0.18", "gitnexus-shared": "file:../gitnexus-shared", @@ -1946,13 +1946,13 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "25.9.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.5.tgz", - "integrity": "sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==", + "version": "26.0.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.0.0.tgz", + "integrity": "sha512-vf2YFi1iY9lHGwNJMs01biZFbKJkrZR1T6/MlzjhJLPdntOHLhTrDSnSVcdtvjihi4VQNlrFRIxLsDBlQpAipA==", "devOptional": true, "license": "MIT", "dependencies": { - "undici-types": ">=7.24.0 <7.24.7" + "undici-types": "~8.3.0" } }, "node_modules/@types/qs": { @@ -3589,9 +3589,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", - "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.0.0.tgz", + "integrity": "sha512-GSvaPUbk1U+FMZ7rJzF+F8e5YVtu7KnD40et/5rBXXRBv2jCO9L3qCewvIDDdudC0QycTFlf6EAA+h3kxBsuUw==", "funding": [ { "type": "github", @@ -3607,7 +3607,7 @@ "argparse": "^2.0.1" }, "bin": { - "js-yaml": "bin/js-yaml.js" + "js-yaml": "bin/js-yaml.mjs" } }, "node_modules/jsesc": { @@ -5510,9 +5510,9 @@ } }, "node_modules/undici-types": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", - "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", "devOptional": true, "license": "MIT" }, diff --git a/gitnexus/package.json b/gitnexus/package.json index fb60f0dbe..fc7e69e2a 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -70,7 +70,7 @@ "graphology-indices": "^0.17.0", "graphology-utils": "^2.3.0", "ignore": "^7.0.5", - "js-yaml": "^4.1.1", + "js-yaml": "^5.0.0", "jsonc-parser": "^3.3.1", "mnemonist": "^0.40.3", "node-addon-api": "^8.0.0", @@ -106,7 +106,7 @@ "@types/cors": "^2.8.17", "@types/express": "^5.0.6", "@types/js-yaml": "^4.0.9", - "@types/node": "^25.6.0", + "@types/node": "^26.0.0", "@types/uuid": "^11.0.0", "@vitest/coverage-v8": "^4.0.18", "gitnexus-shared": "file:../gitnexus-shared",