diff --git a/eval/tests/test_proposer_sandbox.py b/eval/tests/test_proposer_sandbox.py index 338eead52..e5ceb7b2c 100644 --- a/eval/tests/test_proposer_sandbox.py +++ b/eval/tests/test_proposer_sandbox.py @@ -58,6 +58,11 @@ def test_environment_is_allowlisted_and_shell_children_are_credential_free(monke assert settings["sandbox"]["failIfUnavailable"] is True assert settings["sandbox"]["allowUnsandboxedCommands"] is False assert settings["sandbox"]["network"]["deniedDomains"] == ["*"] + # ENV_SCRUB forces "default" mode; the proposer's tools (Bash writes the + # overlay) run headless only because they are explicitly pre-approved. + # Requesting a non-default defaultMode would merely warn, so it must be gone. + assert settings["permissions"]["allow"] == ["Read", "Grep", "Glob", "Bash"] + assert "defaultMode" not in settings["permissions"] @pytest.mark.parametrize( @@ -716,8 +721,10 @@ for line in sys.stdin: "--strict-mcp-config", "--mcp-config", mcp_config, - "--permission-mode", - "dontAsk", + # No --permission-mode: mirrors production (run_proposer). + # ENV_SCRUB forces "default"; Bash runs only because + # settings permissions.allow pre-approves it. This is the + # authoritative empirical gate for that behavior. "--model", "claude-canary-20260718", "--allowedTools", diff --git a/eval/workflow_bench/evolve.py b/eval/workflow_bench/evolve.py index 8cf6be495..d917abe88 100644 --- a/eval/workflow_bench/evolve.py +++ b/eval/workflow_bench/evolve.py @@ -501,7 +501,10 @@ def run_proposer( auth_token=args.auth_token, base_url=args.base_url, ), - permission_mode="dontAsk", + # No permission_mode: CLAUDE_CODE_SUBPROCESS_ENV_SCRUB + # forces "default", so requesting dontAsk only warns. Tools + # are pre-approved via settings permissions.allow + # (proposer_sandbox.build_claude_settings). command_prefix=sandbox.command_prefix, require_pid_namespace=True, bare=True, diff --git a/eval/workflow_bench/proposer_sandbox.py b/eval/workflow_bench/proposer_sandbox.py index 3e492fee9..254906139 100644 --- a/eval/workflow_bench/proposer_sandbox.py +++ b/eval/workflow_bench/proposer_sandbox.py @@ -329,7 +329,14 @@ def build_claude_settings() -> str: }, }, "permissions": { - "defaultMode": "dontAsk", + # CLAUDE_CODE_SUBPROCESS_ENV_SCRUB forces permission mode to + # "default" (allowed_non_write_users hardening), so requesting a + # non-default mode only emits a warning and never takes effect. + # Under "default" a tool runs without a prompt only if it matches an + # allow rule, so pre-approve the proposer's exact tool surface. Bash + # is the only writable tool under --bare (it writes the candidate + # overlay) and stays sandbox-confined by the sandbox.* policy above. + "allow": ["Read", "Grep", "Glob", "Bash"], "disableBypassPermissionsMode": "disable", }, "env": {