diff --git a/eval/tests/test_evolve.py b/eval/tests/test_evolve.py index 5d1ee8e5d..0e0335cb8 100644 --- a/eval/tests/test_evolve.py +++ b/eval/tests/test_evolve.py @@ -560,22 +560,34 @@ def test_proposer_refuses_a_prior_proposal_that_lost_its_trust_boundary(tmp_path ) -def test_run_proposer_keeps_trusted_input_hook_enabled(monkeypatch, tmp_path): +def test_run_proposer_hides_the_hidden_harness_and_keeps_the_full_tool_surface(monkeypatch, tmp_path): + """The proposer writes the artifact the arms are scored with. + + So its clone must be sanitized before the session starts — a proposer that + can read eval/workflow_bench reads the task prompts and hidden oracles it + is about to be graded against, and can encode the answers into the skill. + """ + evidence = tmp_path / "evidence" evidence.mkdir() transcript_projects = tmp_path / "transcript-projects" transcript_projects.mkdir() captured: dict[str, object] = {} + sanitized: list[Path] = [] def fake_make_worktree(_repo, _ref, destination): clone = destination / "clone" clone.mkdir() return clone + def fake_sanitize(clone): + sanitized.append(clone) + return "0" * 40 + class FakeSandbox: claude_bin = "claude" command_prefix: list[str] = [] - settings_json = '{"hooks":{"PreToolUse":[]}}' + settings_json = '{"permissions":{"allow":["Read"]}}' @property def transcript_projects(self): @@ -591,6 +603,7 @@ def test_run_proposer_keeps_trusted_input_hook_enabled(monkeypatch, tmp_path): monkeypatch.setattr(evolve.runner, "make_worktree", fake_make_worktree) monkeypatch.setattr(evolve.runner, "remove_clone", lambda _clone: None) + monkeypatch.setattr(evolve, "sanitize_clone_for_hidden_oracles", fake_sanitize) monkeypatch.setattr(evolve, "prepare_sandbox", fake_prepare_sandbox) monkeypatch.setattr(evolve.runner, "run_claude", fake_run_claude) args = build_parser().parse_args(["--tasks", "tasks.yaml", "--model", "model"]) @@ -605,6 +618,10 @@ def test_run_proposer_keeps_trusted_input_hook_enabled(monkeypatch, tmp_path): ) assert record["ok"] is False + # Sanitization has to happen on the clone the session actually runs in, + # and before the sandbox is prepared around it. + assert [clone.name for clone in sanitized] == ["clone"] + # Not --bare: bare ignores --tools and would cost the proposer Grep/Glob. assert captured.get("bare", False) is False assert captured["allowed_tools"] == evolve.PROPOSER_ALLOWED_TOOLS assert captured["settings_json"] == FakeSandbox.settings_json diff --git a/eval/tests/test_proposer_sandbox.py b/eval/tests/test_proposer_sandbox.py index 6c69c53cb..6a6fe076f 100644 --- a/eval/tests/test_proposer_sandbox.py +++ b/eval/tests/test_proposer_sandbox.py @@ -29,7 +29,6 @@ from workflow_bench.proposer_sandbox import ( SANDBOX_PATH, SANDBOX_PYTHON3, SANDBOX_SHELL_PREFIX, - SANDBOX_TOOL_INPUT_NORMALIZER, SANDBOX_USER_SKILLS, ReadOnlyMount, SandboxError, @@ -70,9 +69,10 @@ def test_environment_is_allowlisted_and_shell_children_are_credential_free(monke assert settings["sandbox"]["allowUnsandboxedCommands"] is False assert settings["sandbox"]["network"]["deniedDomains"] == ["*"] assert SANDBOX_EVIDENCE in settings["sandbox"]["filesystem"]["allowRead"] - normalizer_hook = settings["hooks"]["PreToolUse"][0] - assert normalizer_hook["matcher"] == r"Read|mcp__gitnexus__.*" - assert normalizer_hook["hooks"][0]["command"] == SANDBOX_TOOL_INPUT_NORMALIZER + # Headless `claude -p` (2.1.247) never dispatches PreToolUse from any + # settings source, so a hook here would be confinement theater: it would + # read as a control in review while enforcing nothing at runtime. + assert "hooks" not in settings # ENV_SCRUB forces "default" mode; the proposer's tools (Bash writes the # overlay) run headless only because they are explicitly pre-approved. # Requesting a non-default defaultMode would merely warn, so it must be gone. @@ -178,32 +178,6 @@ def test_sandbox_command_has_minimal_mounts_and_no_host_root_bind(tmp_path: Path assert probe.returncode == 0, probe.stderr assert probe.stdout == f"/home/agent|{SANDBOX_PATH}" - normalizer_index = argv.index(SANDBOX_TOOL_INPUT_NORMALIZER) - normalizer = Path(argv[normalizer_index - 1]) - event = json.dumps( - { - "hook_event_name": "PreToolUse", - "tool_name": "Read", - "tool_input": { - "file_path": "/evidence/rows.json", - "pages": "", - "limit": 100, - }, - } - ) - normalized = subprocess.run( - [sys.executable, normalizer], - input=event, - text=True, - capture_output=True, - check=True, - ) - output = json.loads(normalized.stdout) - assert output["hookSpecificOutput"]["updatedInput"] == { - "file_path": "/evidence/rows.json", - "limit": 100, - } - gitnexus_index = argv.index(SANDBOX_GITNEXUS_CLI) gitnexus_wrapper = Path(argv[gitnexus_index - 1]) assert stat.S_IMODE(gitnexus_wrapper.stat().st_mode) == 0o500 @@ -883,7 +857,7 @@ def test_clone_controlled_mcp_replacement_is_never_executed_or_credentialed(tmp_ os.environ.get("GITNEXUS_REQUIRE_CLAUDE_CANARY") != "1", reason="real Claude/Bash/MCP canary is mandatory in the named Ubuntu CI job", ) -def test_real_claude_hooks_auth_inner_sandbox_and_mcp_permissions(tmp_path: Path) -> None: +def test_real_claude_auth_inner_sandbox_and_mcp_permissions(tmp_path: Path) -> None: """Exercise the exact CLI boundary without contacting a paid model.""" claude = Path(os.environ["CLAUDE_CANARY_BIN"]).resolve() diff --git a/eval/workflow_bench/README.md b/eval/workflow_bench/README.md index de3e3c773..d3934d93f 100644 --- a/eval/workflow_bench/README.md +++ b/eval/workflow_bench/README.md @@ -258,7 +258,12 @@ loser rows, their session transcripts and patches, the rejected `proposal.md` when available (including a workflow seed from a prior run), and the learning queue, then writes ONE bounded candidate overlay plus a reviewer-facing -`proposal.md`. The overlay is re-validated by `candidate_overlay_files` +`proposal.md`. The proposer's clone is sanitized exactly like an arm's before +its session starts: it authors the artifact the arms are scored with, so +letting it read `eval/workflow_bench` would hand it the task prompts and the +hidden oracles it is about to be graded against, and a proposal could win the +gate by encoding the expected behavior into a skill rather than by being a +better skill. The overlay is re-validated by `candidate_overlay_files` (same boundary: Markdown under the plan/work trees, nothing else), frozen, and exercised only by its exact required pairs. Task refs are resolved once before generation zero and the immutable task bindings are forwarded to every diff --git a/eval/workflow_bench/evolve.py b/eval/workflow_bench/evolve.py index a9c9c977a..00cd1f11a 100644 --- a/eval/workflow_bench/evolve.py +++ b/eval/workflow_bench/evolve.py @@ -66,7 +66,7 @@ from .evolution import ( candidate_overlay_files, required_candidate_arms, ) -from .oracle_assets import MAX_CLONE_REFS +from .oracle_assets import MAX_CLONE_REFS, sanitize_clone_for_hidden_oracles from .promotion_apply import ( apply_promoted_overlay as apply_promoted_overlay, committed_destination_base_digests as committed_destination_base_digests, @@ -639,7 +639,7 @@ def stage_proposer_evidence_bundle( # The proposer's exact tool surface. Read/Grep/Glob observe the read-only # evidence bundle and the incumbent skills; Bash writes the candidate overlay. # `--tools` restricts non-bare Claude to this list, so Write/Edit/Skill/Web are -# unavailable while the trusted PreToolUse normalizer remains enabled. Settings +# unavailable, and Grep/Glob stay available (--bare would drop them). Settings # pre-authorize Bash via autoAllowBashIfSandboxed, and the sandbox filesystem # policy confines writes to workspace/tmp/home. Exported so containment tests # exercise the production allowlist without drift. @@ -662,6 +662,14 @@ def run_proposer( clone = runner.make_worktree(REPO_ROOT, "HEAD", Path(tmp)) primary: BaseException | None = None try: + # The proposer authors the skill overlay that the arms are then + # scored with, so it must not see what it is scored against. Its + # clone carries eval/workflow_bench — the task prompts and the + # hidden oracles — which would let a proposal encode the expected + # behavior directly into a skill and win the gate without the + # skill being any better. Strip it from the working tree and from + # recoverable history exactly as the benchmark arms do. + sanitize_clone_for_hidden_oracles(clone) output_root = clone / ".wfbench-output" output_root.mkdir(mode=0o700) internal_overlay = output_root / "overlay" @@ -693,9 +701,9 @@ def run_proposer( # No permission_mode: CLAUDE_CODE_SUBPROCESS_ENV_SCRUB # forces "default", so requesting dontAsk only warns. Tools # are pre-approved via settings permissions.allow - # (proposer_sandbox.build_claude_settings). Do not use - # Claude's --bare flag here: it disables that trusted - # PreToolUse hook along with untrusted hooks/plugins. + # (proposer_sandbox.build_claude_settings). Not --bare: + # bare ignores --tools and imposes its own Bash/Edit/Read + # ceiling, which would cost the proposer Grep and Glob. command_prefix=sandbox.command_prefix, require_pid_namespace=True, settings_json=sandbox.settings_json, diff --git a/eval/workflow_bench/proposer_sandbox.py b/eval/workflow_bench/proposer_sandbox.py index c551b28f8..1eca4819e 100644 --- a/eval/workflow_bench/proposer_sandbox.py +++ b/eval/workflow_bench/proposer_sandbox.py @@ -28,7 +28,6 @@ SANDBOX_CLAUDE = "/opt/claude/claude" SANDBOX_SHELL_PREFIX = "/opt/claude/shell-prefix" SANDBOX_PYTHON3 = "/opt/claude/python3" SANDBOX_GITNEXUS_CLI = "/opt/claude/gitnexus" -SANDBOX_TOOL_INPUT_NORMALIZER = "/opt/claude/normalize-tool-input" SANDBOX_GIT_EXCLUDES = "/opt/claude/git-excludes" SANDBOX_NODE = "/opt/claude/node" SANDBOX_NODE_PREFIX = "/opt/claude/nodejs" @@ -328,7 +327,15 @@ def build_sandbox_environment( def build_claude_settings() -> str: - """Inline settings: one trusted normalizer hook; every Bash stays sandboxed.""" + """Inline settings that keep every Bash sandboxed and pre-approve the tools. + + Deliberately hook-free: headless ``claude -p`` (2.1.247) never dispatches + ``PreToolUse``, whatever source the hook is declared in — inline + ``--settings``, a settings file, project/user/local ``--setting-sources``, + or a trusted project entry in ``~/.claude.json``. Confinement therefore + rests only on mechanisms the CLI honors in this mode: the sandbox policy + below, ``--tools``/``--allowedTools``, and the bwrap mounts. + """ settings = { "sandbox": { @@ -362,20 +369,6 @@ def build_claude_settings() -> str: ], }, }, - "hooks": { - "PreToolUse": [ - { - "matcher": r"Read|mcp__gitnexus__.*", - "hooks": [ - { - "type": "command", - "command": SANDBOX_TOOL_INPUT_NORMALIZER, - "timeout": 5, - } - ], - } - ] - }, "permissions": { # CLAUDE_CODE_SUBPROCESS_ENV_SCRUB forces permission mode to # "default" (allowed_non_write_users hardening), so requesting a @@ -509,36 +502,6 @@ def _create_gitnexus_wrapper(private_root: Path) -> Path: return wrapper -def _create_tool_input_normalizer(private_root: Path) -> Path: - """Create the trusted PreToolUse adapter for strict provider defaults.""" - - wrapper = private_root / "normalize-tool-input" - wrapper.write_text( - f"#!{SANDBOX_PYTHON3}\n" - "import json, sys\n" - "try:\n" - " event = json.load(sys.stdin)\n" - " tool_input = event.get('tool_input')\n" - " if not isinstance(tool_input, dict):\n" - " raise ValueError('tool_input is not an object')\n" - " normalized = {\n" - " key: value for key, value in tool_input.items()\n" - " if not (isinstance(value, str) and not value.strip())\n" - " }\n" - " if normalized != tool_input:\n" - " print(json.dumps({'hookSpecificOutput': {\n" - " 'hookEventName': 'PreToolUse',\n" - " 'permissionDecision': 'allow',\n" - " 'updatedInput': normalized,\n" - " }}, separators=(',', ':')))\n" - "except Exception as exc:\n" - " print(f'tool-input normalizer failed closed: {exc}', file=sys.stderr)\n" - " sys.exit(2)\n" - ) - wrapper.chmod(0o500) - return wrapper - - def _create_git_excludes(private_root: Path) -> Path: """Create the immutable excludes for nested-sandbox mount artifacts.""" @@ -821,7 +784,6 @@ def prepare_sandbox( shell_prefix = _create_shell_prefix_wrapper(private_root) python3_wrapper = _create_python3_wrapper(private_root) gitnexus_wrapper = _create_gitnexus_wrapper(private_root) - tool_input_normalizer = _create_tool_input_normalizer(private_root) git_excludes = _create_git_excludes(private_root) # Claude may discover user-level skills below HOME. Keep the rest of HOME # writable for normal CLI state, but overlay an immutable empty skills root @@ -835,7 +797,6 @@ def prepare_sandbox( ReadOnlyMount(source=shell_prefix, target=SANDBOX_SHELL_PREFIX), ReadOnlyMount(source=python3_wrapper, target=SANDBOX_PYTHON3), ReadOnlyMount(source=gitnexus_wrapper, target=SANDBOX_GITNEXUS_CLI), - ReadOnlyMount(source=tool_input_normalizer, target=SANDBOX_TOOL_INPUT_NORMALIZER), ReadOnlyMount(source=git_excludes, target=SANDBOX_GIT_EXCLUDES), ) primary: BaseException | None = None