diff --git a/eval/tests/test_runner_hardening.py b/eval/tests/test_runner_hardening.py index 8245fdf82..6905891c0 100644 --- a/eval/tests/test_runner_hardening.py +++ b/eval/tests/test_runner_hardening.py @@ -262,3 +262,37 @@ def test_phase_workspace_accepts_new_regular_review_output(tmp_path): artifact.write_text("new review") runner_artifacts.enforce_phase_workspace(tmp_path, before, allowed_artifact=artifact) + + +def test_phase_workspace_ignores_claude_sandbox_bootstrap_noise(tmp_path): + # Reproduced empirically: Claude Code's own enableWeakerNestedSandbox + # bootstrap creates this exact set of paths on every session regardless + # of task or model output (a trivial "say OK" prompt was enough). None + # of it is something the model decided to write, so it must not read as + # an unauthorized planning-phase change. + before = runner_artifacts.workspace_snapshot(tmp_path) + (tmp_path / ".claude" / "agents").mkdir(parents=True) + (tmp_path / ".claude" / "commands").mkdir(parents=True) + (tmp_path / ".claude" / ".cc-writes").write_text("{}") + (tmp_path / ".env").write_text("") + (tmp_path / ".env.development.local").write_text("") + (tmp_path / ".npmrc").write_text("") + (tmp_path / "package.json").write_text("{}") + (tmp_path / "node_modules").mkdir() + (tmp_path / "node_modules" / ".bin").mkdir() + artifact = tmp_path / "review-output.md" + artifact.write_text("new review") + + runner_artifacts.enforce_phase_workspace(tmp_path, before, allowed_artifact=artifact) + + +def test_phase_workspace_still_rejects_a_genuinely_unauthorized_change(tmp_path): + # The bootstrap-noise exclusion must stay narrow: an actual source-file + # edit outside the allowed artifact still has to be caught. + before = runner_artifacts.workspace_snapshot(tmp_path) + (tmp_path / "src.py").write_text("changed") + artifact = tmp_path / "review-output.md" + artifact.write_text("new review") + + with pytest.raises(ValueError, match="unauthorized workspace path"): + runner_artifacts.enforce_phase_workspace(tmp_path, before, allowed_artifact=artifact) diff --git a/eval/workflow_bench/runner_artifacts.py b/eval/workflow_bench/runner_artifacts.py index 0b56a0c9a..da6b9ae58 100644 --- a/eval/workflow_bench/runner_artifacts.py +++ b/eval/workflow_bench/runner_artifacts.py @@ -21,6 +21,40 @@ MAX_WORKSPACE_SNAPSHOT_ENTRIES = 100_000 MAX_WORKSPACE_SNAPSHOT_PATH_BYTES = 16 * 1024 * 1024 MAX_WORKSPACE_SNAPSHOT_FILE_BYTES = 1024 * 1024 * 1024 +# Claude Code's own enableWeakerNestedSandbox bootstrap creates these paths on +# EVERY session regardless of task or model output -- reproduced empirically +# with a trivial "say OK" prompt: a synthetic package.json/lockfiles/ +# node_modules, a full set of .env variants, and .claude/agents, +# .claude/commands, .claude/.cc-writes. None of this is something the model +# decided to write, so it must not count as an "unauthorized" workspace +# change during the planning-phase boundary check (the one thing this +# snapshot is used for -- see workspace_snapshot's callers). Mirrors the +# pre-existing .git exclusion below, which is the same kind of harness/tool +# noise rather than substantive diff. +WORKSPACE_SNAPSHOT_BOOTSTRAP_NOISE = frozenset( + { + ".claude", + ".env", + ".env.development", + ".env.development.local", + ".env.local", + ".env.production", + ".env.production.local", + ".env.test", + ".env.test.local", + ".gitmodules", + ".npmrc", + ".yarnrc", + ".yarnrc.yml", + "bunfig.toml", + "node_modules", + "package-lock.json", + "package.json", + "pnpm-lock.yaml", + "yarn.lock", + } +) + IMPLEMENTATION_ARMS = frozenset( { "workflow", @@ -53,7 +87,9 @@ class VerificationResult: def workspace_snapshot(worktree: Path) -> dict[str, str]: - """Hash the workspace without following links, excluding Git internals.""" + """Hash the workspace without following links, excluding Git internals + and Claude Code's own sandbox-bootstrap noise (see + WORKSPACE_SNAPSHOT_BOOTSTRAP_NOISE).""" root = worktree.expanduser().absolute() mode = root.lstat().st_mode @@ -74,7 +110,7 @@ def workspace_snapshot(worktree: Path) -> dict[str, str]: raise ValueError(f"workspace snapshot directory is unreadable: {directory}: {exc}") from exc for entry in children: relative = relative_dir / entry.name - if relative.parts[0] == ".git": + if relative.parts[0] == ".git" or relative.parts[0] in WORKSPACE_SNAPSHOT_BOOTSTRAP_NOISE: continue entry_count += 1 path_bytes += len(relative.as_posix().encode())