From eb116c8a0748139fec5d08966b29ca2b0057c06a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Tue, 21 Jul 2026 19:19:15 +0100 Subject: [PATCH] fix(eval): exclude Claude Code's own sandbox-bootstrap noise from the planning-phase check (#2615) The first fully successful real workflow_dispatch run on the self-hosted runner (https://github.com/abhigyanpatwari/GitNexus/actions/runs/29843028596) still failed: 17/18 sessions hit error_kind plan-evidence-invalid with "phase changed unauthorized workspace path(s): .claude/.cc-writes, .claude/commands, .env, .env.development, ...". Reproduced directly on the runner (SSM, matching the real sandbox settings exactly, including enableWeakerNestedSandbox): a single trivial "say OK" prompt -- no real task, no real API key even -- is enough to make Claude Code create a synthetic package.json/lockfiles/node_modules, a full set of .env variants, and .claude/agents, .claude/commands, .claude/.cc-writes in the workspace on every single session. None of this is something the model decided to write; it's Claude Code's own internal bootstrap for running inside an already-sandboxed environment, and it happens regardless of task or prompt. enforce_phase_workspace (the planning-phase boundary check: verify the plan session touched only its one plan doc) already excludes .git for exactly this class of reason -- harness/tool noise, not substantive diff. Extends the same exclusion to the empirically-observed bootstrap set. workspace_snapshot has exactly one use (this check, confirmed via every caller), so widening its exclusion list can't hide anything in some other context that actually cares about these paths changing. Co-authored-by: Claude Sonnet 5 --- eval/tests/test_runner_hardening.py | 34 +++++++++++++++++++++ eval/workflow_bench/runner_artifacts.py | 40 +++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/eval/tests/test_runner_hardening.py b/eval/tests/test_runner_hardening.py index 8245fdf82..6905891c0 100644 --- a/eval/tests/test_runner_hardening.py +++ b/eval/tests/test_runner_hardening.py @@ -262,3 +262,37 @@ def test_phase_workspace_accepts_new_regular_review_output(tmp_path): artifact.write_text("new review") runner_artifacts.enforce_phase_workspace(tmp_path, before, allowed_artifact=artifact) + + +def test_phase_workspace_ignores_claude_sandbox_bootstrap_noise(tmp_path): + # Reproduced empirically: Claude Code's own enableWeakerNestedSandbox + # bootstrap creates this exact set of paths on every session regardless + # of task or model output (a trivial "say OK" prompt was enough). None + # of it is something the model decided to write, so it must not read as + # an unauthorized planning-phase change. + before = runner_artifacts.workspace_snapshot(tmp_path) + (tmp_path / ".claude" / "agents").mkdir(parents=True) + (tmp_path / ".claude" / "commands").mkdir(parents=True) + (tmp_path / ".claude" / ".cc-writes").write_text("{}") + (tmp_path / ".env").write_text("") + (tmp_path / ".env.development.local").write_text("") + (tmp_path / ".npmrc").write_text("") + (tmp_path / "package.json").write_text("{}") + (tmp_path / "node_modules").mkdir() + (tmp_path / "node_modules" / ".bin").mkdir() + artifact = tmp_path / "review-output.md" + artifact.write_text("new review") + + runner_artifacts.enforce_phase_workspace(tmp_path, before, allowed_artifact=artifact) + + +def test_phase_workspace_still_rejects_a_genuinely_unauthorized_change(tmp_path): + # The bootstrap-noise exclusion must stay narrow: an actual source-file + # edit outside the allowed artifact still has to be caught. + before = runner_artifacts.workspace_snapshot(tmp_path) + (tmp_path / "src.py").write_text("changed") + artifact = tmp_path / "review-output.md" + artifact.write_text("new review") + + with pytest.raises(ValueError, match="unauthorized workspace path"): + runner_artifacts.enforce_phase_workspace(tmp_path, before, allowed_artifact=artifact) diff --git a/eval/workflow_bench/runner_artifacts.py b/eval/workflow_bench/runner_artifacts.py index 0b56a0c9a..da6b9ae58 100644 --- a/eval/workflow_bench/runner_artifacts.py +++ b/eval/workflow_bench/runner_artifacts.py @@ -21,6 +21,40 @@ MAX_WORKSPACE_SNAPSHOT_ENTRIES = 100_000 MAX_WORKSPACE_SNAPSHOT_PATH_BYTES = 16 * 1024 * 1024 MAX_WORKSPACE_SNAPSHOT_FILE_BYTES = 1024 * 1024 * 1024 +# Claude Code's own enableWeakerNestedSandbox bootstrap creates these paths on +# EVERY session regardless of task or model output -- reproduced empirically +# with a trivial "say OK" prompt: a synthetic package.json/lockfiles/ +# node_modules, a full set of .env variants, and .claude/agents, +# .claude/commands, .claude/.cc-writes. None of this is something the model +# decided to write, so it must not count as an "unauthorized" workspace +# change during the planning-phase boundary check (the one thing this +# snapshot is used for -- see workspace_snapshot's callers). Mirrors the +# pre-existing .git exclusion below, which is the same kind of harness/tool +# noise rather than substantive diff. +WORKSPACE_SNAPSHOT_BOOTSTRAP_NOISE = frozenset( + { + ".claude", + ".env", + ".env.development", + ".env.development.local", + ".env.local", + ".env.production", + ".env.production.local", + ".env.test", + ".env.test.local", + ".gitmodules", + ".npmrc", + ".yarnrc", + ".yarnrc.yml", + "bunfig.toml", + "node_modules", + "package-lock.json", + "package.json", + "pnpm-lock.yaml", + "yarn.lock", + } +) + IMPLEMENTATION_ARMS = frozenset( { "workflow", @@ -53,7 +87,9 @@ class VerificationResult: def workspace_snapshot(worktree: Path) -> dict[str, str]: - """Hash the workspace without following links, excluding Git internals.""" + """Hash the workspace without following links, excluding Git internals + and Claude Code's own sandbox-bootstrap noise (see + WORKSPACE_SNAPSHOT_BOOTSTRAP_NOISE).""" root = worktree.expanduser().absolute() mode = root.lstat().st_mode @@ -74,7 +110,7 @@ def workspace_snapshot(worktree: Path) -> dict[str, str]: raise ValueError(f"workspace snapshot directory is unreadable: {directory}: {exc}") from exc for entry in children: relative = relative_dir / entry.name - if relative.parts[0] == ".git": + if relative.parts[0] == ".git" or relative.parts[0] in WORKSPACE_SNAPSHOT_BOOTSTRAP_NOISE: continue entry_count += 1 path_bytes += len(relative.as_posix().encode())