fix(eval): exclude Claude Code's own sandbox-bootstrap noise from the planning-phase check (#2615)

The first fully successful real workflow_dispatch run on the self-hosted
runner (https://github.com/abhigyanpatwari/GitNexus/actions/runs/29843028596)
still failed: 17/18 sessions hit error_kind plan-evidence-invalid with
"phase changed unauthorized workspace path(s): .claude/.cc-writes,
.claude/commands, .env, .env.development, ...".

Reproduced directly on the runner (SSM, matching the real sandbox settings
exactly, including enableWeakerNestedSandbox): a single trivial "say OK"
prompt -- no real task, no real API key even -- is enough to make Claude
Code create a synthetic package.json/lockfiles/node_modules, a full set
of .env variants, and .claude/agents, .claude/commands, .claude/.cc-writes
in the workspace on every single session. None of this is something the
model decided to write; it's Claude Code's own internal bootstrap for
running inside an already-sandboxed environment, and it happens
regardless of task or prompt.

enforce_phase_workspace (the planning-phase boundary check: verify the
plan session touched only its one plan doc) already excludes .git for
exactly this class of reason -- harness/tool noise, not substantive diff.
Extends the same exclusion to the empirically-observed bootstrap set.
workspace_snapshot has exactly one use (this check, confirmed via every
caller), so widening its exclusion list can't hide anything in some other
context that actually cares about these paths changing.

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

eval/tests/test_runner_hardening.py

@@ -262,3 +262,37 @@ def test_phase_workspace_accepts_new_regular_review_output(tmp_path):
     artifact.write_text("new review")
 
     runner_artifacts.enforce_phase_workspace(tmp_path, before, allowed_artifact=artifact)
+
+
+def test_phase_workspace_ignores_claude_sandbox_bootstrap_noise(tmp_path):
+    # Reproduced empirically: Claude Code's own enableWeakerNestedSandbox
+    # bootstrap creates this exact set of paths on every session regardless
+    # of task or model output (a trivial "say OK" prompt was enough). None
+    # of it is something the model decided to write, so it must not read as
+    # an unauthorized planning-phase change.
+    before = runner_artifacts.workspace_snapshot(tmp_path)
+    (tmp_path / ".claude" / "agents").mkdir(parents=True)
+    (tmp_path / ".claude" / "commands").mkdir(parents=True)
+    (tmp_path / ".claude" / ".cc-writes").write_text("{}")
+    (tmp_path / ".env").write_text("")
+    (tmp_path / ".env.development.local").write_text("")
+    (tmp_path / ".npmrc").write_text("")
+    (tmp_path / "package.json").write_text("{}")
+    (tmp_path / "node_modules").mkdir()
+    (tmp_path / "node_modules" / ".bin").mkdir()
+    artifact = tmp_path / "review-output.md"
+    artifact.write_text("new review")
+
+    runner_artifacts.enforce_phase_workspace(tmp_path, before, allowed_artifact=artifact)
+
+
+def test_phase_workspace_still_rejects_a_genuinely_unauthorized_change(tmp_path):
+    # The bootstrap-noise exclusion must stay narrow: an actual source-file
+    # edit outside the allowed artifact still has to be caught.
+    before = runner_artifacts.workspace_snapshot(tmp_path)
+    (tmp_path / "src.py").write_text("changed")
+    artifact = tmp_path / "review-output.md"
+    artifact.write_text("new review")
+
+    with pytest.raises(ValueError, match="unauthorized workspace path"):
+        runner_artifacts.enforce_phase_workspace(tmp_path, before, allowed_artifact=artifact)

eval/workflow_bench/runner_artifacts.py

@@ -21,6 +21,40 @@ MAX_WORKSPACE_SNAPSHOT_ENTRIES = 100_000
 MAX_WORKSPACE_SNAPSHOT_PATH_BYTES = 16 * 1024 * 1024
 MAX_WORKSPACE_SNAPSHOT_FILE_BYTES = 1024 * 1024 * 1024
 
+# Claude Code's own enableWeakerNestedSandbox bootstrap creates these paths on
+# EVERY session regardless of task or model output -- reproduced empirically
+# with a trivial "say OK" prompt: a synthetic package.json/lockfiles/
+# node_modules, a full set of .env variants, and .claude/agents,
+# .claude/commands, .claude/.cc-writes. None of this is something the model
+# decided to write, so it must not count as an "unauthorized" workspace
+# change during the planning-phase boundary check (the one thing this
+# snapshot is used for -- see workspace_snapshot's callers). Mirrors the
+# pre-existing .git exclusion below, which is the same kind of harness/tool
+# noise rather than substantive diff.
+WORKSPACE_SNAPSHOT_BOOTSTRAP_NOISE = frozenset(
+    {
+        ".claude",
+        ".env",
+        ".env.development",
+        ".env.development.local",
+        ".env.local",
+        ".env.production",
+        ".env.production.local",
+        ".env.test",
+        ".env.test.local",
+        ".gitmodules",
+        ".npmrc",
+        ".yarnrc",
+        ".yarnrc.yml",
+        "bunfig.toml",
+        "node_modules",
+        "package-lock.json",
+        "package.json",
+        "pnpm-lock.yaml",
+        "yarn.lock",
+    }
+)
+
 IMPLEMENTATION_ARMS = frozenset(
     {
         "workflow",
@@ -53,7 +87,9 @@ class VerificationResult:
 
 
 def workspace_snapshot(worktree: Path) -> dict[str, str]:
-    """Hash the workspace without following links, excluding Git internals."""
+    """Hash the workspace without following links, excluding Git internals
+    and Claude Code's own sandbox-bootstrap noise (see
+    WORKSPACE_SNAPSHOT_BOOTSTRAP_NOISE)."""
 
     root = worktree.expanduser().absolute()
     mode = root.lstat().st_mode
@@ -74,7 +110,7 @@ def workspace_snapshot(worktree: Path) -> dict[str, str]:
             raise ValueError(f"workspace snapshot directory is unreadable: {directory}: {exc}") from exc
         for entry in children:
             relative = relative_dir / entry.name
-            if relative.parts[0] == ".git":
+            if relative.parts[0] == ".git" or relative.parts[0] in WORKSPACE_SNAPSHOT_BOOTSTRAP_NOISE:
                 continue
             entry_count += 1
             path_bytes += len(relative.as_posix().encode())