diff --git a/eval/tests/test_proposer_sandbox.py b/eval/tests/test_proposer_sandbox.py index e907c1a0c..fc3ddef4b 100644 --- a/eval/tests/test_proposer_sandbox.py +++ b/eval/tests/test_proposer_sandbox.py @@ -22,6 +22,7 @@ from workflow_bench.proposer_sandbox import ( MAX_EVIDENCE_FILE_BYTES, SANDBOX_NODE, SANDBOX_NODE_PREFIX, + VITE_TEMP_DIR, SANDBOX_PATH, SANDBOX_PYTHON3, SANDBOX_SHELL_PREFIX, @@ -326,6 +327,93 @@ def test_runtime_mounts_skip_the_node_bind_when_node_is_unresolvable(monkeypatch assert SANDBOX_NODE not in args +def test_node_modules_mounts_get_a_writable_vite_temp_overlay(tmp_path: Path) -> None: + # vite writes /.vite-temp/.timestamp-*.mjs before + # loading a TypeScript config, so a read-only dependency mount makes vitest + # fail with EROFS before any test runs -- and every task verify command and + # every hidden oracle ends in "npx vitest run ". Reproduced on the + # self-hosted runner with npx bypassed entirely, proving it is independent + # of the node-prefix mount. + clone = tmp_path / "clone" + clone.mkdir() + deps = tmp_path / "deps" + deps.mkdir() + # task_assets.py captures this directory into the dependency snapshot; the + # overlay is gated on the mount source actually carrying it. + (deps / VITE_TEMP_DIR).mkdir() + executable = tmp_path / "executable" + executable.write_text("#!/bin/sh\nexit 0\n") + executable.chmod(0o755) + + with prepare_sandbox( + clone=clone, + claude_bin=executable, + bwrap_bin=executable, + preflight=False, + read_only_mounts=(ReadOnlyMount(source=deps, target="/workspace/gitnexus/node_modules"),), + ) as sandbox: + argv = sandbox.command_prefix + + bind_index = argv.index("/workspace/gitnexus/node_modules") + assert argv[bind_index - 2 : bind_index + 1] == ["--ro-bind", str(deps), "/workspace/gitnexus/node_modules"] + overlay = f"/workspace/gitnexus/node_modules/{VITE_TEMP_DIR}" + overlay_index = argv.index(overlay) + assert argv[overlay_index - 1] == "--tmpfs" + # the overlay must come AFTER the read-only bind, or the bind would mask it + assert overlay_index > bind_index + + +def test_node_modules_mount_without_a_captured_vite_temp_gets_no_overlay(tmp_path: Path) -> None: + # The trusted GitNexus runtime mounts /opt/gitnexus/node_modules, whose + # source is the built runtime and does NOT carry a .vite-temp. bwrap cannot + # mkdir a mount point inside a read-only bind, so overlaying it would fail + # with "Can't mkdir .../node_modules/.vite-temp: Read-only file system". + # Regression for that CI failure: the overlay must fire only where the + # source actually contains the directory, not for every node_modules mount. + clone = tmp_path / "clone" + clone.mkdir() + runtime = tmp_path / "runtime-node-modules" + runtime.mkdir() # deliberately no .vite-temp + executable = tmp_path / "executable" + executable.write_text("#!/bin/sh\nexit 0\n") + executable.chmod(0o755) + + with prepare_sandbox( + clone=clone, + claude_bin=executable, + bwrap_bin=executable, + preflight=False, + read_only_mounts=(ReadOnlyMount(source=runtime, target="/opt/gitnexus/node_modules"),), + ) as sandbox: + argv = sandbox.command_prefix + + assert "/opt/gitnexus/node_modules" in argv + assert not any(str(item).endswith(f"/{VITE_TEMP_DIR}") for item in argv) + + +def test_non_node_modules_mounts_get_no_vite_temp_overlay(tmp_path: Path) -> None: + # Scoped to dependency mounts: a hidden-oracle or skill mount stays wholly + # read-only, with no writable island inside it. + clone = tmp_path / "clone" + clone.mkdir() + other = tmp_path / "oracle" + other.mkdir() + executable = tmp_path / "executable" + executable.write_text("#!/bin/sh\nexit 0\n") + executable.chmod(0o755) + + with prepare_sandbox( + clone=clone, + claude_bin=executable, + bwrap_bin=executable, + preflight=False, + read_only_mounts=(ReadOnlyMount(source=other, target="/workspace/.wfbench-oracle-abc"),), + ) as sandbox: + argv = sandbox.command_prefix + + assert not any(str(item).endswith(f"/{VITE_TEMP_DIR}") for item in argv) + + def test_stricter_prefix_freezes_evaluated_skills_and_can_unshare_network(tmp_path: Path) -> None: clone = tmp_path / "clone" skill = clone / ".claude" / "skills" / "gitnexus-work" diff --git a/eval/tests/test_task_assets.py b/eval/tests/test_task_assets.py index 4479fce4c..dc5d63ac7 100644 --- a/eval/tests/test_task_assets.py +++ b/eval/tests/test_task_assets.py @@ -9,7 +9,7 @@ from pathlib import Path import pytest -from workflow_bench.proposer_sandbox import SandboxError +from workflow_bench.proposer_sandbox import VITE_TEMP_DIR, SandboxError from workflow_bench.oracle_assets import TaskOracleSnapshot from workflow_bench.runner_tasks import resolve_task_bindings from workflow_bench.task_assets import TaskAssetCache, stage_task_assets @@ -410,3 +410,36 @@ def test_resolved_task_binding_carries_dependency_digests_and_rejects_live_drift (repo / "dependency" / "package.json").write_bytes(b'{"version":2}') with pytest.raises(ValueError, match="definition drifted"): resolve_task_bindings([task], [binding], oracle_snapshots=[oracle]) + + +def test_node_modules_dependency_snapshot_captures_the_vite_temp_mount_point(tmp_path: Path) -> None: + # bwrap cannot mkdir a mount point inside an already-read-only bind, so the + # directory vite needs must exist in the captured dependency bytes. It is + # recorded during capture, which puts it inside the manifest and both + # dependency digests rather than leaving it an untracked mutation of a + # digest-bound snapshot. + repo, _ = _repo_and_task(tmp_path, {"dependency/package.json": b'{"version":1}'}) + task = { + "sandbox_copy": [], + "sandbox_dependencies": [{"source": "dependency", "target": "gitnexus/node_modules"}], + } + with TaskAssetCache(tmp_path / "cache") as cache: + snapshot = cache.prepare(task, repo=repo, resolved_sha=SHA) + captured = {entry.path.as_posix() for entry in snapshot.dependencies[0].entries} + assert f"payload/{VITE_TEMP_DIR}" in captured + vite_temp = next((snapshot.root / "dependencies").glob(f"*/payload/{VITE_TEMP_DIR}")) + assert vite_temp.is_dir() + + +def test_non_node_modules_dependency_snapshot_has_no_vite_temp(tmp_path: Path) -> None: + # The capture is scoped to dependency mounts whose target is node_modules; + # an unrelated vendored dependency is captured byte-for-byte as declared. + repo, _ = _repo_and_task(tmp_path, {"dependency/package.json": b'{"version":1}'}) + task = { + "sandbox_copy": [], + "sandbox_dependencies": [{"source": "dependency", "target": "vendor/dependency"}], + } + with TaskAssetCache(tmp_path / "cache") as cache: + snapshot = cache.prepare(task, repo=repo, resolved_sha=SHA) + captured = {entry.path.as_posix() for entry in snapshot.dependencies[0].entries} + assert not any(path.endswith(VITE_TEMP_DIR) for path in captured) diff --git a/eval/workflow_bench/proposer_sandbox.py b/eval/workflow_bench/proposer_sandbox.py index 7bd46e03c..ea84c545a 100644 --- a/eval/workflow_bench/proposer_sandbox.py +++ b/eval/workflow_bench/proposer_sandbox.py @@ -29,6 +29,14 @@ SANDBOX_SHELL_PREFIX = "/opt/claude/shell-prefix" SANDBOX_PYTHON3 = "/opt/claude/python3" SANDBOX_NODE = "/opt/claude/node" SANDBOX_NODE_PREFIX = "/opt/claude/nodejs" +# Vite transpiles a TypeScript config into /.vite-temp before it +# loads anything, so a read-only dependency mount makes `vitest` die with EROFS +# before a single test runs -- and every task verify command and every hidden +# oracle ends in `npx vitest run `. bwrap cannot create a mount point +# inside an already-read-only bind, so the directory is captured into the +# dependency snapshot (task_assets.py) and a tmpfs is overlaid on it here. +VITE_TEMP_DIR = ".vite-temp" +DEPENDENCY_MOUNT_BASENAME = "node_modules" SANDBOX_PATH = f"/opt/claude:{SANDBOX_NODE_PREFIX}/bin:/usr/local/bin:/usr/bin:/bin" SANDBOX_GITNEXUS = "/opt/gitnexus" SANDBOX_GITNEXUS_SHARED = "/opt/gitnexus-shared" @@ -678,6 +686,20 @@ def _sandbox_command_prefix( ] for mount in mounts: args += ["--ro-bind", str(mount.source), mount.target] + # Overlay an empty writable tmpfs on the one path vite must write. + # Everything else in the mount, and the whole workspace, stays + # read-only, and the overlay lives only inside the sandbox -- it never + # reaches the host clone the credited patch is captured from. + # + # Gate on the mount SOURCE actually containing the directory, not on + # the target name: bwrap cannot create a mount point inside an + # already-read-only bind, so a tmpfs can only be overlaid where the + # directory already exists in the bound bytes. task_assets.py captures + # it into dependency-snapshot node_modules; other node_modules mounts + # (e.g. the trusted GitNexus runtime at /opt/gitnexus/node_modules) do + # not carry it, and overlaying them would fail with EROFS. + if PurePosixPath(mount.target).name == DEPENDENCY_MOUNT_BASENAME and (mount.source / VITE_TEMP_DIR).is_dir(): + args += ["--tmpfs", f"{mount.target}/{VITE_TEMP_DIR}"] args += ["--chdir", SANDBOX_WORKSPACE, "--"] return args diff --git a/eval/workflow_bench/task_assets.py b/eval/workflow_bench/task_assets.py index 14815e0d6..6cd96d84d 100644 --- a/eval/workflow_bench/task_assets.py +++ b/eval/workflow_bench/task_assets.py @@ -25,7 +25,9 @@ from pathlib import Path, PurePosixPath from typing import Any from .proposer_sandbox import ( + DEPENDENCY_MOUNT_BASENAME, SANDBOX_WORKSPACE, + VITE_TEMP_DIR, ReadOnlyMount, SandboxError, _prepare_clone_target, @@ -160,9 +162,11 @@ class TaskAssetSnapshot: source = snapshot_root / Path(*dependency.snapshot_path.parts) metadata = source.lstat() expected_directory = dependency.kind == "directory" - if stat.S_ISLNK(metadata.st_mode) or ( - expected_directory and not stat.S_ISDIR(metadata.st_mode) - ) or (not expected_directory and not stat.S_ISREG(metadata.st_mode)): + if ( + stat.S_ISLNK(metadata.st_mode) + or (expected_directory and not stat.S_ISDIR(metadata.st_mode)) + or (not expected_directory and not stat.S_ISREG(metadata.st_mode)) + ): raise SandboxError(f"dependency snapshot changed: {dependency.source}") target = PurePosixPath(dependency.target) _prepare_clone_target( @@ -213,9 +217,7 @@ class TaskAssetCache: repo_identity = _real_directory(repo, label="task asset repository") declarations, relative_paths = _sandbox_copy_declarations(task) dependency_declarations = _sandbox_dependency_declarations(task) - dependency_identity = tuple( - (declaration.source, declaration.target) for declaration in dependency_declarations - ) + dependency_identity = tuple((declaration.source, declaration.target) for declaration in dependency_declarations) definition = (str(repo_identity), resolved_sha, declarations, dependency_identity) existing = self._by_definition.get(definition) if existing is not None: @@ -258,6 +260,21 @@ class TaskAssetCache: dependency_builder.copy_descriptor(descriptor, PurePosixPath("payload")) finally: os.close(descriptor) + # vitest cannot start against a read-only node_modules: vite + # writes /.vite-temp/.timestamp-*.mjs + # before loading a TypeScript config. bwrap cannot create + # that mount point inside an already-read-only bind, so the + # empty directory is captured here -- before the manifest and + # both dependency digests are computed, so it is part of the + # snapshot rather than an untracked mutation of it. The + # sandbox overlays a tmpfs on it; see VITE_TEMP_DIR. + payload_entry = dependency_builder.entries.get(PurePosixPath("payload")) + if ( + payload_entry is not None + and payload_entry.kind == "directory" + and PurePosixPath(declaration.target).name == DEPENDENCY_MOUNT_BASENAME + ): + dependency_builder.ensure_directory(PurePosixPath("payload") / VITE_TEMP_DIR) dependency_entries = dependency_builder.finished_entries() _validate_dependency_symlinks( container, @@ -462,10 +479,14 @@ class _SnapshotBuilder: destination = self.destination / Path(*relative.parts) os.symlink(target, destination) after = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) - if _mutation_identity(before) != _mutation_identity(after) or os.readlink( - name, - dir_fd=parent_descriptor, - ) != target: + if ( + _mutation_identity(before) != _mutation_identity(after) + or os.readlink( + name, + dir_fd=parent_descriptor, + ) + != target + ): raise SandboxError(f"dependency symlink changed while snapshotting: {relative}") self.total_bytes += len(target_bytes) self.budget.total_bytes += len(target_bytes) @@ -503,6 +524,15 @@ class _SnapshotBuilder: self.entries[entry.path] = entry self.budget.entries += 1 + def ensure_directory(self, relative: PurePosixPath) -> None: + """Record and create one extra directory inside this snapshot. + + Used for harness-owned mount points that must exist in the captured + bytes rather than be created against a read-only bind at runtime. + """ + + self._record_directory(relative) + def finished_entries(self) -> tuple[AssetManifestEntry, ...]: return tuple(sorted(self.entries.values(), key=lambda entry: entry.path.as_posix())) @@ -567,9 +597,7 @@ def _sandbox_dependency_declarations( or declaration.target_path in other.target_path.parents or other.target_path in declaration.target_path.parents ): - raise SandboxError( - f"sandbox dependency targets overlap: {declaration.target} and {other.target}" - ) + raise SandboxError(f"sandbox dependency targets overlap: {declaration.target} and {other.target}") return tuple(declarations) @@ -651,9 +679,7 @@ def _validate_dependency_symlinks( ) if sandbox_resolved != sandbox_boundary and sandbox_boundary not in sandbox_resolved.parents: raise SandboxError(f"dependency symlink escapes the sandbox workspace: {entry.path}") - manifest_resolved = PurePosixPath( - posixpath.normpath((entry.path.parent / target).as_posix()) - ) + manifest_resolved = PurePosixPath(posixpath.normpath((entry.path.parent / target).as_posix())) if manifest_resolved != manifest_boundary and manifest_boundary not in manifest_resolved.parents: continue link = container / Path(*entry.path.parts) @@ -1021,8 +1047,7 @@ def _dependency_mounts( snapshot: TaskAssetSnapshot, ) -> list[ReadOnlyMount]: declarations = tuple( - (declaration.source, declaration.target) - for declaration in _sandbox_dependency_declarations(task) + (declaration.source, declaration.target) for declaration in _sandbox_dependency_declarations(task) ) if snapshot.dependency_declarations != declarations: raise SandboxError("task asset snapshot does not match this dependency declaration")