diff --git a/eval/tests/test_workflow_bench_sessions.py b/eval/tests/test_workflow_bench_sessions.py index 4f6ae459e..e18213fa7 100644 --- a/eval/tests/test_workflow_bench_sessions.py +++ b/eval/tests/test_workflow_bench_sessions.py @@ -355,7 +355,7 @@ def test_mcp_config_uses_only_the_minimal_pinned_harness_runtime(monkeypatch, tm directory.mkdir(parents=True) (runtime / "dist" / "cli" / "index.js").write_text("") (runtime / "hooks" / "claude" / "resolve-analyze-cmd.cjs").write_text("") - (runtime / "package.json").write_text(json.dumps({"version": runner.PINNED_GITNEXUS_VERSION})) + (runtime / "package.json").write_text(json.dumps({"version": "9.9.9-test"})) (runtime / "node_modules" / "gitnexus-shared").symlink_to(shared, target_is_directory=True) (shared / "package.json").write_text(json.dumps({"name": "gitnexus-shared"})) monkeypatch.setattr(runtime_mounts, "HARNESS_ROOT", tmp_path) @@ -379,9 +379,6 @@ def test_mcp_config_uses_only_the_minimal_pinned_harness_runtime(monkeypatch, tm (shared / "package.json", f"{runner.SANDBOX_GITNEXUS_SHARED}/package.json"), (runtime / "hooks" / "claude", f"{runner.SANDBOX_GITNEXUS}/hooks/claude"), ] - package = json.loads((runtime / "package.json").read_text()) - assert package["version"] == runner.PINNED_GITNEXUS_VERSION - mounted_sources = {mount.source for mount in mounts} mounted_targets = {mount.target for mount in mounts} assert runtime not in mounted_sources @@ -458,7 +455,11 @@ def test_real_bubblewrap_runtime_mount_imports_cli_without_exposing_checkout(tmp assert visibility.ok, visibility.stderr_tail assert imported.ok, imported.stderr_tail assert analyze_imported.ok, analyze_imported.stderr_tail - assert imported.stdout_tail.strip() == runner.PINNED_GITNEXUS_VERSION + # The runtime the sandbox sees must be the one this checkout built — + # compared against the checkout itself rather than a constant, so a release + # bump cannot fail a benchmark that is running exactly what it should. + built = json.loads((runtime_mounts.HARNESS_ROOT / "gitnexus" / "package.json").read_text()) + assert imported.stdout_tail.strip() == built["version"] def test_isolated_mcp_registry_contains_only_the_sandbox_clone(tmp_path): diff --git a/eval/workflow_bench/runner.py b/eval/workflow_bench/runner.py index eaf62fdf1..a0edbe399 100644 --- a/eval/workflow_bench/runner.py +++ b/eval/workflow_bench/runner.py @@ -144,7 +144,6 @@ from .runtime_mounts import ( CE_ARMS, CePluginSnapshot, HARNESS_ROOT as HARNESS_ROOT, - PINNED_GITNEXUS_VERSION as PINNED_GITNEXUS_VERSION, ce_plugin_dir_for_arm, ce_plugin_mounts_for_arm, staged_ce_plugin_snapshot, diff --git a/eval/workflow_bench/runtime_mounts.py b/eval/workflow_bench/runtime_mounts.py index 3f2e6fcf2..04bacdf55 100644 --- a/eval/workflow_bench/runtime_mounts.py +++ b/eval/workflow_bench/runtime_mounts.py @@ -28,7 +28,6 @@ from .proposer_sandbox import ( SandboxError, ) -PINNED_GITNEXUS_VERSION = "1.6.9" HARNESS_ROOT = Path(__file__).resolve().parents[2] CE_ARMS = frozenset({"ce_workflow", "ce_workflow_direct", "ce_review"}) @@ -233,11 +232,8 @@ def trusted_gitnexus_runtime_mounts() -> tuple[ReadOnlyMount, ...]: raise SandboxError(f"pinned GitNexus runtime metadata is invalid: {exc}") from exc if stat.S_ISLNK(entrypoint_mode) or not stat.S_ISREG(entrypoint_mode): raise SandboxError(f"pinned GitNexus runtime entrypoint must be regular and non-symlink: {entrypoint}") - if package.get("version") != PINNED_GITNEXUS_VERSION: - raise SandboxError( - "pinned GitNexus runtime version drifted: " - f"expected {PINNED_GITNEXUS_VERSION}, got {package.get('version')!r}" - ) + if not isinstance(package.get("version"), str) or not package["version"]: + raise SandboxError("pinned GitNexus runtime package.json has no version") linked_shared = mounts[2].source / "gitnexus-shared" if not linked_shared.is_symlink() or linked_shared.resolve(strict=True) != shared: