From a522a4fbc22295c5411d42ac8c3fb310e361fbdc Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Sun, 2 Aug 2026 15:37:30 +0000 Subject: [PATCH] fix(eval): stop failing the benchmark when main's version bumps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PINNED_GITNEXUS_VERSION did not select a runtime — the sandbox already mounts gitnexus/dist built from the checkout by the workflow's own `npm run build`, so the benchmark has always run main, not a release. The constant only asserted that the checkout's package.json said "1.6.9" and raised SandboxError otherwise. That makes it a tripwire aimed at the wrong thing. main is 1.6.9 today, so it passes; the next release bumps it and every skill-evolution run hard-fails until someone edits this line — discovered on a Saturday, on a lane that runs unattended once a week, after the box has already been started and the proposer session paid for. What the constant was guarding is still guarded, and better: the sandbox canary now checks the version reported from inside the sandbox against the checkout's own package.json, so it still proves the mounted runtime is the one this checkout built, without a literal that has to be maintained in lockstep with releases. The digest bindings in promotion.json (sandbox_dependency_content_digest, graph and oracle digests) remain what actually pins the substrate a candidate was measured against. The remaining check keeps package.json readable and versioned, so a malformed runtime still fails closed. --- eval/tests/test_workflow_bench_sessions.py | 11 ++++++----- eval/workflow_bench/runner.py | 1 - eval/workflow_bench/runtime_mounts.py | 8 ++------ 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/eval/tests/test_workflow_bench_sessions.py b/eval/tests/test_workflow_bench_sessions.py index 4f6ae459e..e18213fa7 100644 --- a/eval/tests/test_workflow_bench_sessions.py +++ b/eval/tests/test_workflow_bench_sessions.py @@ -355,7 +355,7 @@ def test_mcp_config_uses_only_the_minimal_pinned_harness_runtime(monkeypatch, tm directory.mkdir(parents=True) (runtime / "dist" / "cli" / "index.js").write_text("") (runtime / "hooks" / "claude" / "resolve-analyze-cmd.cjs").write_text("") - (runtime / "package.json").write_text(json.dumps({"version": runner.PINNED_GITNEXUS_VERSION})) + (runtime / "package.json").write_text(json.dumps({"version": "9.9.9-test"})) (runtime / "node_modules" / "gitnexus-shared").symlink_to(shared, target_is_directory=True) (shared / "package.json").write_text(json.dumps({"name": "gitnexus-shared"})) monkeypatch.setattr(runtime_mounts, "HARNESS_ROOT", tmp_path) @@ -379,9 +379,6 @@ def test_mcp_config_uses_only_the_minimal_pinned_harness_runtime(monkeypatch, tm (shared / "package.json", f"{runner.SANDBOX_GITNEXUS_SHARED}/package.json"), (runtime / "hooks" / "claude", f"{runner.SANDBOX_GITNEXUS}/hooks/claude"), ] - package = json.loads((runtime / "package.json").read_text()) - assert package["version"] == runner.PINNED_GITNEXUS_VERSION - mounted_sources = {mount.source for mount in mounts} mounted_targets = {mount.target for mount in mounts} assert runtime not in mounted_sources @@ -458,7 +455,11 @@ def test_real_bubblewrap_runtime_mount_imports_cli_without_exposing_checkout(tmp assert visibility.ok, visibility.stderr_tail assert imported.ok, imported.stderr_tail assert analyze_imported.ok, analyze_imported.stderr_tail - assert imported.stdout_tail.strip() == runner.PINNED_GITNEXUS_VERSION + # The runtime the sandbox sees must be the one this checkout built — + # compared against the checkout itself rather than a constant, so a release + # bump cannot fail a benchmark that is running exactly what it should. + built = json.loads((runtime_mounts.HARNESS_ROOT / "gitnexus" / "package.json").read_text()) + assert imported.stdout_tail.strip() == built["version"] def test_isolated_mcp_registry_contains_only_the_sandbox_clone(tmp_path): diff --git a/eval/workflow_bench/runner.py b/eval/workflow_bench/runner.py index eaf62fdf1..a0edbe399 100644 --- a/eval/workflow_bench/runner.py +++ b/eval/workflow_bench/runner.py @@ -144,7 +144,6 @@ from .runtime_mounts import ( CE_ARMS, CePluginSnapshot, HARNESS_ROOT as HARNESS_ROOT, - PINNED_GITNEXUS_VERSION as PINNED_GITNEXUS_VERSION, ce_plugin_dir_for_arm, ce_plugin_mounts_for_arm, staged_ce_plugin_snapshot, diff --git a/eval/workflow_bench/runtime_mounts.py b/eval/workflow_bench/runtime_mounts.py index 3f2e6fcf2..04bacdf55 100644 --- a/eval/workflow_bench/runtime_mounts.py +++ b/eval/workflow_bench/runtime_mounts.py @@ -28,7 +28,6 @@ from .proposer_sandbox import ( SandboxError, ) -PINNED_GITNEXUS_VERSION = "1.6.9" HARNESS_ROOT = Path(__file__).resolve().parents[2] CE_ARMS = frozenset({"ce_workflow", "ce_workflow_direct", "ce_review"}) @@ -233,11 +232,8 @@ def trusted_gitnexus_runtime_mounts() -> tuple[ReadOnlyMount, ...]: raise SandboxError(f"pinned GitNexus runtime metadata is invalid: {exc}") from exc if stat.S_ISLNK(entrypoint_mode) or not stat.S_ISREG(entrypoint_mode): raise SandboxError(f"pinned GitNexus runtime entrypoint must be regular and non-symlink: {entrypoint}") - if package.get("version") != PINNED_GITNEXUS_VERSION: - raise SandboxError( - "pinned GitNexus runtime version drifted: " - f"expected {PINNED_GITNEXUS_VERSION}, got {package.get('version')!r}" - ) + if not isinstance(package.get("version"), str) or not package["version"]: + raise SandboxError("pinned GitNexus runtime package.json has no version") linked_shared = mounts[2].source / "gitnexus-shared" if not linked_shared.is_symlink() or linked_shared.resolve(strict=True) != shared: