fix(eval): stop failing the benchmark when main's version bumps

PINNED_GITNEXUS_VERSION did not select a runtime — the sandbox already
mounts gitnexus/dist built from the checkout by the workflow's own
`npm run build`, so the benchmark has always run main, not a release. The
constant only asserted that the checkout's package.json said "1.6.9" and
raised SandboxError otherwise.

That makes it a tripwire aimed at the wrong thing. main is 1.6.9 today,
so it passes; the next release bumps it and every skill-evolution run
hard-fails until someone edits this line — discovered on a Saturday, on a
lane that runs unattended once a week, after the box has already been
started and the proposer session paid for.

What the constant was guarding is still guarded, and better: the sandbox
canary now checks the version reported from inside the sandbox against
the checkout's own package.json, so it still proves the mounted runtime
is the one this checkout built, without a literal that has to be
maintained in lockstep with releases. The digest bindings in
promotion.json (sandbox_dependency_content_digest, graph and oracle
digests) remain what actually pins the substrate a candidate was measured
against.

The remaining check keeps package.json readable and versioned, so a
malformed runtime still fails closed.
This commit is contained in:
Gergo Magyar 2026-08-02 15:37:30 +00:00
parent f484e2cf34
commit a522a4fbc2
3 changed files with 8 additions and 12 deletions

View file

@ -355,7 +355,7 @@ def test_mcp_config_uses_only_the_minimal_pinned_harness_runtime(monkeypatch, tm
directory.mkdir(parents=True)
(runtime / "dist" / "cli" / "index.js").write_text("")
(runtime / "hooks" / "claude" / "resolve-analyze-cmd.cjs").write_text("")
(runtime / "package.json").write_text(json.dumps({"version": runner.PINNED_GITNEXUS_VERSION}))
(runtime / "package.json").write_text(json.dumps({"version": "9.9.9-test"}))
(runtime / "node_modules" / "gitnexus-shared").symlink_to(shared, target_is_directory=True)
(shared / "package.json").write_text(json.dumps({"name": "gitnexus-shared"}))
monkeypatch.setattr(runtime_mounts, "HARNESS_ROOT", tmp_path)
@ -379,9 +379,6 @@ def test_mcp_config_uses_only_the_minimal_pinned_harness_runtime(monkeypatch, tm
(shared / "package.json", f"{runner.SANDBOX_GITNEXUS_SHARED}/package.json"),
(runtime / "hooks" / "claude", f"{runner.SANDBOX_GITNEXUS}/hooks/claude"),
]
package = json.loads((runtime / "package.json").read_text())
assert package["version"] == runner.PINNED_GITNEXUS_VERSION
mounted_sources = {mount.source for mount in mounts}
mounted_targets = {mount.target for mount in mounts}
assert runtime not in mounted_sources
@ -458,7 +455,11 @@ def test_real_bubblewrap_runtime_mount_imports_cli_without_exposing_checkout(tmp
assert visibility.ok, visibility.stderr_tail
assert imported.ok, imported.stderr_tail
assert analyze_imported.ok, analyze_imported.stderr_tail
assert imported.stdout_tail.strip() == runner.PINNED_GITNEXUS_VERSION
# The runtime the sandbox sees must be the one this checkout built —
# compared against the checkout itself rather than a constant, so a release
# bump cannot fail a benchmark that is running exactly what it should.
built = json.loads((runtime_mounts.HARNESS_ROOT / "gitnexus" / "package.json").read_text())
assert imported.stdout_tail.strip() == built["version"]
def test_isolated_mcp_registry_contains_only_the_sandbox_clone(tmp_path):

View file

@ -144,7 +144,6 @@ from .runtime_mounts import (
CE_ARMS,
CePluginSnapshot,
HARNESS_ROOT as HARNESS_ROOT,
PINNED_GITNEXUS_VERSION as PINNED_GITNEXUS_VERSION,
ce_plugin_dir_for_arm,
ce_plugin_mounts_for_arm,
staged_ce_plugin_snapshot,

View file

@ -28,7 +28,6 @@ from .proposer_sandbox import (
SandboxError,
)
PINNED_GITNEXUS_VERSION = "1.6.9"
HARNESS_ROOT = Path(__file__).resolve().parents[2]
CE_ARMS = frozenset({"ce_workflow", "ce_workflow_direct", "ce_review"})
@ -233,11 +232,8 @@ def trusted_gitnexus_runtime_mounts() -> tuple[ReadOnlyMount, ...]:
raise SandboxError(f"pinned GitNexus runtime metadata is invalid: {exc}") from exc
if stat.S_ISLNK(entrypoint_mode) or not stat.S_ISREG(entrypoint_mode):
raise SandboxError(f"pinned GitNexus runtime entrypoint must be regular and non-symlink: {entrypoint}")
if package.get("version") != PINNED_GITNEXUS_VERSION:
raise SandboxError(
"pinned GitNexus runtime version drifted: "
f"expected {PINNED_GITNEXUS_VERSION}, got {package.get('version')!r}"
)
if not isinstance(package.get("version"), str) or not package["version"]:
raise SandboxError("pinned GitNexus runtime package.json has no version")
linked_shared = mounts[2].source / "gitnexus-shared"
if not linked_shared.is_symlink() or linked_shared.resolve(strict=True) != shared: