mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-07 02:58:02 +00:00
fix(eval): stop failing the benchmark when main's version bumps
PINNED_GITNEXUS_VERSION did not select a runtime — the sandbox already mounts gitnexus/dist built from the checkout by the workflow's own `npm run build`, so the benchmark has always run main, not a release. The constant only asserted that the checkout's package.json said "1.6.9" and raised SandboxError otherwise. That makes it a tripwire aimed at the wrong thing. main is 1.6.9 today, so it passes; the next release bumps it and every skill-evolution run hard-fails until someone edits this line — discovered on a Saturday, on a lane that runs unattended once a week, after the box has already been started and the proposer session paid for. What the constant was guarding is still guarded, and better: the sandbox canary now checks the version reported from inside the sandbox against the checkout's own package.json, so it still proves the mounted runtime is the one this checkout built, without a literal that has to be maintained in lockstep with releases. The digest bindings in promotion.json (sandbox_dependency_content_digest, graph and oracle digests) remain what actually pins the substrate a candidate was measured against. The remaining check keeps package.json readable and versioned, so a malformed runtime still fails closed.
This commit is contained in:
parent
f484e2cf34
commit
a522a4fbc2
3 changed files with 8 additions and 12 deletions
|
|
@ -355,7 +355,7 @@ def test_mcp_config_uses_only_the_minimal_pinned_harness_runtime(monkeypatch, tm
|
|||
directory.mkdir(parents=True)
|
||||
(runtime / "dist" / "cli" / "index.js").write_text("")
|
||||
(runtime / "hooks" / "claude" / "resolve-analyze-cmd.cjs").write_text("")
|
||||
(runtime / "package.json").write_text(json.dumps({"version": runner.PINNED_GITNEXUS_VERSION}))
|
||||
(runtime / "package.json").write_text(json.dumps({"version": "9.9.9-test"}))
|
||||
(runtime / "node_modules" / "gitnexus-shared").symlink_to(shared, target_is_directory=True)
|
||||
(shared / "package.json").write_text(json.dumps({"name": "gitnexus-shared"}))
|
||||
monkeypatch.setattr(runtime_mounts, "HARNESS_ROOT", tmp_path)
|
||||
|
|
@ -379,9 +379,6 @@ def test_mcp_config_uses_only_the_minimal_pinned_harness_runtime(monkeypatch, tm
|
|||
(shared / "package.json", f"{runner.SANDBOX_GITNEXUS_SHARED}/package.json"),
|
||||
(runtime / "hooks" / "claude", f"{runner.SANDBOX_GITNEXUS}/hooks/claude"),
|
||||
]
|
||||
package = json.loads((runtime / "package.json").read_text())
|
||||
assert package["version"] == runner.PINNED_GITNEXUS_VERSION
|
||||
|
||||
mounted_sources = {mount.source for mount in mounts}
|
||||
mounted_targets = {mount.target for mount in mounts}
|
||||
assert runtime not in mounted_sources
|
||||
|
|
@ -458,7 +455,11 @@ def test_real_bubblewrap_runtime_mount_imports_cli_without_exposing_checkout(tmp
|
|||
assert visibility.ok, visibility.stderr_tail
|
||||
assert imported.ok, imported.stderr_tail
|
||||
assert analyze_imported.ok, analyze_imported.stderr_tail
|
||||
assert imported.stdout_tail.strip() == runner.PINNED_GITNEXUS_VERSION
|
||||
# The runtime the sandbox sees must be the one this checkout built —
|
||||
# compared against the checkout itself rather than a constant, so a release
|
||||
# bump cannot fail a benchmark that is running exactly what it should.
|
||||
built = json.loads((runtime_mounts.HARNESS_ROOT / "gitnexus" / "package.json").read_text())
|
||||
assert imported.stdout_tail.strip() == built["version"]
|
||||
|
||||
|
||||
def test_isolated_mcp_registry_contains_only_the_sandbox_clone(tmp_path):
|
||||
|
|
|
|||
|
|
@ -144,7 +144,6 @@ from .runtime_mounts import (
|
|||
CE_ARMS,
|
||||
CePluginSnapshot,
|
||||
HARNESS_ROOT as HARNESS_ROOT,
|
||||
PINNED_GITNEXUS_VERSION as PINNED_GITNEXUS_VERSION,
|
||||
ce_plugin_dir_for_arm,
|
||||
ce_plugin_mounts_for_arm,
|
||||
staged_ce_plugin_snapshot,
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ from .proposer_sandbox import (
|
|||
SandboxError,
|
||||
)
|
||||
|
||||
PINNED_GITNEXUS_VERSION = "1.6.9"
|
||||
HARNESS_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
CE_ARMS = frozenset({"ce_workflow", "ce_workflow_direct", "ce_review"})
|
||||
|
|
@ -233,11 +232,8 @@ def trusted_gitnexus_runtime_mounts() -> tuple[ReadOnlyMount, ...]:
|
|||
raise SandboxError(f"pinned GitNexus runtime metadata is invalid: {exc}") from exc
|
||||
if stat.S_ISLNK(entrypoint_mode) or not stat.S_ISREG(entrypoint_mode):
|
||||
raise SandboxError(f"pinned GitNexus runtime entrypoint must be regular and non-symlink: {entrypoint}")
|
||||
if package.get("version") != PINNED_GITNEXUS_VERSION:
|
||||
raise SandboxError(
|
||||
"pinned GitNexus runtime version drifted: "
|
||||
f"expected {PINNED_GITNEXUS_VERSION}, got {package.get('version')!r}"
|
||||
)
|
||||
if not isinstance(package.get("version"), str) or not package["version"]:
|
||||
raise SandboxError("pinned GitNexus runtime package.json has no version")
|
||||
|
||||
linked_shared = mounts[2].source / "gitnexus-shared"
|
||||
if not linked_shared.is_symlink() or linked_shared.resolve(strict=True) != shared:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue