From bb09ce28e0d7dc5f1c38777314631aa2ed52f235 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Sat, 1 Aug 2026 16:45:48 +0000 Subject: [PATCH] fix(eval): stop discarding completed benchmark sessions as unverifiable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The evolution loop has not been able to promote anything since it went online. Run 29907431284 (the last green run) reached the gate and threw away 5 of its 18 runs, and the gate requires zero excluded runs in both paired arms — so the generation could never produce a verdict on merit. Two causes, both in the session layer: 1. Claude Code drains background-task bookkeeping after the final result event (`background_tasks_changed`, `task_updated`, `task_notification`, all `type: "system"`). The parent-stream check required the result to be the literal last event, so three sessions that had exited 0 with a complete result and usage payload were recorded as session errors. Trailing `system` events carry no tool_use/tool_result/usage payload and cannot forge skill or cost evidence; anything else after the result still fails closed. 2. The 3600s per-session ceiling killed two `workflow` incumbent runs on inv-bug-pdg-note mid-verification. Successful `workflow` rows in the same run finished in ~1600-2600s across both sessions, so the ceiling moves to 5400s and now lives in one shared constant instead of two argparse defaults that could drift apart. Also marks the activation checklist against reality: the secrets, the Environment, the runner, and the validation dispatch are all in place; the repository variable GITNEXUS_EVOLUTION_ENABLED is the one remaining gap, and until it is set the Saturday cron skips the job in seconds while the EventBridge schedule still starts the runner for the day. --- .../workflows/gitnexus-skill-evolution.yml | 18 +++++++---- eval/tests/test_workflow_bench_sessions.py | 31 +++++++++++++++++++ eval/workflow_bench/evolve.py | 7 ++++- eval/workflow_bench/runner.py | 8 ++++- eval/workflow_bench/runner_sessions.py | 25 ++++++++++++--- 5 files changed, 76 insertions(+), 13 deletions(-) diff --git a/.github/workflows/gitnexus-skill-evolution.yml b/.github/workflows/gitnexus-skill-evolution.yml index bf15b00bc..19bb60d71 100644 --- a/.github/workflows/gitnexus-skill-evolution.yml +++ b/.github/workflows/gitnexus-skill-evolution.yml @@ -5,10 +5,11 @@ # merges without review. # # Activation checklist (the scheduled lane is OFF by default). -# [ ] Configure the repository secret GITNEXUS_BENCH_AUTH_TOKEN (an Anthropic +# [x] Configure the repository secret GITNEXUS_BENCH_AUTH_TOKEN (an Anthropic # API key — benchmark sessions bill real usage; the Claude Code OAuth -# subscription token does not work here). -# [ ] Configure the RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY secrets (the +# subscription token does not work here). Scoped to the +# `gitnexus-evolution` Environment. +# [x] Configure the RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY secrets (the # App that opens the promotion PR). The Mint-App-Token step hard-fails # without them once a promotion is detected. Verify the App installation # is scoped to this repo with only Contents: RW + Pull requests: RW. @@ -40,10 +41,15 @@ # most weekly. Revisit if run frequency increases or the threat model # changes; stopping already bounds the exposure window to the job's own # runtime on 1 day out of 7. -# [ ] Run workflow_dispatch once and confirm: containment preflight passes, +# [x] Run workflow_dispatch once and confirm: containment preflight passes, # the benchmark completes inside the job timeout, the results artifact -# uploads, and a promotion (if any) opens a well-formed PR. -# [ ] Set the repository variable GITNEXUS_EVOLUTION_ENABLED=true. +# uploads, and a promotion (if any) opens a well-formed PR. Run +# 29907431284 (2026-07-22) went green end to end in 14h45m and reached a +# gate decision (`insufficient_evidence`, no promotion). +# [ ] Set the repository variable GITNEXUS_EVOLUTION_ENABLED=true. This is the +# only remaining gap: until it is set the scheduled lane skips the job in +# seconds — while the EventBridge schedule still starts the runner for the +# day — so the Saturday cron costs instance time and produces nothing. # Roll back by setting that variable to false. Note: workflow_dispatch always # runs the full benchmark loop regardless of GITNEXUS_EVOLUTION_ENABLED and # bills real API usage on GITNEXUS_BENCH_AUTH_TOKEN. diff --git a/eval/tests/test_workflow_bench_sessions.py b/eval/tests/test_workflow_bench_sessions.py index c10afa401..1e94644bb 100644 --- a/eval/tests/test_workflow_bench_sessions.py +++ b/eval/tests/test_workflow_bench_sessions.py @@ -967,6 +967,37 @@ def test_final_result_event_must_be_last(monkeypatch, tmp_path): assert "not the last event" in rec["error_detail"]["event_stream_error"] +def test_background_task_teardown_after_the_result_stays_valid_evidence(monkeypatch, tmp_path): + # Claude Code drains background-task bookkeeping after the final result + # event. Those `system` events carry no tool or usage payload, so they must + # not invalidate an otherwise complete session (run 29907431284 lost three + # runs this way, and the gate demands zero excluded runs). + teardown = [ + {"type": "system", "subtype": "background_tasks_changed", "tasks": []}, + {"type": "system", "subtype": "task_updated", "task_id": "bdw43oy7j", "patch": {"status": "killed"}}, + {"type": "system", "subtype": "task_notification", "task_id": "bdw43oy7j", "status": "stopped"}, + ] + stream = ( + "\n".join([*(json.dumps(event) for event in skill_events({"skill": "gitnexus-work"})), VALID_REPORT]) + + "\n" + + "".join(json.dumps(event) + "\n" for event in teardown) + ) + monkeypatch.setattr(runner_sessions, "run_managed", lambda *a, **k: fake_cli_result(stream)) + rec = runner.run_claude( + "task", + tmp_path, + claude_bin="claude", + timeout=5, + expected_skill="gitnexus-work", + ) + + assert rec["ok"] is True + assert rec["error_kind"] is None + assert rec["skill_invoked"] is True + assert rec["transcript_missing"] is False + assert "evidence_diagnostics" not in rec + + def test_snapshot_plan_docs_detects_one_modified_plan_and_rejects_ambiguous_output(tmp_path): plans = tmp_path / "docs" / "plans" plans.mkdir(parents=True) diff --git a/eval/workflow_bench/evolve.py b/eval/workflow_bench/evolve.py index d917abe88..c7d2c22a6 100644 --- a/eval/workflow_bench/evolve.py +++ b/eval/workflow_bench/evolve.py @@ -822,7 +822,12 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument("--out-root", type=Path, default=None) parser.add_argument("--claude-bin", default="claude") - parser.add_argument("--timeout", type=int, default=3600, help="per session, seconds") + parser.add_argument( + "--timeout", + type=int, + default=runner_sessions.SESSION_TIMEOUT_SECONDS, + help="per session, seconds", + ) parser.add_argument("--base-url", default=None) parser.add_argument( "--auth-token", diff --git a/eval/workflow_bench/runner.py b/eval/workflow_bench/runner.py index d67934cd0..24fb6a044 100644 --- a/eval/workflow_bench/runner.py +++ b/eval/workflow_bench/runner.py @@ -119,6 +119,7 @@ from .runner_sessions import ( GITNEXUS_READ_ONLY_TOOLS as GITNEXUS_READ_ONLY_TOOLS, MAX_TRANSCRIPT_BYTES as MAX_TRANSCRIPT_BYTES, SANDBOX_GITNEXUS_ENTRYPOINT as SANDBOX_GITNEXUS_ENTRYPOINT, + SESSION_TIMEOUT_SECONDS, USAGE_FIELDS, allowed_agent_tools, run_claude, @@ -855,7 +856,12 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="exact Compound Engineering plugin version; required for ce_* arms", ) - parser.add_argument("--timeout", type=int, default=3600, help="per session, seconds") + parser.add_argument( + "--timeout", + type=int, + default=SESSION_TIMEOUT_SECONDS, + help="per session, seconds", + ) parser.add_argument("--out", type=Path, default=None) parser.add_argument( "--model", diff --git a/eval/workflow_bench/runner_sessions.py b/eval/workflow_bench/runner_sessions.py index cc1708572..ef092b4c6 100644 --- a/eval/workflow_bench/runner_sessions.py +++ b/eval/workflow_bench/runner_sessions.py @@ -32,6 +32,15 @@ USAGE_FIELDS = ( "output_tokens", ) MAX_TRANSCRIPT_BYTES = 8 * 1024 * 1024 +# Wall-clock ceiling for one headless session, shared by the runner and the +# evolution loop so both CLIs kill a session at the same point. 3600s was too +# tight for the `workflow` arm: run 29907431284 lost two inv-bug-pdg-note +# incumbent runs to SIGTERM at the ceiling while a Bash verification step was +# still going, and the promotion gate demands zero excluded runs in both paired +# arms — so a single ceiling hit costs the whole generation. Successful +# `workflow` rows in that run finished in ~1600-2600s across both sessions, so +# this leaves real headroom over observed work rather than over the timeout. +SESSION_TIMEOUT_SECONDS = 5400 # Provenance tag stamped on every parent-captured transcript artifact. The # evidence preflight (evolve._transcript_artifact_metadata) validates against # this exact value, so producer and consumer stay pinned to one schema. @@ -416,11 +425,17 @@ def run_claude( if proc.stdout_capture_overflow: raise ValueError(f"parent-captured event stream exceeds {MAX_TRANSCRIPT_BYTES} bytes") events = _parse_parent_event_stream(proc.stdout_capture) - result_events = [event for event in events if event.get("type") == "result"] - if len(result_events) != 1: - raise ValueError(f"expected exactly one final result event, observed {len(result_events)}") - data = result_events[0] - if events[-1] is not data: + result_indexes = [index for index, event in enumerate(events) if event.get("type") == "result"] + if len(result_indexes) != 1: + raise ValueError(f"expected exactly one final result event, observed {len(result_indexes)}") + data = events[result_indexes[0]] + # A session that used a background task drains its bookkeeping after + # the final result event (`background_tasks_changed`, `task_updated`, + # `task_notification` — all `type: "system"`), so the result is last + # only among the events that carry evidence. `system` events hold no + # tool_use/tool_result/usage payload and so cannot forge skill or cost + # evidence; any other event after the result still fails closed. + if any(event.get("type") != "system" for event in events[result_indexes[0] + 1 :]): raise ValueError("final result event is not the last event in the captured stream") except (UnicodeError, ValueError) as exc: event_stream_error = str(exc)