diff --git a/.github/workflows/gitnexus-skill-evolution.yml b/.github/workflows/gitnexus-skill-evolution.yml index bf15b00bc..19bb60d71 100644 --- a/.github/workflows/gitnexus-skill-evolution.yml +++ b/.github/workflows/gitnexus-skill-evolution.yml @@ -5,10 +5,11 @@ # merges without review. # # Activation checklist (the scheduled lane is OFF by default). -# [ ] Configure the repository secret GITNEXUS_BENCH_AUTH_TOKEN (an Anthropic +# [x] Configure the repository secret GITNEXUS_BENCH_AUTH_TOKEN (an Anthropic # API key — benchmark sessions bill real usage; the Claude Code OAuth -# subscription token does not work here). -# [ ] Configure the RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY secrets (the +# subscription token does not work here). Scoped to the +# `gitnexus-evolution` Environment. +# [x] Configure the RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY secrets (the # App that opens the promotion PR). The Mint-App-Token step hard-fails # without them once a promotion is detected. Verify the App installation # is scoped to this repo with only Contents: RW + Pull requests: RW. @@ -40,10 +41,15 @@ # most weekly. Revisit if run frequency increases or the threat model # changes; stopping already bounds the exposure window to the job's own # runtime on 1 day out of 7. -# [ ] Run workflow_dispatch once and confirm: containment preflight passes, +# [x] Run workflow_dispatch once and confirm: containment preflight passes, # the benchmark completes inside the job timeout, the results artifact -# uploads, and a promotion (if any) opens a well-formed PR. -# [ ] Set the repository variable GITNEXUS_EVOLUTION_ENABLED=true. +# uploads, and a promotion (if any) opens a well-formed PR. Run +# 29907431284 (2026-07-22) went green end to end in 14h45m and reached a +# gate decision (`insufficient_evidence`, no promotion). +# [ ] Set the repository variable GITNEXUS_EVOLUTION_ENABLED=true. This is the +# only remaining gap: until it is set the scheduled lane skips the job in +# seconds — while the EventBridge schedule still starts the runner for the +# day — so the Saturday cron costs instance time and produces nothing. # Roll back by setting that variable to false. Note: workflow_dispatch always # runs the full benchmark loop regardless of GITNEXUS_EVOLUTION_ENABLED and # bills real API usage on GITNEXUS_BENCH_AUTH_TOKEN. diff --git a/eval/tests/test_workflow_bench_sessions.py b/eval/tests/test_workflow_bench_sessions.py index c10afa401..1e94644bb 100644 --- a/eval/tests/test_workflow_bench_sessions.py +++ b/eval/tests/test_workflow_bench_sessions.py @@ -967,6 +967,37 @@ def test_final_result_event_must_be_last(monkeypatch, tmp_path): assert "not the last event" in rec["error_detail"]["event_stream_error"] +def test_background_task_teardown_after_the_result_stays_valid_evidence(monkeypatch, tmp_path): + # Claude Code drains background-task bookkeeping after the final result + # event. Those `system` events carry no tool or usage payload, so they must + # not invalidate an otherwise complete session (run 29907431284 lost three + # runs this way, and the gate demands zero excluded runs). + teardown = [ + {"type": "system", "subtype": "background_tasks_changed", "tasks": []}, + {"type": "system", "subtype": "task_updated", "task_id": "bdw43oy7j", "patch": {"status": "killed"}}, + {"type": "system", "subtype": "task_notification", "task_id": "bdw43oy7j", "status": "stopped"}, + ] + stream = ( + "\n".join([*(json.dumps(event) for event in skill_events({"skill": "gitnexus-work"})), VALID_REPORT]) + + "\n" + + "".join(json.dumps(event) + "\n" for event in teardown) + ) + monkeypatch.setattr(runner_sessions, "run_managed", lambda *a, **k: fake_cli_result(stream)) + rec = runner.run_claude( + "task", + tmp_path, + claude_bin="claude", + timeout=5, + expected_skill="gitnexus-work", + ) + + assert rec["ok"] is True + assert rec["error_kind"] is None + assert rec["skill_invoked"] is True + assert rec["transcript_missing"] is False + assert "evidence_diagnostics" not in rec + + def test_snapshot_plan_docs_detects_one_modified_plan_and_rejects_ambiguous_output(tmp_path): plans = tmp_path / "docs" / "plans" plans.mkdir(parents=True) diff --git a/eval/workflow_bench/evolve.py b/eval/workflow_bench/evolve.py index d917abe88..c7d2c22a6 100644 --- a/eval/workflow_bench/evolve.py +++ b/eval/workflow_bench/evolve.py @@ -822,7 +822,12 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument("--out-root", type=Path, default=None) parser.add_argument("--claude-bin", default="claude") - parser.add_argument("--timeout", type=int, default=3600, help="per session, seconds") + parser.add_argument( + "--timeout", + type=int, + default=runner_sessions.SESSION_TIMEOUT_SECONDS, + help="per session, seconds", + ) parser.add_argument("--base-url", default=None) parser.add_argument( "--auth-token", diff --git a/eval/workflow_bench/runner.py b/eval/workflow_bench/runner.py index d67934cd0..24fb6a044 100644 --- a/eval/workflow_bench/runner.py +++ b/eval/workflow_bench/runner.py @@ -119,6 +119,7 @@ from .runner_sessions import ( GITNEXUS_READ_ONLY_TOOLS as GITNEXUS_READ_ONLY_TOOLS, MAX_TRANSCRIPT_BYTES as MAX_TRANSCRIPT_BYTES, SANDBOX_GITNEXUS_ENTRYPOINT as SANDBOX_GITNEXUS_ENTRYPOINT, + SESSION_TIMEOUT_SECONDS, USAGE_FIELDS, allowed_agent_tools, run_claude, @@ -855,7 +856,12 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="exact Compound Engineering plugin version; required for ce_* arms", ) - parser.add_argument("--timeout", type=int, default=3600, help="per session, seconds") + parser.add_argument( + "--timeout", + type=int, + default=SESSION_TIMEOUT_SECONDS, + help="per session, seconds", + ) parser.add_argument("--out", type=Path, default=None) parser.add_argument( "--model", diff --git a/eval/workflow_bench/runner_sessions.py b/eval/workflow_bench/runner_sessions.py index cc1708572..ef092b4c6 100644 --- a/eval/workflow_bench/runner_sessions.py +++ b/eval/workflow_bench/runner_sessions.py @@ -32,6 +32,15 @@ USAGE_FIELDS = ( "output_tokens", ) MAX_TRANSCRIPT_BYTES = 8 * 1024 * 1024 +# Wall-clock ceiling for one headless session, shared by the runner and the +# evolution loop so both CLIs kill a session at the same point. 3600s was too +# tight for the `workflow` arm: run 29907431284 lost two inv-bug-pdg-note +# incumbent runs to SIGTERM at the ceiling while a Bash verification step was +# still going, and the promotion gate demands zero excluded runs in both paired +# arms — so a single ceiling hit costs the whole generation. Successful +# `workflow` rows in that run finished in ~1600-2600s across both sessions, so +# this leaves real headroom over observed work rather than over the timeout. +SESSION_TIMEOUT_SECONDS = 5400 # Provenance tag stamped on every parent-captured transcript artifact. The # evidence preflight (evolve._transcript_artifact_metadata) validates against # this exact value, so producer and consumer stay pinned to one schema. @@ -416,11 +425,17 @@ def run_claude( if proc.stdout_capture_overflow: raise ValueError(f"parent-captured event stream exceeds {MAX_TRANSCRIPT_BYTES} bytes") events = _parse_parent_event_stream(proc.stdout_capture) - result_events = [event for event in events if event.get("type") == "result"] - if len(result_events) != 1: - raise ValueError(f"expected exactly one final result event, observed {len(result_events)}") - data = result_events[0] - if events[-1] is not data: + result_indexes = [index for index, event in enumerate(events) if event.get("type") == "result"] + if len(result_indexes) != 1: + raise ValueError(f"expected exactly one final result event, observed {len(result_indexes)}") + data = events[result_indexes[0]] + # A session that used a background task drains its bookkeeping after + # the final result event (`background_tasks_changed`, `task_updated`, + # `task_notification` — all `type: "system"`), so the result is last + # only among the events that carry evidence. `system` events hold no + # tool_use/tool_result/usage payload and so cannot forge skill or cost + # evidence; any other event after the result still fails closed. + if any(event.get("type") != "system" for event in events[result_indexes[0] + 1 :]): raise ValueError("final result event is not the last event in the captured stream") except (UnicodeError, ValueError) as exc: event_stream_error = str(exc)