fix(eval): stop discarding completed benchmark sessions as unverifiable

The evolution loop has not been able to promote anything since it went
online. Run 29907431284 (the last green run) reached the gate and threw
away 5 of its 18 runs, and the gate requires zero excluded runs in both
paired arms — so the generation could never produce a verdict on merit.

Two causes, both in the session layer:

1. Claude Code drains background-task bookkeeping after the final result
   event (`background_tasks_changed`, `task_updated`, `task_notification`,
   all `type: "system"`). The parent-stream check required the result to
   be the literal last event, so three sessions that had exited 0 with a
   complete result and usage payload were recorded as session errors.
   Trailing `system` events carry no tool_use/tool_result/usage payload
   and cannot forge skill or cost evidence; anything else after the
   result still fails closed.

2. The 3600s per-session ceiling killed two `workflow` incumbent runs on
   inv-bug-pdg-note mid-verification. Successful `workflow` rows in the
   same run finished in ~1600-2600s across both sessions, so the ceiling
   moves to 5400s and now lives in one shared constant instead of two
   argparse defaults that could drift apart.

Also marks the activation checklist against reality: the secrets, the
Environment, the runner, and the validation dispatch are all in place;
the repository variable GITNEXUS_EVOLUTION_ENABLED is the one remaining
gap, and until it is set the Saturday cron skips the job in seconds while
the EventBridge schedule still starts the runner for the day.
This commit is contained in:
Gergo Magyar 2026-08-01 16:45:48 +00:00
parent 59ea1ce2c8
commit bb09ce28e0
5 changed files with 76 additions and 13 deletions

View file

@ -5,10 +5,11 @@
# merges without review.
#
# Activation checklist (the scheduled lane is OFF by default).
# [ ] Configure the repository secret GITNEXUS_BENCH_AUTH_TOKEN (an Anthropic
# [x] Configure the repository secret GITNEXUS_BENCH_AUTH_TOKEN (an Anthropic
# API key — benchmark sessions bill real usage; the Claude Code OAuth
# subscription token does not work here).
# [ ] Configure the RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY secrets (the
# subscription token does not work here). Scoped to the
# `gitnexus-evolution` Environment.
# [x] Configure the RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY secrets (the
# App that opens the promotion PR). The Mint-App-Token step hard-fails
# without them once a promotion is detected. Verify the App installation
# is scoped to this repo with only Contents: RW + Pull requests: RW.
@ -40,10 +41,15 @@
# most weekly. Revisit if run frequency increases or the threat model
# changes; stopping already bounds the exposure window to the job's own
# runtime on 1 day out of 7.
# [ ] Run workflow_dispatch once and confirm: containment preflight passes,
# [x] Run workflow_dispatch once and confirm: containment preflight passes,
# the benchmark completes inside the job timeout, the results artifact
# uploads, and a promotion (if any) opens a well-formed PR.
# [ ] Set the repository variable GITNEXUS_EVOLUTION_ENABLED=true.
# uploads, and a promotion (if any) opens a well-formed PR. Run
# 29907431284 (2026-07-22) went green end to end in 14h45m and reached a
# gate decision (`insufficient_evidence`, no promotion).
# [ ] Set the repository variable GITNEXUS_EVOLUTION_ENABLED=true. This is the
# only remaining gap: until it is set the scheduled lane skips the job in
# seconds — while the EventBridge schedule still starts the runner for the
# day — so the Saturday cron costs instance time and produces nothing.
# Roll back by setting that variable to false. Note: workflow_dispatch always
# runs the full benchmark loop regardless of GITNEXUS_EVOLUTION_ENABLED and
# bills real API usage on GITNEXUS_BENCH_AUTH_TOKEN.

View file

@ -967,6 +967,37 @@ def test_final_result_event_must_be_last(monkeypatch, tmp_path):
assert "not the last event" in rec["error_detail"]["event_stream_error"]
def test_background_task_teardown_after_the_result_stays_valid_evidence(monkeypatch, tmp_path):
# Claude Code drains background-task bookkeeping after the final result
# event. Those `system` events carry no tool or usage payload, so they must
# not invalidate an otherwise complete session (run 29907431284 lost three
# runs this way, and the gate demands zero excluded runs).
teardown = [
{"type": "system", "subtype": "background_tasks_changed", "tasks": []},
{"type": "system", "subtype": "task_updated", "task_id": "bdw43oy7j", "patch": {"status": "killed"}},
{"type": "system", "subtype": "task_notification", "task_id": "bdw43oy7j", "status": "stopped"},
]
stream = (
"\n".join([*(json.dumps(event) for event in skill_events({"skill": "gitnexus-work"})), VALID_REPORT])
+ "\n"
+ "".join(json.dumps(event) + "\n" for event in teardown)
)
monkeypatch.setattr(runner_sessions, "run_managed", lambda *a, **k: fake_cli_result(stream))
rec = runner.run_claude(
"task",
tmp_path,
claude_bin="claude",
timeout=5,
expected_skill="gitnexus-work",
)
assert rec["ok"] is True
assert rec["error_kind"] is None
assert rec["skill_invoked"] is True
assert rec["transcript_missing"] is False
assert "evidence_diagnostics" not in rec
def test_snapshot_plan_docs_detects_one_modified_plan_and_rejects_ambiguous_output(tmp_path):
plans = tmp_path / "docs" / "plans"
plans.mkdir(parents=True)

View file

@ -822,7 +822,12 @@ def build_parser() -> argparse.ArgumentParser:
)
parser.add_argument("--out-root", type=Path, default=None)
parser.add_argument("--claude-bin", default="claude")
parser.add_argument("--timeout", type=int, default=3600, help="per session, seconds")
parser.add_argument(
"--timeout",
type=int,
default=runner_sessions.SESSION_TIMEOUT_SECONDS,
help="per session, seconds",
)
parser.add_argument("--base-url", default=None)
parser.add_argument(
"--auth-token",

View file

@ -119,6 +119,7 @@ from .runner_sessions import (
GITNEXUS_READ_ONLY_TOOLS as GITNEXUS_READ_ONLY_TOOLS,
MAX_TRANSCRIPT_BYTES as MAX_TRANSCRIPT_BYTES,
SANDBOX_GITNEXUS_ENTRYPOINT as SANDBOX_GITNEXUS_ENTRYPOINT,
SESSION_TIMEOUT_SECONDS,
USAGE_FIELDS,
allowed_agent_tools,
run_claude,
@ -855,7 +856,12 @@ def build_parser() -> argparse.ArgumentParser:
default=None,
help="exact Compound Engineering plugin version; required for ce_* arms",
)
parser.add_argument("--timeout", type=int, default=3600, help="per session, seconds")
parser.add_argument(
"--timeout",
type=int,
default=SESSION_TIMEOUT_SECONDS,
help="per session, seconds",
)
parser.add_argument("--out", type=Path, default=None)
parser.add_argument(
"--model",

View file

@ -32,6 +32,15 @@ USAGE_FIELDS = (
"output_tokens",
)
MAX_TRANSCRIPT_BYTES = 8 * 1024 * 1024
# Wall-clock ceiling for one headless session, shared by the runner and the
# evolution loop so both CLIs kill a session at the same point. 3600s was too
# tight for the `workflow` arm: run 29907431284 lost two inv-bug-pdg-note
# incumbent runs to SIGTERM at the ceiling while a Bash verification step was
# still going, and the promotion gate demands zero excluded runs in both paired
# arms — so a single ceiling hit costs the whole generation. Successful
# `workflow` rows in that run finished in ~1600-2600s across both sessions, so
# this leaves real headroom over observed work rather than over the timeout.
SESSION_TIMEOUT_SECONDS = 5400
# Provenance tag stamped on every parent-captured transcript artifact. The
# evidence preflight (evolve._transcript_artifact_metadata) validates against
# this exact value, so producer and consumer stay pinned to one schema.
@ -416,11 +425,17 @@ def run_claude(
if proc.stdout_capture_overflow:
raise ValueError(f"parent-captured event stream exceeds {MAX_TRANSCRIPT_BYTES} bytes")
events = _parse_parent_event_stream(proc.stdout_capture)
result_events = [event for event in events if event.get("type") == "result"]
if len(result_events) != 1:
raise ValueError(f"expected exactly one final result event, observed {len(result_events)}")
data = result_events[0]
if events[-1] is not data:
result_indexes = [index for index, event in enumerate(events) if event.get("type") == "result"]
if len(result_indexes) != 1:
raise ValueError(f"expected exactly one final result event, observed {len(result_indexes)}")
data = events[result_indexes[0]]
# A session that used a background task drains its bookkeeping after
# the final result event (`background_tasks_changed`, `task_updated`,
# `task_notification` — all `type: "system"`), so the result is last
# only among the events that carry evidence. `system` events hold no
# tool_use/tool_result/usage payload and so cannot forge skill or cost
# evidence; any other event after the result still fails closed.
if any(event.get("type") != "system" for event in events[result_indexes[0] + 1 :]):
raise ValueError("final result event is not the last event in the captured stream")
except (UnicodeError, ValueError) as exc:
event_stream_error = str(exc)