GitNexus/.github/workflows/release-evaluation.yml
Gergő Magyar c06f3ee32d
fix(eval): validate trial setup and report missing measurements (#3779)
* fix(eval): validate trials and report measurements

Keep pinned task dependencies separate from the candidate runtime. Gate paid workflow sessions on contained reference grading and retain private-safe setup, verification, provenance and provider-token measurements.

* fix(eval): align CI with the release task build

* fix(eval): address review feedback (#3779)

Bind trial dependency bytes before execution and include MCP environment instructions in comparator identity. Preserve verifier states and strengthen preflight coverage.

---------

Co-authored-by: Gergo Magyar <gergomagyar0@gmail.com>
2026-10-31 04:40:37 +00:00

758 lines
39 KiB
YAML

# Compare the exact candidate with the last stable release using trusted main's
# harness. Share the existing evolution runner, protected settings, and power lock.
#
# The shared group keeps only ONE pending run, even with cancel-in-progress:
# false: a newer queued skill-evolution or release-evaluation run cancels this
# run while it is pending. Before relying on a dispatched evaluation for a
# stable publish, confirm its "Start the dedicated EC2 runner" job ran; if the
# run shows Cancelled, dispatch it again once the group is free.
name: Release evaluation
on:
schedule:
- cron: '0 5 * * 3' # Wednesday 05:00 UTC
workflow_dispatch:
inputs:
runner_only:
description: Start, probe, and stop EC2 without paid model calls
type: boolean
default: false
diagnostic_trial:
description: 'Two serial cells, $5 CLI stop each; requires a full candidate SHA, gpt-5.6-sol/xhigh, and no other experiment flags'
type: boolean
default: false
mcp_tool_loading:
description: Add candidate deferred-MCP experiment (12 extra paid cells)
type: boolean
default: false
candidate_ref:
description: 'Full commit SHA or version tag to evaluate; latest-rc resolves the newest published RC'
type: string
default: latest-rc
model:
description: 'Named model for both arms'
type: string
default: gpt-5.6-sol
effort:
description: 'Reasoning effort for both arms'
type: choice
default: xhigh
options: [low, medium, high, xhigh, max]
cli_arm:
description: 'Also run baseline_cli (same tools via a gitnexus shell command, no MCP catalog) on the candidate; adds 12 paid cells, descriptive only, never gates'
type: boolean
default: false
# PR preview inputs are available only to the maintainer-authorized label
# workflow. Manual dispatch cannot select an unmerged PR runtime.
workflow_call:
secrets:
GITNEXUS_EVOLUTION_AWS_ROLE_ARN:
required: true
GITNEXUS_EVOLUTION_EC2_INSTANCE_ID:
required: true
GITNEXUS_BENCH_OPENAI_API_KEY:
required: true
GITNEXUS_BENCH_ANTHROPIC_API_KEY:
required: false
inputs:
candidate_pr:
type: string
required: true
expected_pr_sha:
type: string
required: true
comparison_run_id:
type: string
required: true
model:
type: string
default: gpt-5.6-sol
effort:
type: string
default: xhigh
concurrency:
group: gitnexus-evolution-runner
cancel-in-progress: false
permissions: {}
jobs:
start-runner:
name: Start the dedicated EC2 runner
if: >-
github.repository == 'abhigyanpatwari/GitNexus' &&
github.ref == 'refs/heads/main' &&
(github.event_name == 'workflow_dispatch' ||
(github.event_name == 'pull_request_target' && inputs.candidate_pr != '') ||
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
runs-on: ubuntu-latest
environment: gitnexus-evolution
timeout-minutes: 25
permissions:
contents: read
id-token: write
outputs:
run_attempt: ${{ github.run_attempt }}
cleanup_required: ${{ steps.boot.outputs.cleanup_required }}
stop_deadline_epoch: ${{ steps.boot.outputs.stop_deadline_epoch }}
steps:
- name: Validate diagnostic trial authorization scope
env:
TRIAL: ${{ inputs.diagnostic_trial }}
RUNNER_ONLY: ${{ inputs.runner_only }}
CLI_ARM: ${{ inputs.cli_arm }}
DEFERRED_ARM: ${{ inputs.mcp_tool_loading }}
CANDIDATE_REF: ${{ inputs.candidate_ref }}
MODEL: ${{ inputs.model }}
EFFORT: ${{ inputs.effort }}
run: |
set -euo pipefail
if [[ "$TRIAL" == true ]]; then
if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_RUN_ATTEMPT" != 1 ||
"$RUNNER_ONLY" == true || "$CLI_ARM" == true || "$DEFERRED_ARM" == true ||
"$MODEL" != gpt-5.6-sol || "$EFFORT" != xhigh || ! "$CANDIDATE_REF" =~ ^[0-9a-f]{40}$ ]]; then
echo '::error::Trial requires a fresh manual dispatch, a full candidate SHA, gpt-5.6-sol/xhigh, and no other experiment flags. Paid trials are never retried automatically.'
exit 1
fi
fi
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: ./.github/actions/ec2-runner
id: boot
with:
operation: start
role-arn: ${{ secrets.GITNEXUS_EVOLUTION_AWS_ROLE_ARN }}
instance-id: ${{ secrets.GITNEXUS_EVOLUTION_EC2_INSTANCE_ID }}
region: ${{ vars.GITNEXUS_EVOLUTION_AWS_REGION }}
stop-schedule: ${{ vars.GITNEXUS_EVOLUTION_STOP_SCHEDULE_UTC }}
check-runner:
needs: [start-runner]
name: Bound the wait for the evaluation runner
if: >-
github.repository == 'abhigyanpatwari/GitNexus' &&
github.ref == 'refs/heads/main' &&
(github.event_name == 'workflow_dispatch' ||
(github.event_name == 'pull_request_target' && inputs.candidate_pr != '') ||
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
runs-on: ubuntu-latest
environment: gitnexus-evolution
timeout-minutes: 10
permissions:
contents: read
actions: write # Cancel only this run if its dedicated runner stays offline.
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# EC2 health does not prove the Actions service can accept a job.
- name: Verify the current run's native pickup probe
env:
GH_TOKEN: ${{ github.token }}
run: python3 .github/scripts/evolution-runner-ready.py
# A queued self-hosted job has no pickup timeout. Cancel this run so an
# offline probe cannot remain queued indefinitely and block later runs.
- name: Cancel this run when runner readiness fails
if: failure()
env:
GH_TOKEN: ${{ github.token }}
run: gh api --method POST "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel"
runner-ready:
needs: [start-runner]
name: Verify the runner service is online
if: >-
github.repository == 'abhigyanpatwari/GitNexus' &&
github.ref == 'refs/heads/main' &&
(github.event_name == 'workflow_dispatch' ||
(github.event_name == 'pull_request_target' && inputs.candidate_pr != '') ||
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
# Partial reruns reuse successful startup outputs after cleanup stopped EC2.
runs-on: ${{ fromJSON(needs.start-runner.outputs.run_attempt == github.run_attempt && '["self-hosted", "linux", "x64", "gitnexus-evolution"]' || '["ubuntu-latest"]') }}
environment: gitnexus-evolution
timeout-minutes: 5
permissions: {}
steps:
- name: Require startup in this run attempt
run: |
set -euo pipefail
if [[ '${{ needs.start-runner.outputs.run_attempt }}' != "$GITHUB_RUN_ATTEMPT" ]]; then
echo '::error::The EC2 startup belongs to an earlier attempt. Select Re-run all jobs to start the runner again.'
exit 1
fi
- name: Verify runner pickup
run: |
set -euo pipefail
test -r /proc/uptime
git --version
uname -sm
watch-evaluate-pickup:
name: Bound pickup of the paid evaluation job
needs: [start-runner, check-runner, runner-ready]
if: inputs.runner_only != true
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
actions: write # Cancel this run if evaluate stays queued after the probe.
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Require pickup of this run's paid evaluation job
env:
GH_TOKEN: ${{ github.token }}
run: python3 .github/scripts/evolution-runner-ready.py --job-name "Evaluate candidate and stable quality"
- name: Cancel this run when evaluation pickup fails
if: failure()
env:
GH_TOKEN: ${{ github.token }}
run: gh api --method POST "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel"
evaluate:
name: Evaluate candidate and stable quality
needs: [start-runner, check-runner, runner-ready]
if: >-
github.repository == 'abhigyanpatwari/GitNexus' &&
github.ref == 'refs/heads/main' &&
inputs.runner_only != true &&
(github.event_name == 'workflow_dispatch' ||
(github.event_name == 'pull_request_target' && inputs.candidate_pr != '') ||
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
runs-on: ${{ fromJSON(needs.start-runner.outputs.run_attempt == github.run_attempt && '["self-hosted", "linux", "x64", "gitnexus-evolution"]' || '["ubuntu-latest"]') }}
# Reuse the main-only protected model-secret environment. No tag/fork code
# runs as the harness, and no release or PR write token is available here.
environment: gitnexus-evolution
timeout-minutes: 1260
permissions:
contents: read
actions: read # Authenticate and download the explicitly selected historical measurement.
pull-requests: read
env:
INPUT_REF: ${{ inputs.candidate_ref || 'latest-rc' }}
INPUT_PR: ${{ inputs.candidate_pr }}
INPUT_PR_SHA: ${{ inputs.expected_pr_sha }}
COMPARISON_RUN_ID: ${{ inputs.comparison_run_id }}
MODEL: ${{ inputs.model || 'gpt-5.6-sol' }}
EFFORT: ${{ inputs.effort || 'xhigh' }}
# Manual dispatch only; schedules and PR previews never pay for the extra arm.
CLI_ARM: ${{ inputs.cli_arm == true && inputs.candidate_pr == '' && 'true' || 'false' }}
steps:
- name: Require startup in this run attempt
run: |
set -euo pipefail
if [[ '${{ needs.start-runner.outputs.run_attempt }}' != "$GITHUB_RUN_ATTEMPT" ]]; then
echo '::error::The EC2 startup belongs to an earlier attempt. Select Re-run all jobs to start the runner again.'
exit 1
fi
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
fetch-depth: 0
- name: Resolve immutable runtime revision
id: resolve
working-directory: eval
env:
GH_TOKEN: ${{ github.token }}
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
run: |
set -euo pipefail
if [[ -n "$INPUT_PR" ]]; then
[[ "$COMPARISON_RUN_ID" =~ ^[1-9][0-9]*$ ]] || { echo '::error::PR preview requires comparison_run_id.'; exit 1; }
python3 -m workflow_bench.pr_evaluation resolve \
--repo "$GITHUB_REPOSITORY" --pr "$INPUT_PR" --sha "$INPUT_PR_SHA" >> "$GITHUB_OUTPUT"
exit 0
fi
[[ -z "$INPUT_PR_SHA" && -z "$COMPARISON_RUN_ID" ]] || { echo '::error::PR preview inputs require candidate_pr.'; exit 1; }
# Resolve last stable once; both runtimes use the same trusted tasks.
if [[ "${DIAGNOSTIC_TRIAL:-false}" != true ]]; then
stable="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq '.tag_name')"
[[ "$stable" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo '::error::No stable release found.'; exit 1; }
stable_sha="$(git rev-parse --verify "$stable^{commit}")"
echo "stable_sha=$stable_sha" >> "$GITHUB_OUTPUT"
fi
ref="$INPUT_REF"
if [[ "$ref" == 'latest-rc' ]]; then
ref="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --jq \
'[.[] | select(.prerelease and (.draft | not)) | select(.tag_name | test("^v[0-9]+\\.[0-9]+\\.[0-9]+-rc\\.[0-9]+$"))] | sort_by(.published_at) | last | .tag_name // empty')"
fi
if ! [[ "$ref" =~ ^[0-9a-f]{40}$ || "$ref" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then
echo '::error::No published RC found, or candidate_ref is not a full SHA/version tag.'
exit 1
fi
sha="$(git rev-parse --verify "$ref^{commit}")"
if [[ "$ref" =~ ^[0-9a-f]{40}$ ]]; then
# Manual stable candidates must already be reviewed on main.
git merge-base --is-ancestor "$sha" HEAD
else
# Versioned RC trees are published descendants, not ancestors of
# main. Accept their official published release rather than PR refs.
gh api "repos/$GITHUB_REPOSITORY/releases/tags/$ref" --jq '.draft == false' | grep -qx true
fi
echo "Runtime: $sha"
echo "sha=$sha" >> "$GITHUB_OUTPUT"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ steps.resolve.outputs.sha }}
path: candidate
persist-credentials: false
fetch-depth: 0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
if: inputs.candidate_pr == '' && inputs.diagnostic_trial != true
with:
ref: ${{ steps.resolve.outputs.stable_sha }}
path: stable
persist-credentials: false
fetch-depth: 0
- name: Initialize this run's measurement directory
run: |
rm -rf "$RUNNER_TEMP/release-evaluation"
mkdir -p "$RUNNER_TEMP/release-evaluation/public"
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: '22.18.0'
cache: npm
cache-dependency-path: candidate/gitnexus/package-lock.json
- uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1
with:
version: '0.11.23'
python-version: '3.13'
enable-cache: true
cache-dependency-glob: eval/uv.lock
- name: Authenticate historical PR comparison evidence
if: inputs.candidate_pr != ''
working-directory: eval
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --locked --extra dev python -m workflow_bench.pr_evaluation baseline \
--repo "$GITHUB_REPOSITORY" --run-id "$COMPARISON_RUN_ID" \
--harness-sha "$GITHUB_SHA" --env-out "$GITHUB_ENV" \
--out "$RUNNER_TEMP/release-evaluation/previous.json"
# Both runtimes grade the pinned task source against one dependency tree
# built from that commit's lockfile, never the runtime under test's.
- name: Resolve pinned task revision
id: tasks
working-directory: eval
run: |
set -euo pipefail
sha="$(uv run --locked --extra dev python -m workflow_bench.release_report task-sha)"
[[ "$sha" =~ ^[0-9a-f]{40}$ ]] || { echo '::error::The release task set has no single pinned commit.'; exit 1; }
echo "sha=$sha" >> "$GITHUB_OUTPUT"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ steps.tasks.outputs.sha }}
path: tasks-base
persist-credentials: false
fetch-depth: 0
- name: Install sandbox runtime and pinned Claude CLI
run: |
set -euo pipefail
sudo apt-get -o DPkg::Lock::Timeout=600 update
sudo apt-get -o DPkg::Lock::Timeout=600 install --yes --no-install-recommends bubblewrap ripgrep socat
apparmor_userns=/proc/sys/kernel/apparmor_restrict_unprivileged_userns
if [[ -r "$apparmor_userns" ]] && [[ "$(<"$apparmor_userns")" == '1' ]]; then
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
fi
cli="$RUNNER_TEMP/claude-canary"
install -d -m 0700 "$cli"
install -m 0600 .github/claude-canary-runtime/package{,-lock}.json "$cli/"
npm ci --prefix "$cli" --audit=false --fund=false
test "$("$cli/node_modules/@anthropic-ai/claude-code-linux-x64/claude" --version)" = '2.1.214 (Claude Code)'
- name: Build pinned runtime and task dependencies
working-directory: eval
env:
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
run: |
set -euo pipefail
for checkout in tasks-base stable candidate; do
if [[ ( -n "$INPUT_PR" || "${DIAGNOSTIC_TRIAL:-false}" == true ) && "$checkout" == stable ]]; then continue; fi
uv run --locked --extra dev python -m workflow_bench.release_build \
--repo "$GITHUB_WORKSPACE/$checkout"
done
- name: Prepare pinned task set
working-directory: eval
env:
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
run: |
set -euo pipefail
for runtime in stable candidate; do
if [[ ( -n "$INPUT_PR" || "${DIAGNOSTIC_TRIAL:-false}" == true ) && "$runtime" == stable ]]; then continue; fi
uv run --locked --extra dev python -m workflow_bench.release_report prepare \
--repo "$GITHUB_WORKSPACE/$runtime" --task-repo "$GITHUB_WORKSPACE/tasks-base" \
--out "$RUNNER_TEMP/release-evaluation/$runtime" \
--model "$MODEL" --effort "$EFFORT" --runs 3
done
- name: Prepare the capped diagnostic trial
if: inputs.diagnostic_trial == true
working-directory: eval
env:
RUNTIME_SHA: ${{ steps.resolve.outputs.sha }}
run: |
uv run --locked --extra dev python -m workflow_bench.diagnostic_experiments \
--tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \
--repository "$GITHUB_WORKSPACE/tasks-base" --runtime "$GITHUB_WORKSPACE/candidate" \
--harness-sha "$GITHUB_SHA" --runtime-sha "$RUNTIME_SHA" \
--model "$MODEL" --effort "$EFFORT" --seed 3762 --repeats 1 \
--variants no_graph mcp_eager --task-ids trivial-status-json-alias \
--max-budget-usd 5 --bind-task-dependencies --out "$RUNNER_TEMP/release-evaluation/trial"
- name: Require historical task dependencies before paid PR sessions
if: inputs.candidate_pr != ''
working-directory: eval
run: |
uv run --locked --extra dev python -m workflow_bench.pr_evaluation dependencies \
--tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \
--baseline "$RUNNER_TEMP/release-evaluation/previous.json"
- name: Require real containment before paid sessions
working-directory: eval
env:
GITNEXUS_REQUIRE_BWRAP_CANARY: '1'
GITNEXUS_REQUIRE_CLAUDE_CANARY: '1'
CLAUDE_CANARY_BIN: ${{ runner.temp }}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude
run: >-
uv run --locked --extra dev python -m pytest tests/test_proposer_sandbox.py tests/test_cli_shim.py
tests/test_workflow_bench_sessions.py::test_real_cli_background_completion_is_one_complete_measurement -q
- name: Materialize both release graphs before paid sessions
timeout-minutes: 120
working-directory: eval
env:
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
run: |
set -euo pipefail
if [[ "${DIAGNOSTIC_TRIAL:-false}" == true ]]; then
uv run --locked --extra dev python -m workflow_bench.release_preflight \
--tasks "$RUNNER_TEMP/release-evaluation/trial/cell-0000.tasks.json" \
--grading-controls \
--gitnexus-root "$GITHUB_WORKSPACE/candidate" \
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude"
exit 0
fi
for runtime in stable candidate; do
if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi
uv run --locked --extra dev python -m workflow_bench.release_preflight \
--tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \
--grading-controls \
--gitnexus-root "$GITHUB_WORKSPACE/$runtime" \
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude"
done
- name: Require offline deferred-tool compatibility before paid sessions
if: inputs.mcp_tool_loading == true
working-directory: eval
env:
GITNEXUS_REQUIRE_CLAUDE_CANARY: '1'
CLAUDE_CANARY_BIN: ${{ runner.temp }}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude
run: >-
uv run --locked --extra dev python -m pytest tests/test_mcp_tool_loading.py -q
- name: Recheck PR opt-in and pinned head immediately before paid sessions
if: inputs.candidate_pr != ''
working-directory: eval
env:
GH_TOKEN: ${{ github.token }}
run: |
python3 -m workflow_bench.pr_evaluation resolve \
--repo "$GITHUB_REPOSITORY" --pr "$INPUT_PR" --sha "$INPUT_PR_SHA"
- name: Run repeated paired agent sessions
timeout-minutes: 1140
working-directory: eval
env:
GITNEXUS_BENCH_OPENAI_API_KEY: ${{ secrets.GITNEXUS_BENCH_OPENAI_API_KEY }}
GITNEXUS_BENCH_ANTHROPIC_API_KEY: ${{ secrets.GITNEXUS_BENCH_ANTHROPIC_API_KEY }}
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
run: |
set -euo pipefail
if [[ "${DIAGNOSTIC_TRIAL:-false}" == true ]]; then
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
[[ "$GITHUB_RUN_ATTEMPT" == 1 ]] || { echo '::error::Paid trials cannot be rerun, including failed-job retries.'; exit 1; }
(( remaining > 0 )) || { echo '::error::No time remains before the stop and evidence reserve.'; exit 1; }
(( remaining <= 3600 )) || remaining=3600
timeout --signal=TERM --kill-after=60 "${remaining}s" \
uv run --locked --extra dev python -m workflow_bench.diagnostic_trial --execute \
--plan "$RUNNER_TEMP/release-evaluation/trial/plan.json" \
--runtime "$GITHUB_WORKSPACE/candidate" \
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude"
exit 0
fi
arms=(baseline_nomcp baseline)
if [[ -n "$INPUT_PR" ]]; then arms=(baseline); fi
for runtime in stable candidate; do
if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
(( remaining > 0 )) || { echo '::error::No time remains before the stop and evidence reserve.'; exit 1; }
GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/$runtime/provider-usage.jsonl" \
GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-$runtime" \
timeout --signal=TERM --kill-after=60 "${remaining}s" \
uv run --locked --extra dev python -m workflow_bench.runner \
--tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \
--gitnexus-root "$GITHUB_WORKSPACE/$runtime" \
--arms "${arms[@]}" --runs 3 --include-expensive --mcp-tool-loading eager \
--workers 3 --timeout 3600 --model "$MODEL" --effort "$EFFORT" \
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \
--out "$RUNNER_TEMP/release-evaluation/$runtime/raw"
done
# Descriptive only: its own output directory leaves the paired evidence
# that release_report and release_gate read exactly as without the opt-in.
# Trials reject these opt-ins before startup. Preserve the exact gates
# recognized by the historical measurement-protocol validator.
- name: Run the opt-in CLI-only arm on the candidate
if: env.CLI_ARM == 'true'
continue-on-error: true
timeout-minutes: 600
working-directory: eval
env:
GITNEXUS_BENCH_OPENAI_API_KEY: ${{ secrets.GITNEXUS_BENCH_OPENAI_API_KEY }}
GITNEXUS_BENCH_ANTHROPIC_API_KEY: ${{ secrets.GITNEXUS_BENCH_ANTHROPIC_API_KEY }}
run: |
set -euo pipefail
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
(( remaining > 0 )) || { echo '::warning::No time remains for the CLI-only arm.'; exit 1; }
GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/candidate/cli-provider-usage.jsonl" \
GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-cli" \
timeout --signal=TERM --kill-after=60 "${remaining}s" \
uv run --locked --extra dev python -m workflow_bench.runner \
--tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \
--gitnexus-root "$GITHUB_WORKSPACE/candidate" \
--arms baseline_cli --runs 3 --include-expensive \
--workers 3 --timeout 3600 --model "$MODEL" --effort "$EFFORT" \
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \
--out "$RUNNER_TEMP/release-evaluation/candidate/cli-raw"
- name: Measure candidate with deferred MCP descriptions
if: inputs.mcp_tool_loading == true && inputs.candidate_pr == ''
timeout-minutes: 360
working-directory: eval
env:
GITNEXUS_BENCH_OPENAI_API_KEY: ${{ secrets.GITNEXUS_BENCH_OPENAI_API_KEY }}
GITNEXUS_BENCH_ANTHROPIC_API_KEY: ${{ secrets.GITNEXUS_BENCH_ANTHROPIC_API_KEY }}
run: |
set -euo pipefail
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
(( remaining > 0 )) || { echo '::error::No time remains for deferred sessions before the evidence reserve.'; exit 1; }
GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/candidate/deferred-provider-usage.jsonl" \
GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-deferred" \
timeout --signal=TERM --kill-after=60 "${remaining}s" \
uv run --locked --extra dev python -m workflow_bench.runner \
--tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \
--gitnexus-root "$GITHUB_WORKSPACE/candidate" \
--arms baseline --mcp-tool-loading deferred --runs 3 --include-expensive \
--workers 3 --timeout 3600 --model "$MODEL" --effort "$EFFORT" \
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \
--out "$RUNNER_TEMP/release-evaluation/candidate/deferred"
- name: Summarize complete or incomplete measurements
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
working-directory: eval
run: |
if [[ -n "$INPUT_PR" ]]; then
uv run --locked --extra dev python -m workflow_bench.pr_evaluation summarize \
--results "$RUNNER_TEMP/release-evaluation/candidate/raw/results.jsonl" \
--metadata "$RUNNER_TEMP/release-evaluation/candidate/metadata.json" \
--baseline "$RUNNER_TEMP/release-evaluation/previous.json" \
--sha '${{ steps.resolve.outputs.sha }}' --pr "$INPUT_PR" --run-id "$COMPARISON_RUN_ID" \
--out "$RUNNER_TEMP/release-evaluation/public"
exit 0
fi
failed=0
for runtime in stable candidate; do
if [[ -f "$RUNNER_TEMP/release-evaluation/$runtime/metadata.json" ]]; then
uv run --locked --extra dev python -m workflow_bench.release_report summarize \
--results "$RUNNER_TEMP/release-evaluation/$runtime/raw/results.jsonl" \
--metadata "$RUNNER_TEMP/release-evaluation/$runtime/metadata.json" \
--out "$RUNNER_TEMP/release-evaluation/$runtime/public" || failed=1
else
echo "::error::$runtime evaluation preparation failed; no measurements exist."
failed=1
fi
done
exit "$failed"
- name: Require complete paired evidence and no task regression
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }}
working-directory: eval
run: |
uv run --locked --extra dev python -m workflow_bench.release_gate \
--candidate "$RUNNER_TEMP/release-evaluation/candidate/public/agent-evaluation.json" \
--candidate-sha '${{ steps.resolve.outputs.sha }}' \
--stable "$RUNNER_TEMP/release-evaluation/stable/public/agent-evaluation.json" \
--stable-sha '${{ steps.resolve.outputs.stable_sha }}' \
--out "$RUNNER_TEMP/release-evaluation/public"
- name: Summarize matched MCP loading experiment
if: ${{ always() && inputs.mcp_tool_loading == true && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }}
working-directory: eval
run: |
uv run --locked --extra dev python -m workflow_bench.mcp_loading_report \
--eager "$RUNNER_TEMP/release-evaluation/candidate/raw/results.jsonl" \
--deferred "$RUNNER_TEMP/release-evaluation/candidate/deferred/results.jsonl" \
--metadata "$RUNNER_TEMP/release-evaluation/candidate/metadata.json" \
--out "$RUNNER_TEMP/release-evaluation/public"
- name: Collect sanitized evidence and publish summary
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
run: |
set -euo pipefail
public="$RUNNER_TEMP/release-evaluation/public"
mkdir -p "$public"
if [[ -n "$INPUT_PR" ]]; then
if [[ -f "$public/pr-cost-comparison.md" ]]; then cat "$public/pr-cost-comparison.md" >> "$GITHUB_STEP_SUMMARY"; fi
exit 0
fi
for extension in json md; do
candidate="$RUNNER_TEMP/release-evaluation/candidate/public/agent-evaluation.$extension"
stable="$RUNNER_TEMP/release-evaluation/stable/public/agent-evaluation.$extension"
if [[ -f "$candidate" ]]; then cp "$candidate" "$public/agent-evaluation.$extension"; fi
if [[ -f "$stable" ]]; then cp "$stable" "$public/stable-agent-evaluation.$extension"; fi
done
for report in release-quality-gate agent-evaluation mcp-tool-loading; do
if [[ -f "$public/$report.md" ]]; then cat "$public/$report.md" >> "$GITHUB_STEP_SUMMARY"; fi
done
- name: Summarize the opt-in CLI-only arm
if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
continue-on-error: true
working-directory: eval
run: |
set -euo pipefail
candidate="$RUNNER_TEMP/release-evaluation/candidate"
uv run --locked --extra dev python -m workflow_bench.cli_arm_report \
--paired "$candidate/raw/results.jsonl" --cli "$candidate/cli-raw/results.jsonl" \
--out "$RUNNER_TEMP/release-evaluation/public"
cat "$RUNNER_TEMP/release-evaluation/public/cli-arm-comparison.md" >> "$GITHUB_STEP_SUMMARY"
- name: Summarize time, cost, and tool measurements
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
working-directory: eval
env:
DEFERRED_ARM: ${{ inputs.mcp_tool_loading }}
run: |
set -euo pipefail
root="$RUNNER_TEMP/release-evaluation"
public="$root/public"
mkdir -p "$public"
failed=0
for variant in stable candidate cli deferred; do
case "$variant" in
stable)
if [[ -n "$INPUT_PR" ]]; then continue; fi
results="$root/stable/raw/results.jsonl"
ledger="$root/stable/provider-usage.jsonl" ;;
candidate)
results="$root/candidate/raw/results.jsonl"
ledger="$root/candidate/provider-usage.jsonl" ;;
cli)
if [[ "$CLI_ARM" != true ]]; then continue; fi
results="$root/candidate/cli-raw/results.jsonl"
ledger="$root/candidate/cli-provider-usage.jsonl" ;;
deferred)
if [[ "$DEFERRED_ARM" != true || -n "$INPUT_PR" ]]; then continue; fi
results="$root/candidate/deferred/results.jsonl"
ledger="$root/candidate/deferred-provider-usage.jsonl" ;;
esac
if ! uv run --locked --extra dev python -m workflow_bench.cost_diagnostics \
--results "$results" --provider-usage "$ledger" --allow-incomplete \
--out "$public/$variant-diagnostics.json" --markdown-out "$public/$variant-diagnostics.md"; then
printf '\nDetailed measurements for %s could not be validated; inspect the step log.\n' "$variant" >> "$GITHUB_STEP_SUMMARY"
failed=1
continue
fi
{
printf '\n<details><summary>Detailed measurements: %s</summary>\n\n' "$variant"
cat "$public/$variant-diagnostics.md"
printf '\n</details>\n'
} >> "$GITHUB_STEP_SUMMARY"
done
exit "$failed"
- name: Summarize the diagnostic trial
if: ${{ always() && inputs.diagnostic_trial == true && steps.resolve.outputs.sha != '' }}
working-directory: eval
run: |
set -euo pipefail
root="$RUNNER_TEMP/release-evaluation"
mkdir -p "$root/public"
printf "\n## Two-cell diagnostic trial\n\nOne task, one repetition per arm. \$5 CLI budget stop per cell; provider billing can exceed the planned \$10. This trial checks telemetry and is not release-quality evidence.\n" >> "$GITHUB_STEP_SUMMARY"
failed=0
for cell in cell-0000 cell-0001; do
if ! uv run --locked --extra dev python -m workflow_bench.cost_diagnostics \
--results "$root/trial/results/$cell/results.jsonl" \
--provider-usage "$root/trial/results/$cell.provider.jsonl" --allow-incomplete \
--out "$root/public/trial-$cell-diagnostics.json" \
--markdown-out "$root/public/trial-$cell-diagnostics.md"; then
printf '\nMeasurements for %s could not be validated.\n' "$cell" >> "$GITHUB_STEP_SUMMARY"
failed=1
continue
fi
cat "$root/public/trial-$cell-diagnostics.md" >> "$GITHUB_STEP_SUMMARY"
done
exit "$failed"
- name: Upload sanitized diagnostic trial measurements
if: ${{ always() && inputs.diagnostic_trial == true && steps.resolve.outputs.sha != '' }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: diagnostic-trial-${{ steps.resolve.outputs.sha }}-${{ github.run_id }}
path: |
${{ runner.temp }}/release-evaluation/public/trial-*-diagnostics.json
${{ runner.temp }}/release-evaluation/public/trial-*-diagnostics.md
if-no-files-found: error
retention-days: 30
- name: Upload sanitized release evidence
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-agent-evaluation-${{ steps.resolve.outputs.sha }}
path: |
${{ runner.temp }}/release-evaluation/public/agent-evaluation.json
${{ runner.temp }}/release-evaluation/public/agent-evaluation.md
${{ runner.temp }}/release-evaluation/public/stable-agent-evaluation.json
${{ runner.temp }}/release-evaluation/public/stable-agent-evaluation.md
${{ runner.temp }}/release-evaluation/public/release-quality-gate.json
${{ runner.temp }}/release-evaluation/public/release-quality-gate.md
${{ runner.temp }}/release-evaluation/public/mcp-tool-loading.json
${{ runner.temp }}/release-evaluation/public/mcp-tool-loading.md
${{ runner.temp }}/release-evaluation/public/*-diagnostics.json
${{ runner.temp }}/release-evaluation/public/*-diagnostics.md
if-no-files-found: error
retention-days: 30
- name: Upload the opt-in CLI-only arm comparison
if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-cli-arm-${{ steps.resolve.outputs.sha }}
path: |
${{ runner.temp }}/release-evaluation/public/cli-arm-comparison.json
${{ runner.temp }}/release-evaluation/public/cli-arm-comparison.md
${{ runner.temp }}/release-evaluation/public/cli-diagnostics.json
${{ runner.temp }}/release-evaluation/public/cli-diagnostics.md
if-no-files-found: warn
retention-days: 30
- name: Upload sanitized PR cost comparison
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr != '' && inputs.diagnostic_trial != true }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: pr-cost-comparison-${{ steps.resolve.outputs.sha }}
path: |
${{ runner.temp }}/release-evaluation/public/pr-cost-comparison.json
${{ runner.temp }}/release-evaluation/public/pr-cost-comparison.md
${{ runner.temp }}/release-evaluation/public/*-diagnostics.json
${{ runner.temp }}/release-evaluation/public/*-diagnostics.md
if-no-files-found: error
retention-days: 30
stop-runner:
name: Stop and verify the dedicated EC2 runner
needs: [start-runner, check-runner, runner-ready, watch-evaluate-pickup, evaluate]
if: >-
always() && github.repository == 'abhigyanpatwari/GitNexus' &&
github.ref == 'refs/heads/main' &&
needs.start-runner.outputs.cleanup_required == 'true'
runs-on: ubuntu-latest
environment: gitnexus-evolution
timeout-minutes: 20
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: ./.github/actions/ec2-runner
with:
operation: stop
role-arn: ${{ secrets.GITNEXUS_EVOLUTION_AWS_ROLE_ARN }}
instance-id: ${{ secrets.GITNEXUS_EVOLUTION_EC2_INSTANCE_ID }}
region: ${{ vars.GITNEXUS_EVOLUTION_AWS_REGION }}
stop-schedule: ${{ vars.GITNEXUS_EVOLUTION_STOP_SCHEDULE_UTC }}