# Compare the exact candidate with the last stable release using trusted main's # harness. Share the existing evolution runner, protected settings, and power lock. # # The shared group keeps only ONE pending run, even with cancel-in-progress: # false: a newer queued skill-evolution or release-evaluation run cancels this # run while it is pending. Before relying on a dispatched evaluation for a # stable publish, confirm its "Start the dedicated EC2 runner" job ran; if the # run shows Cancelled, dispatch it again once the group is free. name: Release evaluation on: schedule: - cron: '0 5 * * 3' # Wednesday 05:00 UTC workflow_dispatch: inputs: runner_only: description: Start, probe, and stop EC2 without paid model calls type: boolean default: false candidate_ref: description: 'Full commit SHA or version tag to evaluate; latest-rc resolves the newest published RC' type: string default: latest-rc model: description: 'Named model for both arms' type: string default: gpt-5.6-sol effort: description: 'Reasoning effort for both arms' type: choice default: xhigh options: [low, medium, high, xhigh, max] # PR preview inputs are available only to the maintainer-authorized label # workflow. Manual dispatch cannot select an unmerged PR runtime. workflow_call: secrets: GITNEXUS_EVOLUTION_AWS_ROLE_ARN: required: true GITNEXUS_EVOLUTION_EC2_INSTANCE_ID: required: true GITNEXUS_BENCH_OPENAI_API_KEY: required: true GITNEXUS_BENCH_ANTHROPIC_API_KEY: required: false inputs: candidate_pr: type: string required: true expected_pr_sha: type: string required: true comparison_run_id: type: string required: true model: type: string default: gpt-5.6-sol effort: type: string default: xhigh concurrency: group: gitnexus-evolution-runner cancel-in-progress: false permissions: {} jobs: start-runner: name: Start the dedicated EC2 runner if: >- github.repository == 'abhigyanpatwari/GitNexus' && github.ref == 'refs/heads/main' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_target' && inputs.candidate_pr != '') || (vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3')) runs-on: ubuntu-latest environment: gitnexus-evolution timeout-minutes: 25 permissions: contents: read id-token: write outputs: run_attempt: ${{ github.run_attempt }} cleanup_required: ${{ steps.boot.outputs.cleanup_required }} stop_deadline_epoch: ${{ steps.boot.outputs.stop_deadline_epoch }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: ./.github/actions/ec2-runner id: boot with: operation: start role-arn: ${{ secrets.GITNEXUS_EVOLUTION_AWS_ROLE_ARN }} instance-id: ${{ secrets.GITNEXUS_EVOLUTION_EC2_INSTANCE_ID }} region: ${{ vars.GITNEXUS_EVOLUTION_AWS_REGION }} stop-schedule: ${{ vars.GITNEXUS_EVOLUTION_STOP_SCHEDULE_UTC }} check-runner: needs: [start-runner] name: Bound the wait for the evaluation runner if: >- github.repository == 'abhigyanpatwari/GitNexus' && github.ref == 'refs/heads/main' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_target' && inputs.candidate_pr != '') || (vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3')) runs-on: ubuntu-latest environment: gitnexus-evolution timeout-minutes: 10 permissions: contents: read actions: write # Cancel only this run if its dedicated runner stays offline. steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # EC2 health does not prove the Actions service can accept a job. - name: Verify the current run's native pickup probe env: GH_TOKEN: ${{ github.token }} run: python3 .github/scripts/evolution-runner-ready.py # A queued self-hosted job has no pickup timeout. Cancel this run so an # offline probe cannot remain queued indefinitely and block later runs. - name: Cancel this run when runner readiness fails if: failure() env: GH_TOKEN: ${{ github.token }} run: gh api --method POST "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel" runner-ready: needs: [start-runner] name: Verify the runner service is online if: >- github.repository == 'abhigyanpatwari/GitNexus' && github.ref == 'refs/heads/main' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_target' && inputs.candidate_pr != '') || (vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3')) # Partial reruns reuse successful startup outputs after cleanup stopped EC2. runs-on: ${{ fromJSON(needs.start-runner.outputs.run_attempt == github.run_attempt && '["self-hosted", "linux", "x64", "gitnexus-evolution"]' || '["ubuntu-latest"]') }} environment: gitnexus-evolution timeout-minutes: 5 permissions: {} steps: - name: Require startup in this run attempt run: | set -euo pipefail if [[ '${{ needs.start-runner.outputs.run_attempt }}' != "$GITHUB_RUN_ATTEMPT" ]]; then echo '::error::The EC2 startup belongs to an earlier attempt. Select Re-run all jobs to start the runner again.' exit 1 fi - name: Verify runner pickup run: | set -euo pipefail test -r /proc/uptime git --version uname -sm watch-evaluate-pickup: name: Bound pickup of the paid evaluation job needs: [start-runner, check-runner, runner-ready] if: inputs.runner_only != true runs-on: ubuntu-latest timeout-minutes: 10 permissions: contents: read actions: write # Cancel this run if evaluate stays queued after the probe. steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Require pickup of this run's paid evaluation job env: GH_TOKEN: ${{ github.token }} run: python3 .github/scripts/evolution-runner-ready.py --job-name "Evaluate candidate and stable quality" - name: Cancel this run when evaluation pickup fails if: failure() env: GH_TOKEN: ${{ github.token }} run: gh api --method POST "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel" evaluate: name: Evaluate candidate and stable quality needs: [start-runner, check-runner, runner-ready] if: >- github.repository == 'abhigyanpatwari/GitNexus' && github.ref == 'refs/heads/main' && inputs.runner_only != true && (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_target' && inputs.candidate_pr != '') || (vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3')) runs-on: ${{ fromJSON(needs.start-runner.outputs.run_attempt == github.run_attempt && '["self-hosted", "linux", "x64", "gitnexus-evolution"]' || '["ubuntu-latest"]') }} # Reuse the main-only protected model-secret environment. No tag/fork code # runs as the harness, and no release or PR write token is available here. environment: gitnexus-evolution timeout-minutes: 1260 permissions: contents: read actions: read # Authenticate and download the explicitly selected historical measurement. pull-requests: read env: INPUT_REF: ${{ inputs.candidate_ref || 'latest-rc' }} INPUT_PR: ${{ inputs.candidate_pr }} INPUT_PR_SHA: ${{ inputs.expected_pr_sha }} COMPARISON_RUN_ID: ${{ inputs.comparison_run_id }} MODEL: ${{ inputs.model || 'gpt-5.6-sol' }} EFFORT: ${{ inputs.effort || 'xhigh' }} steps: - name: Require startup in this run attempt run: | set -euo pipefail if [[ '${{ needs.start-runner.outputs.run_attempt }}' != "$GITHUB_RUN_ATTEMPT" ]]; then echo '::error::The EC2 startup belongs to an earlier attempt. Select Re-run all jobs to start the runner again.' exit 1 fi - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 - name: Resolve immutable runtime revision id: resolve working-directory: eval env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail if [[ -n "$INPUT_PR" ]]; then [[ "$COMPARISON_RUN_ID" =~ ^[1-9][0-9]*$ ]] || { echo '::error::PR preview requires comparison_run_id.'; exit 1; } python3 -m workflow_bench.pr_evaluation resolve \ --repo "$GITHUB_REPOSITORY" --pr "$INPUT_PR" --sha "$INPUT_PR_SHA" >> "$GITHUB_OUTPUT" exit 0 fi [[ -z "$INPUT_PR_SHA" && -z "$COMPARISON_RUN_ID" ]] || { echo '::error::PR preview inputs require candidate_pr.'; exit 1; } # Resolve last stable once; both runtimes use the same trusted tasks. stable="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq '.tag_name')" [[ "$stable" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo '::error::No stable release found.'; exit 1; } stable_sha="$(git rev-parse --verify "$stable^{commit}")" echo "stable_sha=$stable_sha" >> "$GITHUB_OUTPUT" ref="$INPUT_REF" if [[ "$ref" == 'latest-rc' ]]; then ref="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --jq \ '[.[] | select(.prerelease and (.draft | not)) | select(.tag_name | test("^v[0-9]+\\.[0-9]+\\.[0-9]+-rc\\.[0-9]+$"))] | sort_by(.published_at) | last | .tag_name // empty')" fi if ! [[ "$ref" =~ ^[0-9a-f]{40}$ || "$ref" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then echo '::error::No published RC found, or candidate_ref is not a full SHA/version tag.' exit 1 fi sha="$(git rev-parse --verify "$ref^{commit}")" if [[ "$ref" =~ ^[0-9a-f]{40}$ ]]; then # Manual stable candidates must already be reviewed on main. git merge-base --is-ancestor "$sha" HEAD else # Versioned RC trees are published descendants, not ancestors of # main. Accept their official published release rather than PR refs. gh api "repos/$GITHUB_REPOSITORY/releases/tags/$ref" --jq '.draft == false' | grep -qx true fi echo "Runtime: $sha" echo "sha=$sha" >> "$GITHUB_OUTPUT" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ steps.resolve.outputs.sha }} path: candidate persist-credentials: false fetch-depth: 0 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: inputs.candidate_pr == '' with: ref: ${{ steps.resolve.outputs.stable_sha }} path: stable persist-credentials: false fetch-depth: 0 - name: Initialize this run's measurement directory run: | rm -rf "$RUNNER_TEMP/release-evaluation" mkdir -p "$RUNNER_TEMP/release-evaluation/public" - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '22.18.0' cache: npm cache-dependency-path: candidate/gitnexus/package-lock.json - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: version: '0.11.23' python-version: '3.13' enable-cache: true cache-dependency-glob: eval/uv.lock - name: Authenticate historical PR comparison evidence if: inputs.candidate_pr != '' working-directory: eval env: GH_TOKEN: ${{ github.token }} run: | uv run --locked --extra dev python -m workflow_bench.pr_evaluation baseline \ --repo "$GITHUB_REPOSITORY" --run-id "$COMPARISON_RUN_ID" \ --harness-sha "$GITHUB_SHA" --model "$MODEL" --effort "$EFFORT" \ --out "$RUNNER_TEMP/release-evaluation/previous.json" # Both runtimes grade the pinned task source against one dependency tree # built from that commit's lockfile, never the runtime under test's. - name: Resolve pinned task revision id: tasks working-directory: eval run: | set -euo pipefail sha="$(uv run --locked --extra dev python -m workflow_bench.release_report task-sha)" [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || { echo '::error::The release task set has no single pinned commit.'; exit 1; } echo "sha=$sha" >> "$GITHUB_OUTPUT" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ steps.tasks.outputs.sha }} path: tasks-base persist-credentials: false fetch-depth: 0 - name: Install sandbox runtime and pinned Claude CLI run: | set -euo pipefail sudo apt-get -o DPkg::Lock::Timeout=600 update sudo apt-get -o DPkg::Lock::Timeout=600 install --yes --no-install-recommends bubblewrap ripgrep socat apparmor_userns=/proc/sys/kernel/apparmor_restrict_unprivileged_userns if [[ -r "$apparmor_userns" ]] && [[ "$(<"$apparmor_userns")" == '1' ]]; then sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 fi cli="$RUNNER_TEMP/claude-canary" install -d -m 0700 "$cli" install -m 0600 .github/claude-canary-runtime/package{,-lock}.json "$cli/" npm ci --prefix "$cli" --audit=false --fund=false test "$("$cli/node_modules/@anthropic-ai/claude-code-linux-x64/claude" --version)" = '2.1.214 (Claude Code)' - name: Build pinned runtime and task dependencies working-directory: eval run: | set -euo pipefail for checkout in tasks-base stable candidate; do if [[ -n "$INPUT_PR" && "$checkout" == stable ]]; then continue; fi uv run --locked --extra dev python -m workflow_bench.release_build \ --repo "$GITHUB_WORKSPACE/$checkout" done - name: Prepare pinned task set working-directory: eval run: | set -euo pipefail for runtime in stable candidate; do if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi uv run --locked --extra dev python -m workflow_bench.release_report prepare \ --repo "$GITHUB_WORKSPACE/$runtime" --task-repo "$GITHUB_WORKSPACE/tasks-base" \ --out "$RUNNER_TEMP/release-evaluation/$runtime" \ --model "$MODEL" --effort "$EFFORT" --runs 3 done - name: Require historical task dependencies before paid PR sessions if: inputs.candidate_pr != '' working-directory: eval run: | uv run --locked --extra dev python -m workflow_bench.pr_evaluation dependencies \ --tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \ --baseline "$RUNNER_TEMP/release-evaluation/previous.json" - name: Require real containment before paid sessions working-directory: eval env: GITNEXUS_REQUIRE_BWRAP_CANARY: '1' GITNEXUS_REQUIRE_CLAUDE_CANARY: '1' CLAUDE_CANARY_BIN: ${{ runner.temp }}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude run: >- uv run --locked --extra dev python -m pytest tests/test_proposer_sandbox.py tests/test_workflow_bench_sessions.py::test_real_cli_background_completion_is_one_complete_measurement -q - name: Materialize both release graphs before paid sessions timeout-minutes: 120 working-directory: eval run: | set -euo pipefail for runtime in stable candidate; do if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi uv run --locked --extra dev python -m workflow_bench.release_preflight \ --tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \ --gitnexus-root "$GITHUB_WORKSPACE/$runtime" \ --claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" done - name: Recheck PR opt-in and pinned head immediately before paid sessions if: inputs.candidate_pr != '' working-directory: eval env: GH_TOKEN: ${{ github.token }} run: | python3 -m workflow_bench.pr_evaluation resolve \ --repo "$GITHUB_REPOSITORY" --pr "$INPUT_PR" --sha "$INPUT_PR_SHA" - name: Run repeated paired agent sessions timeout-minutes: 1140 working-directory: eval env: GITNEXUS_BENCH_OPENAI_API_KEY: ${{ secrets.GITNEXUS_BENCH_OPENAI_API_KEY }} GITNEXUS_BENCH_ANTHROPIC_API_KEY: ${{ secrets.GITNEXUS_BENCH_ANTHROPIC_API_KEY }} run: | set -euo pipefail arms=(baseline_nomcp baseline) if [[ -n "$INPUT_PR" ]]; then arms=(baseline); fi for runtime in stable candidate; do if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 )) (( remaining > 0 )) || { echo '::error::No time remains before the stop and evidence reserve.'; exit 1; } timeout --signal=TERM --kill-after=60 "${remaining}s" \ uv run --locked --extra dev python -m workflow_bench.runner \ --tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \ --gitnexus-root "$GITHUB_WORKSPACE/$runtime" \ --arms "${arms[@]}" --runs 3 --include-expensive \ --workers 3 --timeout 3600 --model "$MODEL" --effort "$EFFORT" \ --claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \ --out "$RUNNER_TEMP/release-evaluation/$runtime/raw" done - name: Summarize complete or incomplete measurements if: ${{ always() && steps.resolve.outputs.sha != '' }} working-directory: eval run: | if [[ -n "$INPUT_PR" ]]; then uv run --locked --extra dev python -m workflow_bench.pr_evaluation summarize \ --results "$RUNNER_TEMP/release-evaluation/candidate/raw/results.jsonl" \ --metadata "$RUNNER_TEMP/release-evaluation/candidate/metadata.json" \ --baseline "$RUNNER_TEMP/release-evaluation/previous.json" \ --sha '${{ steps.resolve.outputs.sha }}' --pr "$INPUT_PR" --run-id "$COMPARISON_RUN_ID" \ --out "$RUNNER_TEMP/release-evaluation/public" exit 0 fi failed=0 for runtime in stable candidate; do if [[ -f "$RUNNER_TEMP/release-evaluation/$runtime/metadata.json" ]]; then uv run --locked --extra dev python -m workflow_bench.release_report summarize \ --results "$RUNNER_TEMP/release-evaluation/$runtime/raw/results.jsonl" \ --metadata "$RUNNER_TEMP/release-evaluation/$runtime/metadata.json" \ --out "$RUNNER_TEMP/release-evaluation/$runtime/public" || failed=1 else echo "::error::$runtime evaluation preparation failed; no measurements exist." failed=1 fi done exit "$failed" - name: Require complete paired evidence and no task regression if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' }} working-directory: eval run: | uv run --locked --extra dev python -m workflow_bench.release_gate \ --candidate "$RUNNER_TEMP/release-evaluation/candidate/public/agent-evaluation.json" \ --candidate-sha '${{ steps.resolve.outputs.sha }}' \ --stable "$RUNNER_TEMP/release-evaluation/stable/public/agent-evaluation.json" \ --stable-sha '${{ steps.resolve.outputs.stable_sha }}' \ --out "$RUNNER_TEMP/release-evaluation/public" - name: Collect sanitized evidence and publish summary if: ${{ always() && steps.resolve.outputs.sha != '' }} run: | set -euo pipefail public="$RUNNER_TEMP/release-evaluation/public" mkdir -p "$public" if [[ -n "$INPUT_PR" ]]; then if [[ -f "$public/pr-cost-comparison.md" ]]; then cat "$public/pr-cost-comparison.md" >> "$GITHUB_STEP_SUMMARY"; fi exit 0 fi for extension in json md; do candidate="$RUNNER_TEMP/release-evaluation/candidate/public/agent-evaluation.$extension" stable="$RUNNER_TEMP/release-evaluation/stable/public/agent-evaluation.$extension" if [[ -f "$candidate" ]]; then cp "$candidate" "$public/agent-evaluation.$extension"; fi if [[ -f "$stable" ]]; then cp "$stable" "$public/stable-agent-evaluation.$extension"; fi done for report in release-quality-gate agent-evaluation; do if [[ -f "$public/$report.md" ]]; then cat "$public/$report.md" >> "$GITHUB_STEP_SUMMARY"; fi done - name: Upload sanitized release evidence if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-agent-evaluation-${{ steps.resolve.outputs.sha }} path: | ${{ runner.temp }}/release-evaluation/public/agent-evaluation.json ${{ runner.temp }}/release-evaluation/public/agent-evaluation.md ${{ runner.temp }}/release-evaluation/public/stable-agent-evaluation.json ${{ runner.temp }}/release-evaluation/public/stable-agent-evaluation.md ${{ runner.temp }}/release-evaluation/public/release-quality-gate.json ${{ runner.temp }}/release-evaluation/public/release-quality-gate.md if-no-files-found: error retention-days: 30 - name: Upload sanitized PR cost comparison if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr != '' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: pr-cost-comparison-${{ steps.resolve.outputs.sha }} path: | ${{ runner.temp }}/release-evaluation/public/pr-cost-comparison.json ${{ runner.temp }}/release-evaluation/public/pr-cost-comparison.md if-no-files-found: error retention-days: 30 stop-runner: name: Stop and verify the dedicated EC2 runner needs: [start-runner, check-runner, runner-ready, watch-evaluate-pickup, evaluate] if: >- always() && github.repository == 'abhigyanpatwari/GitNexus' && github.ref == 'refs/heads/main' && needs.start-runner.outputs.cleanup_required == 'true' runs-on: ubuntu-latest environment: gitnexus-evolution timeout-minutes: 20 permissions: contents: read id-token: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: ./.github/actions/ec2-runner with: operation: stop role-arn: ${{ secrets.GITNEXUS_EVOLUTION_AWS_ROLE_ARN }} instance-id: ${{ secrets.GITNEXUS_EVOLUTION_EC2_INSTANCE_ID }} region: ${{ vars.GITNEXUS_EVOLUTION_AWS_REGION }} stop-schedule: ${{ vars.GITNEXUS_EVOLUTION_STOP_SCHEDULE_UTC }}