mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-11-16 09:22:00 +00:00
* fix(eval): validate trials and report measurements Keep pinned task dependencies separate from the candidate runtime. Gate paid workflow sessions on contained reference grading and retain private-safe setup, verification, provenance and provider-token measurements. * fix(eval): align CI with the release task build * fix(eval): address review feedback (#3779) Bind trial dependency bytes before execution and include MCP environment instructions in comparator identity. Preserve verifier states and strengthen preflight coverage. --------- Co-authored-by: Gergo Magyar <gergomagyar0@gmail.com>
758 lines
39 KiB
YAML
758 lines
39 KiB
YAML
# Compare the exact candidate with the last stable release using trusted main's
|
|
# harness. Share the existing evolution runner, protected settings, and power lock.
|
|
#
|
|
# The shared group keeps only ONE pending run, even with cancel-in-progress:
|
|
# false: a newer queued skill-evolution or release-evaluation run cancels this
|
|
# run while it is pending. Before relying on a dispatched evaluation for a
|
|
# stable publish, confirm its "Start the dedicated EC2 runner" job ran; if the
|
|
# run shows Cancelled, dispatch it again once the group is free.
|
|
name: Release evaluation
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 5 * * 3' # Wednesday 05:00 UTC
|
|
workflow_dispatch:
|
|
inputs:
|
|
runner_only:
|
|
description: Start, probe, and stop EC2 without paid model calls
|
|
type: boolean
|
|
default: false
|
|
diagnostic_trial:
|
|
description: 'Two serial cells, $5 CLI stop each; requires a full candidate SHA, gpt-5.6-sol/xhigh, and no other experiment flags'
|
|
type: boolean
|
|
default: false
|
|
mcp_tool_loading:
|
|
description: Add candidate deferred-MCP experiment (12 extra paid cells)
|
|
type: boolean
|
|
default: false
|
|
candidate_ref:
|
|
description: 'Full commit SHA or version tag to evaluate; latest-rc resolves the newest published RC'
|
|
type: string
|
|
default: latest-rc
|
|
model:
|
|
description: 'Named model for both arms'
|
|
type: string
|
|
default: gpt-5.6-sol
|
|
effort:
|
|
description: 'Reasoning effort for both arms'
|
|
type: choice
|
|
default: xhigh
|
|
options: [low, medium, high, xhigh, max]
|
|
cli_arm:
|
|
description: 'Also run baseline_cli (same tools via a gitnexus shell command, no MCP catalog) on the candidate; adds 12 paid cells, descriptive only, never gates'
|
|
type: boolean
|
|
default: false
|
|
# PR preview inputs are available only to the maintainer-authorized label
|
|
# workflow. Manual dispatch cannot select an unmerged PR runtime.
|
|
workflow_call:
|
|
secrets:
|
|
GITNEXUS_EVOLUTION_AWS_ROLE_ARN:
|
|
required: true
|
|
GITNEXUS_EVOLUTION_EC2_INSTANCE_ID:
|
|
required: true
|
|
GITNEXUS_BENCH_OPENAI_API_KEY:
|
|
required: true
|
|
GITNEXUS_BENCH_ANTHROPIC_API_KEY:
|
|
required: false
|
|
inputs:
|
|
candidate_pr:
|
|
type: string
|
|
required: true
|
|
expected_pr_sha:
|
|
type: string
|
|
required: true
|
|
comparison_run_id:
|
|
type: string
|
|
required: true
|
|
model:
|
|
type: string
|
|
default: gpt-5.6-sol
|
|
effort:
|
|
type: string
|
|
default: xhigh
|
|
|
|
concurrency:
|
|
group: gitnexus-evolution-runner
|
|
cancel-in-progress: false
|
|
|
|
permissions: {}
|
|
|
|
jobs:
|
|
start-runner:
|
|
name: Start the dedicated EC2 runner
|
|
if: >-
|
|
github.repository == 'abhigyanpatwari/GitNexus' &&
|
|
github.ref == 'refs/heads/main' &&
|
|
(github.event_name == 'workflow_dispatch' ||
|
|
(github.event_name == 'pull_request_target' && inputs.candidate_pr != '') ||
|
|
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
|
|
runs-on: ubuntu-latest
|
|
environment: gitnexus-evolution
|
|
timeout-minutes: 25
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
outputs:
|
|
run_attempt: ${{ github.run_attempt }}
|
|
cleanup_required: ${{ steps.boot.outputs.cleanup_required }}
|
|
stop_deadline_epoch: ${{ steps.boot.outputs.stop_deadline_epoch }}
|
|
steps:
|
|
- name: Validate diagnostic trial authorization scope
|
|
env:
|
|
TRIAL: ${{ inputs.diagnostic_trial }}
|
|
RUNNER_ONLY: ${{ inputs.runner_only }}
|
|
CLI_ARM: ${{ inputs.cli_arm }}
|
|
DEFERRED_ARM: ${{ inputs.mcp_tool_loading }}
|
|
CANDIDATE_REF: ${{ inputs.candidate_ref }}
|
|
MODEL: ${{ inputs.model }}
|
|
EFFORT: ${{ inputs.effort }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ "$TRIAL" == true ]]; then
|
|
if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_RUN_ATTEMPT" != 1 ||
|
|
"$RUNNER_ONLY" == true || "$CLI_ARM" == true || "$DEFERRED_ARM" == true ||
|
|
"$MODEL" != gpt-5.6-sol || "$EFFORT" != xhigh || ! "$CANDIDATE_REF" =~ ^[0-9a-f]{40}$ ]]; then
|
|
echo '::error::Trial requires a fresh manual dispatch, a full candidate SHA, gpt-5.6-sol/xhigh, and no other experiment flags. Paid trials are never retried automatically.'
|
|
exit 1
|
|
fi
|
|
fi
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
- uses: ./.github/actions/ec2-runner
|
|
id: boot
|
|
with:
|
|
operation: start
|
|
role-arn: ${{ secrets.GITNEXUS_EVOLUTION_AWS_ROLE_ARN }}
|
|
instance-id: ${{ secrets.GITNEXUS_EVOLUTION_EC2_INSTANCE_ID }}
|
|
region: ${{ vars.GITNEXUS_EVOLUTION_AWS_REGION }}
|
|
stop-schedule: ${{ vars.GITNEXUS_EVOLUTION_STOP_SCHEDULE_UTC }}
|
|
|
|
check-runner:
|
|
needs: [start-runner]
|
|
name: Bound the wait for the evaluation runner
|
|
if: >-
|
|
github.repository == 'abhigyanpatwari/GitNexus' &&
|
|
github.ref == 'refs/heads/main' &&
|
|
(github.event_name == 'workflow_dispatch' ||
|
|
(github.event_name == 'pull_request_target' && inputs.candidate_pr != '') ||
|
|
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
|
|
runs-on: ubuntu-latest
|
|
environment: gitnexus-evolution
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
actions: write # Cancel only this run if its dedicated runner stays offline.
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
# EC2 health does not prove the Actions service can accept a job.
|
|
- name: Verify the current run's native pickup probe
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: python3 .github/scripts/evolution-runner-ready.py
|
|
# A queued self-hosted job has no pickup timeout. Cancel this run so an
|
|
# offline probe cannot remain queued indefinitely and block later runs.
|
|
- name: Cancel this run when runner readiness fails
|
|
if: failure()
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: gh api --method POST "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel"
|
|
|
|
runner-ready:
|
|
needs: [start-runner]
|
|
name: Verify the runner service is online
|
|
if: >-
|
|
github.repository == 'abhigyanpatwari/GitNexus' &&
|
|
github.ref == 'refs/heads/main' &&
|
|
(github.event_name == 'workflow_dispatch' ||
|
|
(github.event_name == 'pull_request_target' && inputs.candidate_pr != '') ||
|
|
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
|
|
# Partial reruns reuse successful startup outputs after cleanup stopped EC2.
|
|
runs-on: ${{ fromJSON(needs.start-runner.outputs.run_attempt == github.run_attempt && '["self-hosted", "linux", "x64", "gitnexus-evolution"]' || '["ubuntu-latest"]') }}
|
|
environment: gitnexus-evolution
|
|
timeout-minutes: 5
|
|
permissions: {}
|
|
steps:
|
|
- name: Require startup in this run attempt
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ '${{ needs.start-runner.outputs.run_attempt }}' != "$GITHUB_RUN_ATTEMPT" ]]; then
|
|
echo '::error::The EC2 startup belongs to an earlier attempt. Select Re-run all jobs to start the runner again.'
|
|
exit 1
|
|
fi
|
|
- name: Verify runner pickup
|
|
run: |
|
|
set -euo pipefail
|
|
test -r /proc/uptime
|
|
git --version
|
|
uname -sm
|
|
|
|
watch-evaluate-pickup:
|
|
name: Bound pickup of the paid evaluation job
|
|
needs: [start-runner, check-runner, runner-ready]
|
|
if: inputs.runner_only != true
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
actions: write # Cancel this run if evaluate stays queued after the probe.
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
- name: Require pickup of this run's paid evaluation job
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: python3 .github/scripts/evolution-runner-ready.py --job-name "Evaluate candidate and stable quality"
|
|
- name: Cancel this run when evaluation pickup fails
|
|
if: failure()
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: gh api --method POST "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel"
|
|
|
|
evaluate:
|
|
name: Evaluate candidate and stable quality
|
|
needs: [start-runner, check-runner, runner-ready]
|
|
if: >-
|
|
github.repository == 'abhigyanpatwari/GitNexus' &&
|
|
github.ref == 'refs/heads/main' &&
|
|
inputs.runner_only != true &&
|
|
(github.event_name == 'workflow_dispatch' ||
|
|
(github.event_name == 'pull_request_target' && inputs.candidate_pr != '') ||
|
|
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
|
|
runs-on: ${{ fromJSON(needs.start-runner.outputs.run_attempt == github.run_attempt && '["self-hosted", "linux", "x64", "gitnexus-evolution"]' || '["ubuntu-latest"]') }}
|
|
# Reuse the main-only protected model-secret environment. No tag/fork code
|
|
# runs as the harness, and no release or PR write token is available here.
|
|
environment: gitnexus-evolution
|
|
timeout-minutes: 1260
|
|
permissions:
|
|
contents: read
|
|
actions: read # Authenticate and download the explicitly selected historical measurement.
|
|
pull-requests: read
|
|
env:
|
|
INPUT_REF: ${{ inputs.candidate_ref || 'latest-rc' }}
|
|
INPUT_PR: ${{ inputs.candidate_pr }}
|
|
INPUT_PR_SHA: ${{ inputs.expected_pr_sha }}
|
|
COMPARISON_RUN_ID: ${{ inputs.comparison_run_id }}
|
|
MODEL: ${{ inputs.model || 'gpt-5.6-sol' }}
|
|
EFFORT: ${{ inputs.effort || 'xhigh' }}
|
|
# Manual dispatch only; schedules and PR previews never pay for the extra arm.
|
|
CLI_ARM: ${{ inputs.cli_arm == true && inputs.candidate_pr == '' && 'true' || 'false' }}
|
|
steps:
|
|
- name: Require startup in this run attempt
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ '${{ needs.start-runner.outputs.run_attempt }}' != "$GITHUB_RUN_ATTEMPT" ]]; then
|
|
echo '::error::The EC2 startup belongs to an earlier attempt. Select Re-run all jobs to start the runner again.'
|
|
exit 1
|
|
fi
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
fetch-depth: 0
|
|
- name: Resolve immutable runtime revision
|
|
id: resolve
|
|
working-directory: eval
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ -n "$INPUT_PR" ]]; then
|
|
[[ "$COMPARISON_RUN_ID" =~ ^[1-9][0-9]*$ ]] || { echo '::error::PR preview requires comparison_run_id.'; exit 1; }
|
|
python3 -m workflow_bench.pr_evaluation resolve \
|
|
--repo "$GITHUB_REPOSITORY" --pr "$INPUT_PR" --sha "$INPUT_PR_SHA" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
fi
|
|
[[ -z "$INPUT_PR_SHA" && -z "$COMPARISON_RUN_ID" ]] || { echo '::error::PR preview inputs require candidate_pr.'; exit 1; }
|
|
# Resolve last stable once; both runtimes use the same trusted tasks.
|
|
if [[ "${DIAGNOSTIC_TRIAL:-false}" != true ]]; then
|
|
stable="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq '.tag_name')"
|
|
[[ "$stable" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo '::error::No stable release found.'; exit 1; }
|
|
stable_sha="$(git rev-parse --verify "$stable^{commit}")"
|
|
echo "stable_sha=$stable_sha" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
ref="$INPUT_REF"
|
|
if [[ "$ref" == 'latest-rc' ]]; then
|
|
ref="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --jq \
|
|
'[.[] | select(.prerelease and (.draft | not)) | select(.tag_name | test("^v[0-9]+\\.[0-9]+\\.[0-9]+-rc\\.[0-9]+$"))] | sort_by(.published_at) | last | .tag_name // empty')"
|
|
fi
|
|
if ! [[ "$ref" =~ ^[0-9a-f]{40}$ || "$ref" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then
|
|
echo '::error::No published RC found, or candidate_ref is not a full SHA/version tag.'
|
|
exit 1
|
|
fi
|
|
sha="$(git rev-parse --verify "$ref^{commit}")"
|
|
if [[ "$ref" =~ ^[0-9a-f]{40}$ ]]; then
|
|
# Manual stable candidates must already be reviewed on main.
|
|
git merge-base --is-ancestor "$sha" HEAD
|
|
else
|
|
# Versioned RC trees are published descendants, not ancestors of
|
|
# main. Accept their official published release rather than PR refs.
|
|
gh api "repos/$GITHUB_REPOSITORY/releases/tags/$ref" --jq '.draft == false' | grep -qx true
|
|
fi
|
|
echo "Runtime: $sha"
|
|
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ steps.resolve.outputs.sha }}
|
|
path: candidate
|
|
persist-credentials: false
|
|
fetch-depth: 0
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
if: inputs.candidate_pr == '' && inputs.diagnostic_trial != true
|
|
with:
|
|
ref: ${{ steps.resolve.outputs.stable_sha }}
|
|
path: stable
|
|
persist-credentials: false
|
|
fetch-depth: 0
|
|
- name: Initialize this run's measurement directory
|
|
run: |
|
|
rm -rf "$RUNNER_TEMP/release-evaluation"
|
|
mkdir -p "$RUNNER_TEMP/release-evaluation/public"
|
|
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
|
with:
|
|
node-version: '22.18.0'
|
|
cache: npm
|
|
cache-dependency-path: candidate/gitnexus/package-lock.json
|
|
- uses: astral-sh/setup-uv@18d369b23da25a81e9458561248e1aafff6a70fa # v10.3.1
|
|
with:
|
|
version: '0.11.23'
|
|
python-version: '3.13'
|
|
enable-cache: true
|
|
cache-dependency-glob: eval/uv.lock
|
|
- name: Authenticate historical PR comparison evidence
|
|
if: inputs.candidate_pr != ''
|
|
working-directory: eval
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
uv run --locked --extra dev python -m workflow_bench.pr_evaluation baseline \
|
|
--repo "$GITHUB_REPOSITORY" --run-id "$COMPARISON_RUN_ID" \
|
|
--harness-sha "$GITHUB_SHA" --env-out "$GITHUB_ENV" \
|
|
--out "$RUNNER_TEMP/release-evaluation/previous.json"
|
|
# Both runtimes grade the pinned task source against one dependency tree
|
|
# built from that commit's lockfile, never the runtime under test's.
|
|
- name: Resolve pinned task revision
|
|
id: tasks
|
|
working-directory: eval
|
|
run: |
|
|
set -euo pipefail
|
|
sha="$(uv run --locked --extra dev python -m workflow_bench.release_report task-sha)"
|
|
[[ "$sha" =~ ^[0-9a-f]{40}$ ]] || { echo '::error::The release task set has no single pinned commit.'; exit 1; }
|
|
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ steps.tasks.outputs.sha }}
|
|
path: tasks-base
|
|
persist-credentials: false
|
|
fetch-depth: 0
|
|
- name: Install sandbox runtime and pinned Claude CLI
|
|
run: |
|
|
set -euo pipefail
|
|
sudo apt-get -o DPkg::Lock::Timeout=600 update
|
|
sudo apt-get -o DPkg::Lock::Timeout=600 install --yes --no-install-recommends bubblewrap ripgrep socat
|
|
apparmor_userns=/proc/sys/kernel/apparmor_restrict_unprivileged_userns
|
|
if [[ -r "$apparmor_userns" ]] && [[ "$(<"$apparmor_userns")" == '1' ]]; then
|
|
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
|
fi
|
|
cli="$RUNNER_TEMP/claude-canary"
|
|
install -d -m 0700 "$cli"
|
|
install -m 0600 .github/claude-canary-runtime/package{,-lock}.json "$cli/"
|
|
npm ci --prefix "$cli" --audit=false --fund=false
|
|
test "$("$cli/node_modules/@anthropic-ai/claude-code-linux-x64/claude" --version)" = '2.1.214 (Claude Code)'
|
|
- name: Build pinned runtime and task dependencies
|
|
working-directory: eval
|
|
env:
|
|
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
|
|
run: |
|
|
set -euo pipefail
|
|
for checkout in tasks-base stable candidate; do
|
|
if [[ ( -n "$INPUT_PR" || "${DIAGNOSTIC_TRIAL:-false}" == true ) && "$checkout" == stable ]]; then continue; fi
|
|
uv run --locked --extra dev python -m workflow_bench.release_build \
|
|
--repo "$GITHUB_WORKSPACE/$checkout"
|
|
done
|
|
- name: Prepare pinned task set
|
|
working-directory: eval
|
|
env:
|
|
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
|
|
run: |
|
|
set -euo pipefail
|
|
for runtime in stable candidate; do
|
|
if [[ ( -n "$INPUT_PR" || "${DIAGNOSTIC_TRIAL:-false}" == true ) && "$runtime" == stable ]]; then continue; fi
|
|
uv run --locked --extra dev python -m workflow_bench.release_report prepare \
|
|
--repo "$GITHUB_WORKSPACE/$runtime" --task-repo "$GITHUB_WORKSPACE/tasks-base" \
|
|
--out "$RUNNER_TEMP/release-evaluation/$runtime" \
|
|
--model "$MODEL" --effort "$EFFORT" --runs 3
|
|
done
|
|
- name: Prepare the capped diagnostic trial
|
|
if: inputs.diagnostic_trial == true
|
|
working-directory: eval
|
|
env:
|
|
RUNTIME_SHA: ${{ steps.resolve.outputs.sha }}
|
|
run: |
|
|
uv run --locked --extra dev python -m workflow_bench.diagnostic_experiments \
|
|
--tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \
|
|
--repository "$GITHUB_WORKSPACE/tasks-base" --runtime "$GITHUB_WORKSPACE/candidate" \
|
|
--harness-sha "$GITHUB_SHA" --runtime-sha "$RUNTIME_SHA" \
|
|
--model "$MODEL" --effort "$EFFORT" --seed 3762 --repeats 1 \
|
|
--variants no_graph mcp_eager --task-ids trivial-status-json-alias \
|
|
--max-budget-usd 5 --bind-task-dependencies --out "$RUNNER_TEMP/release-evaluation/trial"
|
|
- name: Require historical task dependencies before paid PR sessions
|
|
if: inputs.candidate_pr != ''
|
|
working-directory: eval
|
|
run: |
|
|
uv run --locked --extra dev python -m workflow_bench.pr_evaluation dependencies \
|
|
--tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \
|
|
--baseline "$RUNNER_TEMP/release-evaluation/previous.json"
|
|
- name: Require real containment before paid sessions
|
|
working-directory: eval
|
|
env:
|
|
GITNEXUS_REQUIRE_BWRAP_CANARY: '1'
|
|
GITNEXUS_REQUIRE_CLAUDE_CANARY: '1'
|
|
CLAUDE_CANARY_BIN: ${{ runner.temp }}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude
|
|
run: >-
|
|
uv run --locked --extra dev python -m pytest tests/test_proposer_sandbox.py tests/test_cli_shim.py
|
|
tests/test_workflow_bench_sessions.py::test_real_cli_background_completion_is_one_complete_measurement -q
|
|
- name: Materialize both release graphs before paid sessions
|
|
timeout-minutes: 120
|
|
working-directory: eval
|
|
env:
|
|
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ "${DIAGNOSTIC_TRIAL:-false}" == true ]]; then
|
|
uv run --locked --extra dev python -m workflow_bench.release_preflight \
|
|
--tasks "$RUNNER_TEMP/release-evaluation/trial/cell-0000.tasks.json" \
|
|
--grading-controls \
|
|
--gitnexus-root "$GITHUB_WORKSPACE/candidate" \
|
|
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude"
|
|
exit 0
|
|
fi
|
|
for runtime in stable candidate; do
|
|
if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi
|
|
uv run --locked --extra dev python -m workflow_bench.release_preflight \
|
|
--tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \
|
|
--grading-controls \
|
|
--gitnexus-root "$GITHUB_WORKSPACE/$runtime" \
|
|
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude"
|
|
done
|
|
- name: Require offline deferred-tool compatibility before paid sessions
|
|
if: inputs.mcp_tool_loading == true
|
|
working-directory: eval
|
|
env:
|
|
GITNEXUS_REQUIRE_CLAUDE_CANARY: '1'
|
|
CLAUDE_CANARY_BIN: ${{ runner.temp }}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude
|
|
run: >-
|
|
uv run --locked --extra dev python -m pytest tests/test_mcp_tool_loading.py -q
|
|
- name: Recheck PR opt-in and pinned head immediately before paid sessions
|
|
if: inputs.candidate_pr != ''
|
|
working-directory: eval
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
python3 -m workflow_bench.pr_evaluation resolve \
|
|
--repo "$GITHUB_REPOSITORY" --pr "$INPUT_PR" --sha "$INPUT_PR_SHA"
|
|
- name: Run repeated paired agent sessions
|
|
timeout-minutes: 1140
|
|
working-directory: eval
|
|
env:
|
|
GITNEXUS_BENCH_OPENAI_API_KEY: ${{ secrets.GITNEXUS_BENCH_OPENAI_API_KEY }}
|
|
GITNEXUS_BENCH_ANTHROPIC_API_KEY: ${{ secrets.GITNEXUS_BENCH_ANTHROPIC_API_KEY }}
|
|
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ "${DIAGNOSTIC_TRIAL:-false}" == true ]]; then
|
|
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
|
|
[[ "$GITHUB_RUN_ATTEMPT" == 1 ]] || { echo '::error::Paid trials cannot be rerun, including failed-job retries.'; exit 1; }
|
|
(( remaining > 0 )) || { echo '::error::No time remains before the stop and evidence reserve.'; exit 1; }
|
|
(( remaining <= 3600 )) || remaining=3600
|
|
timeout --signal=TERM --kill-after=60 "${remaining}s" \
|
|
uv run --locked --extra dev python -m workflow_bench.diagnostic_trial --execute \
|
|
--plan "$RUNNER_TEMP/release-evaluation/trial/plan.json" \
|
|
--runtime "$GITHUB_WORKSPACE/candidate" \
|
|
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude"
|
|
exit 0
|
|
fi
|
|
arms=(baseline_nomcp baseline)
|
|
if [[ -n "$INPUT_PR" ]]; then arms=(baseline); fi
|
|
for runtime in stable candidate; do
|
|
if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi
|
|
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
|
|
(( remaining > 0 )) || { echo '::error::No time remains before the stop and evidence reserve.'; exit 1; }
|
|
GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/$runtime/provider-usage.jsonl" \
|
|
GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-$runtime" \
|
|
timeout --signal=TERM --kill-after=60 "${remaining}s" \
|
|
uv run --locked --extra dev python -m workflow_bench.runner \
|
|
--tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \
|
|
--gitnexus-root "$GITHUB_WORKSPACE/$runtime" \
|
|
--arms "${arms[@]}" --runs 3 --include-expensive --mcp-tool-loading eager \
|
|
--workers 3 --timeout 3600 --model "$MODEL" --effort "$EFFORT" \
|
|
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \
|
|
--out "$RUNNER_TEMP/release-evaluation/$runtime/raw"
|
|
done
|
|
# Descriptive only: its own output directory leaves the paired evidence
|
|
# that release_report and release_gate read exactly as without the opt-in.
|
|
# Trials reject these opt-ins before startup. Preserve the exact gates
|
|
# recognized by the historical measurement-protocol validator.
|
|
- name: Run the opt-in CLI-only arm on the candidate
|
|
if: env.CLI_ARM == 'true'
|
|
continue-on-error: true
|
|
timeout-minutes: 600
|
|
working-directory: eval
|
|
env:
|
|
GITNEXUS_BENCH_OPENAI_API_KEY: ${{ secrets.GITNEXUS_BENCH_OPENAI_API_KEY }}
|
|
GITNEXUS_BENCH_ANTHROPIC_API_KEY: ${{ secrets.GITNEXUS_BENCH_ANTHROPIC_API_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
|
|
(( remaining > 0 )) || { echo '::warning::No time remains for the CLI-only arm.'; exit 1; }
|
|
GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/candidate/cli-provider-usage.jsonl" \
|
|
GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-cli" \
|
|
timeout --signal=TERM --kill-after=60 "${remaining}s" \
|
|
uv run --locked --extra dev python -m workflow_bench.runner \
|
|
--tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \
|
|
--gitnexus-root "$GITHUB_WORKSPACE/candidate" \
|
|
--arms baseline_cli --runs 3 --include-expensive \
|
|
--workers 3 --timeout 3600 --model "$MODEL" --effort "$EFFORT" \
|
|
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \
|
|
--out "$RUNNER_TEMP/release-evaluation/candidate/cli-raw"
|
|
- name: Measure candidate with deferred MCP descriptions
|
|
if: inputs.mcp_tool_loading == true && inputs.candidate_pr == ''
|
|
timeout-minutes: 360
|
|
working-directory: eval
|
|
env:
|
|
GITNEXUS_BENCH_OPENAI_API_KEY: ${{ secrets.GITNEXUS_BENCH_OPENAI_API_KEY }}
|
|
GITNEXUS_BENCH_ANTHROPIC_API_KEY: ${{ secrets.GITNEXUS_BENCH_ANTHROPIC_API_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
|
|
(( remaining > 0 )) || { echo '::error::No time remains for deferred sessions before the evidence reserve.'; exit 1; }
|
|
GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/candidate/deferred-provider-usage.jsonl" \
|
|
GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-deferred" \
|
|
timeout --signal=TERM --kill-after=60 "${remaining}s" \
|
|
uv run --locked --extra dev python -m workflow_bench.runner \
|
|
--tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \
|
|
--gitnexus-root "$GITHUB_WORKSPACE/candidate" \
|
|
--arms baseline --mcp-tool-loading deferred --runs 3 --include-expensive \
|
|
--workers 3 --timeout 3600 --model "$MODEL" --effort "$EFFORT" \
|
|
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \
|
|
--out "$RUNNER_TEMP/release-evaluation/candidate/deferred"
|
|
- name: Summarize complete or incomplete measurements
|
|
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
|
|
working-directory: eval
|
|
run: |
|
|
if [[ -n "$INPUT_PR" ]]; then
|
|
uv run --locked --extra dev python -m workflow_bench.pr_evaluation summarize \
|
|
--results "$RUNNER_TEMP/release-evaluation/candidate/raw/results.jsonl" \
|
|
--metadata "$RUNNER_TEMP/release-evaluation/candidate/metadata.json" \
|
|
--baseline "$RUNNER_TEMP/release-evaluation/previous.json" \
|
|
--sha '${{ steps.resolve.outputs.sha }}' --pr "$INPUT_PR" --run-id "$COMPARISON_RUN_ID" \
|
|
--out "$RUNNER_TEMP/release-evaluation/public"
|
|
exit 0
|
|
fi
|
|
failed=0
|
|
for runtime in stable candidate; do
|
|
if [[ -f "$RUNNER_TEMP/release-evaluation/$runtime/metadata.json" ]]; then
|
|
uv run --locked --extra dev python -m workflow_bench.release_report summarize \
|
|
--results "$RUNNER_TEMP/release-evaluation/$runtime/raw/results.jsonl" \
|
|
--metadata "$RUNNER_TEMP/release-evaluation/$runtime/metadata.json" \
|
|
--out "$RUNNER_TEMP/release-evaluation/$runtime/public" || failed=1
|
|
else
|
|
echo "::error::$runtime evaluation preparation failed; no measurements exist."
|
|
failed=1
|
|
fi
|
|
done
|
|
exit "$failed"
|
|
- name: Require complete paired evidence and no task regression
|
|
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }}
|
|
working-directory: eval
|
|
run: |
|
|
uv run --locked --extra dev python -m workflow_bench.release_gate \
|
|
--candidate "$RUNNER_TEMP/release-evaluation/candidate/public/agent-evaluation.json" \
|
|
--candidate-sha '${{ steps.resolve.outputs.sha }}' \
|
|
--stable "$RUNNER_TEMP/release-evaluation/stable/public/agent-evaluation.json" \
|
|
--stable-sha '${{ steps.resolve.outputs.stable_sha }}' \
|
|
--out "$RUNNER_TEMP/release-evaluation/public"
|
|
- name: Summarize matched MCP loading experiment
|
|
if: ${{ always() && inputs.mcp_tool_loading == true && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }}
|
|
working-directory: eval
|
|
run: |
|
|
uv run --locked --extra dev python -m workflow_bench.mcp_loading_report \
|
|
--eager "$RUNNER_TEMP/release-evaluation/candidate/raw/results.jsonl" \
|
|
--deferred "$RUNNER_TEMP/release-evaluation/candidate/deferred/results.jsonl" \
|
|
--metadata "$RUNNER_TEMP/release-evaluation/candidate/metadata.json" \
|
|
--out "$RUNNER_TEMP/release-evaluation/public"
|
|
- name: Collect sanitized evidence and publish summary
|
|
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
|
|
run: |
|
|
set -euo pipefail
|
|
public="$RUNNER_TEMP/release-evaluation/public"
|
|
mkdir -p "$public"
|
|
if [[ -n "$INPUT_PR" ]]; then
|
|
if [[ -f "$public/pr-cost-comparison.md" ]]; then cat "$public/pr-cost-comparison.md" >> "$GITHUB_STEP_SUMMARY"; fi
|
|
exit 0
|
|
fi
|
|
for extension in json md; do
|
|
candidate="$RUNNER_TEMP/release-evaluation/candidate/public/agent-evaluation.$extension"
|
|
stable="$RUNNER_TEMP/release-evaluation/stable/public/agent-evaluation.$extension"
|
|
if [[ -f "$candidate" ]]; then cp "$candidate" "$public/agent-evaluation.$extension"; fi
|
|
if [[ -f "$stable" ]]; then cp "$stable" "$public/stable-agent-evaluation.$extension"; fi
|
|
done
|
|
for report in release-quality-gate agent-evaluation mcp-tool-loading; do
|
|
if [[ -f "$public/$report.md" ]]; then cat "$public/$report.md" >> "$GITHUB_STEP_SUMMARY"; fi
|
|
done
|
|
- name: Summarize the opt-in CLI-only arm
|
|
if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
|
|
continue-on-error: true
|
|
working-directory: eval
|
|
run: |
|
|
set -euo pipefail
|
|
candidate="$RUNNER_TEMP/release-evaluation/candidate"
|
|
uv run --locked --extra dev python -m workflow_bench.cli_arm_report \
|
|
--paired "$candidate/raw/results.jsonl" --cli "$candidate/cli-raw/results.jsonl" \
|
|
--out "$RUNNER_TEMP/release-evaluation/public"
|
|
cat "$RUNNER_TEMP/release-evaluation/public/cli-arm-comparison.md" >> "$GITHUB_STEP_SUMMARY"
|
|
- name: Summarize time, cost, and tool measurements
|
|
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
|
|
working-directory: eval
|
|
env:
|
|
DEFERRED_ARM: ${{ inputs.mcp_tool_loading }}
|
|
run: |
|
|
set -euo pipefail
|
|
root="$RUNNER_TEMP/release-evaluation"
|
|
public="$root/public"
|
|
mkdir -p "$public"
|
|
failed=0
|
|
for variant in stable candidate cli deferred; do
|
|
case "$variant" in
|
|
stable)
|
|
if [[ -n "$INPUT_PR" ]]; then continue; fi
|
|
results="$root/stable/raw/results.jsonl"
|
|
ledger="$root/stable/provider-usage.jsonl" ;;
|
|
candidate)
|
|
results="$root/candidate/raw/results.jsonl"
|
|
ledger="$root/candidate/provider-usage.jsonl" ;;
|
|
cli)
|
|
if [[ "$CLI_ARM" != true ]]; then continue; fi
|
|
results="$root/candidate/cli-raw/results.jsonl"
|
|
ledger="$root/candidate/cli-provider-usage.jsonl" ;;
|
|
deferred)
|
|
if [[ "$DEFERRED_ARM" != true || -n "$INPUT_PR" ]]; then continue; fi
|
|
results="$root/candidate/deferred/results.jsonl"
|
|
ledger="$root/candidate/deferred-provider-usage.jsonl" ;;
|
|
esac
|
|
if ! uv run --locked --extra dev python -m workflow_bench.cost_diagnostics \
|
|
--results "$results" --provider-usage "$ledger" --allow-incomplete \
|
|
--out "$public/$variant-diagnostics.json" --markdown-out "$public/$variant-diagnostics.md"; then
|
|
printf '\nDetailed measurements for %s could not be validated; inspect the step log.\n' "$variant" >> "$GITHUB_STEP_SUMMARY"
|
|
failed=1
|
|
continue
|
|
fi
|
|
{
|
|
printf '\n<details><summary>Detailed measurements: %s</summary>\n\n' "$variant"
|
|
cat "$public/$variant-diagnostics.md"
|
|
printf '\n</details>\n'
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
done
|
|
exit "$failed"
|
|
- name: Summarize the diagnostic trial
|
|
if: ${{ always() && inputs.diagnostic_trial == true && steps.resolve.outputs.sha != '' }}
|
|
working-directory: eval
|
|
run: |
|
|
set -euo pipefail
|
|
root="$RUNNER_TEMP/release-evaluation"
|
|
mkdir -p "$root/public"
|
|
printf "\n## Two-cell diagnostic trial\n\nOne task, one repetition per arm. \$5 CLI budget stop per cell; provider billing can exceed the planned \$10. This trial checks telemetry and is not release-quality evidence.\n" >> "$GITHUB_STEP_SUMMARY"
|
|
failed=0
|
|
for cell in cell-0000 cell-0001; do
|
|
if ! uv run --locked --extra dev python -m workflow_bench.cost_diagnostics \
|
|
--results "$root/trial/results/$cell/results.jsonl" \
|
|
--provider-usage "$root/trial/results/$cell.provider.jsonl" --allow-incomplete \
|
|
--out "$root/public/trial-$cell-diagnostics.json" \
|
|
--markdown-out "$root/public/trial-$cell-diagnostics.md"; then
|
|
printf '\nMeasurements for %s could not be validated.\n' "$cell" >> "$GITHUB_STEP_SUMMARY"
|
|
failed=1
|
|
continue
|
|
fi
|
|
cat "$root/public/trial-$cell-diagnostics.md" >> "$GITHUB_STEP_SUMMARY"
|
|
done
|
|
exit "$failed"
|
|
- name: Upload sanitized diagnostic trial measurements
|
|
if: ${{ always() && inputs.diagnostic_trial == true && steps.resolve.outputs.sha != '' }}
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
with:
|
|
name: diagnostic-trial-${{ steps.resolve.outputs.sha }}-${{ github.run_id }}
|
|
path: |
|
|
${{ runner.temp }}/release-evaluation/public/trial-*-diagnostics.json
|
|
${{ runner.temp }}/release-evaluation/public/trial-*-diagnostics.md
|
|
if-no-files-found: error
|
|
retention-days: 30
|
|
- name: Upload sanitized release evidence
|
|
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }}
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
with:
|
|
name: release-agent-evaluation-${{ steps.resolve.outputs.sha }}
|
|
path: |
|
|
${{ runner.temp }}/release-evaluation/public/agent-evaluation.json
|
|
${{ runner.temp }}/release-evaluation/public/agent-evaluation.md
|
|
${{ runner.temp }}/release-evaluation/public/stable-agent-evaluation.json
|
|
${{ runner.temp }}/release-evaluation/public/stable-agent-evaluation.md
|
|
${{ runner.temp }}/release-evaluation/public/release-quality-gate.json
|
|
${{ runner.temp }}/release-evaluation/public/release-quality-gate.md
|
|
${{ runner.temp }}/release-evaluation/public/mcp-tool-loading.json
|
|
${{ runner.temp }}/release-evaluation/public/mcp-tool-loading.md
|
|
${{ runner.temp }}/release-evaluation/public/*-diagnostics.json
|
|
${{ runner.temp }}/release-evaluation/public/*-diagnostics.md
|
|
if-no-files-found: error
|
|
retention-days: 30
|
|
- name: Upload the opt-in CLI-only arm comparison
|
|
if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
with:
|
|
name: release-cli-arm-${{ steps.resolve.outputs.sha }}
|
|
path: |
|
|
${{ runner.temp }}/release-evaluation/public/cli-arm-comparison.json
|
|
${{ runner.temp }}/release-evaluation/public/cli-arm-comparison.md
|
|
${{ runner.temp }}/release-evaluation/public/cli-diagnostics.json
|
|
${{ runner.temp }}/release-evaluation/public/cli-diagnostics.md
|
|
if-no-files-found: warn
|
|
retention-days: 30
|
|
- name: Upload sanitized PR cost comparison
|
|
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr != '' && inputs.diagnostic_trial != true }}
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
with:
|
|
name: pr-cost-comparison-${{ steps.resolve.outputs.sha }}
|
|
path: |
|
|
${{ runner.temp }}/release-evaluation/public/pr-cost-comparison.json
|
|
${{ runner.temp }}/release-evaluation/public/pr-cost-comparison.md
|
|
${{ runner.temp }}/release-evaluation/public/*-diagnostics.json
|
|
${{ runner.temp }}/release-evaluation/public/*-diagnostics.md
|
|
if-no-files-found: error
|
|
retention-days: 30
|
|
|
|
stop-runner:
|
|
name: Stop and verify the dedicated EC2 runner
|
|
needs: [start-runner, check-runner, runner-ready, watch-evaluate-pickup, evaluate]
|
|
if: >-
|
|
always() && github.repository == 'abhigyanpatwari/GitNexus' &&
|
|
github.ref == 'refs/heads/main' &&
|
|
needs.start-runner.outputs.cleanup_required == 'true'
|
|
runs-on: ubuntu-latest
|
|
environment: gitnexus-evolution
|
|
timeout-minutes: 20
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
- uses: ./.github/actions/ec2-runner
|
|
with:
|
|
operation: stop
|
|
role-arn: ${{ secrets.GITNEXUS_EVOLUTION_AWS_ROLE_ARN }}
|
|
instance-id: ${{ secrets.GITNEXUS_EVOLUTION_EC2_INSTANCE_ID }}
|
|
region: ${{ vars.GITNEXUS_EVOLUTION_AWS_REGION }}
|
|
stop-schedule: ${{ vars.GITNEXUS_EVOLUTION_STOP_SCHEDULE_UTC }}
|