mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-31 06:52:49 +00:00
252a64573e
Validate continuation evidence, sum per-turn tokens and turns, retain final cumulative cost, and include background wait in elapsed time. Add an offline pinned-CLI canary before paid evaluation and allow 60-minute sessions within the existing deadline and cleanup reserve.
411 lines
19 KiB
YAML
411 lines
19 KiB
YAML
# Compare the exact candidate with the last stable release using trusted main's
|
|
# harness. Share the existing evolution runner, protected settings, and power lock.
|
|
#
|
|
# The shared group keeps only ONE pending run, even with cancel-in-progress:
|
|
# false: a newer queued skill-evolution or release-evaluation run cancels this
|
|
# run while it is pending. Before relying on a dispatched evaluation for a
|
|
# stable publish, confirm its "Start the dedicated EC2 runner" job ran; if the
|
|
# run shows Cancelled, dispatch it again once the group is free.
|
|
name: Release evaluation
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 5 * * 3' # Wednesday 05:00 UTC
|
|
workflow_dispatch:
|
|
inputs:
|
|
runner_only:
|
|
description: Start, probe, and stop EC2 without paid model calls
|
|
type: boolean
|
|
default: false
|
|
candidate_ref:
|
|
description: 'Full commit SHA or version tag to evaluate; latest-rc resolves the newest published RC'
|
|
type: string
|
|
default: latest-rc
|
|
model:
|
|
description: 'Named model for both arms'
|
|
type: string
|
|
default: gpt-5.6-sol
|
|
effort:
|
|
description: 'Reasoning effort for both arms'
|
|
type: choice
|
|
default: xhigh
|
|
options: [low, medium, high, xhigh, max]
|
|
|
|
concurrency:
|
|
group: gitnexus-evolution-runner
|
|
cancel-in-progress: false
|
|
|
|
permissions: {}
|
|
|
|
jobs:
|
|
start-runner:
|
|
name: Start the dedicated EC2 runner
|
|
if: >-
|
|
github.repository == 'abhigyanpatwari/GitNexus' &&
|
|
github.ref == 'refs/heads/main' &&
|
|
(github.event_name == 'workflow_dispatch' ||
|
|
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
|
|
runs-on: ubuntu-latest
|
|
environment: gitnexus-evolution
|
|
timeout-minutes: 25
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
outputs:
|
|
run_attempt: ${{ github.run_attempt }}
|
|
cleanup_required: ${{ steps.boot.outputs.cleanup_required }}
|
|
stop_deadline_epoch: ${{ steps.boot.outputs.stop_deadline_epoch }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
- uses: ./.github/actions/ec2-runner
|
|
id: boot
|
|
with:
|
|
operation: start
|
|
role-arn: ${{ secrets.GITNEXUS_EVOLUTION_AWS_ROLE_ARN }}
|
|
instance-id: ${{ secrets.GITNEXUS_EVOLUTION_EC2_INSTANCE_ID }}
|
|
region: ${{ vars.GITNEXUS_EVOLUTION_AWS_REGION }}
|
|
stop-schedule: ${{ vars.GITNEXUS_EVOLUTION_STOP_SCHEDULE_UTC }}
|
|
|
|
check-runner:
|
|
needs: [start-runner]
|
|
name: Bound the wait for the evaluation runner
|
|
if: >-
|
|
github.repository == 'abhigyanpatwari/GitNexus' &&
|
|
github.ref == 'refs/heads/main' &&
|
|
(github.event_name == 'workflow_dispatch' ||
|
|
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
|
|
runs-on: ubuntu-latest
|
|
environment: gitnexus-evolution
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
actions: write # Cancel only this run if its dedicated runner stays offline.
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
# EC2 health does not prove the Actions service can accept a job.
|
|
- name: Verify the current run's native pickup probe
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: python3 .github/scripts/evolution-runner-ready.py
|
|
# A queued self-hosted job has no pickup timeout. Cancel this run so an
|
|
# offline probe cannot remain queued indefinitely and block later runs.
|
|
- name: Cancel this run when runner readiness fails
|
|
if: failure()
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: gh api --method POST "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel"
|
|
|
|
runner-ready:
|
|
needs: [start-runner]
|
|
name: Verify the runner service is online
|
|
if: >-
|
|
github.repository == 'abhigyanpatwari/GitNexus' &&
|
|
github.ref == 'refs/heads/main' &&
|
|
(github.event_name == 'workflow_dispatch' ||
|
|
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
|
|
# Partial reruns reuse successful startup outputs after cleanup stopped EC2.
|
|
runs-on: ${{ fromJSON(needs.start-runner.outputs.run_attempt == github.run_attempt && '["self-hosted", "linux", "x64", "gitnexus-evolution"]' || '["ubuntu-latest"]') }}
|
|
environment: gitnexus-evolution
|
|
timeout-minutes: 5
|
|
permissions: {}
|
|
steps:
|
|
- name: Require startup in this run attempt
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ '${{ needs.start-runner.outputs.run_attempt }}' != "$GITHUB_RUN_ATTEMPT" ]]; then
|
|
echo '::error::The EC2 startup belongs to an earlier attempt. Select Re-run all jobs to start the runner again.'
|
|
exit 1
|
|
fi
|
|
- name: Verify runner pickup
|
|
run: |
|
|
set -euo pipefail
|
|
test -r /proc/uptime
|
|
git --version
|
|
uname -sm
|
|
|
|
watch-evaluate-pickup:
|
|
name: Bound pickup of the paid evaluation job
|
|
needs: [start-runner, check-runner, runner-ready]
|
|
if: inputs.runner_only != true
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
actions: write # Cancel this run if evaluate stays queued after the probe.
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
- name: Require pickup of this run's paid evaluation job
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: python3 .github/scripts/evolution-runner-ready.py --job-name "Evaluate candidate and stable quality"
|
|
- name: Cancel this run when evaluation pickup fails
|
|
if: failure()
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: gh api --method POST "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel"
|
|
|
|
evaluate:
|
|
name: Evaluate candidate and stable quality
|
|
needs: [start-runner, check-runner, runner-ready]
|
|
if: >-
|
|
github.repository == 'abhigyanpatwari/GitNexus' &&
|
|
github.ref == 'refs/heads/main' &&
|
|
inputs.runner_only != true &&
|
|
(github.event_name == 'workflow_dispatch' ||
|
|
(vars.GITNEXUS_EVOLUTION_ENABLED == 'true' && vars.GITNEXUS_EVOLUTION_WORKERS == '3'))
|
|
runs-on: ${{ fromJSON(needs.start-runner.outputs.run_attempt == github.run_attempt && '["self-hosted", "linux", "x64", "gitnexus-evolution"]' || '["ubuntu-latest"]') }}
|
|
# Reuse the main-only protected model-secret environment. No tag/fork code
|
|
# runs as the harness, and no release or PR write token is available here.
|
|
environment: gitnexus-evolution
|
|
timeout-minutes: 1260
|
|
permissions:
|
|
contents: read
|
|
env:
|
|
INPUT_REF: ${{ inputs.candidate_ref || 'latest-rc' }}
|
|
MODEL: ${{ inputs.model || 'gpt-5.6-sol' }}
|
|
EFFORT: ${{ inputs.effort || 'xhigh' }}
|
|
steps:
|
|
- name: Require startup in this run attempt
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ '${{ needs.start-runner.outputs.run_attempt }}' != "$GITHUB_RUN_ATTEMPT" ]]; then
|
|
echo '::error::The EC2 startup belongs to an earlier attempt. Select Re-run all jobs to start the runner again.'
|
|
exit 1
|
|
fi
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
fetch-depth: 0
|
|
- name: Resolve immutable runtime revision
|
|
id: resolve
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
set -euo pipefail
|
|
# Resolve last stable once; both runtimes use the same trusted tasks.
|
|
stable="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq '.tag_name')"
|
|
[[ "$stable" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo '::error::No stable release found.'; exit 1; }
|
|
stable_sha="$(git rev-parse --verify "$stable^{commit}")"
|
|
echo "stable_sha=$stable_sha" >> "$GITHUB_OUTPUT"
|
|
ref="$INPUT_REF"
|
|
if [[ "$ref" == 'latest-rc' ]]; then
|
|
ref="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --jq \
|
|
'[.[] | select(.prerelease and (.draft | not)) | select(.tag_name | test("^v[0-9]+\\.[0-9]+\\.[0-9]+-rc\\.[0-9]+$"))] | sort_by(.published_at) | last | .tag_name // empty')"
|
|
fi
|
|
if ! [[ "$ref" =~ ^[0-9a-f]{40}$ || "$ref" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then
|
|
echo '::error::No published RC found, or candidate_ref is not a full SHA/version tag.'
|
|
exit 1
|
|
fi
|
|
sha="$(git rev-parse --verify "$ref^{commit}")"
|
|
if [[ "$ref" =~ ^[0-9a-f]{40}$ ]]; then
|
|
# Manual stable candidates must already be reviewed on main.
|
|
git merge-base --is-ancestor "$sha" HEAD
|
|
else
|
|
# Versioned RC trees are published descendants, not ancestors of
|
|
# main. Accept their official published release rather than PR refs.
|
|
gh api "repos/$GITHUB_REPOSITORY/releases/tags/$ref" --jq '.draft == false' | grep -qx true
|
|
fi
|
|
echo "Runtime: $sha"
|
|
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ steps.resolve.outputs.sha }}
|
|
path: candidate
|
|
persist-credentials: false
|
|
fetch-depth: 0
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ steps.resolve.outputs.stable_sha }}
|
|
path: stable
|
|
persist-credentials: false
|
|
fetch-depth: 0
|
|
- name: Initialize this run's measurement directory
|
|
run: |
|
|
rm -rf "$RUNNER_TEMP/release-evaluation"
|
|
mkdir -p "$RUNNER_TEMP/release-evaluation/public"
|
|
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
|
with:
|
|
node-version: '22.18.0'
|
|
cache: npm
|
|
cache-dependency-path: candidate/gitnexus/package-lock.json
|
|
- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
|
with:
|
|
version: '0.11.23'
|
|
python-version: '3.13'
|
|
enable-cache: true
|
|
cache-dependency-glob: eval/uv.lock
|
|
# Both runtimes grade the pinned task source against one dependency tree
|
|
# built from that commit's lockfile, never the runtime under test's.
|
|
- name: Resolve pinned task revision
|
|
id: tasks
|
|
working-directory: eval
|
|
run: |
|
|
set -euo pipefail
|
|
sha="$(uv run --locked --extra dev python -m workflow_bench.release_report task-sha)"
|
|
[[ "$sha" =~ ^[0-9a-f]{40}$ ]] || { echo '::error::The release task set has no single pinned commit.'; exit 1; }
|
|
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ steps.tasks.outputs.sha }}
|
|
path: tasks-base
|
|
persist-credentials: false
|
|
fetch-depth: 0
|
|
- name: Install sandbox runtime and pinned Claude CLI
|
|
run: |
|
|
set -euo pipefail
|
|
sudo apt-get -o DPkg::Lock::Timeout=600 update
|
|
sudo apt-get -o DPkg::Lock::Timeout=600 install --yes --no-install-recommends bubblewrap ripgrep socat
|
|
apparmor_userns=/proc/sys/kernel/apparmor_restrict_unprivileged_userns
|
|
if [[ -r "$apparmor_userns" ]] && [[ "$(<"$apparmor_userns")" == '1' ]]; then
|
|
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
|
fi
|
|
cli="$RUNNER_TEMP/claude-canary"
|
|
install -d -m 0700 "$cli"
|
|
install -m 0600 .github/claude-canary-runtime/package{,-lock}.json "$cli/"
|
|
npm ci --prefix "$cli" --audit=false --fund=false
|
|
test "$("$cli/node_modules/@anthropic-ai/claude-code-linux-x64/claude" --version)" = '2.1.214 (Claude Code)'
|
|
- name: Build pinned runtime and task dependencies
|
|
working-directory: eval
|
|
run: |
|
|
set -euo pipefail
|
|
for checkout in tasks-base stable candidate; do
|
|
uv run --locked --extra dev python -m workflow_bench.release_build \
|
|
--repo "$GITHUB_WORKSPACE/$checkout"
|
|
done
|
|
- name: Prepare pinned task set
|
|
working-directory: eval
|
|
run: |
|
|
set -euo pipefail
|
|
for runtime in stable candidate; do
|
|
uv run --locked --extra dev python -m workflow_bench.release_report prepare \
|
|
--repo "$GITHUB_WORKSPACE/$runtime" --task-repo "$GITHUB_WORKSPACE/tasks-base" \
|
|
--out "$RUNNER_TEMP/release-evaluation/$runtime" \
|
|
--model "$MODEL" --effort "$EFFORT" --runs 3
|
|
done
|
|
- name: Require real containment before paid sessions
|
|
working-directory: eval
|
|
env:
|
|
GITNEXUS_REQUIRE_BWRAP_CANARY: '1'
|
|
GITNEXUS_REQUIRE_CLAUDE_CANARY: '1'
|
|
CLAUDE_CANARY_BIN: ${{ runner.temp }}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude
|
|
run: >-
|
|
uv run --locked --extra dev python -m pytest tests/test_proposer_sandbox.py
|
|
tests/test_workflow_bench_sessions.py::test_real_cli_background_completion_is_one_complete_measurement -q
|
|
- name: Materialize both release graphs before paid sessions
|
|
timeout-minutes: 120
|
|
working-directory: eval
|
|
run: |
|
|
set -euo pipefail
|
|
for runtime in stable candidate; do
|
|
uv run --locked --extra dev python -m workflow_bench.release_preflight \
|
|
--tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \
|
|
--gitnexus-root "$GITHUB_WORKSPACE/$runtime" \
|
|
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude"
|
|
done
|
|
- name: Run repeated paired agent sessions
|
|
timeout-minutes: 1140
|
|
working-directory: eval
|
|
env:
|
|
GITNEXUS_BENCH_OPENAI_API_KEY: ${{ secrets.GITNEXUS_BENCH_OPENAI_API_KEY }}
|
|
GITNEXUS_BENCH_ANTHROPIC_API_KEY: ${{ secrets.GITNEXUS_BENCH_ANTHROPIC_API_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
for runtime in stable candidate; do
|
|
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
|
|
(( remaining > 0 )) || { echo '::error::No time remains before the stop and evidence reserve.'; exit 1; }
|
|
timeout --signal=TERM --kill-after=60 "${remaining}s" \
|
|
uv run --locked --extra dev python -m workflow_bench.runner \
|
|
--tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \
|
|
--gitnexus-root "$GITHUB_WORKSPACE/$runtime" \
|
|
--arms baseline_nomcp baseline --runs 3 --include-expensive \
|
|
--workers 3 --timeout 3600 --model "$MODEL" --effort "$EFFORT" \
|
|
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \
|
|
--out "$RUNNER_TEMP/release-evaluation/$runtime/raw"
|
|
done
|
|
- name: Summarize complete or incomplete measurements
|
|
if: ${{ always() && steps.resolve.outputs.sha != '' }}
|
|
working-directory: eval
|
|
run: |
|
|
failed=0
|
|
for runtime in stable candidate; do
|
|
if [[ -f "$RUNNER_TEMP/release-evaluation/$runtime/metadata.json" ]]; then
|
|
uv run --locked --extra dev python -m workflow_bench.release_report summarize \
|
|
--results "$RUNNER_TEMP/release-evaluation/$runtime/raw/results.jsonl" \
|
|
--metadata "$RUNNER_TEMP/release-evaluation/$runtime/metadata.json" \
|
|
--out "$RUNNER_TEMP/release-evaluation/$runtime/public" || failed=1
|
|
else
|
|
echo "::error::$runtime evaluation preparation failed; no measurements exist."
|
|
failed=1
|
|
fi
|
|
done
|
|
exit "$failed"
|
|
- name: Require complete paired evidence and no task regression
|
|
if: ${{ always() && steps.resolve.outputs.sha != '' }}
|
|
working-directory: eval
|
|
run: |
|
|
uv run --locked --extra dev python -m workflow_bench.release_gate \
|
|
--candidate "$RUNNER_TEMP/release-evaluation/candidate/public/agent-evaluation.json" \
|
|
--candidate-sha '${{ steps.resolve.outputs.sha }}' \
|
|
--stable "$RUNNER_TEMP/release-evaluation/stable/public/agent-evaluation.json" \
|
|
--stable-sha '${{ steps.resolve.outputs.stable_sha }}' \
|
|
--out "$RUNNER_TEMP/release-evaluation/public"
|
|
- name: Collect sanitized evidence and publish summary
|
|
if: ${{ always() && steps.resolve.outputs.sha != '' }}
|
|
run: |
|
|
set -euo pipefail
|
|
public="$RUNNER_TEMP/release-evaluation/public"
|
|
mkdir -p "$public"
|
|
for extension in json md; do
|
|
candidate="$RUNNER_TEMP/release-evaluation/candidate/public/agent-evaluation.$extension"
|
|
stable="$RUNNER_TEMP/release-evaluation/stable/public/agent-evaluation.$extension"
|
|
if [[ -f "$candidate" ]]; then cp "$candidate" "$public/agent-evaluation.$extension"; fi
|
|
if [[ -f "$stable" ]]; then cp "$stable" "$public/stable-agent-evaluation.$extension"; fi
|
|
done
|
|
for report in release-quality-gate agent-evaluation; do
|
|
if [[ -f "$public/$report.md" ]]; then cat "$public/$report.md" >> "$GITHUB_STEP_SUMMARY"; fi
|
|
done
|
|
- name: Upload sanitized release evidence
|
|
if: ${{ always() && steps.resolve.outputs.sha != '' }}
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
with:
|
|
name: release-agent-evaluation-${{ steps.resolve.outputs.sha }}
|
|
path: |
|
|
${{ runner.temp }}/release-evaluation/public/agent-evaluation.json
|
|
${{ runner.temp }}/release-evaluation/public/agent-evaluation.md
|
|
${{ runner.temp }}/release-evaluation/public/stable-agent-evaluation.json
|
|
${{ runner.temp }}/release-evaluation/public/stable-agent-evaluation.md
|
|
${{ runner.temp }}/release-evaluation/public/release-quality-gate.json
|
|
${{ runner.temp }}/release-evaluation/public/release-quality-gate.md
|
|
if-no-files-found: error
|
|
retention-days: 30
|
|
|
|
stop-runner:
|
|
name: Stop and verify the dedicated EC2 runner
|
|
needs: [start-runner, check-runner, runner-ready, watch-evaluate-pickup, evaluate]
|
|
if: >-
|
|
always() && github.repository == 'abhigyanpatwari/GitNexus' &&
|
|
github.ref == 'refs/heads/main' &&
|
|
needs.start-runner.outputs.cleanup_required == 'true'
|
|
runs-on: ubuntu-latest
|
|
environment: gitnexus-evolution
|
|
timeout-minutes: 20
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
- uses: ./.github/actions/ec2-runner
|
|
with:
|
|
operation: stop
|
|
role-arn: ${{ secrets.GITNEXUS_EVOLUTION_AWS_ROLE_ARN }}
|
|
instance-id: ${{ secrets.GITNEXUS_EVOLUTION_EC2_INSTANCE_ID }}
|
|
region: ${{ vars.GITNEXUS_EVOLUTION_AWS_REGION }}
|
|
stop-schedule: ${{ vars.GITNEXUS_EVOLUTION_STOP_SCHEDULE_UTC }}
|