diff --git a/.github/workflows/gitnexus-skill-evolution.yml b/.github/workflows/gitnexus-skill-evolution.yml index 329640e78..7c9adda5a 100644 --- a/.github/workflows/gitnexus-skill-evolution.yml +++ b/.github/workflows/gitnexus-skill-evolution.yml @@ -74,6 +74,11 @@ on: required: false default: '3' type: string + workers: + description: 'Benchmark cells of one task to run at once — raise only to match the runner’s vCPUs' + required: false + default: '1' + type: string model: description: 'Model for the benchmark arms (match the model your skill users run)' required: false @@ -129,6 +134,11 @@ jobs: env: GENERATIONS: ${{ inputs.generations || '1' }} RUNS: ${{ inputs.runs || '3' }} + # Serial by default. A cell that loses CPU to its siblings takes longer, + # and a session that reaches its timeout is an excluded run the promotion + # gate refuses to work with — so this only goes up when the runner has the + # vCPUs to back it (the box is sized for one cell at a time today). + WORKERS: ${{ inputs.workers || '1' }} MODEL: ${{ inputs.model || 'claude-sonnet-5' }} PROPOSER_MODEL: ${{ inputs.proposer_model || 'claude-opus-4-8' }} INCLUDE_EXPENSIVE: ${{ inputs.include_expensive && '1' || '' }} @@ -303,6 +313,7 @@ jobs: --proposer-model "${PROPOSER_MODEL}" \ --generations "${GENERATIONS}" \ --runs "${RUNS}" \ + --workers "${WORKERS}" \ --claude-bin "${RUNNER_TEMP}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \ --out-root "${OUT_ROOT}" \ --apply \ diff --git a/eval/workflow_bench/evolve.py b/eval/workflow_bench/evolve.py index 02481673b..ac1c56723 100644 --- a/eval/workflow_bench/evolve.py +++ b/eval/workflow_bench/evolve.py @@ -620,6 +620,8 @@ def runner_argv( str(args.tasks), "--runs", str(args.runs), + "--workers", + str(args.workers), "--model", args.model, "--claude-bin", @@ -792,6 +794,13 @@ def build_parser() -> argparse.ArgumentParser: "quality matters more than cost here, so a stronger model is fine", ) parser.add_argument("--runs", type=int, default=3, help="per arm per task; the gate needs ≥3") + parser.add_argument( + "--workers", + type=int, + default=1, + help="benchmark cells of one task to run at once (default 1, fully " + "serial); size it to the machine — see workflow_bench.runner --workers", + ) parser.add_argument("--generations", type=int, default=1) parser.add_argument( "--arms", @@ -857,6 +866,8 @@ def main() -> int: parser.error("--generations must be positive") if args.runs < 1 or args.timeout < 1: parser.error("--runs and --timeout must be positive") + if args.workers < 1: + parser.error("--workers must be positive") try: args.model = runner.normalized_model_identifier(args.model) args.proposer_model = runner.normalized_model_identifier( diff --git a/gitnexus/test/unit/skill-evolution-workflow.test.ts b/gitnexus/test/unit/skill-evolution-workflow.test.ts index ea484abf3..9549e3ec0 100644 --- a/gitnexus/test/unit/skill-evolution-workflow.test.ts +++ b/gitnexus/test/unit/skill-evolution-workflow.test.ts @@ -18,6 +18,7 @@ const workflowDocument = load(workflow) as { string, { environment?: unknown; + env?: Record; 'timeout-minutes'?: unknown; steps?: Array<{ name?: string; @@ -47,6 +48,14 @@ describe('gitnexus skill-evolution workflow contract', () => { expect(loop).toContain('--apply'); }); + it('passes the cell concurrency through to the benchmark', () => { + // The lane is serial unless told otherwise: concurrency only pays off when + // the runner has the vCPUs for it, and a cell starved of CPU drifts toward + // its session timeout, which the gate counts as an excluded run. + expect(stepRun('Run the propose → benchmark → gate loop')).toContain('--workers "${WORKERS}"'); + expect(evolveJob?.env?.WORKERS).toBe("${{ inputs.workers || '1' }}"); + }); + it('runs the proposer on its own model, separate from the benchmark arms', () => { const loop = stepRun('Run the propose → benchmark → gate loop'); // The benchmark arms match the production model; the proposer/diagnosis