feat(ci): expose benchmark cell concurrency to the evolution lane

`--workers` reaches the sweep from evolve.py and from a workflow_dispatch
input. Both default to 1, so nothing about the scheduled lane changes:
the runner is sized for one cell at a time, and a cell starved of CPU
drifts toward its session timeout, which the gate counts as an excluded
run and refuses to decide on.

generation_timeout_seconds is left alone deliberately — it is a
worst-case sum-of-every-timeout bound (843h at current settings), already
far looser than any real run, and concurrency only makes it looser.

Raising the input is gated on the runner resize; the contract test pins
the default so the lane cannot start running 3-way on a 2-vCPU box by
accident.
This commit is contained in:
Gergo Magyar 2026-08-01 18:47:04 +00:00
parent 63d29c384f
commit edb24da1e8
3 changed files with 31 additions and 0 deletions

View file

@ -74,6 +74,11 @@ on:
required: false
default: '3'
type: string
workers:
description: 'Benchmark cells of one task to run at once — raise only to match the runners vCPUs'
required: false
default: '1'
type: string
model:
description: 'Model for the benchmark arms (match the model your skill users run)'
required: false
@ -129,6 +134,11 @@ jobs:
env:
GENERATIONS: ${{ inputs.generations || '1' }}
RUNS: ${{ inputs.runs || '3' }}
# Serial by default. A cell that loses CPU to its siblings takes longer,
# and a session that reaches its timeout is an excluded run the promotion
# gate refuses to work with — so this only goes up when the runner has the
# vCPUs to back it (the box is sized for one cell at a time today).
WORKERS: ${{ inputs.workers || '1' }}
MODEL: ${{ inputs.model || 'claude-sonnet-5' }}
PROPOSER_MODEL: ${{ inputs.proposer_model || 'claude-opus-4-8' }}
INCLUDE_EXPENSIVE: ${{ inputs.include_expensive && '1' || '' }}
@ -303,6 +313,7 @@ jobs:
--proposer-model "${PROPOSER_MODEL}" \
--generations "${GENERATIONS}" \
--runs "${RUNS}" \
--workers "${WORKERS}" \
--claude-bin "${RUNNER_TEMP}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \
--out-root "${OUT_ROOT}" \
--apply \

View file

@ -620,6 +620,8 @@ def runner_argv(
str(args.tasks),
"--runs",
str(args.runs),
"--workers",
str(args.workers),
"--model",
args.model,
"--claude-bin",
@ -792,6 +794,13 @@ def build_parser() -> argparse.ArgumentParser:
"quality matters more than cost here, so a stronger model is fine",
)
parser.add_argument("--runs", type=int, default=3, help="per arm per task; the gate needs ≥3")
parser.add_argument(
"--workers",
type=int,
default=1,
help="benchmark cells of one task to run at once (default 1, fully "
"serial); size it to the machine — see workflow_bench.runner --workers",
)
parser.add_argument("--generations", type=int, default=1)
parser.add_argument(
"--arms",
@ -857,6 +866,8 @@ def main() -> int:
parser.error("--generations must be positive")
if args.runs < 1 or args.timeout < 1:
parser.error("--runs and --timeout must be positive")
if args.workers < 1:
parser.error("--workers must be positive")
try:
args.model = runner.normalized_model_identifier(args.model)
args.proposer_model = runner.normalized_model_identifier(

View file

@ -18,6 +18,7 @@ const workflowDocument = load(workflow) as {
string,
{
environment?: unknown;
env?: Record<string, string>;
'timeout-minutes'?: unknown;
steps?: Array<{
name?: string;
@ -47,6 +48,14 @@ describe('gitnexus skill-evolution workflow contract', () => {
expect(loop).toContain('--apply');
});
it('passes the cell concurrency through to the benchmark', () => {
// The lane is serial unless told otherwise: concurrency only pays off when
// the runner has the vCPUs for it, and a cell starved of CPU drifts toward
// its session timeout, which the gate counts as an excluded run.
expect(stepRun('Run the propose → benchmark → gate loop')).toContain('--workers "${WORKERS}"');
expect(evolveJob?.env?.WORKERS).toBe("${{ inputs.workers || '1' }}");
});
it('runs the proposer on its own model, separate from the benchmark arms', () => {
const loop = stepRun('Run the propose → benchmark → gate loop');
// The benchmark arms match the production model; the proposer/diagnosis