feat(eval): run DeepSWE with Claude Code through a LiteLLM gateway

Default agent is now pier's claude-code, pinned to the release
evaluation's CLI (2.1.214). A non-Claude model (default gpt-5.6-sol)
goes through a LiteLLM gateway container, routed like
workflow_bench.model_gateway (Anthropic Messages -> OpenAI Responses).
It runs on host port 80 because pier's egress proxy only allows 80/443,
and is reached by a hostname the proxy can resolve (host.docker.internal,
else the EC2 private DNS name). anthropic/ or claude- models use
ANTHROPIC_API_KEY directly.

mini-swe-agent stays available, pinned to eval/uv.lock, with `mcp`
installed because litellm's Responses path imports it. report.py now
records an arm with no job directory as a problem instead of crashing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
abhigyantrumio 2026-10-30 05:07:23 +05:30
parent 16b274a8fa
commit 7b1d9a3ee2
4 changed files with 126 additions and 27 deletions

View file

@ -42,6 +42,8 @@ cd ~/GitNexus && eval/deepswe/run.sh --n-tasks 10 --concurrency 4
It prints the `scp` and `terminate-instances` commands for when you're done.
Task images are about 4 GB each, plus pier's agent layer and the GitNexus layer, so 400 GB fits roughly 40–50 tasks across both arms. For all 113 tasks, pass `--volume-gb 1000` or run `docker image prune -a` between batches.
`run.sh` performs these steps:
1. Pins `deep-swe` to a commit.

View file

@ -85,6 +85,8 @@ def load_trial(trial_dir: Path) -> dict[str, Any] | None:
def load_job(job_dir: Path) -> list[dict[str, Any]]:
if not job_dir.is_dir():
return []
trials = [load_trial(path) for path in sorted(job_dir.iterdir()) if path.is_dir()]
return [trial for trial in trials if trial is not None]
@ -105,6 +107,8 @@ def build_report(jobs: dict[str, Path]) -> dict[str, Any]:
keyed: dict[str, dict[tuple[str, int], dict[str, Any]]] = {}
for arm in ARMS:
trials = load_job(jobs[arm])
if not trials:
report["problems"].append(f"{arm}: no trial results under {jobs[arm]}")
keyed[arm] = _keyed(trials)
graded = [row for row in trials if row["graded"]]
ungraded = [row for row in trials if not row["graded"]]

View file

@ -3,16 +3,24 @@
#
# OPENAI_API_KEY=... eval/deepswe/run.sh --n-tasks 10 --concurrency 4
#
# Needs docker, git, node/npm 22.18+, python3.11+ and pier (`uv tool install
# datacurve-pier==0.3.1`). Everything lands under --work (default ~/deepswe-work).
# The default agent is Claude Code. A non-Claude model (default gpt-5.6-sol)
# goes through a local LiteLLM gateway, as in workflow_bench; an `anthropic/`
# or `claude-` model uses ANTHROPIC_API_KEY directly. Needs docker, git,
# python3.11+, pier (`uv tool install datacurve-pier==0.3.1`) and, to build the
# package, node/npm 22.18+. Everything lands under --work (default ~/deepswe-work).
set -euo pipefail
DEEP_SWE_REPO=https://github.com/datacurve-ai/deep-swe
DEEP_SWE_COMMIT=0b9fabbb63b9104d678fe965e1632f2dd9eaa2ea
PIER_VERSION=0.3.1
CLAUDE_CODE_VERSION=2.1.214 # the release evaluation's pinned CLI
LITELLM_VERSION=1.100.4 # eval/uv.lock
MINI_SWE_AGENT_VERSION=2.2.7 # eval/uv.lock
GATEWAY_CONTAINER=deepswe-litellm-gateway
GATEWAY_PORT=80 # pier's egress proxy only allows ports 80 and 443
AGENT=mini-swe-agent
MODEL=openai/gpt-5.6-sol
AGENT=claude-code
MODEL=gpt-5.6-sol
EFFORT=low
N_TASKS=10
SEED=0
@ -20,23 +28,27 @@ CONCURRENCY=2
COST_LIMIT=5
WORK="$HOME/deepswe-work"
PACKAGE=""
GATEWAY_HOST=""
TASKS=()
ARMS=(baseline gitnexus)
usage() {
cat <<EOF
Usage: $0 [options]
--n-tasks N seeded random subset size (default $N_TASKS; 0 = all 113)
--seed S sample seed (default $SEED)
--task ID run this task id; repeatable, overrides --n-tasks
--concurrency C pier --n-concurrent per arm (default $CONCURRENCY; each trial uses 2 CPUs / 8 GB)
--agent A pier agent (default $AGENT)
--model M pier model (default $MODEL)
--effort E reasoning effort (default $EFFORT)
--cost-limit USD per-trial agent cost cap, same for both arms (default $COST_LIMIT; 0 = none)
--package TGZ prebuilt \`npm pack\` tarball of gitnexus/ (default: build from this checkout)
--arms "A B" arms to run (default: "${ARMS[*]}")
--work DIR working directory (default $WORK)
--n-tasks N seeded random subset size (default $N_TASKS; 0 = all 113)
--seed S sample seed (default $SEED)
--task ID run this task id; repeatable, overrides --n-tasks
--concurrency C pier --n-concurrent per arm (default $CONCURRENCY; each trial uses 2 CPUs / 8 GB)
--agent A claude-code (default) or mini-swe-agent
--model M default $MODEL; claude-code: a gateway model name or anthropic/<claude model>;
mini-swe-agent: a litellm name such as openai/gpt-5.6-sol
--effort E reasoning effort (default $EFFORT)
--cost-limit USD per-trial agent cost cap, same for both arms (default $COST_LIMIT; 0 = none)
--package TGZ prebuilt \`npm pack\` tarball of gitnexus/ (default: build from this checkout)
--gateway-host H hostname the trial egress proxy uses to reach this host's gateway
(default: host.docker.internal if it resolves, else the EC2 private DNS name)
--arms "A B" arms to run (default: "${ARMS[*]}")
--work DIR working directory (default $WORK)
EOF
}
@ -51,6 +63,7 @@ while [ $# -gt 0 ]; do
--effort) EFFORT="$2"; shift 2 ;;
--cost-limit) COST_LIMIT="$2"; shift 2 ;;
--package) PACKAGE="$2"; shift 2 ;;
--gateway-host) GATEWAY_HOST="$2"; shift 2 ;;
--arms) read -r -a ARMS <<<"$2"; shift 2 ;;
--work) WORK="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
@ -60,19 +73,91 @@ done
EVAL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
REPO_ROOT="$(cd "$EVAL_DIR/.." && pwd)"
needed=(docker git python3 pier)
needed=(docker git python3 pier curl)
[[ " ${ARMS[*]} " == *" gitnexus "* ]] && [ -z "$PACKAGE" ] && needed+=(node npm) # only to build the package
for tool in "${needed[@]}"; do
command -v "$tool" >/dev/null || { echo "missing $tool on PATH" >&2; exit 1; }
done
pier_version="$(pier --version 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)"
[ "$pier_version" = "$PIER_VERSION" ] || echo "warning: pier $pier_version, pinned $PIER_VERSION" >&2
case "$MODEL" in
openai/*) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required for $MODEL}" ;;
anthropic/*) : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required for $MODEL}" ;;
agent_args=(--agent "$AGENT" --model "$MODEL")
use_gateway=0
case "$AGENT" in
claude-code)
agent_args+=(--ak "version=$CLAUDE_CODE_VERSION" --ak "reasoning_effort=$EFFORT")
[ "$COST_LIMIT" != 0 ] && agent_args+=(--ak "max_budget_usd=$COST_LIMIT")
case "$MODEL" in
anthropic/*|claude-*) : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required for $MODEL}" ;;
*/*) echo "claude-code through the gateway takes a bare model name, not $MODEL" >&2; exit 2 ;;
*) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required for the $MODEL gateway}"; use_gateway=1 ;;
esac
;;
mini-swe-agent)
# litellm's Responses path imports `mcp`, which a bare mini-swe-agent install lacks.
agent_args+=(--ak "version=$MINI_SWE_AGENT_VERSION" --ak "reasoning_effort=$EFFORT"
--ak "cost_limit=$COST_LIMIT" --ak 'extra_python_packages=["mcp"]')
case "$MODEL" in
openai/*) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required for $MODEL}" ;;
anthropic/*) : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required for $MODEL}" ;;
esac
;;
*) echo "unsupported agent: $AGENT" >&2; exit 2 ;;
esac
mkdir -p "$WORK"
start_gateway() {
if [ -z "$GATEWAY_HOST" ]; then
if getent hosts host.docker.internal >/dev/null; then
GATEWAY_HOST=host.docker.internal
else
imds_token="$(curl -fsS -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' \
http://169.254.169.254/latest/api/token 2>/dev/null || true)"
GATEWAY_HOST="$(curl -fsS -H "X-aws-ec2-metadata-token: $imds_token" \
http://169.254.169.254/latest/meta-data/local-hostname 2>/dev/null || hostname -f)"
fi
fi
local dir="$WORK/gateway"
mkdir -p "$dir"
# Same routing as workflow_bench.model_gateway: Claude Code speaks Anthropic
# Messages; LiteLLM serves them from OpenAI's Responses API.
cat > "$dir/litellm.yaml" <<EOF
model_list:
- model_name: $MODEL
litellm_params:
model: openai/$MODEL
api_key: os.environ/OPENAI_API_KEY
timeout: 900
model_info:
mode: responses
litellm_settings:
request_timeout: 900
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
EOF
GATEWAY_KEY="sk-deepswe-$(python3 -c 'import secrets; print(secrets.token_hex(16))')"
docker rm -f "$GATEWAY_CONTAINER" >/dev/null 2>&1 || true
docker run -d --name "$GATEWAY_CONTAINER" -p "$GATEWAY_PORT:4000" \
-e OPENAI_API_KEY -e LITELLM_MASTER_KEY="$GATEWAY_KEY" -e LITELLM_LOCAL_MODEL_COST_MAP=True \
-v "$dir:/cfg:ro" python:3.12-slim \
sh -c "pip install --quiet --no-cache-dir 'litellm[proxy]==$LITELLM_VERSION' && exec litellm --config /cfg/litellm.yaml --host 0.0.0.0 --port 4000" \
>/dev/null
trap 'docker rm -f "$GATEWAY_CONTAINER" >/dev/null 2>&1 || true' EXIT
echo "waiting for the LiteLLM gateway on http://$GATEWAY_HOST:$GATEWAY_PORT"
for _ in $(seq 1 180); do
if curl -fsS -o /dev/null -H "Authorization: Bearer $GATEWAY_KEY" \
"http://$GATEWAY_HOST:$GATEWAY_PORT/health/liveliness" 2>/dev/null; then
agent_args+=(--ae "ANTHROPIC_BASE_URL=http://$GATEWAY_HOST:$GATEWAY_PORT" --ae "ANTHROPIC_AUTH_TOKEN=$GATEWAY_KEY")
return 0
fi
docker inspect -f '{{.State.Running}}' "$GATEWAY_CONTAINER" 2>/dev/null | grep -q true || break
sleep 2
done
echo "gateway did not become ready; last log lines:" >&2
docker logs --tail 20 "$GATEWAY_CONTAINER" >&2 || true
exit 1
}
if [ ! -d "$WORK/deep-swe/.git" ]; then
git clone --quiet "$DEEP_SWE_REPO" "$WORK/deep-swe"
fi
@ -100,14 +185,9 @@ package_args=()
(cd "$EVAL_DIR" && python3 -m deepswe.prepare --deep-swe "$WORK/deep-swe" --out "$WORK/tasks" \
--arms "${ARMS[@]}" "${package_args[@]}" "${select_args[@]}")
[ "$use_gateway" = 1 ] && start_gateway
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
agent_args=(--agent "$AGENT" --model "$MODEL")
if [ "$AGENT" = mini-swe-agent ]; then
agent_args+=(--ak "reasoning_effort=$EFFORT" --ak "cost_limit=$COST_LIMIT")
elif [ "$AGENT" = claude-code ]; then
agent_args+=(--ak "reasoning_effort=$EFFORT")
[ "$COST_LIMIT" != 0 ] && agent_args+=(--ak "max_budget_usd=$COST_LIMIT")
fi
for arm in "${ARMS[@]}"; do
echo "== pier: $arm"
# A failed trial is recorded in the job and reported; keep going to the other arm.

View file

@ -190,17 +190,30 @@ def test_report_marks_missing_billing_incomplete(tmp_path):
assert "| gitnexus | 2/2 | incomplete |" in report.render_markdown(result)
def test_report_survives_an_arm_that_never_produced_a_job(tmp_path):
_trial(tmp_path / "b", "x__1", "x")
result = report.build_report({"baseline": tmp_path / "b", "gitnexus": tmp_path / "never-created"})
assert any(problem.startswith("gitnexus: no trial results") for problem in result["problems"])
assert result["pairs"] == {"graded_in_both": 0, "expected": 1} and result["changes"] == {}
assert "| gitnexus | 0/0 |" in report.render_markdown(result)
def test_scripts_pin_the_same_versions_and_fill_every_bootstrap_placeholder():
run_sh = (DEEPSWE_DIR / "run.sh").read_text()
bootstrap = (DEEPSWE_DIR / "ec2" / "bootstrap.sh").read_text()
launch = (DEEPSWE_DIR / "ec2" / "launch.sh").read_text()
def pin(text: str, name: str) -> str:
match = re.search(rf"^{name}=(\S+)$", text, re.MULTILINE)
match = re.search(rf"^{name}=(\S+)", text, re.MULTILINE)
assert match, name
return match.group(1)
assert pin(run_sh, "PIER_VERSION") == pin(bootstrap, "PIER_VERSION")
# The gateway and the optional mini-swe-agent use the versions eval/ is locked to.
lock = (DEEPSWE_DIR.parent / "uv.lock").read_text()
for package, name in (("litellm", "LITELLM_VERSION"), ("mini-swe-agent", "MINI_SWE_AGENT_VERSION")):
assert f'name = "{package}"\nversion = "{pin(run_sh, name)}"' in lock, package
assert re.fullmatch(r"\d+\.\d+\.\d+", pin(run_sh, "CLAUDE_CODE_VERSION"))
assert pin(bootstrap, "NODE_VERSION") == prepare.NODE_VERSION
assert re.fullmatch(r"[0-9a-f]{40}", pin(run_sh, "DEEP_SWE_COMMIT"))
placeholders = set(re.findall(r"@([A-Z_]+)@", bootstrap))