From 7b1d9a3ee2677fa7f849979ebfdcd206f26cb52a Mon Sep 17 00:00:00 2001 From: abhigyantrumio Date: Fri, 30 Oct 2026 05:07:23 +0530 Subject: [PATCH] feat(eval): run DeepSWE with Claude Code through a LiteLLM gateway Default agent is now pier's claude-code, pinned to the release evaluation's CLI (2.1.214). A non-Claude model (default gpt-5.6-sol) goes through a LiteLLM gateway container, routed like workflow_bench.model_gateway (Anthropic Messages -> OpenAI Responses). It runs on host port 80 because pier's egress proxy only allows 80/443, and is reached by a hostname the proxy can resolve (host.docker.internal, else the EC2 private DNS name). anthropic/ or claude- models use ANTHROPIC_API_KEY directly. mini-swe-agent stays available, pinned to eval/uv.lock, with `mcp` installed because litellm's Responses path imports it. report.py now records an arm with no job directory as a problem instead of crashing. Co-Authored-By: Claude Opus 5.5 --- eval/deepswe/README.md | 2 + eval/deepswe/report.py | 4 ++ eval/deepswe/run.sh | 132 +++++++++++++++++++++++++++++-------- eval/tests/test_deepswe.py | 15 ++++- 4 files changed, 126 insertions(+), 27 deletions(-) diff --git a/eval/deepswe/README.md b/eval/deepswe/README.md index 3cda210cc..7c7ac34a1 100644 --- a/eval/deepswe/README.md +++ b/eval/deepswe/README.md @@ -42,6 +42,8 @@ cd ~/GitNexus && eval/deepswe/run.sh --n-tasks 10 --concurrency 4 It prints the `scp` and `terminate-instances` commands for when you're done. +Task images are about 4 GB each, plus pier's agent layer and the GitNexus layer, so 400 GB fits roughly 40–50 tasks across both arms. For all 113 tasks, pass `--volume-gb 1000` or run `docker image prune -a` between batches. + `run.sh` performs these steps: 1. Pins `deep-swe` to a commit. diff --git a/eval/deepswe/report.py b/eval/deepswe/report.py index 4d4b772cd..65d569956 100644 --- a/eval/deepswe/report.py +++ b/eval/deepswe/report.py @@ -85,6 +85,8 @@ def load_trial(trial_dir: Path) -> dict[str, Any] | None: def load_job(job_dir: Path) -> list[dict[str, Any]]: + if not job_dir.is_dir(): + return [] trials = [load_trial(path) for path in sorted(job_dir.iterdir()) if path.is_dir()] return [trial for trial in trials if trial is not None] @@ -105,6 +107,8 @@ def build_report(jobs: dict[str, Path]) -> dict[str, Any]: keyed: dict[str, dict[tuple[str, int], dict[str, Any]]] = {} for arm in ARMS: trials = load_job(jobs[arm]) + if not trials: + report["problems"].append(f"{arm}: no trial results under {jobs[arm]}") keyed[arm] = _keyed(trials) graded = [row for row in trials if row["graded"]] ungraded = [row for row in trials if not row["graded"]] diff --git a/eval/deepswe/run.sh b/eval/deepswe/run.sh index 9f8750473..facd7c3de 100755 --- a/eval/deepswe/run.sh +++ b/eval/deepswe/run.sh @@ -3,16 +3,24 @@ # # OPENAI_API_KEY=... eval/deepswe/run.sh --n-tasks 10 --concurrency 4 # -# Needs docker, git, node/npm 22.18+, python3.11+ and pier (`uv tool install -# datacurve-pier==0.3.1`). Everything lands under --work (default ~/deepswe-work). +# The default agent is Claude Code. A non-Claude model (default gpt-5.6-sol) +# goes through a local LiteLLM gateway, as in workflow_bench; an `anthropic/` +# or `claude-` model uses ANTHROPIC_API_KEY directly. Needs docker, git, +# python3.11+, pier (`uv tool install datacurve-pier==0.3.1`) and, to build the +# package, node/npm 22.18+. Everything lands under --work (default ~/deepswe-work). set -euo pipefail DEEP_SWE_REPO=https://github.com/datacurve-ai/deep-swe DEEP_SWE_COMMIT=0b9fabbb63b9104d678fe965e1632f2dd9eaa2ea PIER_VERSION=0.3.1 +CLAUDE_CODE_VERSION=2.1.214 # the release evaluation's pinned CLI +LITELLM_VERSION=1.100.4 # eval/uv.lock +MINI_SWE_AGENT_VERSION=2.2.7 # eval/uv.lock +GATEWAY_CONTAINER=deepswe-litellm-gateway +GATEWAY_PORT=80 # pier's egress proxy only allows ports 80 and 443 -AGENT=mini-swe-agent -MODEL=openai/gpt-5.6-sol +AGENT=claude-code +MODEL=gpt-5.6-sol EFFORT=low N_TASKS=10 SEED=0 @@ -20,23 +28,27 @@ CONCURRENCY=2 COST_LIMIT=5 WORK="$HOME/deepswe-work" PACKAGE="" +GATEWAY_HOST="" TASKS=() ARMS=(baseline gitnexus) usage() { cat <; + mini-swe-agent: a litellm name such as openai/gpt-5.6-sol + --effort E reasoning effort (default $EFFORT) + --cost-limit USD per-trial agent cost cap, same for both arms (default $COST_LIMIT; 0 = none) + --package TGZ prebuilt \`npm pack\` tarball of gitnexus/ (default: build from this checkout) + --gateway-host H hostname the trial egress proxy uses to reach this host's gateway + (default: host.docker.internal if it resolves, else the EC2 private DNS name) + --arms "A B" arms to run (default: "${ARMS[*]}") + --work DIR working directory (default $WORK) EOF } @@ -51,6 +63,7 @@ while [ $# -gt 0 ]; do --effort) EFFORT="$2"; shift 2 ;; --cost-limit) COST_LIMIT="$2"; shift 2 ;; --package) PACKAGE="$2"; shift 2 ;; + --gateway-host) GATEWAY_HOST="$2"; shift 2 ;; --arms) read -r -a ARMS <<<"$2"; shift 2 ;; --work) WORK="$2"; shift 2 ;; -h|--help) usage; exit 0 ;; @@ -60,19 +73,91 @@ done EVAL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" REPO_ROOT="$(cd "$EVAL_DIR/.." && pwd)" -needed=(docker git python3 pier) +needed=(docker git python3 pier curl) [[ " ${ARMS[*]} " == *" gitnexus "* ]] && [ -z "$PACKAGE" ] && needed+=(node npm) # only to build the package for tool in "${needed[@]}"; do command -v "$tool" >/dev/null || { echo "missing $tool on PATH" >&2; exit 1; } done pier_version="$(pier --version 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" [ "$pier_version" = "$PIER_VERSION" ] || echo "warning: pier $pier_version, pinned $PIER_VERSION" >&2 -case "$MODEL" in - openai/*) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required for $MODEL}" ;; - anthropic/*) : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required for $MODEL}" ;; + +agent_args=(--agent "$AGENT" --model "$MODEL") +use_gateway=0 +case "$AGENT" in + claude-code) + agent_args+=(--ak "version=$CLAUDE_CODE_VERSION" --ak "reasoning_effort=$EFFORT") + [ "$COST_LIMIT" != 0 ] && agent_args+=(--ak "max_budget_usd=$COST_LIMIT") + case "$MODEL" in + anthropic/*|claude-*) : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required for $MODEL}" ;; + */*) echo "claude-code through the gateway takes a bare model name, not $MODEL" >&2; exit 2 ;; + *) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required for the $MODEL gateway}"; use_gateway=1 ;; + esac + ;; + mini-swe-agent) + # litellm's Responses path imports `mcp`, which a bare mini-swe-agent install lacks. + agent_args+=(--ak "version=$MINI_SWE_AGENT_VERSION" --ak "reasoning_effort=$EFFORT" + --ak "cost_limit=$COST_LIMIT" --ak 'extra_python_packages=["mcp"]') + case "$MODEL" in + openai/*) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required for $MODEL}" ;; + anthropic/*) : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required for $MODEL}" ;; + esac + ;; + *) echo "unsupported agent: $AGENT" >&2; exit 2 ;; esac mkdir -p "$WORK" +start_gateway() { + if [ -z "$GATEWAY_HOST" ]; then + if getent hosts host.docker.internal >/dev/null; then + GATEWAY_HOST=host.docker.internal + else + imds_token="$(curl -fsS -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' \ + http://169.254.169.254/latest/api/token 2>/dev/null || true)" + GATEWAY_HOST="$(curl -fsS -H "X-aws-ec2-metadata-token: $imds_token" \ + http://169.254.169.254/latest/meta-data/local-hostname 2>/dev/null || hostname -f)" + fi + fi + local dir="$WORK/gateway" + mkdir -p "$dir" + # Same routing as workflow_bench.model_gateway: Claude Code speaks Anthropic + # Messages; LiteLLM serves them from OpenAI's Responses API. + cat > "$dir/litellm.yaml" </dev/null 2>&1 || true + docker run -d --name "$GATEWAY_CONTAINER" -p "$GATEWAY_PORT:4000" \ + -e OPENAI_API_KEY -e LITELLM_MASTER_KEY="$GATEWAY_KEY" -e LITELLM_LOCAL_MODEL_COST_MAP=True \ + -v "$dir:/cfg:ro" python:3.12-slim \ + sh -c "pip install --quiet --no-cache-dir 'litellm[proxy]==$LITELLM_VERSION' && exec litellm --config /cfg/litellm.yaml --host 0.0.0.0 --port 4000" \ + >/dev/null + trap 'docker rm -f "$GATEWAY_CONTAINER" >/dev/null 2>&1 || true' EXIT + echo "waiting for the LiteLLM gateway on http://$GATEWAY_HOST:$GATEWAY_PORT" + for _ in $(seq 1 180); do + if curl -fsS -o /dev/null -H "Authorization: Bearer $GATEWAY_KEY" \ + "http://$GATEWAY_HOST:$GATEWAY_PORT/health/liveliness" 2>/dev/null; then + agent_args+=(--ae "ANTHROPIC_BASE_URL=http://$GATEWAY_HOST:$GATEWAY_PORT" --ae "ANTHROPIC_AUTH_TOKEN=$GATEWAY_KEY") + return 0 + fi + docker inspect -f '{{.State.Running}}' "$GATEWAY_CONTAINER" 2>/dev/null | grep -q true || break + sleep 2 + done + echo "gateway did not become ready; last log lines:" >&2 + docker logs --tail 20 "$GATEWAY_CONTAINER" >&2 || true + exit 1 +} + if [ ! -d "$WORK/deep-swe/.git" ]; then git clone --quiet "$DEEP_SWE_REPO" "$WORK/deep-swe" fi @@ -100,14 +185,9 @@ package_args=() (cd "$EVAL_DIR" && python3 -m deepswe.prepare --deep-swe "$WORK/deep-swe" --out "$WORK/tasks" \ --arms "${ARMS[@]}" "${package_args[@]}" "${select_args[@]}") +[ "$use_gateway" = 1 ] && start_gateway + stamp="$(date -u +%Y%m%dT%H%M%SZ)" -agent_args=(--agent "$AGENT" --model "$MODEL") -if [ "$AGENT" = mini-swe-agent ]; then - agent_args+=(--ak "reasoning_effort=$EFFORT" --ak "cost_limit=$COST_LIMIT") -elif [ "$AGENT" = claude-code ]; then - agent_args+=(--ak "reasoning_effort=$EFFORT") - [ "$COST_LIMIT" != 0 ] && agent_args+=(--ak "max_budget_usd=$COST_LIMIT") -fi for arm in "${ARMS[@]}"; do echo "== pier: $arm" # A failed trial is recorded in the job and reported; keep going to the other arm. diff --git a/eval/tests/test_deepswe.py b/eval/tests/test_deepswe.py index e381fec77..f9d10ad8f 100644 --- a/eval/tests/test_deepswe.py +++ b/eval/tests/test_deepswe.py @@ -190,17 +190,30 @@ def test_report_marks_missing_billing_incomplete(tmp_path): assert "| gitnexus | 2/2 | incomplete |" in report.render_markdown(result) +def test_report_survives_an_arm_that_never_produced_a_job(tmp_path): + _trial(tmp_path / "b", "x__1", "x") + result = report.build_report({"baseline": tmp_path / "b", "gitnexus": tmp_path / "never-created"}) + assert any(problem.startswith("gitnexus: no trial results") for problem in result["problems"]) + assert result["pairs"] == {"graded_in_both": 0, "expected": 1} and result["changes"] == {} + assert "| gitnexus | 0/0 |" in report.render_markdown(result) + + def test_scripts_pin_the_same_versions_and_fill_every_bootstrap_placeholder(): run_sh = (DEEPSWE_DIR / "run.sh").read_text() bootstrap = (DEEPSWE_DIR / "ec2" / "bootstrap.sh").read_text() launch = (DEEPSWE_DIR / "ec2" / "launch.sh").read_text() def pin(text: str, name: str) -> str: - match = re.search(rf"^{name}=(\S+)$", text, re.MULTILINE) + match = re.search(rf"^{name}=(\S+)", text, re.MULTILINE) assert match, name return match.group(1) assert pin(run_sh, "PIER_VERSION") == pin(bootstrap, "PIER_VERSION") + # The gateway and the optional mini-swe-agent use the versions eval/ is locked to. + lock = (DEEPSWE_DIR.parent / "uv.lock").read_text() + for package, name in (("litellm", "LITELLM_VERSION"), ("mini-swe-agent", "MINI_SWE_AGENT_VERSION")): + assert f'name = "{package}"\nversion = "{pin(run_sh, name)}"' in lock, package + assert re.fullmatch(r"\d+\.\d+\.\d+", pin(run_sh, "CLAUDE_CODE_VERSION")) assert pin(bootstrap, "NODE_VERSION") == prepare.NODE_VERSION assert re.fullmatch(r"[0-9a-f]{40}", pin(run_sh, "DEEP_SWE_COMMIT")) placeholders = set(re.findall(r"@([A-Z_]+)@", bootstrap))