mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-11-16 09:22:00 +00:00
feat(eval): run DeepSWE with Claude Code through a LiteLLM gateway
Default agent is now pier's claude-code, pinned to the release evaluation's CLI (2.1.214). A non-Claude model (default gpt-5.6-sol) goes through a LiteLLM gateway container, routed like workflow_bench.model_gateway (Anthropic Messages -> OpenAI Responses). It runs on host port 80 because pier's egress proxy only allows 80/443, and is reached by a hostname the proxy can resolve (host.docker.internal, else the EC2 private DNS name). anthropic/ or claude- models use ANTHROPIC_API_KEY directly. mini-swe-agent stays available, pinned to eval/uv.lock, with `mcp` installed because litellm's Responses path imports it. report.py now records an arm with no job directory as a problem instead of crashing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
16b274a8fa
commit
7b1d9a3ee2
4 changed files with 126 additions and 27 deletions
|
|
@ -42,6 +42,8 @@ cd ~/GitNexus && eval/deepswe/run.sh --n-tasks 10 --concurrency 4
|
|||
|
||||
It prints the `scp` and `terminate-instances` commands for when you're done.
|
||||
|
||||
Task images are about 4 GB each, plus pier's agent layer and the GitNexus layer, so 400 GB fits roughly 40–50 tasks across both arms. For all 113 tasks, pass `--volume-gb 1000` or run `docker image prune -a` between batches.
|
||||
|
||||
`run.sh` performs these steps:
|
||||
|
||||
1. Pins `deep-swe` to a commit.
|
||||
|
|
|
|||
|
|
@ -85,6 +85,8 @@ def load_trial(trial_dir: Path) -> dict[str, Any] | None:
|
|||
|
||||
|
||||
def load_job(job_dir: Path) -> list[dict[str, Any]]:
|
||||
if not job_dir.is_dir():
|
||||
return []
|
||||
trials = [load_trial(path) for path in sorted(job_dir.iterdir()) if path.is_dir()]
|
||||
return [trial for trial in trials if trial is not None]
|
||||
|
||||
|
|
@ -105,6 +107,8 @@ def build_report(jobs: dict[str, Path]) -> dict[str, Any]:
|
|||
keyed: dict[str, dict[tuple[str, int], dict[str, Any]]] = {}
|
||||
for arm in ARMS:
|
||||
trials = load_job(jobs[arm])
|
||||
if not trials:
|
||||
report["problems"].append(f"{arm}: no trial results under {jobs[arm]}")
|
||||
keyed[arm] = _keyed(trials)
|
||||
graded = [row for row in trials if row["graded"]]
|
||||
ungraded = [row for row in trials if not row["graded"]]
|
||||
|
|
|
|||
|
|
@ -3,16 +3,24 @@
|
|||
#
|
||||
# OPENAI_API_KEY=... eval/deepswe/run.sh --n-tasks 10 --concurrency 4
|
||||
#
|
||||
# Needs docker, git, node/npm 22.18+, python3.11+ and pier (`uv tool install
|
||||
# datacurve-pier==0.3.1`). Everything lands under --work (default ~/deepswe-work).
|
||||
# The default agent is Claude Code. A non-Claude model (default gpt-5.6-sol)
|
||||
# goes through a local LiteLLM gateway, as in workflow_bench; an `anthropic/`
|
||||
# or `claude-` model uses ANTHROPIC_API_KEY directly. Needs docker, git,
|
||||
# python3.11+, pier (`uv tool install datacurve-pier==0.3.1`) and, to build the
|
||||
# package, node/npm 22.18+. Everything lands under --work (default ~/deepswe-work).
|
||||
set -euo pipefail
|
||||
|
||||
DEEP_SWE_REPO=https://github.com/datacurve-ai/deep-swe
|
||||
DEEP_SWE_COMMIT=0b9fabbb63b9104d678fe965e1632f2dd9eaa2ea
|
||||
PIER_VERSION=0.3.1
|
||||
CLAUDE_CODE_VERSION=2.1.214 # the release evaluation's pinned CLI
|
||||
LITELLM_VERSION=1.100.4 # eval/uv.lock
|
||||
MINI_SWE_AGENT_VERSION=2.2.7 # eval/uv.lock
|
||||
GATEWAY_CONTAINER=deepswe-litellm-gateway
|
||||
GATEWAY_PORT=80 # pier's egress proxy only allows ports 80 and 443
|
||||
|
||||
AGENT=mini-swe-agent
|
||||
MODEL=openai/gpt-5.6-sol
|
||||
AGENT=claude-code
|
||||
MODEL=gpt-5.6-sol
|
||||
EFFORT=low
|
||||
N_TASKS=10
|
||||
SEED=0
|
||||
|
|
@ -20,23 +28,27 @@ CONCURRENCY=2
|
|||
COST_LIMIT=5
|
||||
WORK="$HOME/deepswe-work"
|
||||
PACKAGE=""
|
||||
GATEWAY_HOST=""
|
||||
TASKS=()
|
||||
ARMS=(baseline gitnexus)
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 [options]
|
||||
--n-tasks N seeded random subset size (default $N_TASKS; 0 = all 113)
|
||||
--seed S sample seed (default $SEED)
|
||||
--task ID run this task id; repeatable, overrides --n-tasks
|
||||
--concurrency C pier --n-concurrent per arm (default $CONCURRENCY; each trial uses 2 CPUs / 8 GB)
|
||||
--agent A pier agent (default $AGENT)
|
||||
--model M pier model (default $MODEL)
|
||||
--effort E reasoning effort (default $EFFORT)
|
||||
--cost-limit USD per-trial agent cost cap, same for both arms (default $COST_LIMIT; 0 = none)
|
||||
--package TGZ prebuilt \`npm pack\` tarball of gitnexus/ (default: build from this checkout)
|
||||
--arms "A B" arms to run (default: "${ARMS[*]}")
|
||||
--work DIR working directory (default $WORK)
|
||||
--n-tasks N seeded random subset size (default $N_TASKS; 0 = all 113)
|
||||
--seed S sample seed (default $SEED)
|
||||
--task ID run this task id; repeatable, overrides --n-tasks
|
||||
--concurrency C pier --n-concurrent per arm (default $CONCURRENCY; each trial uses 2 CPUs / 8 GB)
|
||||
--agent A claude-code (default) or mini-swe-agent
|
||||
--model M default $MODEL; claude-code: a gateway model name or anthropic/<claude model>;
|
||||
mini-swe-agent: a litellm name such as openai/gpt-5.6-sol
|
||||
--effort E reasoning effort (default $EFFORT)
|
||||
--cost-limit USD per-trial agent cost cap, same for both arms (default $COST_LIMIT; 0 = none)
|
||||
--package TGZ prebuilt \`npm pack\` tarball of gitnexus/ (default: build from this checkout)
|
||||
--gateway-host H hostname the trial egress proxy uses to reach this host's gateway
|
||||
(default: host.docker.internal if it resolves, else the EC2 private DNS name)
|
||||
--arms "A B" arms to run (default: "${ARMS[*]}")
|
||||
--work DIR working directory (default $WORK)
|
||||
EOF
|
||||
}
|
||||
|
||||
|
|
@ -51,6 +63,7 @@ while [ $# -gt 0 ]; do
|
|||
--effort) EFFORT="$2"; shift 2 ;;
|
||||
--cost-limit) COST_LIMIT="$2"; shift 2 ;;
|
||||
--package) PACKAGE="$2"; shift 2 ;;
|
||||
--gateway-host) GATEWAY_HOST="$2"; shift 2 ;;
|
||||
--arms) read -r -a ARMS <<<"$2"; shift 2 ;;
|
||||
--work) WORK="$2"; shift 2 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
|
|
@ -60,19 +73,91 @@ done
|
|||
|
||||
EVAL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
REPO_ROOT="$(cd "$EVAL_DIR/.." && pwd)"
|
||||
needed=(docker git python3 pier)
|
||||
needed=(docker git python3 pier curl)
|
||||
[[ " ${ARMS[*]} " == *" gitnexus "* ]] && [ -z "$PACKAGE" ] && needed+=(node npm) # only to build the package
|
||||
for tool in "${needed[@]}"; do
|
||||
command -v "$tool" >/dev/null || { echo "missing $tool on PATH" >&2; exit 1; }
|
||||
done
|
||||
pier_version="$(pier --version 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)"
|
||||
[ "$pier_version" = "$PIER_VERSION" ] || echo "warning: pier $pier_version, pinned $PIER_VERSION" >&2
|
||||
case "$MODEL" in
|
||||
openai/*) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required for $MODEL}" ;;
|
||||
anthropic/*) : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required for $MODEL}" ;;
|
||||
|
||||
agent_args=(--agent "$AGENT" --model "$MODEL")
|
||||
use_gateway=0
|
||||
case "$AGENT" in
|
||||
claude-code)
|
||||
agent_args+=(--ak "version=$CLAUDE_CODE_VERSION" --ak "reasoning_effort=$EFFORT")
|
||||
[ "$COST_LIMIT" != 0 ] && agent_args+=(--ak "max_budget_usd=$COST_LIMIT")
|
||||
case "$MODEL" in
|
||||
anthropic/*|claude-*) : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required for $MODEL}" ;;
|
||||
*/*) echo "claude-code through the gateway takes a bare model name, not $MODEL" >&2; exit 2 ;;
|
||||
*) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required for the $MODEL gateway}"; use_gateway=1 ;;
|
||||
esac
|
||||
;;
|
||||
mini-swe-agent)
|
||||
# litellm's Responses path imports `mcp`, which a bare mini-swe-agent install lacks.
|
||||
agent_args+=(--ak "version=$MINI_SWE_AGENT_VERSION" --ak "reasoning_effort=$EFFORT"
|
||||
--ak "cost_limit=$COST_LIMIT" --ak 'extra_python_packages=["mcp"]')
|
||||
case "$MODEL" in
|
||||
openai/*) : "${OPENAI_API_KEY:?OPENAI_API_KEY is required for $MODEL}" ;;
|
||||
anthropic/*) : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required for $MODEL}" ;;
|
||||
esac
|
||||
;;
|
||||
*) echo "unsupported agent: $AGENT" >&2; exit 2 ;;
|
||||
esac
|
||||
mkdir -p "$WORK"
|
||||
|
||||
start_gateway() {
|
||||
if [ -z "$GATEWAY_HOST" ]; then
|
||||
if getent hosts host.docker.internal >/dev/null; then
|
||||
GATEWAY_HOST=host.docker.internal
|
||||
else
|
||||
imds_token="$(curl -fsS -X PUT -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' \
|
||||
http://169.254.169.254/latest/api/token 2>/dev/null || true)"
|
||||
GATEWAY_HOST="$(curl -fsS -H "X-aws-ec2-metadata-token: $imds_token" \
|
||||
http://169.254.169.254/latest/meta-data/local-hostname 2>/dev/null || hostname -f)"
|
||||
fi
|
||||
fi
|
||||
local dir="$WORK/gateway"
|
||||
mkdir -p "$dir"
|
||||
# Same routing as workflow_bench.model_gateway: Claude Code speaks Anthropic
|
||||
# Messages; LiteLLM serves them from OpenAI's Responses API.
|
||||
cat > "$dir/litellm.yaml" <<EOF
|
||||
model_list:
|
||||
- model_name: $MODEL
|
||||
litellm_params:
|
||||
model: openai/$MODEL
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
timeout: 900
|
||||
model_info:
|
||||
mode: responses
|
||||
litellm_settings:
|
||||
request_timeout: 900
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
EOF
|
||||
GATEWAY_KEY="sk-deepswe-$(python3 -c 'import secrets; print(secrets.token_hex(16))')"
|
||||
docker rm -f "$GATEWAY_CONTAINER" >/dev/null 2>&1 || true
|
||||
docker run -d --name "$GATEWAY_CONTAINER" -p "$GATEWAY_PORT:4000" \
|
||||
-e OPENAI_API_KEY -e LITELLM_MASTER_KEY="$GATEWAY_KEY" -e LITELLM_LOCAL_MODEL_COST_MAP=True \
|
||||
-v "$dir:/cfg:ro" python:3.12-slim \
|
||||
sh -c "pip install --quiet --no-cache-dir 'litellm[proxy]==$LITELLM_VERSION' && exec litellm --config /cfg/litellm.yaml --host 0.0.0.0 --port 4000" \
|
||||
>/dev/null
|
||||
trap 'docker rm -f "$GATEWAY_CONTAINER" >/dev/null 2>&1 || true' EXIT
|
||||
echo "waiting for the LiteLLM gateway on http://$GATEWAY_HOST:$GATEWAY_PORT"
|
||||
for _ in $(seq 1 180); do
|
||||
if curl -fsS -o /dev/null -H "Authorization: Bearer $GATEWAY_KEY" \
|
||||
"http://$GATEWAY_HOST:$GATEWAY_PORT/health/liveliness" 2>/dev/null; then
|
||||
agent_args+=(--ae "ANTHROPIC_BASE_URL=http://$GATEWAY_HOST:$GATEWAY_PORT" --ae "ANTHROPIC_AUTH_TOKEN=$GATEWAY_KEY")
|
||||
return 0
|
||||
fi
|
||||
docker inspect -f '{{.State.Running}}' "$GATEWAY_CONTAINER" 2>/dev/null | grep -q true || break
|
||||
sleep 2
|
||||
done
|
||||
echo "gateway did not become ready; last log lines:" >&2
|
||||
docker logs --tail 20 "$GATEWAY_CONTAINER" >&2 || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ ! -d "$WORK/deep-swe/.git" ]; then
|
||||
git clone --quiet "$DEEP_SWE_REPO" "$WORK/deep-swe"
|
||||
fi
|
||||
|
|
@ -100,14 +185,9 @@ package_args=()
|
|||
(cd "$EVAL_DIR" && python3 -m deepswe.prepare --deep-swe "$WORK/deep-swe" --out "$WORK/tasks" \
|
||||
--arms "${ARMS[@]}" "${package_args[@]}" "${select_args[@]}")
|
||||
|
||||
[ "$use_gateway" = 1 ] && start_gateway
|
||||
|
||||
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
agent_args=(--agent "$AGENT" --model "$MODEL")
|
||||
if [ "$AGENT" = mini-swe-agent ]; then
|
||||
agent_args+=(--ak "reasoning_effort=$EFFORT" --ak "cost_limit=$COST_LIMIT")
|
||||
elif [ "$AGENT" = claude-code ]; then
|
||||
agent_args+=(--ak "reasoning_effort=$EFFORT")
|
||||
[ "$COST_LIMIT" != 0 ] && agent_args+=(--ak "max_budget_usd=$COST_LIMIT")
|
||||
fi
|
||||
for arm in "${ARMS[@]}"; do
|
||||
echo "== pier: $arm"
|
||||
# A failed trial is recorded in the job and reported; keep going to the other arm.
|
||||
|
|
|
|||
|
|
@ -190,17 +190,30 @@ def test_report_marks_missing_billing_incomplete(tmp_path):
|
|||
assert "| gitnexus | 2/2 | incomplete |" in report.render_markdown(result)
|
||||
|
||||
|
||||
def test_report_survives_an_arm_that_never_produced_a_job(tmp_path):
|
||||
_trial(tmp_path / "b", "x__1", "x")
|
||||
result = report.build_report({"baseline": tmp_path / "b", "gitnexus": tmp_path / "never-created"})
|
||||
assert any(problem.startswith("gitnexus: no trial results") for problem in result["problems"])
|
||||
assert result["pairs"] == {"graded_in_both": 0, "expected": 1} and result["changes"] == {}
|
||||
assert "| gitnexus | 0/0 |" in report.render_markdown(result)
|
||||
|
||||
|
||||
def test_scripts_pin_the_same_versions_and_fill_every_bootstrap_placeholder():
|
||||
run_sh = (DEEPSWE_DIR / "run.sh").read_text()
|
||||
bootstrap = (DEEPSWE_DIR / "ec2" / "bootstrap.sh").read_text()
|
||||
launch = (DEEPSWE_DIR / "ec2" / "launch.sh").read_text()
|
||||
|
||||
def pin(text: str, name: str) -> str:
|
||||
match = re.search(rf"^{name}=(\S+)$", text, re.MULTILINE)
|
||||
match = re.search(rf"^{name}=(\S+)", text, re.MULTILINE)
|
||||
assert match, name
|
||||
return match.group(1)
|
||||
|
||||
assert pin(run_sh, "PIER_VERSION") == pin(bootstrap, "PIER_VERSION")
|
||||
# The gateway and the optional mini-swe-agent use the versions eval/ is locked to.
|
||||
lock = (DEEPSWE_DIR.parent / "uv.lock").read_text()
|
||||
for package, name in (("litellm", "LITELLM_VERSION"), ("mini-swe-agent", "MINI_SWE_AGENT_VERSION")):
|
||||
assert f'name = "{package}"\nversion = "{pin(run_sh, name)}"' in lock, package
|
||||
assert re.fullmatch(r"\d+\.\d+\.\d+", pin(run_sh, "CLAUDE_CODE_VERSION"))
|
||||
assert pin(bootstrap, "NODE_VERSION") == prepare.NODE_VERSION
|
||||
assert re.fullmatch(r"[0-9a-f]{40}", pin(run_sh, "DEEP_SWE_COMMIT"))
|
||||
placeholders = set(re.findall(r"@([A-Z_]+)@", bootstrap))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue