From 6add3c0be33d496ecf749c3feb34130fd4e565f4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 28 Apr 2026 23:34:50 +0000 Subject: [PATCH] compat-matrix: fixes from VM end-to-end validation Tier 1 (single anthropic cell) and tier 2 (full basic_messaging_non_streaming row across 5 providers) now run cleanly from `run_daily.sh` on the GCP VM. Five issues showed up during validation; each is fixed in this commit. 1. uv version pin ---------------- The litellm worktree pins an exact uv version in `pyproject.toml`'s `[tool.uv] required-version` field. The cron VM's system uv (currently 0.11.8) refused to sync against the v1.83.10-stable lockfile (which pins ==0.10.9). Fix: parse the pinned version out of the worktree's pyproject.toml, download the matching standalone binary into `/.uv-bin/uv-`, and use it for sync/run/proxy. Cached across runs. 2. Missing proxy + dev extras --------------------------- `uv sync --frozen` only installed the base dependency set, so `uv run litellm` died at startup with `ModuleNotFoundError: No module named 'websockets'`. Per the repo's AGENTS.md the canonical incantation is `uv sync --frozen --group proxy-dev --extra proxy`. 3. Wrong env vars for the test driver ----------------------------------- The script was setting `ANTHROPIC_BASE_URL` and `ANTHROPIC_AUTH_TOKEN`, which is what Claude Code itself reads, but the test files read `LITELLM_PROXY_BASE_URL` and `LITELLM_PROXY_API_KEY` (search the test_config-driven test files for `PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"`). Tests were marking themselves `fail` with "missing required env: set LITELLM_PROXY_BASE_URL...". Fix: rename the two env vars in the pytest invocation. The driver still propagates them onward as ANTHROPIC_BASE_URL/AUTH_TOKEN to Claude. 4. Cleanup couldn't find the proxy -------------------------------- The previous setup did `( ... && setsid uv run litellm ... ) &; PROXY_PID=$!`. `setsid` detaches the inner uv into its own session, but `$!` records the PID of the outer subshell, not the long-lived python proxy. So `kill -TERM "-${PROXY_PID}"` in the EXIT trap targeted the wrong pgid and the proxy survived as an orphan whenever the script was killed externally. Fix: replace the subshell with `setsid bash -c '...'` that writes $$ to a known pid file before exec'ing the proxy. The cleanup trap reads that file and uses it as the pgid. Belt-and-braces: the trap also `pgrep -f`s by port number and SIGKILLs survivors. Trap now fires on `INT TERM` too, not just `EXIT`. 5. .uv-bin cache survives git clean --------------------------------- The original `git clean -fdx -e .venv` wiped `.uv-bin/` between runs, forcing re-download of the pinned uv binary on every invocation. Now excluded. Things that worked first try ---------------------------- * Worktree clone + checkout to the resolved tag. * gh auth on this VM (mateo-berri account, collaborator on BerriAI/litellm-docs). * The matrix builder produced a well-formed `compatibility-matrix.json` with the right per-cell aggregation even when 4 of 5 cells failed (tier 2 was: anthropic=pass, bedrock_invoke=fail, bedrock_converse=fail, vertex_ai=fail with a real 403 from GCP for insufficient scopes, azure=fail with timeout). --- tests/claude_code/cron_vm/run_daily.sh | 105 +++++++++++++++++++------ 1 file changed, 79 insertions(+), 26 deletions(-) diff --git a/tests/claude_code/cron_vm/run_daily.sh b/tests/claude_code/cron_vm/run_daily.sh index 3b53507ae7f..cf0f3a21ccc 100755 --- a/tests/claude_code/cron_vm/run_daily.sh +++ b/tests/claude_code/cron_vm/run_daily.sh @@ -43,25 +43,44 @@ PYTEST_K="${PYTEST_K:-}" POPULATOR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" WORKDIR="$(mktemp -d -t litellm-compat-matrix.XXXXXX)" -PROXY_PID="" +PROXY_PID_FILE="${WORKDIR}/proxy.pid" +# Cleanup is intentionally aggressive: it can run on normal exit, on a +# signal received by the script, or after a partial failure where the +# proxy is up but ${PROXY_PID_FILE} is stale. We try four things in +# order and stop as soon as the proxy port is free: +# +# 1. SIGTERM the pid recorded in proxy.pid. +# 2. SIGKILL anything from `pgrep -f "litellm.*--port ${PROXY_PORT}"` +# that survived. This catches the common case where the recorded +# pid was the sh wrapper, not the long-lived python child. +# 3. ss -K on the port (kernel kills sockets but not processes; +# mostly useful for catching lingering CLOSE_WAITs). +# 4. wipe ${WORKDIR}. cleanup() { local rc=$? set +e - if [[ -n "${PROXY_PID}" ]] && kill -0 "${PROXY_PID}" 2>/dev/null; then - # Negative pid = process group; the proxy spawns workers that don't - # forward signals from a parent. - kill -TERM "-${PROXY_PID}" 2>/dev/null || true - for _ in 1 2 3 4 5; do - kill -0 "${PROXY_PID}" 2>/dev/null || break - sleep 1 - done - kill -KILL "-${PROXY_PID}" 2>/dev/null || true + local proxy_pid + if [[ -f "${PROXY_PID_FILE}" ]]; then + proxy_pid="$(cat "${PROXY_PID_FILE}")" + if [[ -n "${proxy_pid}" ]]; then + kill -TERM "-${proxy_pid}" 2>/dev/null || kill -TERM "${proxy_pid}" 2>/dev/null || true + for _ in 1 2 3 4 5; do + kill -0 "${proxy_pid}" 2>/dev/null || break + sleep 1 + done + fi fi + # Belt-and-braces: any python or uv talking to ${PROXY_PORT} that + # survived the SIGTERM gets SIGKILL'd by name. + pgrep -f "litellm.*--port[ =]?${PROXY_PORT}\b" 2>/dev/null \ + | xargs -r kill -KILL 2>/dev/null || true + pgrep -f "${WORKTREE}/.uv-bin/uv.*run litellm" 2>/dev/null \ + | xargs -r kill -KILL 2>/dev/null || true rm -rf "${WORKDIR}" exit "${rc}" } -trap cleanup EXIT +trap cleanup EXIT INT TERM log() { printf '==> %s\n' "$*" >&2; } die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } @@ -117,13 +136,42 @@ fi log "updating worktree to ${LITELLM_VERSION}" git -C "${WORKTREE}" fetch --tags --force git -C "${WORKTREE}" reset --hard -# Keep the venv around — uv sync will reconcile it. Drop everything else -# (compat-results.json, __pycache__, etc.) so each run starts clean. -git -C "${WORKTREE}" clean -fdx -e .venv +# Keep the venv and the .uv-bin cache around — uv sync will reconcile +# the venv and we don't want to re-download the pinned uv binary on +# every run. Drop everything else (compat-results.json, __pycache__, +# etc.) so each run starts clean. +git -C "${WORKTREE}" clean -fdx -e .venv -e .uv-bin git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}" -log "uv sync --frozen" -(cd "${WORKTREE}" && uv sync --frozen) +# litellm pins an exact uv version in pyproject.toml's [tool.uv] +# `required-version` field, so a system uv that's newer or older +# refuses to sync. We pin our own local copy at the version the +# checked-out tag asks for, cached under .uv-bin/ inside the worktree +# so subsequent runs skip the download. +PINNED_UV_VERSION="$( + awk -F'"' '/^required-version[[:space:]]*=/{ for (i=2; i<=NF; i++) if ($i ~ /^[=<>!~]+/) { gsub(/^[=<>!~]+/, "", $i); print $i; exit } }' \ + "${WORKTREE}/pyproject.toml" +)" +if [[ -z "${PINNED_UV_VERSION}" ]]; then + log "no uv version pin in pyproject.toml; using system uv" + WORKTREE_UV="$(command -v uv)" +else + WORKTREE_UV="${WORKTREE}/.uv-bin/uv-${PINNED_UV_VERSION}" + if [[ ! -x "${WORKTREE_UV}" ]]; then + log "downloading uv ${PINNED_UV_VERSION} for the worktree" + mkdir -p "${WORKTREE}/.uv-bin" + curl -fsSL \ + "https://github.com/astral-sh/uv/releases/download/${PINNED_UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \ + | tar -xzO "uv-x86_64-unknown-linux-gnu/uv" >"${WORKTREE_UV}.tmp" + chmod +x "${WORKTREE_UV}.tmp" + mv "${WORKTREE_UV}.tmp" "${WORKTREE_UV}" + fi +fi +# `--extra proxy` pulls fastapi/uvicorn/etc. so `uv run litellm` can +# actually serve. `--group proxy-dev` brings in pytest and the rest of +# what tests/claude_code/ needs. +log "uv sync --frozen --group proxy-dev --extra proxy (uv ${PINNED_UV_VERSION:-system})" +(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy) PROXY_CONFIG="${WORKTREE}/tests/claude_code/test_config.yaml" [[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (does ${LITELLM_VERSION} predate the compat matrix work?)" @@ -133,12 +181,17 @@ PROXY_CONFIG="${WORKTREE}/tests/claude_code/test_config.yaml" # --------------------------------------------------------------------------- log "starting proxy on :${PROXY_PORT}" -( - cd "${WORKTREE}" \ - && setsid uv run litellm --config "${PROXY_CONFIG}" --port "${PROXY_PORT}" \ - >"${WORKDIR}/proxy.log" 2>&1 -) & -PROXY_PID=$! +# `setsid` puts the proxy in its own session+pgroup so cleanup() can +# SIGTERM the whole tree by passing the pgid as a negative pid. We +# write that pid to a file so cleanup() doesn't need to remember a +# variable that might be stale by the time the trap fires. +setsid bash -c ' + echo "$$" > "$0" + cd "$1" + exec "$2" run litellm --config "$3" --port "$4" +' "${PROXY_PID_FILE}" "${WORKTREE}" "${WORKTREE_UV}" "${PROXY_CONFIG}" "${PROXY_PORT}" \ + >"${WORKDIR}/proxy.log" 2>&1 & +disown HEALTH_URL="http://127.0.0.1:${PROXY_PORT}/health/liveliness" for _ in $(seq 1 45); do @@ -171,10 +224,10 @@ log "running pytest" set +e ( cd "${WORKTREE}" \ - && ANTHROPIC_BASE_URL="http://127.0.0.1:${PROXY_PORT}" \ - ANTHROPIC_AUTH_TOKEN="${PROXY_API_KEY}" \ + && LITELLM_PROXY_BASE_URL="http://127.0.0.1:${PROXY_PORT}" \ + LITELLM_PROXY_API_KEY="${PROXY_API_KEY}" \ COMPAT_RESULTS_PATH="${RESULTS_JSON}" \ - uv run pytest "${PYTEST_ARGS[@]}" + "${WORKTREE_UV}" run pytest "${PYTEST_ARGS[@]}" ) PYTEST_EXIT=$? set -e @@ -189,7 +242,7 @@ MATRIX_JSON="${WORKDIR}/compatibility-matrix.json" log "building ${MATRIX_JSON}" ( cd "${WORKTREE}" \ - && uv run python "${POPULATOR_DIR}/build_matrix.py" \ + && "${WORKTREE_UV}" run python "${POPULATOR_DIR}/build_matrix.py" \ --manifest "${WORKTREE}/tests/claude_code/manifest.yaml" \ --results "${RESULTS_JSON}" \ --output "${MATRIX_JSON}" \