diff --git a/.github/workflows/compat-matrix-image.yml b/.github/workflows/compat-matrix-image.yml index c554096cd7e..f08792c904c 100644 --- a/.github/workflows/compat-matrix-image.yml +++ b/.github/workflows/compat-matrix-image.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - tests/e2e/claude_code/cron_vm/** + - tests/e2e/claude_code/pr_gate_version_resolver.py - .github/workflows/compat-matrix-image.yml workflow_dispatch: @@ -28,6 +29,14 @@ jobs: - name: Build the Render cron image run: docker build -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix:${{ github.sha }} tests/e2e - - name: Run the pinned binaries as the cron user + - name: Resolve and install the Claude Code CLI as the cron user run: | - docker run --rm compat-matrix:${{ github.sha }} bash -c 'set -e; whoami; claude --version; gh --version; uv --version' + docker run --rm compat-matrix:${{ github.sha }} bash -c ' + set -euo pipefail + whoami + gh --version + uv --version + version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/claude_code/pr_gate_version_resolver.py)" + /opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli + /tmp/claude-cli/claude --version + ' diff --git a/tests/e2e/claude_code/cli_driver.py b/tests/e2e/claude_code/cli_driver.py index a01d8ab3e7c..6996849de80 100644 --- a/tests/e2e/claude_code/cli_driver.py +++ b/tests/e2e/claude_code/cli_driver.py @@ -297,6 +297,7 @@ def run_claude( } env["ANTHROPIC_BASE_URL"] = base_url env["ANTHROPIC_AUTH_TOKEN"] = api_key + env["DISABLE_AUTOUPDATER"] = "1" # Hand the CLI a fresh empty HOME so a compromised claude package # or a model-directed Read tool call can't see the runtime user's # real dotfiles. Created here, removed in the `finally` below diff --git a/tests/e2e/claude_code/cron_vm/Dockerfile b/tests/e2e/claude_code/cron_vm/Dockerfile index 623d6b1840a..05c1387c5a2 100644 --- a/tests/e2e/claude_code/cron_vm/Dockerfile +++ b/tests/e2e/claude_code/cron_vm/Dockerfile @@ -4,8 +4,6 @@ ARG GH_VERSION=2.101.0 ARG GH_SHA256=9bca2d1c16825f109907a23307628a2f0698fbf99662b73a5cf0b020293072b8 ARG UV_VERSION=0.10.9 ARG UV_SHA256=20d79708222611fa540b5c9ed84f352bcd3937740e51aacc0f8b15b271c57594 -ARG CLAUDE_CODE_VERSION=2.1.228 -ARG CLAUDE_CODE_SHA256=d535985e6941a3eb00179ccd7f52ceb0c6623a0305a518ebc4e6514f84a94c99 SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -23,11 +21,6 @@ RUN curl -fsSLo /tmp/uv.tar.gz "https://github.com/astral-sh/uv/releases/downloa && tar -xzf /tmp/uv.tar.gz -C /usr/local/bin --strip-components=1 uv-x86_64-unknown-linux-gnu/uv \ && rm /tmp/uv.tar.gz -RUN curl -fsSLo /tmp/claude "https://downloads.claude.ai/claude-code-releases/${CLAUDE_CODE_VERSION}/linux-x64/claude" \ - && echo "${CLAUDE_CODE_SHA256} /tmp/claude" | sha256sum -c - \ - && install -m 0755 /tmp/claude /usr/local/bin/claude \ - && rm /tmp/claude - RUN groupadd --gid 1000 populator && useradd --uid 1000 --gid 1000 --create-home populator ENV HOME=/home/populator \ diff --git a/tests/e2e/claude_code/cron_vm/README.md b/tests/e2e/claude_code/cron_vm/README.md index ed2bf4ab436..4c16b6e237a 100644 --- a/tests/e2e/claude_code/cron_vm/README.md +++ b/tests/e2e/claude_code/cron_vm/README.md @@ -16,16 +16,18 @@ than as a GitHub Action or on a dedicated VM. Trade-offs: clone of litellm plus a cold `uv sync`. That adds a few minutes on top of the ~10 minute test run; the job's 12 hour ceiling is nowhere near. -- ⚠️ The Claude Code CLI version under test is pinned in the - `Dockerfile` (`CLAUDE_CODE_VERSION` + its checksum). Bumping it is a - PR, see the gotchas below. +- ✅ The Claude Code CLI under test is chosen on every run (the newest + npm release published at least 3 days ago) and downloaded + checksum-verified, so the matrix follows CLI releases without a PR; + see the gotchas for pinning a run. ## Layout | File | Purpose | | --- | --- | -| `Dockerfile` | The image Render builds: Debian bookworm-slim plus pinned, checksum-verified `gh`, `uv`, and the Claude Code CLI, with this `tests/e2e/` tree copied to `/opt/litellm/tests/e2e/`. Runs as the non-root user `populator` (uid/gid 1000, which is what Render's secret files are readable by). | -| `run_daily.sh` | The actual cron job. Resolves versions, clones the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. | +| `Dockerfile` | The image Render builds: Debian bookworm-slim plus pinned, checksum-verified `gh` and `uv`, with this `tests/e2e/` tree copied to `/opt/litellm/tests/e2e/`. Runs as the non-root user `populator` (uid/gid 1000, which is what Render's secret files are readable by). | +| `run_daily.sh` | The actual cron job. Resolves versions, clones the worktree, installs the Claude Code CLI under test, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. | +| `install_claude_code.sh` | Downloads one Claude Code release (` `) from the vendor's native release channel, verifies it against the sha256 in that release's `manifest.json`, and refuses a binary whose `--version` disagrees. Run by the cron and by the `compat-matrix-image` GitHub workflow. | | `build_matrix.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. | | `check_regressions.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.find_regressions`. Diffs the freshly built matrix against the currently-published one and exits `3` if any cell flipped green→red, which gates auto-merge. | | `litellm-compat-matrix.env.example` | The service's env vars, one per line, with what each is for. | @@ -35,10 +37,7 @@ than as a GitHub Action or on a dedicated VM. Trade-offs: 1. **Resolves the latest LiteLLM final release tag** (newest bare `vX.Y.Z`, skipping `-rc.N`/`-dev.N` pre-releases) by paging the GitHub Releases API (`curl | jq`). -2. **Reads the Claude Code CLI version** via `claude --version`. That - is whatever the `Dockerfile` pins; the job never upgrades it on its - own. -3. **Clones the worktree** at `~/litellm-cron-worktree/` (a +2. **Clones the worktree** at `~/litellm-cron-worktree/` (a `--filter=blob:none` clone, so only the checked-out tag's blobs are fetched), `git checkout --force `, then `uv sync --frozen --no-install-project` against a uv-managed CPython 3.12 followed by @@ -52,12 +51,20 @@ than as a GitHub Action or on a dedicated VM. Trade-offs: `claude_code/` so the tree's EKS-harness `conftest.py` (whose imports the stable venv doesn't install) is never loaded. The tag's own `tests/e2e/` is deliberately not used. +3. **Resolves and installs the Claude Code CLI under test**: + `pr_gate_version_resolver.py` (run on the venv, the image has no + Python of its own) picks the newest `@anthropic-ai/claude-code` npm + release published at least 3 days ago, the same buffer the PR gate + uses, unless `CLAUDE_CODE_VERSION` pins one, and + `install_claude_code.sh` downloads that release's `linux-x64` binary + into the run's scratch dir, verified against the release manifest. 4. **Boots the proxy** as a `setsid` background process on port `4100` bound to loopback, then polls `/health/liveliness` until it's up. -5. **Runs pytest** on `tests/e2e/claude_code/` with `LITELLM_PROXY_URL` - pointed at the proxy and `COMPAT_RESULTS_PATH` set so the conftest - hook writes the per-test results artifact. Test failures become - `fail` cells in the JSON, not script errors. +5. **Runs pytest** on `tests/e2e/claude_code/` with that CLI first on + `PATH`, `LITELLM_PROXY_URL` pointed at the proxy, and + `COMPAT_RESULTS_PATH` set so the conftest hook writes the per-test + results artifact. Test failures become `fail` cells in the JSON, not + script errors. 6. **Builds `compatibility-matrix.json`** by handing the artifact + manifest to `build_matrix.py`. 7. **Opens or updates a docs PR**: `gh repo clone` of `litellm-docs` @@ -70,7 +77,8 @@ than as a GitHub Action or on a dedicated VM. Trade-offs: branch ... already exists" is treated as success). If the JSON is byte-identical to what `main` already publishes, the push is skipped entirely. These PRs are not gated on a second human review. -8. **Gates auto-merge on a regression check**: before enabling + + **Auto-merge is gated on a regression check**: before enabling auto-merge, `check_regressions.py` diffs the new matrix against the one currently on `main`. Auto-merge (`gh pr merge --auto --squash`) is only enabled when **no cell flipped green→red** — i.e. every @@ -82,7 +90,7 @@ than as a GitHub Action or on a dedicated VM. Trade-offs: auto-merge a prior same-day run enabled is explicitly disabled — so a human reviews before it lands on the public table. The check fails *closed*: if it errors, auto-merge is withheld. -9. **Sweeps stale compat-matrix PRs**: once today's PR exists, every +8. **Sweeps stale compat-matrix PRs**: once today's PR exists, every other open `compat-matrix/*` PR that the publishing account opened from a branch on the docs repo itself is closed (and its bot-owned branch deleted), so at most one compat-matrix PR is ever open — the @@ -165,7 +173,8 @@ curl -fsS -X POST "https://api.render.com/v1/services/${CRON_ID}/deploys" \ curl -fsS "https://api.render.com/v1/services/${CRON_ID}/deploys?limit=1" \ -H "Authorization: Bearer ${RENDER_API_KEY}" -# A run that does NOT open a PR (first-time validation, CLI bumps): +# A run that does NOT open a PR (first-time validation, a CLI pinned +# with CLAUDE_CODE_VERSION): # set SKIP_PUBLISH=1 on the service, trigger a run, then remove it. # The matrix JSON is printed at the end of the run's log (nothing on # the container's disk outlives the run) and saved to @@ -217,21 +226,25 @@ docker run --rm --platform linux/amd64 \ or fine-grained Contents:RW + Pull requests:RW). It is delivered as a file, not an env var, so pytest, the proxy, and the claude CLI never inherit it; manual runs export `GITHUB_TOKEN` instead. -- **Bumping the Claude Code CLI is a PR.** Change `CLAUDE_CODE_VERSION` - in the `Dockerfile` and set `CLAUDE_CODE_SHA256` to the `linux-x64` - checksum from - `https://downloads.claude.ai/claude-code-releases//manifest.json`. - The first run on a new CLI is the riskiest one: if the new CLI - changes its wire format the matrix run can produce systematic - failures, so trigger a `SKIP_PUBLISH=1` run before the next scheduled - fire. `gh` and `uv` bump the same way, with the checksum from the - release's `gh__checksums.txt` and the tarball's `.sha256` - sidecar respectively. +- **The Claude Code CLI is chosen per run, not pinned.** Each run + tests the newest `@anthropic-ai/claude-code` npm release published + at least 3 days ago, downloaded from + `https://downloads.claude.ai/claude-code-releases//linux-x64/claude` + and verified against the sha256 in that release's `manifest.json`. + A CLI release that breaks a cell shows up as a green→red flip, which + withholds auto-merge on that day's docs PR for review. To rerun the + matrix on one specific CLI, set `CLAUDE_CODE_VERSION` on the run. + `gh` and `uv` stay pinned in the `Dockerfile`; bump them in a PR with + the checksum from the release's `gh__checksums.txt` and the + tarball's `.sha256` sidecar respectively. - **A local build on Apple silicon only proves the image assembles.** Under QEMU the Claude Code binary (a Bun executable) dies with - `CPU lacks AVX support` and `gh` panics in the Go runtime, so - `claude --version` and a full run are verified with a - `SKIP_PUBLISH=1` run on Render, not locally. + `CPU lacks AVX support` and `gh` panics in the Go runtime, so the CLI + download and `claude --version` are verified by the + `compat-matrix-image` GitHub workflow (an x86 runner that builds the + image and runs `install_claude_code.sh` in it on every PR touching + this directory) and a full run with a `SKIP_PUBLISH=1` run on Render, + not locally. - **Nothing persists between runs.** A failed run leaves no half-installed venv behind, but also no cache: don't expect a rerun to be faster than the first one. diff --git a/tests/e2e/claude_code/cron_vm/install_claude_code.sh b/tests/e2e/claude_code/cron_vm/install_claude_code.sh new file mode 100755 index 00000000000..123401cc78a --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/install_claude_code.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +RELEASES_URL="https://downloads.claude.ai/claude-code-releases" + +log() { printf '==> %s\n' "$*" >&2; } +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +[[ $# -eq 2 ]] || die "usage: $(basename "$0") " +VERSION="$1" +DEST_DIR="$2" +[[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "not a Claude Code release version: '${VERSION}'" + +mkdir -p "${DEST_DIR}" +MANIFEST="${DEST_DIR}/manifest.json" +curl -fsSL --retry 3 --retry-all-errors --output "${MANIFEST}" "${RELEASES_URL}/${VERSION}/manifest.json" \ + || die "no release manifest for claude code ${VERSION} at ${RELEASES_URL}" +CHECKSUM="$(jq -r '.platforms["linux-x64"].checksum // empty' "${MANIFEST}")" +[[ "${CHECKSUM}" =~ ^[0-9a-f]{64}$ ]] || die "manifest for claude code ${VERSION} carries no linux-x64 sha256" + +log "downloading claude code ${VERSION} (linux-x64)" +DOWNLOAD="${DEST_DIR}/claude.download" +curl -fsSL --retry 3 --retry-all-errors --output "${DOWNLOAD}" "${RELEASES_URL}/${VERSION}/linux-x64/claude" +echo "${CHECKSUM} ${DOWNLOAD}" | sha256sum -c - >/dev/null \ + || die "claude code ${VERSION} sha256 mismatch; refusing to install" +chmod 0755 "${DOWNLOAD}" +mv "${DOWNLOAD}" "${DEST_DIR}/claude" + +PROBE_HOME="$(mktemp -d -t claude-probe-home.XXXXXX)" +trap 'rm -rf "${PROBE_HOME}"' EXIT +REPORTED="$( + env -i HOME="${PROBE_HOME}" PATH="${PATH}" DISABLE_AUTOUPDATER=1 \ + "${DEST_DIR}/claude" --version | awk '{print $1}' +)" || die "claude code ${VERSION} could not run --version" +[[ "${REPORTED}" == "${VERSION}" ]] \ + || die "installed claude code reports '${REPORTED}', expected ${VERSION}" +log "installed claude code ${VERSION} at ${DEST_DIR}/claude" diff --git a/tests/e2e/claude_code/cron_vm/run_daily.sh b/tests/e2e/claude_code/cron_vm/run_daily.sh index 172dd03b614..af01057eff7 100755 --- a/tests/e2e/claude_code/cron_vm/run_daily.sh +++ b/tests/e2e/claude_code/cron_vm/run_daily.sh @@ -7,15 +7,21 @@ # 1. Resolve the latest LiteLLM final release tag from the GitHub # Releases API. # 2. Update a long-lived worktree at $WORKTREE to that tag and `uv sync` it. -# 3. Boot the proxy as a background subprocess on $PROXY_PORT (default +# 3. Resolve the Claude Code CLI version under test (the newest npm +# release at least 3 days old, via pr_gate_version_resolver.py, or +# $CLAUDE_CODE_VERSION when set) and download that release's +# linux-x64 binary into the run's scratch dir, checksum-verified +# against the vendor's release manifest (install_claude_code.sh). +# 4. Boot the proxy as a background subprocess on $PROXY_PORT (default # 4100; a separate port from the human-tended :4000 proxy). -# 4. Run `pytest tests/e2e/claude_code/` against the proxy. Test -# failures become `fail` cells in the JSON, not script errors. -# 5. Hand the per-test results artifact + manifest to a small Python +# 5. Run `pytest tests/e2e/claude_code/` against the proxy with that +# CLI first on PATH. Test failures become `fail` cells in the JSON, +# not script errors. +# 6. Hand the per-test results artifact + manifest to a small Python # CLI (`build_matrix.py`) that wraps the existing # `matrix_builder.build_from_paths` to produce the published # compatibility-matrix.json. -# 6. `gh repo clone` litellm-docs, write the JSON to a deterministic +# 7. `gh repo clone` litellm-docs, write the JSON to a deterministic # branch (`compat-matrix/--`), commit, # push the branch straight to BerriAI/litellm-docs (mateo-berri has # write access), `gh pr create`, then — *only if no cell regressed @@ -23,7 +29,7 @@ # auto-merge so the PR merges itself once required checks pass. A # green→red regression leaves auto-merge off for human review; an # already-red cell (red→red) does not block. -# 7. Sweep stale compat-matrix PRs: once today's PR exists, close any +# 8. Sweep stale compat-matrix PRs: once today's PR exists, close any # other open `compat-matrix/*` PR (and delete its bot-owned branch) # so at most ONE compat-matrix PR is ever open — the newest. A # gate-withheld PR that nobody triages is superseded by the next @@ -33,7 +39,7 @@ # rather than spawning a new one. If the JSON is byte-identical to the # docs branch, we skip the push entirely. # -# Required commands on $PATH: git, uv, gh, jq, curl, claude. +# Required commands on $PATH: git, uv, gh, jq, curl. # Required state: a litellm checkout at $LITELLM_REPO (this file lives in # it); $WORKTREE is created on first run. # @@ -51,6 +57,9 @@ DOCS_BRANCH="${DOCS_BRANCH:-main}" DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}" SKIP_PUBLISH="${SKIP_PUBLISH:-0}" PYTEST_K="${PYTEST_K:-}" +# Empty means "resolve it": the newest @anthropic-ai/claude-code npm +# release published at least 3 days ago. Set it to pin a manual run. +CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-}" # The e2e suite uses PEP 695 `type` aliases, so the venv needs Python # >= 3.12 (also what repo CI runs) even when the host's system python is # older. uv fetches a managed CPython of this version on first use -- @@ -108,7 +117,7 @@ trap cleanup EXIT INT TERM log() { printf '==> %s\n' "$*" >&2; } die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } -for cmd in git uv gh jq curl claude; do +for cmd in git uv gh jq curl; do command -v "${cmd}" >/dev/null 2>&1 || die "missing required command: ${cmd}" done @@ -199,10 +208,6 @@ LITELLM_VERSION="$( [[ -n "${LITELLM_VERSION}" ]] || die "could not resolve latest PEP 440 final release (vX.Y.Z) in 5 pages of releases" log "resolved litellm: ${LITELLM_VERSION}" -CLAUDE_CODE_VERSION="$(claude --version 2>/dev/null | awk '{print $1}')" -[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "could not read 'claude --version'" -log "local claude code: ${CLAUDE_CODE_VERSION}" - # --------------------------------------------------------------------------- # 2. Update the worktree to that tag # --------------------------------------------------------------------------- @@ -314,7 +319,27 @@ PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml" [[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (shim incomplete?)" # --------------------------------------------------------------------------- -# 3. Boot the proxy +# 3. Resolve and install the Claude Code CLI under test +# --------------------------------------------------------------------------- + +# The resolver is stdlib-only, but the image ships no python of its +# own, so it runs on the venv the sync above just built. Its 3-day +# publish-age buffer (PRD #26476) keeps a release that gets pulled or +# patched within days from ever driving the published matrix. +if [[ -z "${CLAUDE_CODE_VERSION}" ]]; then + CLAUDE_CODE_VERSION="$( + cd "${WORKTREE}" \ + && "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/../pr_gate_version_resolver.py" + )" || die "could not resolve the Claude Code version to test" + log "resolved claude code: ${CLAUDE_CODE_VERSION}" +else + log "CLAUDE_CODE_VERSION set; testing claude code ${CLAUDE_CODE_VERSION}" +fi +CLAUDE_CLI_DIR="${WORKDIR}/claude-cli" +"${POPULATOR_DIR}/install_claude_code.sh" "${CLAUDE_CODE_VERSION}" "${CLAUDE_CLI_DIR}" + +# --------------------------------------------------------------------------- +# 4. Boot the proxy # --------------------------------------------------------------------------- log "starting proxy on 127.0.0.1:${PROXY_PORT}" @@ -350,7 +375,7 @@ curl -fsS "${HEALTH_URL}" >/dev/null \ || { tail -50 "${WORKDIR}/proxy.log" >&2; die "proxy did not become healthy"; } # --------------------------------------------------------------------------- -# 4. Run pytest +# 5. Run pytest # --------------------------------------------------------------------------- RESULTS_JSON="${WORKDIR}/compat-results.json" @@ -374,6 +399,7 @@ set +e && LITELLM_PROXY_URL="http://127.0.0.1:${PROXY_PORT}" \ LITELLM_MASTER_KEY="${PROXY_API_KEY}" \ COMPAT_RESULTS_PATH="${RESULTS_JSON}" \ + PATH="${CLAUDE_CLI_DIR}:${PATH}" \ "${WORKTREE_UV}" run --no-sync pytest "${PYTEST_ARGS[@]}" ) PYTEST_EXIT=$? @@ -386,7 +412,7 @@ log "pytest exit code: ${PYTEST_EXIT} (failures become 'fail' cells, not script [[ -f "${RESULTS_JSON}" ]] || die "pytest did not produce ${RESULTS_JSON}" # --------------------------------------------------------------------------- -# 5. Build the matrix JSON +# 6. Build the matrix JSON # --------------------------------------------------------------------------- MATRIX_JSON="${WORKDIR}/compatibility-matrix.json" @@ -402,7 +428,7 @@ log "building ${MATRIX_JSON}" ) # --------------------------------------------------------------------------- -# 6. Open a docs-repo PR +# 7. Open a docs-repo PR # --------------------------------------------------------------------------- if [[ "${SKIP_PUBLISH}" == "1" ]]; then @@ -634,7 +660,9 @@ else || die "auto-merge still armed on ${BRANCH_NAME} (enabled ${AUTOMERGE_ARMED}) after --disable-auto" fi -# --- Stale-PR sweep ---------------------------------------------------------- +# --------------------------------------------------------------------------- +# 8. Sweep stale compat-matrix PRs +# --------------------------------------------------------------------------- # Keep at most ONE compat-matrix PR open: today's. Any other open # `compat-matrix/*` PR is a leftover from a day whose regression gate # withheld auto-merge and nobody triaged it; the PR we just opened or