diff --git a/.github/workflows/compat-matrix-image.yml b/.github/workflows/compat-matrix-image.yml new file mode 100644 index 00000000000..c554096cd7e --- /dev/null +++ b/.github/workflows/compat-matrix-image.yml @@ -0,0 +1,33 @@ +name: Compat Matrix Image + +on: + pull_request: + paths: + - tests/e2e/claude_code/cron_vm/** + - .github/workflows/compat-matrix-image.yml + workflow_dispatch: + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + compat-matrix-image: + name: compat-matrix-image + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Build the Render cron image + run: docker build -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix:${{ github.sha }} tests/e2e + + - name: Run the pinned binaries as the cron user + run: | + docker run --rm compat-matrix:${{ github.sha }} bash -c 'set -e; whoami; claude --version; gh --version; uv --version' diff --git a/tests/e2e/claude_code/cron_vm/Dockerfile b/tests/e2e/claude_code/cron_vm/Dockerfile new file mode 100644 index 00000000000..623d6b1840a --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/Dockerfile @@ -0,0 +1,41 @@ +FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251 + +ARG GH_VERSION=2.101.0 +ARG GH_SHA256=9bca2d1c16825f109907a23307628a2f0698fbf99662b73a5cf0b020293072b8 +ARG UV_VERSION=0.10.9 +ARG UV_SHA256=20d79708222611fa540b5c9ed84f352bcd3937740e51aacc0f8b15b271c57594 +ARG CLAUDE_CODE_VERSION=2.1.228 +ARG CLAUDE_CODE_SHA256=d535985e6941a3eb00179ccd7f52ceb0c6623a0305a518ebc4e6514f84a94c99 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git jq procps iproute2 \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -fsSLo /tmp/gh.tar.gz "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" \ + && echo "${GH_SHA256} /tmp/gh.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/gh.tar.gz -C /usr/local/bin --strip-components=2 "gh_${GH_VERSION}_linux_amd64/bin/gh" \ + && rm /tmp/gh.tar.gz + +RUN curl -fsSLo /tmp/uv.tar.gz "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \ + && echo "${UV_SHA256} /tmp/uv.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/uv.tar.gz -C /usr/local/bin --strip-components=1 uv-x86_64-unknown-linux-gnu/uv \ + && rm /tmp/uv.tar.gz + +RUN curl -fsSLo /tmp/claude "https://downloads.claude.ai/claude-code-releases/${CLAUDE_CODE_VERSION}/linux-x64/claude" \ + && echo "${CLAUDE_CODE_SHA256} /tmp/claude" | sha256sum -c - \ + && install -m 0755 /tmp/claude /usr/local/bin/claude \ + && rm /tmp/claude + +RUN groupadd --gid 1000 populator && useradd --uid 1000 --gid 1000 --create-home populator + +ENV HOME=/home/populator \ + LITELLM_REPO=/opt/litellm \ + DISABLE_AUTOUPDATER=1 + +COPY --chown=populator:populator . /opt/litellm/tests/e2e/ + +USER populator +WORKDIR /home/populator +CMD ["/opt/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh"] diff --git a/tests/e2e/claude_code/cron_vm/README.md b/tests/e2e/claude_code/cron_vm/README.md index f120c30605b..ed2bf4ab436 100644 --- a/tests/e2e/claude_code/cron_vm/README.md +++ b/tests/e2e/claude_code/cron_vm/README.md @@ -1,66 +1,59 @@ -# Cron VM setup for the Claude Code compatibility-matrix populator +# Render cron job for the Claude Code compatibility-matrix populator -The populator runs daily on a dedicated GCP VM -(`litellm-compatibility-matrix-populator`) rather than as a GitHub -Action. Trade-offs: +The populator runs daily as the Render cron job `litellm-compat-matrix` +(Docker runtime, built from the `Dockerfile` in this directory) rather +than as a GitHub Action or on a dedicated VM. Trade-offs: -- ✅ Real VM means we can `gh auth login` against an account that's - already a collaborator on `BerriAI/litellm-docs`, instead of - provisioning a GitHub App with `pull-requests: write`. -- ✅ Persistent state (a single `~/litellm-cron-worktree/` and its `.venv`) - is reused across runs, so each daily run does a fast `git checkout` + - incremental `uv sync` rather than a fresh clone + cold sync. -- ✅ No Docker dependency — the proxy runs directly via `uv run litellm`. -- ⚠️ The VM has to actually be on. systemd's `Persistent=true` recovers - from short outages, but a multi-day outage means the matrix goes - stale until the VM is back. -- ⚠️ Provider credentials live on the VM filesystem - (`/etc/litellm-compat-matrix.env`) instead of GitHub secrets. Treat - the VM as an environment with comparable blast radius to a CI runner. - -This directory used to live at `tests/claude_code/cron_vm/` (paired with -the standalone `tests/claude_code/` suite); it now runs the maintained -`tests/e2e/claude_code/` suite instead. The pytest env interface changed -accordingly: the runner exports `LITELLM_PROXY_URL` / `LITELLM_MASTER_KEY` -(previously `LITELLM_PROXY_BASE_URL` / `LITELLM_PROXY_API_KEY`), the azure -column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously -`AZURE_FOUNDRY_*`), and the GPT columns need `OPENAI_API_KEY` and -`AZURE_API_BASE` / `AZURE_API_KEY` — see `litellm-compat-matrix.env.example`. +- ✅ No machine to keep on or patch. Render builds the image from this + directory on every push to `main` that touches `tests/e2e/**` and + runs it on the schedule. +- ✅ Credentials live in Render env vars and secret files, scoped to + this one service, instead of on a VM filesystem. +- ✅ The publish token still uses the `mateo-berri` account, which is a + collaborator on `BerriAI/litellm-docs`, so no GitHub App with + `pull-requests: write` has to be provisioned. +- ⚠️ The disk is ephemeral, so every run starts from a fresh (blobless) + clone of litellm plus a cold `uv sync`. That adds a few minutes on + top of the ~10 minute test run; the job's 12 hour ceiling is nowhere + near. +- ⚠️ The Claude Code CLI version under test is pinned in the + `Dockerfile` (`CLAUDE_CODE_VERSION` + its checksum). Bumping it is a + PR, see the gotchas below. ## Layout | File | Purpose | | --- | --- | -| `run_daily.sh` | The actual cron job. Resolves versions, updates the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. | +| `Dockerfile` | The image Render builds: Debian bookworm-slim plus pinned, checksum-verified `gh`, `uv`, and the Claude Code CLI, with this `tests/e2e/` tree copied to `/opt/litellm/tests/e2e/`. Runs as the non-root user `populator` (uid/gid 1000, which is what Render's secret files are readable by). | +| `run_daily.sh` | The actual cron job. Resolves versions, clones the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. | | `build_matrix.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. | | `check_regressions.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.find_regressions`. Diffs the freshly built matrix against the currently-published one and exits `3` if any cell flipped green→red, which gates auto-merge. | -| `litellm-compat-matrix.service` | systemd oneshot that invokes `run_daily.sh`. | -| `litellm-compat-matrix.timer` | `OnCalendar=*-*-* 06:00:00 UTC`, `Persistent=true`. | -| `litellm-compat-matrix.env.example` | Template for `/etc/litellm-compat-matrix.env`. | +| `litellm-compat-matrix.env.example` | The service's env vars, one per line, with what each is for. | ## What `run_daily.sh` does 1. **Resolves the latest LiteLLM final release tag** (newest bare `vX.Y.Z`, skipping `-rc.N`/`-dev.N` pre-releases) by paging the GitHub Releases API (`curl | jq`). -2. **Reads the local Claude Code CLI version** via `claude --version`. - The cron does not auto-upgrade the CLI — operators do that - out-of-band by running `npm install -g @anthropic-ai/claude-code@latest`. -3. **Updates the persistent worktree** at `~/litellm-cron-worktree/`: - `git fetch --tags --force`, `git reset --hard`, - `git clean -fdx -e .venv -e .uv-bin`, `git checkout --force `. - The `.venv` is preserved across runs so `uv sync --frozen` is - incremental. Then **shims the test suite**: `tests/e2e/` in the - worktree is rebuilt from the dev checkout — the `claude_code/` suite - plus the five shared transport helpers it imports (`proxy_client.py`, - `e2e_http.py`, `models.py`, `e2e_config.py`, `transport.py`) — so the - cron always runs *today's* tests against the latest stable proxy. The - tag's own `tests/e2e/` tree (including the EKS-harness `conftest.py`, - whose imports the stable venv doesn't install) is deliberately not - used. +2. **Reads the Claude Code CLI version** via `claude --version`. That + is whatever the `Dockerfile` pins; the job never upgrades it on its + own. +3. **Clones the worktree** at `~/litellm-cron-worktree/` (a + `--filter=blob:none` clone, so only the checked-out tag's blobs are + fetched), `git checkout --force `, then `uv sync --frozen + --no-install-project` against a uv-managed CPython 3.12 followed by + `uv pip install --no-build litellm==`, so the proxy under + test is the published PyPI wheel (what users install) rather than a + source build: the tag builds a Rust extension through maturin, and + the image ships no C or Rust toolchain. Then **shims the test suite**: + `tests/e2e/` in the worktree is replaced by the image's copy of this + whole tree, so the cron always runs *today's* tests against the + latest stable proxy, and pytest runs with `--confcutdir` pointed at + `claude_code/` so the tree's EKS-harness `conftest.py` (whose imports + the stable venv doesn't install) is never loaded. The tag's own + `tests/e2e/` is deliberately not used. 4. **Boots the proxy** as a `setsid` background process on port `4100` - (so it can't collide with a developer's `:4000`), then polls - `/health/liveliness` until it's up. + bound to loopback, then polls `/health/liveliness` until it's up. 5. **Runs pytest** on `tests/e2e/claude_code/` with `LITELLM_PROXY_URL` pointed at the proxy and `COMPAT_RESULTS_PATH` set so the conftest hook writes the per-test results artifact. Test failures become @@ -74,8 +67,9 @@ column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously `mateo-berri` token has write access, so this is a same-repo branch, not a fork), `gh pr create`. A re-run on the same day fast-forwards the existing branch and `gh pr create` no-ops ("a pull request for - branch ... already exists" is treated as success). These PRs are no - longer gated on a second human review. + branch ... already exists" is treated as success). If the JSON is + byte-identical to what `main` already publishes, the push is skipped + entirely. These PRs are not gated on a second human review. 8. **Gates auto-merge on a regression check**: before enabling auto-merge, `check_regressions.py` diffs the new matrix against the one currently on `main`. Auto-merge (`gh pr merge --auto --squash`) @@ -89,107 +83,155 @@ column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously human reviews before it lands on the public table. The check fails *closed*: if it errors, auto-merge is withheld. 9. **Sweeps stale compat-matrix PRs**: once today's PR exists, every - other open `compat-matrix/*` PR on the docs repo is closed (and its - bot-owned branch deleted), so at most one compat-matrix PR is ever - open — the newest. + other open `compat-matrix/*` PR that the publishing account opened + from a branch on the docs repo itself is closed (and its bot-owned + branch deleted), so at most one compat-matrix PR is ever open — the + newest. A contributor's PR under that prefix is never touched. -## One-time VM setup +## The Render service -Run as `mateo` on the cron VM: +Everything below is what the live service is set to; recreate it with +the same values if it ever has to be rebuilt. + +| Setting | Value | +| --- | --- | +| Workspace | Litellm (the one that already builds the other litellm services) | +| Type | Cron job, Docker runtime | +| Repo / branch | `BerriAI/litellm` @ `main` | +| Dockerfile path | `tests/e2e/claude_code/cron_vm/Dockerfile` | +| Docker build context | `tests/e2e` (the repo root `.dockerignore` excludes `tests`, so the context has to start below it) | +| Build filter | included paths `tests/e2e/**` | +| Schedule | `0 6 * * *` (06:00 UTC daily) | +| Plan / region | `4c-16g` (4 CPU, 16 GB, what the dashboard calls Pro Max; the suite fans out to ~75 concurrent CLI calls) / Oregon | +| Env vars | every key in `litellm-compat-matrix.env.example` | +| Secret files | `github-token` (the publish PAT, one line) and `vertex-service-account.json` (the Vertex service-account key) | + +Render mounts secret files at `/etc/secrets/`, which is where +`CREDENTIALS_DIRECTORY` and `GOOGLE_APPLICATION_CREDENTIALS` in the env +example point. Render also passes env vars to `docker build` as build +args, which is why the `Dockerfile` declares no `ARG` that could ever +be given a secret's name. + +Creating it through the API looks like this (fill `envVars` and +`secretFiles` from the env example and the two secrets; `ownerId` is +the workspace id from `GET /v1/owners`): ```bash -# 1. Toolchain -sudo apt-get update -sudo apt-get install -y git nodejs npm jq curl -curl -LsSf https://astral.sh/uv/install.sh | sh -sudo apt-get install -y gh # or follow https://cli.github.com/ - -# 2. Claude Code CLI (the cron does NOT auto-upgrade this; rerun this -# line out-of-band when you want a fresh CLI to be tested) -sudo npm install -g @anthropic-ai/claude-code@latest - -# 3. Litellm checkout. Used by systemd's WorkingDirectory and as the -# source of the .service / .timer files. The cron itself runs out -# of the separate worktree at ~/litellm-cron-worktree/. -mkdir -p ~/litellm -git clone https://github.com/BerriAI/litellm.git ~/litellm/litellm -git -C ~/litellm/litellm checkout litellm_internal_staging - -# 4. gh auth — must be a collaborator on BerriAI/litellm-docs. -gh auth login # follow prompts; pick HTTPS + token paste flow - -# 5. Provider credentials + the publish token. -sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example \ - /etc/litellm-compat-matrix.env -sudoedit /etc/litellm-compat-matrix.env # fill in real values -sudo chmod 0600 /etc/litellm-compat-matrix.env -# The mateo-berri PAT lives in its own file, mapped into the service via -# systemd LoadCredential so it stays out of the test processes' env -# (see the env.example comment for why). -sudo install -m 0600 /dev/null /etc/litellm-compat-matrix-github-token -sudoedit /etc/litellm-compat-matrix-github-token # single line: the PAT - -# 6. systemd units. -sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ -sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ -sudo systemctl daemon-reload -sudo systemctl enable --now litellm-compat-matrix.timer +curl -fsS https://api.render.com/v1/services \ + -H "Authorization: Bearer ${RENDER_API_KEY}" \ + -H 'Content-Type: application/json' \ + -d '{ + "type": "cron_job", + "name": "litellm-compat-matrix", + "ownerId": "", + "repo": "https://github.com/BerriAI/litellm", + "branch": "main", + "autoDeploy": "yes", + "buildFilter": {"paths": ["tests/e2e/**"], "ignoredPaths": []}, + "envVars": [{"key": "ANTHROPIC_API_KEY", "value": "..."}], + "secretFiles": [{"name": "github-token", "content": "..."}, + {"name": "vertex-service-account.json", "content": "..."}], + "serviceDetails": { + "runtime": "docker", + "schedule": "0 6 * * *", + "plan": "4c-16g", + "region": "oregon", + "envSpecificDetails": { + "dockerfilePath": "tests/e2e/claude_code/cron_vm/Dockerfile", + "dockerContext": "tests/e2e" + } + } + }' ``` ## Operating it ```bash -# When does it run next? -systemctl list-timers litellm-compat-matrix.timer +# Trigger a real run right now (PRs to litellm-docs). The id is the +# service id (`crn-...`) from the dashboard URL or `GET /v1/services`. +curl -fsS -X POST "https://api.render.com/v1/cron-jobs/${CRON_ID}/runs" \ + -H "Authorization: Bearer ${RENDER_API_KEY}" -# Trigger a real run right now (PRs to litellm-docs). -sudo systemctl start litellm-compat-matrix.service +# Follow a run: the Logs tab on the service, or the API. +curl -fsS "https://api.render.com/v1/logs?ownerId=${OWNER_ID}&resource=${CRON_ID}&limit=100" \ + -H "Authorization: Bearer ${RENDER_API_KEY}" -# Trigger a run that does NOT open a PR (good for first-time validation). -SKIP_PUBLISH=1 ~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh +# Rebuild the image after a merge that touches tests/e2e/** (see the +# auto-deploy gotcha below). The deploy is done once its status is +# `live`; a run triggered before that still uses the previous image. +curl -fsS -X POST "https://api.render.com/v1/services/${CRON_ID}/deploys" \ + -H "Authorization: Bearer ${RENDER_API_KEY}" \ + -H 'Content-Type: application/json' -d '{"clearCache": "do_not_clear"}' +curl -fsS "https://api.render.com/v1/services/${CRON_ID}/deploys?limit=1" \ + -H "Authorization: Bearer ${RENDER_API_KEY}" -# Narrow to one cell while debugging. -SKIP_PUBLISH=1 PYTEST_K='basic_messaging_non_streaming and anthropic' \ - ~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh +# A run that does NOT open a PR (first-time validation, CLI bumps): +# set SKIP_PUBLISH=1 on the service, trigger a run, then remove it. +# The matrix JSON is printed at the end of the run's log (nothing on +# the container's disk outlives the run) and saved to +# ~/compatibility-matrix.json for a local docker run. +# PYTEST_K='basic_messaging_non_streaming and anthropic' narrows the +# run to one cell the same way. -# Watch the most recent run. -journalctl -u litellm-compat-matrix.service -f - -# Read older runs. -journalctl -u litellm-compat-matrix.service --since '2 days ago' - -# Disable until further notice (e.g. while debugging). -sudo systemctl disable --now litellm-compat-matrix.timer +# Build and run the image locally (docker on Apple silicon needs the +# platform flag; the context is tests/e2e, see the table above). +docker build --platform linux/amd64 \ + -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix tests/e2e +docker run --rm --platform linux/amd64 \ + --env-file litellm-compat-matrix.env -e SKIP_PUBLISH=1 \ + -v "$PWD/secrets:/etc/secrets:ro" compat-matrix ``` ## Gotchas - **The venv is pinned to Python 3.12 (`CRON_PYTHON_VERSION`).** The - e2e suite uses PEP 695 `type` aliases, which the VM's system Python - (3.11) can't parse; `run_daily.sh` has uv fetch a managed CPython + e2e suite uses PEP 695 `type` aliases, which the image's Debian + Python can't parse; `run_daily.sh` has uv fetch a managed CPython into `~/litellm-cron-worktree/.uv-python/` and syncs the venv against - it. The first run after a version bump is a cold venv rebuild. -- **The proxy port is `4100`, not `4000`.** This is so a developer SSH'd - into the same VM with their own `:4000` proxy doesn't collide with a - cron run. Override with `PROXY_PORT=...` in `/etc/litellm-compat-matrix.env` - if you need to. + it. +- **The proxy port is `4100`, not `4000`.** Kept from the VM days so a + developer running the script locally next to their own `:4000` proxy + doesn't collide. Override with `PROXY_PORT=...`. - **`uv sync --frozen` requires the resolved tag to be tagged on - GitHub.** If the latest stable release was made but not pushed as a - git tag, the `git checkout` step fails. Push the tag, then rerun. + GitHub, and the wheel install requires it on PyPI.** If the latest + stable release was made but not pushed as a git tag, the `git + checkout` step fails; push the tag, then rerun. PyPI has had every + stable version days before its GitHub release so far (1.102.0 was + uploaded 2026-09-20, released on GitHub 2026-09-22), so the + `--no-build` install failing means the wheel is genuinely missing, + not late. +- **Pushes do not redeploy the service; deploy by hand.** `autoDeploy` + is `yes` on the service, but Render only hears about pushes through + its GitHub app, which is not installed on the `BerriAI` org (an org + admin step), so no push to the branch has ever started a deploy. + After a merge that changes anything under `tests/e2e/**`, run the + deploy command from the operating section (or "Manual Deploy" on the + dashboard) and wait for `live` before triggering a run, otherwise + the next scheduled run still executes the old image. - **Publish-token rotation is your problem.** The cron does not - refresh the token; if `mateo-berri`'s PAT in - `/etc/litellm-compat-matrix-github-token` expires, the run fails at - the `git push`/`gh pr create` step with a 401 ("Bad credentials" / - "Authentication failed"). Mint a fresh PAT and update that file. - The token needs write access to `BerriAI/litellm-docs` (classic - `repo` scope, or fine-grained Contents:RW + Pull requests:RW). It is - delivered via systemd `LoadCredential`, not the env file, so pytest, - the proxy, and the claude CLI never inherit it; manual runs export - `GITHUB_TOKEN` instead. -- **First run after upgrading the Claude Code CLI is the riskiest one.** - If the new CLI changes its wire format the matrix run can produce - systematic failures. Always run with `SKIP_PUBLISH=1` after a CLI - upgrade before letting the next scheduled fire happen. -- **Disk:** the worktree's `.venv` is ~1.3 GB and the `.git` directory - is ~1 GB. Plan for at least 5 GB free on the VM, otherwise - `uv sync` will fail mid-run and leave you with a half-installed venv. + refresh the token; if `mateo-berri`'s PAT in the `github-token` + secret file expires, the run fails at the `git push`/`gh pr create` + step with a 401 ("Bad credentials" / "Authentication failed"). Mint + a fresh PAT and replace the secret file on the service. The token + needs write access to `BerriAI/litellm-docs` (classic `repo` scope, + or fine-grained Contents:RW + Pull requests:RW). It is delivered as + a file, not an env var, so pytest, the proxy, and the claude CLI + never inherit it; manual runs export `GITHUB_TOKEN` instead. +- **Bumping the Claude Code CLI is a PR.** Change `CLAUDE_CODE_VERSION` + in the `Dockerfile` and set `CLAUDE_CODE_SHA256` to the `linux-x64` + checksum from + `https://downloads.claude.ai/claude-code-releases//manifest.json`. + The first run on a new CLI is the riskiest one: if the new CLI + changes its wire format the matrix run can produce systematic + failures, so trigger a `SKIP_PUBLISH=1` run before the next scheduled + fire. `gh` and `uv` bump the same way, with the checksum from the + release's `gh__checksums.txt` and the tarball's `.sha256` + sidecar respectively. +- **A local build on Apple silicon only proves the image assembles.** + Under QEMU the Claude Code binary (a Bun executable) dies with + `CPU lacks AVX support` and `gh` panics in the Go runtime, so + `claude --version` and a full run are verified with a + `SKIP_PUBLISH=1` run on Render, not locally. +- **Nothing persists between runs.** A failed run leaves no + half-installed venv behind, but also no cache: don't expect a rerun + to be faster than the first one. diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example index d15561e96cd..579752f1ea8 100644 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example @@ -1,9 +1,8 @@ -# Environment file consumed by `litellm-compat-matrix.service`. +# Environment variables of the Render cron job `litellm-compat-matrix`. # -# Install at `/etc/litellm-compat-matrix.env` and chmod 0600. -# `EnvironmentFile=-` in the unit means the service is allowed to start -# even if this file is missing, but the populator will fail at the -# first provider request without these credentials. +# Set every value here on the Render service (Environment tab, or the +# `envVars` list of the create-service call in README.md). A local run +# passes a filled-in copy with `docker run --env-file`. # Anthropic ANTHROPIC_API_KEY= @@ -17,11 +16,12 @@ AWS_BEARER_TOKEN_BEDROCK= AWS_REGION_NAME=us-east-1 # Vertex AI (vertex_ai + vertex_ai_gpt columns). -# On the GCP VM, the default service-account ADC from the metadata server -# is used -- no JSON key file is needed. If you ever need to run outside -# GCP, also export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa.json. +# The service-account key JSON is the Render secret file +# `vertex-service-account.json`, mounted at /etc/secrets, and +# GOOGLE_APPLICATION_CREDENTIALS points google-auth at it. VERTEXAI_PROJECT= VERTEXAI_LOCATION=global +GOOGLE_APPLICATION_CREDENTIALS=/etc/secrets/vertex-service-account.json # Azure AI Foundry (azure column — Claude models on Foundry) AZURE_AI_API_KEY= @@ -35,19 +35,20 @@ AZURE_API_BASE= AZURE_API_KEY= # The publish PAT (mateo-berri, write access on BerriAI/litellm-docs) -# deliberately does NOT live in this file. Everything here lands in the -# process environment of pytest, the proxy, and the model-driven claude -# CLI, where any same-UID reader can lift it from /proc//environ. -# Instead, install the token at /etc/litellm-compat-matrix-github-token -# (chmod 0600, single line); the service maps it in via systemd -# LoadCredential and run_daily.sh keeps it out of every child process -# env. Used to (a) resolve the latest stable release, (b) push the -# daily compat-matrix branch directly to BerriAI/litellm-docs, (c) open -# the same-repo PR, and (d) enable squash auto-merge on it. Scopes: +# deliberately is NOT an env var. Everything here lands in the process +# environment of pytest, the proxy, and the model-driven claude CLI, +# where any same-UID reader can lift it from /proc//environ. +# Instead, the token is the Render secret file `github-token` (single +# line), mounted under CREDENTIALS_DIRECTORY, and run_daily.sh reads it +# from there and keeps it out of every child process env. Used to +# (a) resolve the latest stable release, (b) push the daily +# compat-matrix branch directly to BerriAI/litellm-docs, (c) open the +# same-repo PR, and (d) enable squash auto-merge on it. Scopes: # classic `repo` + `workflow`, or fine-grained on BerriAI/litellm-docs # with Contents:RW + Pull requests:RW + Workflows:RW. # Manual runs export GITHUB_TOKEN instead, or skip publishing entirely # with SKIP_PUBLISH=1 (only writes the matrix JSON locally). +CREDENTIALS_DIRECTORY=/etc/secrets # Optional: the bedrock_mantle column is opt-in because the AWS account # needs the Mantle (OpenAI-on-Bedrock) models enabled. Without this the @@ -59,9 +60,9 @@ AZURE_API_KEY= # usually run them. Skipped cells are recorded as not_tested. # COMPAT_OPENAI_GPT_CELLS=1 -# Optional overrides; defaults are sensible for the cron VM. +# Optional overrides; defaults are sensible for the cron job. # PROXY_PORT=4100 -# LITELLM_WORKTREE=/home/mateo/litellm-cron-worktree +# LITELLM_WORKTREE=/home/populator/litellm-cron-worktree # DOCS_REPO=BerriAI/litellm-docs # DOCS_BRANCH=main # DOCS_TARGET_PATH=src/data/compatibility-matrix.json diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service deleted file mode 100644 index 6c74b3b04bb..00000000000 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service +++ /dev/null @@ -1,113 +0,0 @@ -# systemd service for the Claude Code compatibility-matrix populator. -# -# Triggered by `litellm-compat-matrix.timer`; not started directly. The -# unit is a `Type=oneshot` so the timer's `OnCalendar=` semantics -# describe "run once per day" cleanly — there's no long-lived daemon to -# supervise; each invocation runs the populator end-to-end and exits. -# -# Install -# ------- -# -# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ -# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ -# sudo systemctl daemon-reload -# sudo systemctl enable --now litellm-compat-matrix.timer -# -# Paths are hard-coded to /home/mateo rather than using systemd's %h -# specifier. Why: in *system* units (this one), %h is expanded at -# parse time against the *manager's* home -- which is /root for PID 1 -# -- and *not* against the User= directive. That mismatch makes -# ReadWritePaths point at /root/.cache (which doesn't exist), causing -# the namespace setup to fail with status=226/NAMESPACE before the -# script ever runs. The runtime user (`User=mateo`) must: -# -# * have a checkout of `BerriAI/litellm` at `~/litellm/litellm` so the -# publisher module is importable; -# * have a uv venv at `~/litellm/litellm/.venv` (created by -# `uv sync --frozen` inside that checkout once); -# * have `gh` already authenticated against an account with -# `pull-requests: write` on `BerriAI/litellm-docs`; -# * have provider credentials exported in `/etc/litellm-compat-matrix.env` -# (see `litellm-compat-matrix.env.example` in this directory); -# * have the mateo-berri publish PAT at -# `/etc/litellm-compat-matrix-github-token` (chmod 0600, single -# line), delivered via `LoadCredential=` below. - -[Unit] -Description=Claude Code compatibility-matrix populator (oneshot) -Documentation=file:///home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/README.md -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -User=mateo -Group=mateo - -# Provider credentials + any gh/PROXY_PORT overrides live here. Format -# is the standard `KEY=value` one line per env var. -EnvironmentFile=-/etc/litellm-compat-matrix.env - -# The mateo-berri publish PAT is mapped in via the credential store, NOT -# the EnvironmentFile, so it never lands in the process environment that -# pytest, the proxy, and the model-driven claude CLI inherit (any -# same-UID process can read /proc//environ). run_daily.sh reads -# ${CREDENTIALS_DIRECTORY}/github-token and hands it to gh per call. -# Unlike EnvironmentFile= above, this is deliberately NOT optional: a -# missing token file fails the unit at start instead of 30 minutes in. -LoadCredential=github-token:/etc/litellm-compat-matrix-github-token - -# systemd starts with a minimal PATH (~/usr/local/bin:/usr/bin:/bin). -# `uv` and `claude` are installed under the runtime user's `~/.local/bin` -# so we have to prepend it explicitly; otherwise run_daily.sh fails at -# the up-front command-presence check. -Environment=PATH=/home/mateo/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - -# `HOME` is auto-set to /home/mateo when User=mateo is honored, but be -# explicit so anything that reads $HOME (e.g. uv's cache lookup, the -# claude CLI's per-session dir) sees the right value even if a future -# refactor flips DynamicUser= or PrivateUsers= on. -Environment=HOME=/home/mateo - -WorkingDirectory=/home/mateo/litellm/litellm - -ExecStart=/home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh - -# 90 minutes is generous: cold runs do `git clone` + `uv sync` of a new -# tag's lockfile, which can take a couple of minutes on a 2-vCPU VM, -# plus the full feature x provider grid of pytest cells hitting several -# cloud providers. -TimeoutStartSec=90min - -# A failed run shouldn't restart automatically — the next timer fire is -# the right retry. Reruns of the same day's matrix are idempotent. -Restart=no - -# Security hardening: the populator only reads the litellm checkout and -# the env-file; everything else it writes lives in either the worktree -# (managed) or `/tmp` (cleaned up by tempfile). -# -# ReadWritePaths whitelist: -# * litellm-cron-worktree - the long-lived stable-tag checkout + -# its `.venv` (`uv sync` rewrites every -# run) + `.uv-bin` (pinned `uv` binary -# cache). -# * .cache - uv's wheel cache (~/.cache/uv) so we -# don't redownload pinned deps each run. -# * .claude - `claude` CLI's per-session state under -# `~/.claude/projects//`; created -# on every `claude --print` invocation. -# * .config/gh - `gh` CLI host config; technically not -# needed when we pass GH_TOKEN inline, -# but cheap to whitelist and prevents -# future regressions if a code path -# ever falls back to the host config. -# * /tmp - mktemp -d workdir + proxy logs. -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=read-only -ReadWritePaths=/home/mateo/litellm-cron-worktree /home/mateo/.cache /home/mateo/.claude /home/mateo/.config/gh /tmp -PrivateTmp=true - -[Install] -WantedBy=multi-user.target diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer deleted file mode 100644 index ee22538c6ed..00000000000 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer +++ /dev/null @@ -1,25 +0,0 @@ -# Daily timer for the compatibility-matrix populator. -# -# 06:00 UTC matches the original GitHub Actions cron schedule; chosen so -# operators in US/EU timezones see fresh PRs at the start of their work -# day. -# -# `Persistent=true` causes a missed run (VM was off / suspended) to -# fire the next time the timer is started, which is the property we -# want for a once-a-day job: the matrix should refresh as soon as the -# VM is reachable again, not wait another 24h. -# -# `RandomizedDelaySec=10min` smears load if multiple matrix-style -# pipelines are ever colocated on the same VM in the future. - -[Unit] -Description=Run the Claude Code compatibility-matrix populator daily - -[Timer] -OnCalendar=*-*-* 06:00:00 UTC -Persistent=true -RandomizedDelaySec=10min -Unit=litellm-compat-matrix.service - -[Install] -WantedBy=timers.target diff --git a/tests/e2e/claude_code/cron_vm/run_daily.sh b/tests/e2e/claude_code/cron_vm/run_daily.sh index e878007d8a3..172dd03b614 100755 --- a/tests/e2e/claude_code/cron_vm/run_daily.sh +++ b/tests/e2e/claude_code/cron_vm/run_daily.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Daily Claude Code compatibility-matrix populator. # -# Runs from the GCP VM `litellm-compatibility-matrix-populator` via the -# systemd timer in this directory. The flow is: +# Runs daily as the Render cron job `litellm-compat-matrix`, built from +# the Dockerfile in this directory (see README.md). The flow is: # # 1. Resolve the latest LiteLLM final release tag from the GitHub # Releases API. @@ -33,12 +33,12 @@ # rather than spawning a new one. If the JSON is byte-identical to the # docs branch, we skip the push entirely. # -# Required commands on $PATH: git, uv, gh, jq, curl, claude, npm. +# Required commands on $PATH: git, uv, gh, jq, curl, claude. # Required state: a litellm checkout at $LITELLM_REPO (this file lives in -# it), $WORKTREE is created on first run, gh is already authenticated. +# it); $WORKTREE is created on first run. # -# Override any default by setting the matching env var; see the systemd -# unit for the production wiring. +# Override any default by setting the matching env var; see README.md +# for the production wiring. set -Eeuo pipefail @@ -52,12 +52,12 @@ DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}" SKIP_PUBLISH="${SKIP_PUBLISH:-0}" PYTEST_K="${PYTEST_K:-}" # The e2e suite uses PEP 695 `type` aliases, so the venv needs Python -# >= 3.12 (also what repo CI runs) even when the VM's system python is +# >= 3.12 (also what repo CI runs) even when the host's system python is # older. uv fetches a managed CPython of this version on first use -- # checksum-verified against the manifest baked into the pinned uv # binary -- and installs it under ${WORKTREE}/.uv-python (see -# UV_PYTHON_INSTALL_DIR below) so it lives inside the one tree the -# systemd sandbox lets us write to. +# UV_PYTHON_INSTALL_DIR below) so everything the run writes lives inside +# the worktree. CRON_PYTHON_VERSION="${CRON_PYTHON_VERSION:-3.12}" # Merge method for auto-merge. BerriAI/litellm-docs only allows squash # merges (merge-commit and rebase are disabled at the repo level), so @@ -113,9 +113,9 @@ for cmd in git uv gh jq curl claude; do done # Publishing pushes the branch straight to BerriAI/litellm-docs and opens -# the PR as mateo-berri, who has write access on the docs repo. Under -# systemd the PAT arrives as a file via LoadCredential=, NOT via the -# EnvironmentFile: several suite cells let the model-driven claude CLI +# the PR as mateo-berri, who has write access on the docs repo. On +# Render the PAT arrives as a secret file under ${CREDENTIALS_DIRECTORY}, +# NOT via an env var: several suite cells let the model-driven claude CLI # read arbitrary files as this user, and /proc//environ of the # script, pytest, and the proxy would hand an env-borne token to any # same-UID reader. Kept as an unexported shell variable and passed per @@ -125,13 +125,18 @@ done # quota. if [[ -z "${GITHUB_TOKEN:-}" && -n "${CREDENTIALS_DIRECTORY:-}" && -f "${CREDENTIALS_DIRECTORY}/github-token" ]]; then GITHUB_TOKEN="$(<"${CREDENTIALS_DIRECTORY}/github-token")" - log "publish token source: systemd credential store" + log "publish token source: ${CREDENTIALS_DIRECTORY}/github-token" elif [[ -n "${GITHUB_TOKEN:-}" ]]; then log "publish token source: process environment" fi if [[ "${SKIP_PUBLISH}" != "1" ]]; then [[ -n "${GITHUB_TOKEN:-}" ]] \ - || die "publish token required: /etc/litellm-compat-matrix-github-token via LoadCredential under systemd, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)" + || die "publish token required: the github-token secret file under CREDENTIALS_DIRECTORY, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)" + # The stale-PR sweep below closes only PRs this account opened, so the + # login is resolved from the token once rather than hardcoded. + PUBLISH_LOGIN="$(GH_TOKEN="${GITHUB_TOKEN}" gh api user --jq .login)" \ + || die "could not resolve the publishing account from the github token" + log "publishing as ${PUBLISH_LOGIN}" fi # --------------------------------------------------------------------------- @@ -205,7 +210,7 @@ log "local claude code: ${CLAUDE_CODE_VERSION}" if [[ ! -d "${WORKTREE}/.git" ]]; then log "first run: cloning litellm into ${WORKTREE}" mkdir -p "$(dirname "${WORKTREE}")" - git clone https://github.com/BerriAI/litellm.git "${WORKTREE}" + git clone --filter=blob:none https://github.com/BerriAI/litellm.git "${WORKTREE}" fi log "updating worktree to ${LITELLM_VERSION}" @@ -221,40 +226,27 @@ git -C "${WORKTREE}" clean -fdx -e .venv -e .uv-bin -e .uv-python git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}" # Always rebuild tests/e2e/ in the worktree from the dev checkout, -# regardless of what the resolved ${LITELLM_VERSION} tag ships. Two -# reasons: +# regardless of what the resolved ${LITELLM_VERSION} tag ships: the +# matrix populator's job is to exercise *today's* tests against the +# latest stable proxy, and the dev checkout carries the most recent +# test fixes that haven't yet rolled into a stable release. # -# * The matrix populator's job is to exercise *today's* tests against -# the latest stable proxy. The dev checkout carries the most recent -# test fixes that haven't yet rolled into a stable release, and we -# want every cron run to pick those up the moment they land on -# ${LITELLM_REPO}, not whenever the next stable release happens. -# * The tag's own tests/e2e/ ships the full EKS e2e harness, whose -# top-level conftest.py imports modules (e2e_db, lifecycle, -# otel_client, ...) that the stable venv does not install. Copying -# the whole tree would make pytest collection blow up on those -# imports. -# -# So the shim is a fresh `rm -rf` of tests/e2e/ followed by copying ONLY -# the claude_code suite plus the shared transport helpers it imports. +# The whole tree is copied rather than an allowlist of the helpers the +# suite imports: the helpers import each other (proxy_client -> +# e2e_config -> fixture_mode -> ...), so a new edge in that graph turned +# an allowlist into a ModuleNotFoundError at conftest load. The tree's +# top-level conftest.py pulls in the full EKS harness (e2e_db, +# lifecycle, ...), which the stable venv does not install, so the pytest +# run below points --confcutdir at claude_code/ and never loads it. # pytest puts tests/e2e/ itself on sys.path (it has no __init__.py, while # claude_code/ does), which is what resolves both the `claude_code.*` # and the bare `proxy_client` / `e2e_http` imports inside the suite. -E2E_HELPER_FILES=(proxy_client.py e2e_http.py models.py e2e_config.py transport.py) -if [[ ! -d "${LITELLM_REPO}/tests/e2e/claude_code" ]]; then - die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code" -fi -for helper in "${E2E_HELPER_FILES[@]}"; do - [[ -f "${LITELLM_REPO}/tests/e2e/${helper}" ]] \ - || die "missing shim helper: ${LITELLM_REPO}/tests/e2e/${helper}" -done -log "shimming tests/e2e/claude_code/ + helpers from ${LITELLM_REPO} (always-overwrite)" +[[ -d "${LITELLM_REPO}/tests/e2e/claude_code" ]] \ + || die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code" +log "shimming tests/e2e/ from ${LITELLM_REPO} (always-overwrite)" rm -rf "${WORKTREE}/tests/e2e" mkdir -p "${WORKTREE}/tests/e2e" -cp -r "${LITELLM_REPO}/tests/e2e/claude_code" "${WORKTREE}/tests/e2e/" -for helper in "${E2E_HELPER_FILES[@]}"; do - cp "${LITELLM_REPO}/tests/e2e/${helper}" "${WORKTREE}/tests/e2e/" -done +cp -r "${LITELLM_REPO}/tests/e2e/." "${WORKTREE}/tests/e2e/" # litellm pins an exact uv version in pyproject.toml's [tool.uv] # `required-version` field, so a system uv that's newer or older @@ -305,10 +297,18 @@ fi # actually serve. `--group proxy-dev` brings in pytest and the rest of # what tests/e2e/claude_code/ needs. `--python` pins the venv to # ${CRON_PYTHON_VERSION}; the first run after a version bump recreates -# the venv from scratch (a one-time cold sync). +# the venv from scratch (a one-time cold sync). `--no-install-project` +# leaves litellm itself out: the tag builds a Rust extension through +# maturin, which needs a C and Rust toolchain the image does not carry, +# so the published PyPI wheel (what users install) goes in right after, +# and every later `uv run` passes `--no-sync` so uv never tries to put +# the source build back. export UV_PYTHON_INSTALL_DIR="${WORKTREE}/.uv-python" -log "uv sync --frozen --group proxy-dev --extra proxy --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})" -(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --python "${CRON_PYTHON_VERSION}") +log "uv sync --frozen --group proxy-dev --extra proxy --no-install-project --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})" +(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --no-install-project --python "${CRON_PYTHON_VERSION}") +LITELLM_WHEEL_VERSION="${LITELLM_VERSION#v}" +log "installing the published litellm==${LITELLM_WHEEL_VERSION} wheel from PyPI" +"${WORKTREE_UV}" pip install --python "${WORKTREE}/.venv/bin/python" --no-deps --no-build "litellm==${LITELLM_WHEEL_VERSION}" PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml" [[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (shim incomplete?)" @@ -321,10 +321,10 @@ log "starting proxy on 127.0.0.1:${PROXY_PORT}" # Bind the proxy to loopback only. The populator proxy is talked to # exclusively by the pytest run on the same host (the health check and # the test env set `LITELLM_PROXY_URL=http://127.0.0.1:...`), -# so there's no reason to expose it on the VM's external interfaces. +# so there's no reason to expose it on the container's external interfaces. # Without `--host`, `litellm` defaults to 0.0.0.0, which combined with # the predictable default `LITELLM_MASTER_KEY=sk-cron-matrix` would -# allow anything that can reach :${PROXY_PORT} on the VM to authenticate +# allow anything that can reach :${PROXY_PORT} on the host to authenticate # and burn upstream provider credentials. # # `setsid` puts the proxy in its own session+pgroup so cleanup() can @@ -334,7 +334,7 @@ log "starting proxy on 127.0.0.1:${PROXY_PORT}" setsid env LITELLM_MASTER_KEY="${PROXY_API_KEY}" bash -c ' echo "$$" > "$0" cd "$1" - exec "$2" run litellm --config "$3" --host 127.0.0.1 --port "$4" + exec "$2" run --no-sync litellm --config "$3" --host 127.0.0.1 --port "$4" ' "${PROXY_PID_FILE}" "${WORKTREE}" "${WORKTREE_UV}" "${PROXY_CONFIG}" "${PROXY_PORT}" \ >"${WORKDIR}/proxy.log" 2>&1 & disown @@ -359,6 +359,7 @@ RESULTS_JSON="${WORKDIR}/compat-results.json" # the cron skips them if/when they land in the suite. PYTEST_ARGS=( tests/e2e/claude_code/ + --confcutdir=tests/e2e/claude_code "--ignore-glob=*_unit_tests*" ) if [[ -n "${PYTEST_K}" ]]; then @@ -373,7 +374,7 @@ set +e && LITELLM_PROXY_URL="http://127.0.0.1:${PROXY_PORT}" \ LITELLM_MASTER_KEY="${PROXY_API_KEY}" \ COMPAT_RESULTS_PATH="${RESULTS_JSON}" \ - "${WORKTREE_UV}" run pytest "${PYTEST_ARGS[@]}" + "${WORKTREE_UV}" run --no-sync pytest "${PYTEST_ARGS[@]}" ) PYTEST_EXIT=$? set -e @@ -392,7 +393,7 @@ MATRIX_JSON="${WORKDIR}/compatibility-matrix.json" log "building ${MATRIX_JSON}" ( cd "${WORKTREE}" \ - && "${WORKTREE_UV}" run python "${POPULATOR_DIR}/build_matrix.py" \ + && "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/build_matrix.py" \ --manifest "${WORKTREE}/tests/e2e/claude_code/manifest.yaml" \ --results "${RESULTS_JSON}" \ --output "${MATRIX_JSON}" \ @@ -405,8 +406,9 @@ log "building ${MATRIX_JSON}" # --------------------------------------------------------------------------- if [[ "${SKIP_PUBLISH}" == "1" ]]; then - cp "${MATRIX_JSON}" "${LITELLM_REPO}/compatibility-matrix.json" - log "SKIP_PUBLISH=1; matrix written to ${LITELLM_REPO}/compatibility-matrix.json" + cp "${MATRIX_JSON}" "${HOME}/compatibility-matrix.json" + log "SKIP_PUBLISH=1; matrix saved to ${HOME}/compatibility-matrix.json and printed below" + cat "${MATRIX_JSON}" exit 0 fi @@ -415,7 +417,7 @@ BRANCH_NAME="compat-matrix/${LITELLM_VERSION}-${CLAUDE_CODE_VERSION}-${DATE_UTC} DOCS_CLONE="${WORKDIR}/litellm-docs" log "cloning ${DOCS_REPO}@${DOCS_BRANCH}" -gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}" +GH_TOKEN="${GITHUB_TOKEN}" gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}" cd "${DOCS_CLONE}" git config user.email "litellm-bot@berri.ai" @@ -452,7 +454,7 @@ log "checking for green->red regressions vs the published matrix" set +e REGRESSION_REPORT="$( cd "${WORKTREE}" \ - && "${WORKTREE_UV}" run python "${POPULATOR_DIR}/check_regressions.py" \ + && "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/check_regressions.py" \ --old "${PUBLISHED_MATRIX}" \ --new "${MATRIX_JSON}" )" @@ -492,7 +494,7 @@ git commit -m "${COMMIT_MSG}" # # Plain --force (not --force-with-lease) is acceptable here: the # compat-matrix/* branch is bot-owned, only this script ever writes to -# it, and runs are serialized by the systemd timer. --force-with-lease +# it, and runs are serialized by the cron schedule. --force-with-lease # would require a fetch to populate the remote-tracking ref before each # push and adds no safety in this single-writer setup. PUBLISH_PUSH_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${DOCS_REPO}.git" @@ -553,7 +555,7 @@ Generated by \`tests/e2e/claude_code/cron_vm/run_daily.sh\`. Close without mergi EOF )" -log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as mateo-berri)" +log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as ${PUBLISH_LOGIN})" # GH_TOKEN is mateo-berri's write-scoped token, the same identity used # for release-listing above. The branch lives on ${DOCS_REPO} itself, so # --head is a bare branch name (a same-repo PR), not `OWNER:BRANCH`. @@ -644,15 +646,25 @@ fi # # Non-fatal: a sweep failure (rate limit, transient API error) leaves # stale PRs for the next run to retry; it must not fail the pipeline. +# +# The docs repo carries a few hundred open PRs, so the list has to page +# past gh's default 30 (and the earlier 100, which never reached a +# week-old compat-matrix PR and left it open for good). +# +# `compat-matrix/` is only a naming convention, so the prefix alone does +# not make a PR this job's: a contributor can open a fork PR under that +# name. Only PRs the publishing account itself opened from a branch on +# the docs repo qualify; anything else stays untouched. log "sweeping stale compat-matrix PRs (keeping ${BRANCH_NAME})" set +e STALE_PRS="$( GH_TOKEN="${GITHUB_TOKEN}" gh pr list \ --repo "${DOCS_REPO}" \ --state open \ - --limit 100 \ - --json number,headRefName \ - --jq '.[] | select(.headRefName | startswith("compat-matrix/")) | "\(.number)\t\(.headRefName)"' + --author "${PUBLISH_LOGIN}" \ + --limit 1000 \ + --json number,headRefName,isCrossRepository \ + --jq '.[] | select((.headRefName | startswith("compat-matrix/")) and (.isCrossRepository | not)) | "\(.number)\t\(.headRefName)"' )" while IFS=$'\t' read -r stale_pr stale_head; do [[ -z "${stale_pr}" ]] && continue @@ -660,7 +672,7 @@ while IFS=$'\t' read -r stale_pr stale_head; do GH_TOKEN="${GITHUB_TOKEN}" gh pr close "${stale_pr}" \ --repo "${DOCS_REPO}" \ --delete-branch \ - --comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open." 2>&1 | sed 's/^/ /' + --comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open" 2>&1 | sed 's/^/ /' if [[ ${PIPESTATUS[0]} -eq 0 ]]; then log "closed stale compat-matrix PR #${stale_pr} (${stale_head})" else