diff --git a/.github/workflows/claude_code_compat_matrix.yml b/.github/workflows/claude_code_compat_matrix.yml deleted file mode 100644 index f5892276cfd..00000000000 --- a/.github/workflows/claude_code_compat_matrix.yml +++ /dev/null @@ -1,130 +0,0 @@ -name: Claude Code Compatibility Matrix (daily cron) - -# Slice 4 of the Claude Code Compatibility Matrix (PRD #26476, issue #26480). -# -# Three triggers per the PRD's "Daily Cron" section: -# - Daily cron (06:00 UTC) — picks up newly-published Claude Code releases. -# - `release` of a `v*-stable` tag on this repo — re-runs the matrix the -# moment a new stable LiteLLM ships. -# - Manual dispatch — operators can re-run the publisher on demand. -# -# The job runs on a GitHub-hosted ubuntu-latest runner, which gives us a -# fresh VM per run and is "isolated from the main CI environment" in the -# sense that nothing else on this runner survives the run. Since the -# always-latest Claude Code CLI is only installed inside this ephemeral -# VM, a malicious or broken Claude Code release cannot affect the trusted -# build infrastructure used by the PR gate (which lives in CircleCI and -# uses a `latest minus 3 days` Claude Code pin). -# -# Cross-repo authentication (per "Cross-repo authentication" in the PRD): -# A GitHub App installed on `BerriAI/litellm-docs` only, scoped to -# `contents: write` (so the publisher can push the head branch) and -# `pull-requests: write` (so `gh pr create` can open the docs PR), mints -# an installation token at job-start. The token is only ever used by the -# publisher, which only ever writes `compatibility-matrix.json` -# (enforced by `select_files_to_commit`) and only ever opens PRs against -# `litellm-docs` (enforced by the `--repo` flag passed to `gh`). - -on: - schedule: - - cron: "0 6 * * *" # daily at 06:00 UTC - release: - types: [published] - workflow_dispatch: - inputs: - skip_publish: - description: "Run the test pipeline but skip the docs-repo push." - required: false - type: boolean - default: false - -permissions: - contents: read - -jobs: - publish-matrix: - # Skip release runs that aren't tagged `v*-stable`. Plain `v1.84.0-rc1` - # or `v1.84.0` releases must NOT republish the matrix — only the - # latest *stable* tag is reflected on the docs page. - if: | - github.repository == 'BerriAI/litellm' && ( - github.event_name != 'release' || - endsWith(github.event.release.tag_name, '-stable') - ) - runs-on: ubuntu-latest - timeout-minutes: 90 - - steps: - - name: Checkout litellm - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - version: "0.10.9" - enable-cache: false - - - name: Set up Node (for the Claude Code CLI) - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 - with: - node-version: "20" - - - name: Mint docs-repo installation token from GitHub App - id: docs-token - uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 - with: - app-id: ${{ secrets.COMPAT_MATRIX_APP_ID }} - private-key: ${{ secrets.COMPAT_MATRIX_APP_PRIVATE_KEY }} - owner: BerriAI - repositories: litellm-docs - - - name: Install LiteLLM dev deps - run: uv sync --frozen - - - name: Run matrix publisher - env: - # Token used to direct-push compatibility-matrix.json to the docs - # repo's main branch. Comes from the GitHub App installation token - # minted above; scoped to litellm-docs only. - DOCS_REPO_TOKEN: ${{ steps.docs-token.outputs.token }} - # Token used by the resolver to lift the unauthenticated GitHub - # rate limit on the Releases API. The default GITHUB_TOKEN is - # sufficient for read-only access to public release metadata. - GITHUB_TOKEN: ${{ github.token }} - # Real provider credentials needed by the per-cell tests. These - # are the same secrets the LLM-translation workflow uses. - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }} - VERTEXAI_PROJECT: ${{ secrets.VERTEXAI_PROJECT }} - VERTEXAI_LOCATION: ${{ secrets.VERTEXAI_LOCATION }} - GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }} - AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} - AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} - SKIP_PUBLISH: ${{ inputs.skip_publish }} - run: | - set -euo pipefail - if [ "${SKIP_PUBLISH:-false}" = "true" ]; then - uv run python -m tests.claude_code.publisher --skip-publish - else - uv run python -m tests.claude_code.publisher - fi - - - name: Upload compat-results.json artifact (debugging) - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: compat-results-${{ github.run_id }} - path: | - compat-results.json - compatibility-matrix.json - if-no-files-found: ignore - retention-days: 30 diff --git a/tests/claude_code/_publisher_unit_tests/test_publisher.py b/tests/claude_code/_publisher_unit_tests/test_publisher.py index 6d62b08ff1f..d1116ded85e 100644 --- a/tests/claude_code/_publisher_unit_tests/test_publisher.py +++ b/tests/claude_code/_publisher_unit_tests/test_publisher.py @@ -1,25 +1,23 @@ """Unit tests for the daily-cron matrix publisher. -The publisher orchestrates Docker, git, npm, gh and pytest — per the -PRD's "Testing Decisions" section, the orchestration itself is too thin -to warrant heavy mocking. These tests cover the small pure helpers that -do warrant test coverage: +The publisher orchestrates git, uv, gh and pytest on a dedicated GCP VM — +the orchestration itself is too thin to warrant heavy mocking. These +tests cover the small pure helpers that do warrant test coverage: -- `commit_message_for_matrix`: deterministic commit message containing - the LiteLLM and Claude Code versions plus `generated_at`, so the docs - repo's git log shows what produced each push. -- `docker_image_for_tag`: maps a `v*-stable` tag to its `ghcr.io` image. -- `select_files_to_commit`: enforces the "only `compatibility-matrix.json` - is pushed" guarantee that the GitHub App's broad `contents: write` scope - doesn't enforce on its own (per PRD: "File-level restriction is enforced - by script correctness"). -- `pr_branch_name`: deterministic head-branch name for the docs PR. +- ``commit_message_for_matrix``: deterministic commit message containing + the LiteLLM and Claude Code versions plus ``generated_at``, so the + docs repo's git log shows what produced each push. +- ``select_files_to_commit``: enforces the "only + ``compatibility-matrix.json`` is pushed" guarantee. Even with PR + review in front of the docs branch, the publisher refuses to stage + any other file as defence in depth. +- ``pr_branch_name``: deterministic head-branch name for the docs PR. Two cron runs on the same UTC day with the same resolved versions must collide on this branch so the second run updates the existing PR rather than spawning a new one. -- `pr_title_for_matrix` / `pr_body_for_matrix`: the strings the publisher - hands to `gh pr create`. Tested for content (not formatting trivia) - to keep the tests resilient to copy edits. +- ``pr_title_for_matrix`` / ``pr_body_for_matrix``: the strings the + publisher hands to ``gh pr create``. Tested for content (not formatting + trivia) to keep the tests resilient to copy edits. """ from __future__ import annotations @@ -30,7 +28,6 @@ from tests.claude_code.publisher import ( DOCS_TARGET_BASENAME, PR_BRANCH_PREFIX, commit_message_for_matrix, - docker_image_for_tag, pr_body_for_matrix, pr_branch_name, pr_title_for_matrix, @@ -67,18 +64,6 @@ def test_commit_message_is_deterministic(): assert commit_message_for_matrix(matrix) == commit_message_for_matrix(matrix) -def test_docker_image_for_tag_targets_berriai_ghcr(): - assert ( - docker_image_for_tag("v1.83.0-stable") - == "ghcr.io/berriai/litellm:v1.83.0-stable" - ) - - -def test_docker_image_for_tag_rejects_empty_tag(): - with pytest.raises(ValueError, match="tag must be a non-empty string"): - docker_image_for_tag("") - - def test_select_files_to_commit_drops_anything_other_than_matrix(): """The script's safety net: even if pytest leaves stray artifacts in the docs-repo checkout, only `compatibility-matrix.json` ever ships. diff --git a/tests/claude_code/cron_vm/README.md b/tests/claude_code/cron_vm/README.md new file mode 100644 index 00000000000..d4f0fd1a618 --- /dev/null +++ b/tests/claude_code/cron_vm/README.md @@ -0,0 +1,126 @@ +# Cron VM setup for the Claude Code compatibility-matrix populator + +The populator runs daily on a dedicated GCP VM +(`litellm-compatibility-matrix-populator`) rather than as a GitHub +Action. Trade-offs: + +- ✅ Real VM means we can `gh auth login` against a human/bot account + that's already a collaborator on `BerriAI/litellm-docs`, instead of + provisioning a GitHub App with `pull-requests: write`. +- ✅ Persistent state (a single `~/litellm-cron-worktree/` and its `.venv`) + is reused across runs, so each daily run does a fast `git checkout` + + incremental `uv sync` rather than a fresh clone + cold sync. +- ✅ No Docker dependency — proxy is run directly via `uv run litellm`. +- ⚠️ The VM has to actually be on. systemd's `Persistent=true` recovers + from short outages, but a multi-day outage means the matrix goes + stale until the VM is back. +- ⚠️ Provider credentials live on the VM filesystem + (`/etc/litellm-compat-matrix.env`) instead of GitHub secrets. Treat + the VM as an environment with comparable blast radius to a CI runner. + +## What the populator does, end to end + +`tests/claude_code/publisher.py` (`python -m tests.claude_code.publisher`): + +1. Resolves the latest `v*-stable` tag of `BerriAI/litellm` via the + GitHub Releases API (`tests/claude_code/resolver.py`). +2. Reads the locally installed Claude Code CLI version + (`claude --version`). +3. Updates the persistent worktree at `~/litellm-cron-worktree/` to + that tag, and `uv sync --frozen`s its `.venv`. `git clean -fdx -e .venv` + wipes any cruft from previous runs while keeping the venv around. +4. Boots the LiteLLM proxy as a subprocess on port `4100` (override + with `PROXY_PORT`), using + `tests/claude_code/test_config.yaml` from the checked-out tag. +5. Runs `pytest tests/claude_code/` with `ANTHROPIC_BASE_URL` pointed + at the proxy and `COMPAT_RESULTS_PATH` set so the conftest hook + writes the per-test results artifact. +6. Builds `compatibility-matrix.json` from the artifact via + `tests.claude_code.matrix_builder.build_from_paths`. +7. Clones the docs repo (`gh repo clone BerriAI/litellm-docs`) into a + temp dir, checks out a deterministic head branch + (`compat-matrix/--`), + commits the JSON, force-with-lease pushes, and opens a PR with + `gh pr create`. + +Re-running on the same day with the same versions is idempotent: the +branch name collides, the force-with-lease updates the existing branch, +and `gh pr create` no-ops because the PR already exists. + +## One-time VM setup + +Run as `mateo` on the cron VM: + +```bash +# 1. Toolchain +sudo apt-get update +sudo apt-get install -y git nodejs npm +curl -LsSf https://astral.sh/uv/install.sh | sh +sudo apt-get install -y gh # or follow https://cli.github.com/ + +# 2. Claude Code CLI (the cron does NOT auto-upgrade this; rerun this +# line out-of-band when you want a fresh CLI to be tested) +sudo npm install -g @anthropic-ai/claude-code@latest + +# 3. Litellm dev checkout. Used as the launcher for the publisher +# module; the populator mutates a separate worktree under +# ~/litellm-cron-worktree/. +mkdir -p ~/litellm +git clone https://github.com/BerriAI/litellm.git ~/litellm/litellm +cd ~/litellm/litellm && uv sync --frozen + +# 4. gh auth — must be a collaborator on BerriAI/litellm-docs. +gh auth login # follow prompts; pick HTTPS + token paste flow + +# 5. Provider credentials. +sudo cp tests/claude_code/cron_vm/litellm-compat-matrix.env.example \ + /etc/litellm-compat-matrix.env +sudoedit /etc/litellm-compat-matrix.env # fill in real values +sudo chmod 0600 /etc/litellm-compat-matrix.env + +# 6. systemd units. +sudo cp tests/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ +sudo cp tests/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now litellm-compat-matrix.timer +``` + +## Operating it + +```bash +# When does it run next? +systemctl list-timers litellm-compat-matrix.timer + +# Trigger a run right now (still PRs to litellm-docs). +sudo systemctl start litellm-compat-matrix.service + +# Trigger a run that does NOT open a PR (good for first-time validation). +cd ~/litellm/litellm +uv run python -m tests.claude_code.publisher --skip-publish + +# Watch the most recent run. +journalctl -u litellm-compat-matrix.service -f + +# Read older runs. +journalctl -u litellm-compat-matrix.service --since '2 days ago' + +# Disable until further notice (e.g. while debugging). +sudo systemctl disable --now litellm-compat-matrix.timer +``` + +## Gotchas + +- **The proxy port is `4100`, not `4000`.** This is so a developer SSH'd + into the same VM with their own `:4000` proxy doesn't collide with a + cron run. Override with `PROXY_PORT=...` in + `/etc/litellm-compat-matrix.env` if you need to. +- **`uv sync --frozen` requires the resolved tag to be tagged on + GitHub.** If the latest stable release was made but not pushed as a + git tag, the run will `git checkout` fail. Push the tag, then rerun. +- **`gh auth` token rotation is your problem.** The cron does not + refresh the token; if the bot account's PAT expires the run will + fail at `gh repo clone` with a 401. Re-run `gh auth login`. +- **First run after upgrading the Claude Code CLI is the riskiest one.** + If the new CLI changes its wire format the matrix run can produce + systematic failures. Always run `--skip-publish` after a CLI upgrade + to inspect the JSON before the next scheduled fire. diff --git a/tests/claude_code/cron_vm/litellm-compat-matrix.env.example b/tests/claude_code/cron_vm/litellm-compat-matrix.env.example new file mode 100644 index 00000000000..861057e0dfd --- /dev/null +++ b/tests/claude_code/cron_vm/litellm-compat-matrix.env.example @@ -0,0 +1,34 @@ +# Environment file consumed by `litellm-compat-matrix.service`. +# +# Install at `/etc/litellm-compat-matrix.env` and chmod 0600. +# `EnvironmentFile=-` in the unit means the service is allowed to start +# even if this file is missing, but the populator will fail at the +# first provider request without these credentials. + +# Anthropic +ANTHROPIC_API_KEY= + +# Bedrock (invoke + converse columns) +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_REGION_NAME=us-east-1 + +# Vertex AI +VERTEXAI_PROJECT= +VERTEXAI_LOCATION=us-east5 +GOOGLE_APPLICATION_CREDENTIALS=/etc/litellm-compat-matrix.gcp.json + +# Microsoft Foundry (Azure column) +AZURE_FOUNDRY_API_KEY= +AZURE_FOUNDRY_API_BASE= + +# Optional: lifts the unauthenticated rate limit on the GitHub Releases +# API used by `resolver.py`. Not required. +# GITHUB_TOKEN= + +# Optional overrides; defaults are sensible for the cron VM. +# PROXY_PORT=4100 +# LITELLM_WORKTREE=/home/mateo/litellm-cron-worktree +# DOCS_REPO=BerriAI/litellm-docs +# DOCS_BRANCH=main +# DOCS_TARGET_PATH=src/data/compatibility-matrix.json diff --git a/tests/claude_code/cron_vm/litellm-compat-matrix.service b/tests/claude_code/cron_vm/litellm-compat-matrix.service new file mode 100644 index 00000000000..43d8a8326bf --- /dev/null +++ b/tests/claude_code/cron_vm/litellm-compat-matrix.service @@ -0,0 +1,72 @@ +# systemd service for the Claude Code compatibility-matrix populator. +# +# Triggered by `litellm-compat-matrix.timer`; not started directly. The +# unit is a `Type=oneshot` so the timer's `OnCalendar=` semantics +# describe "run once per day" cleanly — there's no long-lived daemon to +# supervise; each invocation runs the populator end-to-end and exits. +# +# Install +# ------- +# +# sudo cp tests/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ +# sudo cp tests/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ +# sudo systemctl daemon-reload +# sudo systemctl enable --now litellm-compat-matrix.timer +# +# The %h specifier expands to the runtime user's home directory, which +# avoids hard-coding /home/mateo into a unit that should be portable +# across operator handoffs. The runtime user (`User=mateo`) must: +# +# * have a checkout of `BerriAI/litellm` at `~/litellm/litellm` so the +# publisher module is importable; +# * have a uv venv at `~/litellm/litellm/.venv` (created by +# `uv sync --frozen` inside that checkout once); +# * have `gh` already authenticated against an account with +# `pull-requests: write` on `BerriAI/litellm-docs`; +# * have provider credentials exported in `/etc/litellm-compat-matrix.env` +# (see `litellm-compat-matrix.env.example` in this directory). + +[Unit] +Description=Claude Code compatibility-matrix populator (oneshot) +Documentation=file://%h/litellm/litellm/tests/claude_code/cron_vm/README.md +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=mateo +Group=mateo + +# Provider credentials + any gh/PROXY_PORT overrides live here. Format +# is the standard `KEY=value` one line per env var. +EnvironmentFile=-/etc/litellm-compat-matrix.env + +# Run from the dev checkout so `python -m tests.claude_code.publisher` +# resolves; the publisher itself manages the separate worktree it +# mutates per run. +WorkingDirectory=%h/litellm/litellm + +# `uv run --frozen` reuses the dev checkout's venv. The publisher then +# bootstraps its own worktree + venv for the proxy. +ExecStart=/usr/bin/env -S uv run --frozen python -m tests.claude_code.publisher + +# 90 minutes is generous: cold runs do `git clone` + `uv sync` of a new +# tag's lockfile, which can take a couple of minutes on a 2-vCPU VM, +# plus 30 cells of pytest hitting four cloud providers. +TimeoutStartSec=90min + +# A failed run shouldn't restart automatically — the next timer fire is +# the right retry. Reruns of the same day's matrix are idempotent. +Restart=no + +# Security hardening: the populator only reads the litellm checkout and +# the env-file; everything else it writes lives in either the worktree +# (managed) or `/tmp` (cleaned up by tempfile). +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths=%h/litellm-cron-worktree %h/.cache /tmp +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/tests/claude_code/cron_vm/litellm-compat-matrix.timer b/tests/claude_code/cron_vm/litellm-compat-matrix.timer new file mode 100644 index 00000000000..ee22538c6ed --- /dev/null +++ b/tests/claude_code/cron_vm/litellm-compat-matrix.timer @@ -0,0 +1,25 @@ +# Daily timer for the compatibility-matrix populator. +# +# 06:00 UTC matches the original GitHub Actions cron schedule; chosen so +# operators in US/EU timezones see fresh PRs at the start of their work +# day. +# +# `Persistent=true` causes a missed run (VM was off / suspended) to +# fire the next time the timer is started, which is the property we +# want for a once-a-day job: the matrix should refresh as soon as the +# VM is reachable again, not wait another 24h. +# +# `RandomizedDelaySec=10min` smears load if multiple matrix-style +# pipelines are ever colocated on the same VM in the future. + +[Unit] +Description=Run the Claude Code compatibility-matrix populator daily + +[Timer] +OnCalendar=*-*-* 06:00:00 UTC +Persistent=true +RandomizedDelaySec=10min +Unit=litellm-compat-matrix.service + +[Install] +WantedBy=timers.target diff --git a/tests/claude_code/publisher.py b/tests/claude_code/publisher.py index 0ee8bd5b4ce..4c3c40bc43b 100644 --- a/tests/claude_code/publisher.py +++ b/tests/claude_code/publisher.py @@ -1,37 +1,50 @@ -"""Daily-cron matrix publisher. +"""Daily-cron matrix publisher (GCP VM edition). -End-to-end orchestrator that runs on the isolated cron VM (per the PRD's -"Two CI environments / Daily Cron" section). The flow is: +End-to-end orchestrator that runs on the dedicated cron VM +`litellm-compatibility-matrix-populator`. The flow is: - 1. Resolve the latest LiteLLM `v*-stable` tag via `resolver.py`. - 2. Pull the corresponding Docker image and start it as the proxy. - 3. Install the absolute latest Claude Code CLI from npm. - 4. Run `pytest tests/claude_code/` against the proxy. - 5. Build `compatibility-matrix.json` from the per-test results artifact - using the Matrix JSON Builder (`matrix_builder.py`). - 6. Open (or update) a pull request against the docs repo with the JSON - change, using the GitHub App installation token mounted as - `DOCS_REPO_TOKEN`. This is intentionally a PR rather than a direct - push so docs maintainers get to review each matrix update before it - ships to readers. + 1. Resolve the latest LiteLLM ``v*-stable`` tag via ``resolver.py``. + 2. Update a long-lived git worktree of ``BerriAI/litellm`` to that tag, + run ``uv sync --frozen`` against it, and boot the proxy as a + subprocess on a non-conflicting local port. Reusing the same worktree + across runs (rather than a fresh tempdir) keeps disk footprint + bounded — ``uv sync`` removes packages no longer pinned and ``git + checkout`` mutates the same files in place. + 3. Run ``pytest tests/claude_code/`` against the proxy. The locally + installed Claude Code CLI is exercised as-is — there is no + ``npm install`` step, so the operator controls when the CLI is + upgraded by running ``npm install -g @anthropic-ai/claude-code@latest`` + out-of-band (typically baked into the VM image or a separate cron). + 4. Build ``compatibility-matrix.json`` from the per-test results + artifact using the Matrix JSON Builder (``matrix_builder.py``). + 5. Open (or update) a pull request against the docs repo with the JSON + change, using a ``gh`` CLI that has been pre-authenticated on the VM + against an account with ``pull-requests: write`` on + ``BerriAI/litellm-docs``. -The orchestration is thin glue over Docker, git, npm, gh and subprocess — -per the PRD's "Testing Decisions" section, it intentionally ships without -a unit-test harness; the daily-cron failure surface is itself the test. -The pure helpers below (commit message, image-name builder, file -allowlist, PR title/body/branch builders) are unit-tested under -`_publisher_unit_tests/`. +Why no Docker +------------- -The "only `compatibility-matrix.json` is ever committed" guarantee is -enforced by `select_files_to_commit` rather than by token scope, since -GitHub Apps cannot scope `contents: write` to a single file path. +The original design pulled ``ghcr.io/berriai/litellm:`` per run on +GitHub-hosted runners. The cron VM does not run docker — installing it +would just trade one set of moving parts (docker daemon, image pulls, +networking) for the simpler "one git checkout + one ``uv sync``" we +already use to start the proxy interactively. Removing the docker code +also halves the publisher's surface area. -Idempotency: re-runs on the same UTC day with the same resolved versions -land on the same branch (`compat-matrix/--`). If the JSON is byte-identical to the docs repo's -`main`, the script exits before pushing. If a PR is already open for the -branch, `gh pr create` no-ops with a non-fatal message; we treat that as -success. +Idempotency +----------- + +Re-runs on the same UTC day with the same resolved versions land on the +same head branch (``compat-matrix/--``). If the JSON is byte-identical to the docs repo's +target branch, the script exits before pushing. If a PR is already open +for the branch, ``gh pr create`` no-ops with a non-fatal message; we +treat that as success. + +The "only ``compatibility-matrix.json`` is ever committed" guarantee is +enforced by ``select_files_to_commit`` rather than by token scope, since +GitHub does not expose file-path-scoped tokens. """ from __future__ import annotations @@ -39,10 +52,14 @@ from __future__ import annotations import argparse import datetime import os +import shutil +import signal import subprocess import sys import tempfile import time +import urllib.error +import urllib.request from pathlib import Path from typing import Any, List, Mapping, Optional, Sequence @@ -52,16 +69,31 @@ from tests.claude_code.resolver import latest_stable_litellm_tag DOCS_REPO_DEFAULT = "BerriAI/litellm-docs" DOCS_TARGET_BASENAME = "compatibility-matrix.json" DOCS_TARGET_PATH_DEFAULT = f"static/data/{DOCS_TARGET_BASENAME}" -DOCKER_IMAGE_BASE = "ghcr.io/berriai/litellm" -DEFAULT_PROXY_PORT = 4000 -DEFAULT_PROXY_API_KEY = "sk-cron-matrix" # only used inside the ephemeral VM PR_BRANCH_PREFIX = "compat-matrix" +# Sourced from the same config the human-tended dev proxy uses, but on a +# different port so a developer running the proxy on :4000 doesn't +# collide with the cron run. +DEFAULT_PROXY_PORT = 4100 +DEFAULT_PROXY_API_KEY = "sk-cron-matrix" # the proxy never sees real auth +DEFAULT_PROXY_HEALTH_TIMEOUT_SECONDS = 90 + +# Location of a persistent litellm checkout that the cron mutates each +# run (``git checkout `` + ``uv sync``). Persisting across runs +# keeps disk usage bounded. Override via the ``LITELLM_WORKTREE`` env var +# or ``--worktree`` so the cron can target a deliberate path on the VM. +DEFAULT_WORKTREE = Path.home() / "litellm-cron-worktree" + REPO_ROOT = Path(__file__).resolve().parents[2] DEFAULT_MANIFEST = REPO_ROOT / "tests" / "claude_code" / "manifest.yaml" DEFAULT_RESULTS = REPO_ROOT / "compat-results.json" +# --------------------------------------------------------------------------- +# Pure helpers — unit-tested under _publisher_unit_tests/. +# --------------------------------------------------------------------------- + + def commit_message_for_matrix(matrix: Mapping[str, Any]) -> str: """Build a deterministic commit message for the docs-repo push. @@ -88,16 +120,15 @@ def pr_branch_name( Two cron runs that resolve to the same (litellm_version, claude_code_version, UTC date) land on the same branch and therefore - the same PR — the second push is a fast-forward update of the - existing branch and `gh pr create` no-ops. This is the idempotency - contract the PRD's "Daily Cron" section requires. + the same PR — the second push is a force-with-lease update of the + existing branch and ``gh pr create`` no-ops. Each component is required because: - - `litellm_version` distinguishes consecutive stable tags - - `claude_code_version` distinguishes a Claude-Code-only refresh - - `date_utc` lets us still produce a fresh branch when neither - upstream version moved but a maintainer manually re-ran the - workflow on a later day to recover from a transient failure + - ``litellm_version`` distinguishes consecutive stable tags + - ``claude_code_version`` distinguishes a Claude-Code-only refresh + - ``date_utc`` lets us still produce a fresh branch when neither + upstream version moved but a maintainer manually re-ran on a + later day to recover from a transient failure """ if not litellm_version: raise ValueError("litellm_version must be a non-empty string") @@ -129,7 +160,7 @@ def pr_body_for_matrix(matrix: Mapping[str, Any]) -> str: Renders one line per feature with the per-provider statuses inline, so reviewers don't have to diff the JSON to see what changed since the last refresh. The status column ordering follows the manifest - (which the matrix already reflects in `providers`), keeping the + (which the matrix already reflects in ``providers``), keeping the table stable across days. """ litellm_version = matrix.get("litellm_version", "") @@ -175,82 +206,150 @@ def pr_body_for_matrix(matrix: Mapping[str, Any]) -> str: return "\n".join(lines) -def docker_image_for_tag(tag: str) -> str: - """Return the ghcr.io image reference for a `v*-stable` tag.""" - if not tag: - raise ValueError("tag must be a non-empty string") - return f"{DOCKER_IMAGE_BASE}:{tag}" - - def select_files_to_commit( staged_paths: Sequence[str], allowed_basename: str ) -> List[str]: """Return only the paths whose basename matches the allowlist. - The cron VM's GitHub App holds `contents: write` on the entire docs - repo (GitHub does not support file-path-scoped tokens), so the - "only ship the matrix JSON" property is enforced here instead. Any - stray file in the working tree is dropped before the commit step. + Even though the docs-repo PR is reviewable, the publisher still + enforces a one-file allowlist as defence in depth — a stray file in + the working tree from a future feature must not be smuggled into the + PR by accident. """ return [p for p in staged_paths if os.path.basename(p) == allowed_basename] +# --------------------------------------------------------------------------- +# Subprocess + filesystem glue. Side-effectful, deliberately not unit-tested +# (the daily cron's failure surface is the test). +# --------------------------------------------------------------------------- + + def _now_utc_iso() -> str: - """ISO-8601 UTC timestamp with `Z` suffix, matching the v1 schema.""" + """ISO-8601 UTC timestamp with ``Z`` suffix, matching the v1 schema.""" return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +def _today_utc_date() -> str: + """Fallback UTC date string used when ``generated_at`` is missing.""" + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") + + def _run(cmd: Sequence[str], **kwargs: Any) -> subprocess.CompletedProcess: """Print + run subprocess; raise on nonzero exit unless caller opts out.""" - print("+ " + " ".join(cmd), flush=True) + print("+ " + " ".join(str(part) for part in cmd), flush=True) return subprocess.run(cmd, check=True, **kwargs) def _get_claude_code_version() -> str: - """Return the version string printed by `claude --version`.""" + """Return the version string printed by ``claude --version``.""" completed = subprocess.run( ["claude", "--version"], capture_output=True, text=True, check=True ) - # `claude --version` prints e.g. "2.1.120 (Claude Code)"; we keep the - # raw first whitespace-delimited token, which is the version. + # ``claude --version`` prints e.g. "2.1.120 (Claude Code)"; the first + # whitespace-delimited token is the version. out = (completed.stdout or "").strip() return out.split()[0] if out else "" -def _start_proxy(image: str, port: int) -> str: - """Start the LiteLLM proxy via `docker run -d`; returns the container id.""" - completed = subprocess.run( +def _ensure_worktree(worktree: Path) -> None: + """Make sure ``worktree`` is a working litellm checkout. + + On first run we ``git clone`` ``BerriAI/litellm`` into the worktree + path. On subsequent runs we reuse what's already there — the run + just needs ``git fetch && git checkout ``. + + Why a clone instead of a ``git worktree`` of the dev checkout: the + dev checkout (``~/litellm/litellm``) is where humans iterate and may + sit on uncommitted changes or arbitrary feature branches. A separate + clone keeps the cron's ``git checkout `` from disturbing that. + """ + if (worktree / ".git").exists(): + return + worktree.parent.mkdir(parents=True, exist_ok=True) + _run( [ - "docker", - "run", - "-d", - "-p", - f"{port}:4000", - "--name", - "litellm-compat-matrix-proxy", - image, - "--port", - "4000", - ], - capture_output=True, - text=True, - check=True, + "git", + "clone", + "https://github.com/BerriAI/litellm.git", + str(worktree), + ] ) - container_id = (completed.stdout or "").strip() - if not container_id: - raise RuntimeError("docker run did not return a container id") - return container_id -def _stop_proxy(container_id: str) -> None: - subprocess.run(["docker", "rm", "-f", container_id], check=False) +def _checkout_tag_in_worktree(worktree: Path, tag: str) -> None: + """Fetch and ``git checkout `` inside ``worktree``. + + Uses ``git checkout --force`` so any cruft left behind by a previous + run (e.g. ``compat-results.json``, ``__pycache__``) is wiped before + a fresh ``uv sync``. The cron has no use for that state — every run + starts from the published tag. + """ + _run(["git", "fetch", "--tags", "--force"], cwd=worktree) + _run(["git", "reset", "--hard"], cwd=worktree) + _run(["git", "clean", "-fdx", "-e", ".venv"], cwd=worktree) + _run(["git", "checkout", "--force", tag], cwd=worktree) -def _wait_for_proxy(port: int, timeout_seconds: int = 60) -> None: - """Poll the proxy's /health endpoint until it returns 200 or we time out.""" - import urllib.error - import urllib.request +def _uv_sync(worktree: Path) -> None: + """Bring the worktree's ``.venv`` in line with the checked-out tag. + ``uv sync --frozen`` is deterministic: it installs exactly what the + lockfile says and removes anything no longer referenced. That's the + "doesn't blow up storage" property the operator requires — the venv + can never grow unboundedly across runs. + """ + _run(["uv", "sync", "--frozen"], cwd=worktree) + + +def _start_proxy(worktree: Path, port: int, config_path: Path) -> subprocess.Popen: + """Start the LiteLLM proxy as a subprocess; returns the Popen handle. + + Started in its own process group so we can SIGTERM the whole tree + on shutdown — the proxy itself spawns worker subprocesses that + don't otherwise propagate signals from a parent. + """ + cmd = [ + "uv", + "run", + "litellm", + "--config", + str(config_path), + "--port", + str(port), + ] + print("+ " + " ".join(cmd), flush=True) + return subprocess.Popen( + cmd, + cwd=worktree, + stdout=sys.stdout, + stderr=sys.stderr, + start_new_session=True, + ) + + +def _stop_proxy(proc: subprocess.Popen) -> None: + """SIGTERM the proxy's process group; SIGKILL if it doesn't exit.""" + if proc.poll() is not None: + return + try: + os.killpg(proc.pid, signal.SIGTERM) + except ProcessLookupError: + return + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait(timeout=5) + + +def _wait_for_proxy( + port: int, timeout_seconds: int = DEFAULT_PROXY_HEALTH_TIMEOUT_SECONDS +) -> None: + """Poll ``/health/liveliness`` until it returns 200 or we time out.""" url = f"http://127.0.0.1:{port}/health/liveliness" deadline = time.time() + timeout_seconds last_err: Optional[BaseException] = None @@ -267,12 +366,16 @@ def _wait_for_proxy(port: int, timeout_seconds: int = 60) -> None: ) +# --------------------------------------------------------------------------- +# PR creation against the docs repo. +# --------------------------------------------------------------------------- + + def publish( *, docs_repo: str, docs_branch: str, docs_target_path: str, - docs_token: str, manifest_path: Path, results_path: Path, matrix_output_path: Path, @@ -282,17 +385,11 @@ def publish( ) -> None: """Build the matrix JSON and open a PR against the docs repo. - Only `docs_target_path` is staged from the docs-repo working tree — - any other file produced by the build is dropped via - `select_files_to_commit`. The PR base is `docs_branch` (typically - `main`); the head branch name is deterministic per - `pr_branch_name(...)` so re-runs on the same UTC day update the - same PR rather than spawning a new one. - - `docs_token` is the GitHub App installation token. It must be scoped - to `contents: write` AND `pull-requests: write` on `docs_repo`. The - workflow's `actions/create-github-app-token` step is responsible for - requesting both permissions. + Auth uses whatever ``gh auth login`` has stashed on the cron VM — + the GCP edition does not pass an explicit token. The account ``gh`` + is logged in as must be a collaborator on ``docs_repo`` with + ``pull-requests: write`` (the same permission ``gh pr create`` + needs interactively). """ matrix = build_from_paths( manifest_path=manifest_path, @@ -311,17 +408,20 @@ def publish( with tempfile.TemporaryDirectory(prefix="docs-repo-") as workdir: workdir_path = Path(workdir) - clone_url = f"https://x-access-token:{docs_token}@github.com/{docs_repo}.git" + # ``gh repo clone`` reuses the VM's ``gh auth`` state, so we + # don't need to construct an authenticated URL by hand. _run( [ - "git", + "gh", + "repo", "clone", + docs_repo, + str(workdir_path), + "--", "--depth", "1", "--branch", docs_branch, - clone_url, - str(workdir_path), ] ) _run( @@ -334,7 +434,7 @@ def publish( ) # Branch always starts from the freshly-cloned base. If the # remote branch already exists from an earlier run on the same - # day, the later `git push --force-with-lease` reconciles — + # day, the later ``git push --force-with-lease`` reconciles — # we'd rather present the latest matrix JSON than preserve a # stale intermediate state. _run(["git", "checkout", "-b", branch_name], cwd=workdir_path) @@ -343,8 +443,8 @@ def publish( target_in_docs.parent.mkdir(parents=True, exist_ok=True) target_in_docs.write_text(matrix_output_path.read_text()) - # Defense in depth: even if some other tool dropped a file in the - # working tree, only the matrix JSON is staged. + # Defence in depth: even if some other tool dropped a file in + # the working tree, only the matrix JSON is staged. staged = [docs_target_path] keep = select_files_to_commit(staged, DOCS_TARGET_BASENAME) if not keep: @@ -356,7 +456,7 @@ def publish( _run(["git", "add", path], cwd=workdir_path) # Skip the push entirely if the JSON is byte-identical to what's - # already on `docs_branch` — keeps the docs-repo PR list clean + # already on ``docs_branch`` — keeps the docs-repo PR list clean # during idempotent reruns of the cron. diff = subprocess.run( ["git", "diff", "--cached", "--quiet"], @@ -371,7 +471,7 @@ def publish( ["git", "commit", "-m", commit_message_for_matrix(matrix)], cwd=workdir_path, ) - # `--force-with-lease` so a same-day rerun updates the existing + # ``--force-with-lease`` so a same-day rerun updates the existing # branch (and therefore the existing PR) safely; the lease check # ensures we never overwrite a docs-maintainer's manual fixup # commit on the same branch. @@ -394,20 +494,9 @@ def publish( title=pr_title_for_matrix(matrix), body=pr_body_for_matrix(matrix), cwd=workdir_path, - token=docs_token, ) -def _today_utc_date() -> str: - """Fallback UTC date string used when `generated_at` is missing. - - Centralised so the branch-naming helper stays a pure function — it - refuses to inject the clock itself, which makes it trivially - unit-testable. - """ - return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") - - def _open_or_update_pr( *, docs_repo: str, @@ -416,17 +505,8 @@ def _open_or_update_pr( title: str, body: str, cwd: Path, - token: str, ) -> None: - """Open a PR via the `gh` CLI; treat 'already exists' as success. - - `gh` is preinstalled on `ubuntu-latest` runners and is also the - pattern other workflows in this repo follow (e.g. - `auto_update_price_and_context_window.yml`). We pass the GitHub App - installation token through `GH_TOKEN` so `gh` doesn't fall back to - the runner's default `GITHUB_TOKEN`, which is scoped to this repo - and would not have write access on `litellm-docs`. - """ + """Open a PR via the ``gh`` CLI; treat 'already exists' as success.""" completed = subprocess.run( [ "gh", @@ -444,7 +524,6 @@ def _open_or_update_pr( body, ], cwd=cwd, - env={**os.environ, "GH_TOKEN": token}, capture_output=True, text=True, check=False, @@ -453,7 +532,7 @@ def _open_or_update_pr( print(completed.stdout.strip(), flush=True) return stderr = completed.stderr or "" - # `gh pr create` exits non-zero when a PR already exists for the + # ``gh pr create`` exits non-zero when a PR already exists for the # head branch. That's the idempotent re-run path and not an error: # the branch was already force-pushed above, so the existing PR now # carries the freshest matrix JSON. @@ -467,6 +546,11 @@ def _open_or_update_pr( raise RuntimeError(f"gh pr create failed with exit code {completed.returncode}") +# --------------------------------------------------------------------------- +# Top-level orchestrator. +# --------------------------------------------------------------------------- + + def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -477,13 +561,22 @@ def main(argv: Optional[Sequence[str]] = None) -> int: parser.add_argument( "--docs-branch", default=os.environ.get("DOCS_BRANCH", "main"), - help="Branch on the docs repo to push to.", + help="Base branch on the docs repo to PR against.", ) parser.add_argument( "--docs-target-path", default=os.environ.get("DOCS_TARGET_PATH", DOCS_TARGET_PATH_DEFAULT), help="Path inside the docs repo where the matrix JSON lives.", ) + parser.add_argument( + "--worktree", + type=Path, + default=Path(os.environ.get("LITELLM_WORKTREE", str(DEFAULT_WORKTREE))), + help=( + "Persistent litellm checkout the cron mutates each run " + f"(default: {DEFAULT_WORKTREE})." + ), + ) parser.add_argument( "--proxy-port", type=int, @@ -493,6 +586,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: "--manifest", type=Path, default=DEFAULT_MANIFEST, + help="Manifest the matrix JSON is built against.", ) parser.add_argument( "--results", @@ -505,78 +599,76 @@ def main(argv: Optional[Sequence[str]] = None) -> int: type=Path, default=REPO_ROOT / DOCS_TARGET_BASENAME, ) - parser.add_argument( - "--skip-proxy", - action="store_true", - help=( - "Skip Docker/proxy/CLI/pytest steps and go straight to publish — " - "useful when the workflow runs those steps in separate jobs." - ), - ) parser.add_argument( "--skip-publish", action="store_true", - help="Run the test pipeline but do not push to the docs repo.", + help="Run the test pipeline but do not open a PR.", ) args = parser.parse_args(argv) - docs_token = os.environ.get("DOCS_REPO_TOKEN", "") - if not args.skip_publish and not docs_token: - print( - "DOCS_REPO_TOKEN is required to push to the docs repo " - "(GitHub App installation token)", - file=sys.stderr, - ) - return 2 - - container_id: Optional[str] = None - litellm_version: str - claude_code_version: str + proxy_proc: Optional[subprocess.Popen] = None try: litellm_version = latest_stable_litellm_tag( token=os.environ.get("GITHUB_TOKEN") ) print(f"resolved latest stable litellm: {litellm_version}", flush=True) - if not args.skip_proxy: - image = docker_image_for_tag(litellm_version) - _run(["docker", "pull", image]) - container_id = _start_proxy(image, args.proxy_port) - _wait_for_proxy(args.proxy_port) - - _run(["npm", "install", "-g", "@anthropic-ai/claude-code@latest"]) - claude_code_version = _get_claude_code_version() - print(f"installed claude code cli: {claude_code_version}", flush=True) - - env = { - **os.environ, - "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{args.proxy_port}", - "ANTHROPIC_AUTH_TOKEN": DEFAULT_PROXY_API_KEY, - "COMPAT_RESULTS_PATH": str(args.results), - } - _run( - [ - "pytest", - "tests/claude_code/", - "--ignore=tests/claude_code/_driver_unit_tests", - "--ignore=tests/claude_code/_builder_unit_tests", - "--ignore=tests/claude_code/_publisher_unit_tests", - ], - env=env, - check=False, + claude_code_version = _get_claude_code_version() + if not claude_code_version: + print( + "could not read 'claude --version'; is the CLI installed?", + file=sys.stderr, ) - else: - claude_code_version = os.environ.get("CLAUDE_CODE_VERSION", "") + return 2 + print(f"local claude code cli: {claude_code_version}", flush=True) + + _ensure_worktree(args.worktree) + _checkout_tag_in_worktree(args.worktree, litellm_version) + _uv_sync(args.worktree) + + config_path = args.worktree / "tests" / "claude_code" / "test_config.yaml" + if not config_path.exists(): + raise RuntimeError( + f"proxy config not found at {config_path}; the resolved tag " + f"{litellm_version} may predate the compat matrix work" + ) + + proxy_proc = _start_proxy(args.worktree, args.proxy_port, config_path) + _wait_for_proxy(args.proxy_port) + + env = { + **os.environ, + "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{args.proxy_port}", + "ANTHROPIC_AUTH_TOKEN": DEFAULT_PROXY_API_KEY, + "COMPAT_RESULTS_PATH": str(args.results), + } + # Run pytest from inside the worktree so it picks up the + # checked-out tag's test code (and its conftest hook), not the + # current process's working directory. + subprocess.run( + [ + "uv", + "run", + "pytest", + "tests/claude_code/", + "--ignore=tests/claude_code/_driver_unit_tests", + "--ignore=tests/claude_code/_builder_unit_tests", + "--ignore=tests/claude_code/_publisher_unit_tests", + "--ignore=tests/claude_code/_pr_gate_unit_tests", + ], + env=env, + cwd=args.worktree, + check=False, + ) if args.skip_publish: - print("skip-publish: not pushing to docs repo", flush=True) + print("skip-publish: not opening a PR", flush=True) return 0 publish( docs_repo=args.docs_repo, docs_branch=args.docs_branch, docs_target_path=args.docs_target_path, - docs_token=docs_token, manifest_path=args.manifest, results_path=args.results, matrix_output_path=args.matrix_output, @@ -586,8 +678,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ) return 0 finally: - if container_id is not None: - _stop_proxy(container_id) + if proxy_proc is not None: + _stop_proxy(proxy_proc) if __name__ == "__main__": @@ -598,10 +690,11 @@ __all__ = [ "DOCS_REPO_DEFAULT", "DOCS_TARGET_BASENAME", "DOCS_TARGET_PATH_DEFAULT", - "DOCKER_IMAGE_BASE", + "DEFAULT_PROXY_PORT", + "DEFAULT_PROXY_API_KEY", + "DEFAULT_WORKTREE", "PR_BRANCH_PREFIX", "commit_message_for_matrix", - "docker_image_for_tag", "select_files_to_commit", "pr_branch_name", "pr_title_for_matrix",