diff --git a/tests/claude_code/_publisher_unit_tests/__init__.py b/tests/claude_code/_publisher_unit_tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/claude_code/_publisher_unit_tests/test_publisher.py b/tests/claude_code/_publisher_unit_tests/test_publisher.py deleted file mode 100644 index d1116ded85e..00000000000 --- a/tests/claude_code/_publisher_unit_tests/test_publisher.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Unit tests for the daily-cron matrix publisher. - -The publisher orchestrates git, uv, gh and pytest on a dedicated GCP VM — -the orchestration itself is too thin to warrant heavy mocking. These -tests cover the small pure helpers that do warrant test coverage: - -- ``commit_message_for_matrix``: deterministic commit message containing - the LiteLLM and Claude Code versions plus ``generated_at``, so the - docs repo's git log shows what produced each push. -- ``select_files_to_commit``: enforces the "only - ``compatibility-matrix.json`` is pushed" guarantee. Even with PR - review in front of the docs branch, the publisher refuses to stage - any other file as defence in depth. -- ``pr_branch_name``: deterministic head-branch name for the docs PR. - Two cron runs on the same UTC day with the same resolved versions - must collide on this branch so the second run updates the existing - PR rather than spawning a new one. -- ``pr_title_for_matrix`` / ``pr_body_for_matrix``: the strings the - publisher hands to ``gh pr create``. Tested for content (not formatting - trivia) to keep the tests resilient to copy edits. -""" - -from __future__ import annotations - -import pytest - -from tests.claude_code.publisher import ( - DOCS_TARGET_BASENAME, - PR_BRANCH_PREFIX, - commit_message_for_matrix, - pr_body_for_matrix, - pr_branch_name, - pr_title_for_matrix, - select_files_to_commit, -) - - -def test_commit_message_includes_versions_and_timestamp(): - matrix = { - "schema_version": "1", - "generated_at": "2026-04-25T06:00:00Z", - "litellm_version": "v1.83.0-stable", - "claude_code_version": "2.1.120", - "providers": ["anthropic"], - "features": [], - } - message = commit_message_for_matrix(matrix) - # Headline is short and identifies the artifact. - headline, _, body = message.partition("\n") - assert "compatibility matrix" in headline.lower() - # Body must surface the three pieces of provenance the docs banner shows. - assert "v1.83.0-stable" in body - assert "2.1.120" in body - assert "2026-04-25T06:00:00Z" in body - - -def test_commit_message_is_deterministic(): - """Same matrix in → same message out (no clock, no randomness).""" - matrix = { - "litellm_version": "v1.83.0-stable", - "claude_code_version": "2.1.120", - "generated_at": "2026-04-25T06:00:00Z", - } - assert commit_message_for_matrix(matrix) == commit_message_for_matrix(matrix) - - -def test_select_files_to_commit_drops_anything_other_than_matrix(): - """The script's safety net: even if pytest leaves stray artifacts in - the docs-repo checkout, only `compatibility-matrix.json` ever ships. - - Acceptance criterion: "the script does not write any other files". - """ - staged = [ - "static/data/compatibility-matrix.json", - "static/data/notes.txt", - ".github/workflows/secret.yml", - "compat-results.json", - ] - assert select_files_to_commit(staged, DOCS_TARGET_BASENAME) == [ - "static/data/compatibility-matrix.json" - ] - - -def test_select_files_to_commit_returns_empty_when_nothing_matches(): - assert select_files_to_commit(["other.json"], DOCS_TARGET_BASENAME) == [] - - -def test_select_files_to_commit_basename_match_not_substring(): - """`compatibility-matrix.json.bak` must not be treated as the allowed file.""" - assert ( - select_files_to_commit( - ["static/data/compatibility-matrix.json.bak"], DOCS_TARGET_BASENAME - ) - == [] - ) - - -# --------------------------------------------------------------------------- -# PR-creation helpers -# --------------------------------------------------------------------------- - - -def test_pr_branch_name_is_deterministic_per_inputs(): - """Same (litellm, claude, date) -> same branch -> same PR. - - This is the idempotency contract for re-runs on the same UTC day: - the second push lands on the existing branch, `gh pr create` no-ops - because the PR already exists, and the docs-repo PR list stays - clean. - """ - a = pr_branch_name( - litellm_version="v1.83.0-stable", - claude_code_version="2.1.120", - date_utc="2026-04-25", - ) - b = pr_branch_name( - litellm_version="v1.83.0-stable", - claude_code_version="2.1.120", - date_utc="2026-04-25", - ) - assert a == b - assert a.startswith(f"{PR_BRANCH_PREFIX}/") - # All three components must appear so a maintainer can read the - # provenance off the branch name without opening the PR body. - assert "v1.83.0-stable" in a - assert "2.1.120" in a - assert "2026-04-25" in a - - -def test_pr_branch_name_changes_when_any_component_changes(): - """A bump in any component must produce a fresh branch (and thus PR).""" - base = pr_branch_name( - litellm_version="v1.83.0-stable", - claude_code_version="2.1.120", - date_utc="2026-04-25", - ) - new_litellm = pr_branch_name( - litellm_version="v1.84.0-stable", - claude_code_version="2.1.120", - date_utc="2026-04-25", - ) - new_claude = pr_branch_name( - litellm_version="v1.83.0-stable", - claude_code_version="2.1.121", - date_utc="2026-04-25", - ) - new_date = pr_branch_name( - litellm_version="v1.83.0-stable", - claude_code_version="2.1.120", - date_utc="2026-04-26", - ) - assert len({base, new_litellm, new_claude, new_date}) == 4 - - -@pytest.mark.parametrize( - "kwargs", - [ - { - "litellm_version": "", - "claude_code_version": "2.1.120", - "date_utc": "2026-04-25", - }, - { - "litellm_version": "v1.83.0-stable", - "claude_code_version": "", - "date_utc": "2026-04-25", - }, - { - "litellm_version": "v1.83.0-stable", - "claude_code_version": "2.1.120", - "date_utc": "", - }, - ], -) -def test_pr_branch_name_rejects_empty_components(kwargs): - """Empty inputs would silently collapse two distinct PRs onto one branch.""" - with pytest.raises(ValueError): - pr_branch_name(**kwargs) - - -def test_pr_title_includes_versions_inline(): - """The title is what shows up in notification emails / the PR list, - so the LiteLLM and Claude Code versions must be inline.""" - matrix = { - "litellm_version": "v1.83.0-stable", - "claude_code_version": "2.1.120", - "generated_at": "2026-04-25T06:00:00Z", - } - title = pr_title_for_matrix(matrix) - assert "v1.83.0-stable" in title - assert "2.1.120" in title - # Single-line — newlines in a PR title would render as a literal `\n`. - assert "\n" not in title - - -def test_pr_body_renders_per_feature_status_table(): - """Reviewers triage the PR off the body without opening the diff, - so each feature must list its per-provider status, in manifest order.""" - matrix = { - "litellm_version": "v1.83.0-stable", - "claude_code_version": "2.1.120", - "generated_at": "2026-04-25T06:00:00Z", - "providers": ["anthropic", "bedrock_invoke"], - "features": [ - { - "id": "basic_messaging_non_streaming", - "name": "Basic messaging (non-streaming)", - "providers": { - "anthropic": {"status": "pass"}, - "bedrock_invoke": {"status": "fail", "error": "boom"}, - }, - }, - { - "id": "tool_use", - "name": "Tool use", - "providers": { - "anthropic": {"status": "pass"}, - "bedrock_invoke": { - "status": "not_applicable", - "reason": "tier mismatch", - }, - }, - }, - ], - } - body = pr_body_for_matrix(matrix) - # Provenance header. - assert "v1.83.0-stable" in body - assert "2.1.120" in body - assert "2026-04-25T06:00:00Z" in body - # Per-feature lines surface the names and statuses inline. - assert "Basic messaging (non-streaming)" in body - assert "anthropic=pass" in body - assert "bedrock_invoke=fail" in body - assert "Tool use" in body - assert "bedrock_invoke=not_applicable" in body - # Provider order in each row follows the `providers` list, not dict - # iteration of `.providers` — so a renamed provider in the matrix - # without a manifest update would visibly be absent rather than - # silently drift. - feature_line = next( - line for line in body.splitlines() if line.startswith("- **Basic") - ) - assert feature_line.index("anthropic=") < feature_line.index("bedrock_invoke=") - - -def test_pr_body_handles_missing_providers_in_a_feature(): - """A feature that didn't run on a manifest provider must still appear - in that column — as `not_tested` — so the table stays rectangular.""" - matrix = { - "litellm_version": "v1.83.0-stable", - "claude_code_version": "2.1.120", - "generated_at": "2026-04-25T06:00:00Z", - "providers": ["anthropic", "bedrock_invoke"], - "features": [ - { - "id": "vision", - "name": "Vision", - "providers": {"anthropic": {"status": "pass"}}, - } - ], - } - body = pr_body_for_matrix(matrix) - assert "anthropic=pass" in body - assert "bedrock_invoke=not_tested" in body diff --git a/tests/claude_code/_publisher_unit_tests/test_resolver.py b/tests/claude_code/_publisher_unit_tests/test_resolver.py deleted file mode 100644 index 91a89c9b271..00000000000 --- a/tests/claude_code/_publisher_unit_tests/test_resolver.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Unit tests for the Latest Stable LiteLLM Resolver. - -The resolver is the smallest piece of cron-pipeline glue: it asks the GitHub -Releases API for `BerriAI/litellm` and returns the newest tag matching the -`v*-stable` pattern. Tests inject a fake HTTP getter so they run offline. - -Per the PRD's "Testing Decisions" section, version resolvers were officially -deferred from v0 — but the resolver is also small enough that a few cheap -unit tests are worth more than the daily-cron failing loudly. -""" - -from __future__ import annotations - -import json -from typing import List - -import pytest - -from tests.claude_code.resolver import ( - GITHUB_RELEASES_URL, - ResolverError, - latest_stable_litellm_tag, -) - - -def _fake_getter(payload): - """Return a callable that records calls and returns the given JSON payload. - - Mirrors the `runner=` injection seam used by `cli_driver` tests so the - unit tests don't depend on `urllib`. - """ - - captured: List[dict] = [] - - def getter(url, *, token=None): - captured.append({"url": url, "token": token}) - return json.dumps(payload) - - getter.captured = captured # type: ignore[attr-defined] - return getter - - -def test_resolver_returns_newest_stable_tag(): - """Given a list of releases, the newest `v*-stable` wins.""" - getter = _fake_getter( - [ - {"tag_name": "v1.81.0-stable"}, - {"tag_name": "v1.83.0-stable"}, - {"tag_name": "v1.82.5-stable"}, - ] - ) - assert latest_stable_litellm_tag(http_get=getter) == "v1.83.0-stable" - - -def test_resolver_ignores_non_stable_tags(): - """RC, alpha, beta, plain `v1.83.0`, and noise tags are filtered out.""" - getter = _fake_getter( - [ - {"tag_name": "v1.84.0-rc1"}, - {"tag_name": "v1.84.0-stable.draft"}, - {"tag_name": "v1.84.0"}, - {"tag_name": "v1.83.0-stable"}, - {"tag_name": "stable-2026-04-25"}, - {"tag_name": "v1.85.0-alpha"}, - ] - ) - assert latest_stable_litellm_tag(http_get=getter) == "v1.83.0-stable" - - -def test_resolver_uses_numeric_version_sort_not_lexicographic(): - """v1.10.0-stable must be newer than v1.9.0-stable (numeric, not string).""" - getter = _fake_getter( - [ - {"tag_name": "v1.9.0-stable"}, - {"tag_name": "v1.10.0-stable"}, - {"tag_name": "v1.9.5-stable"}, - ] - ) - assert latest_stable_litellm_tag(http_get=getter) == "v1.10.0-stable" - - -def test_resolver_raises_when_no_stable_tags(): - getter = _fake_getter([{"tag_name": "v1.84.0-rc1"}, {"tag_name": "v1.83.0"}]) - with pytest.raises(ResolverError, match="no v\\*-stable tags found"): - latest_stable_litellm_tag(http_get=getter) - - -def test_resolver_raises_when_response_is_not_a_list(): - getter = _fake_getter({"message": "API rate limit exceeded"}) - with pytest.raises(ResolverError, match="not a list"): - latest_stable_litellm_tag(http_get=getter) - - -def test_resolver_calls_github_releases_endpoint_with_optional_token(): - """The resolver must call the BerriAI/litellm releases API and forward - the auth token (if provided) to the http getter so callers can lift - the unauthenticated rate limit when running on the cron VM.""" - getter = _fake_getter([{"tag_name": "v1.83.0-stable"}]) - latest_stable_litellm_tag(http_get=getter, token="ghs_xxx") - assert getter.captured == [{"url": GITHUB_RELEASES_URL, "token": "ghs_xxx"}] - - -def test_resolver_skips_releases_with_missing_tag_name(): - """Defensive: GitHub draft releases can omit `tag_name`; don't crash.""" - getter = _fake_getter( - [ - {"name": "untagged draft"}, - {"tag_name": None}, - {"tag_name": "v1.83.0-stable"}, - ] - ) - assert latest_stable_litellm_tag(http_get=getter) == "v1.83.0-stable" diff --git a/tests/claude_code/cron_vm/README.md b/tests/claude_code/cron_vm/README.md index d4f0fd1a618..7c05be00cf2 100644 --- a/tests/claude_code/cron_vm/README.md +++ b/tests/claude_code/cron_vm/README.md @@ -4,13 +4,13 @@ The populator runs daily on a dedicated GCP VM (`litellm-compatibility-matrix-populator`) rather than as a GitHub Action. Trade-offs: -- ✅ Real VM means we can `gh auth login` against a human/bot account - that's already a collaborator on `BerriAI/litellm-docs`, instead of +- ✅ Real VM means we can `gh auth login` against an account that's + already a collaborator on `BerriAI/litellm-docs`, instead of provisioning a GitHub App with `pull-requests: write`. - ✅ Persistent state (a single `~/litellm-cron-worktree/` and its `.venv`) is reused across runs, so each daily run does a fast `git checkout` + incremental `uv sync` rather than a fresh clone + cold sync. -- ✅ No Docker dependency — proxy is run directly via `uv run litellm`. +- ✅ No Docker dependency — the proxy runs directly via `uv run litellm`. - ⚠️ The VM has to actually be on. systemd's `Persistent=true` recovers from short outages, but a multi-day outage means the matrix goes stale until the VM is back. @@ -18,34 +18,44 @@ Action. Trade-offs: (`/etc/litellm-compat-matrix.env`) instead of GitHub secrets. Treat the VM as an environment with comparable blast radius to a CI runner. -## What the populator does, end to end +## Layout -`tests/claude_code/publisher.py` (`python -m tests.claude_code.publisher`): +| File | Purpose | +| --- | --- | +| `run_daily.sh` | The actual cron job. Resolves versions, updates the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR. | +| `build_matrix.py` | Tiny Python CLI that wraps `tests.claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. | +| `litellm-compat-matrix.service` | systemd oneshot that invokes `run_daily.sh`. | +| `litellm-compat-matrix.timer` | `OnCalendar=*-*-* 06:00:00 UTC`, `Persistent=true`. | +| `litellm-compat-matrix.env.example` | Template for `/etc/litellm-compat-matrix.env`. | -1. Resolves the latest `v*-stable` tag of `BerriAI/litellm` via the - GitHub Releases API (`tests/claude_code/resolver.py`). -2. Reads the locally installed Claude Code CLI version - (`claude --version`). -3. Updates the persistent worktree at `~/litellm-cron-worktree/` to - that tag, and `uv sync --frozen`s its `.venv`. `git clean -fdx -e .venv` - wipes any cruft from previous runs while keeping the venv around. -4. Boots the LiteLLM proxy as a subprocess on port `4100` (override - with `PROXY_PORT`), using - `tests/claude_code/test_config.yaml` from the checked-out tag. -5. Runs `pytest tests/claude_code/` with `ANTHROPIC_BASE_URL` pointed - at the proxy and `COMPAT_RESULTS_PATH` set so the conftest hook - writes the per-test results artifact. -6. Builds `compatibility-matrix.json` from the artifact via - `tests.claude_code.matrix_builder.build_from_paths`. -7. Clones the docs repo (`gh repo clone BerriAI/litellm-docs`) into a - temp dir, checks out a deterministic head branch +## What `run_daily.sh` does + +1. **Resolves the latest LiteLLM `v*-stable` tag** by hitting the + GitHub Releases API (`curl | jq`). +2. **Reads the local Claude Code CLI version** via `claude --version`. + The cron does not auto-upgrade the CLI — operators do that + out-of-band by running `npm install -g @anthropic-ai/claude-code@latest`. +3. **Updates the persistent worktree** at `~/litellm-cron-worktree/`: + `git fetch --tags --force`, `git reset --hard`, + `git clean -fdx -e .venv`, `git checkout --force `. The + `.venv` is preserved across runs so `uv sync --frozen` is + incremental. +4. **Boots the proxy** as a `setsid` background process on port `4100` + (so it can't collide with a developer's `:4000`), then polls + `/health/liveliness` until it's up. +5. **Runs pytest** with `ANTHROPIC_BASE_URL` pointed at the proxy and + `COMPAT_RESULTS_PATH` set so the conftest hook writes the per-test + results artifact. Test failures become `fail` cells in the JSON, + not script errors. +6. **Builds `compatibility-matrix.json`** by handing the artifact + + manifest to `build_matrix.py`. +7. **Opens or updates a docs PR**: `gh repo clone` of `litellm-docs` + into a tempdir, deterministic head branch (`compat-matrix/--`), - commits the JSON, force-with-lease pushes, and opens a PR with - `gh pr create`. - -Re-running on the same day with the same versions is idempotent: the -branch name collides, the force-with-lease updates the existing branch, -and `gh pr create` no-ops because the PR already exists. + `--force-with-lease` push, `gh pr create`. A re-run on the same + day fast-forwards the existing branch and `gh pr create` no-ops + ("a pull request for branch ... already exists" is treated as + success). ## One-time VM setup @@ -54,7 +64,7 @@ Run as `mateo` on the cron VM: ```bash # 1. Toolchain sudo apt-get update -sudo apt-get install -y git nodejs npm +sudo apt-get install -y git nodejs npm jq curl curl -LsSf https://astral.sh/uv/install.sh | sh sudo apt-get install -y gh # or follow https://cli.github.com/ @@ -62,25 +72,24 @@ sudo apt-get install -y gh # or follow https://cli.github.com/ # line out-of-band when you want a fresh CLI to be tested) sudo npm install -g @anthropic-ai/claude-code@latest -# 3. Litellm dev checkout. Used as the launcher for the publisher -# module; the populator mutates a separate worktree under -# ~/litellm-cron-worktree/. +# 3. Litellm checkout. Used by systemd's WorkingDirectory and as the +# source of the .service / .timer files. The cron itself runs out +# of the separate worktree at ~/litellm-cron-worktree/. mkdir -p ~/litellm git clone https://github.com/BerriAI/litellm.git ~/litellm/litellm -cd ~/litellm/litellm && uv sync --frozen # 4. gh auth — must be a collaborator on BerriAI/litellm-docs. gh auth login # follow prompts; pick HTTPS + token paste flow # 5. Provider credentials. -sudo cp tests/claude_code/cron_vm/litellm-compat-matrix.env.example \ +sudo cp ~/litellm/litellm/tests/claude_code/cron_vm/litellm-compat-matrix.env.example \ /etc/litellm-compat-matrix.env sudoedit /etc/litellm-compat-matrix.env # fill in real values sudo chmod 0600 /etc/litellm-compat-matrix.env # 6. systemd units. -sudo cp tests/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ -sudo cp tests/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ +sudo cp ~/litellm/litellm/tests/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ +sudo cp ~/litellm/litellm/tests/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ sudo systemctl daemon-reload sudo systemctl enable --now litellm-compat-matrix.timer ``` @@ -91,12 +100,15 @@ sudo systemctl enable --now litellm-compat-matrix.timer # When does it run next? systemctl list-timers litellm-compat-matrix.timer -# Trigger a run right now (still PRs to litellm-docs). +# Trigger a real run right now (PRs to litellm-docs). sudo systemctl start litellm-compat-matrix.service # Trigger a run that does NOT open a PR (good for first-time validation). -cd ~/litellm/litellm -uv run python -m tests.claude_code.publisher --skip-publish +SKIP_PUBLISH=1 ~/litellm/litellm/tests/claude_code/cron_vm/run_daily.sh + +# Narrow to one cell while debugging. +SKIP_PUBLISH=1 PYTEST_K='basic_messaging_non_streaming and anthropic' \ + ~/litellm/litellm/tests/claude_code/cron_vm/run_daily.sh # Watch the most recent run. journalctl -u litellm-compat-matrix.service -f @@ -112,15 +124,18 @@ sudo systemctl disable --now litellm-compat-matrix.timer - **The proxy port is `4100`, not `4000`.** This is so a developer SSH'd into the same VM with their own `:4000` proxy doesn't collide with a - cron run. Override with `PROXY_PORT=...` in - `/etc/litellm-compat-matrix.env` if you need to. + cron run. Override with `PROXY_PORT=...` in `/etc/litellm-compat-matrix.env` + if you need to. - **`uv sync --frozen` requires the resolved tag to be tagged on GitHub.** If the latest stable release was made but not pushed as a - git tag, the run will `git checkout` fail. Push the tag, then rerun. + git tag, the `git checkout` step fails. Push the tag, then rerun. - **`gh auth` token rotation is your problem.** The cron does not refresh the token; if the bot account's PAT expires the run will fail at `gh repo clone` with a 401. Re-run `gh auth login`. - **First run after upgrading the Claude Code CLI is the riskiest one.** If the new CLI changes its wire format the matrix run can produce - systematic failures. Always run `--skip-publish` after a CLI upgrade - to inspect the JSON before the next scheduled fire. + systematic failures. Always run with `SKIP_PUBLISH=1` after a CLI + upgrade before letting the next scheduled fire happen. +- **Disk:** the worktree's `.venv` is ~1.3 GB and the `.git` directory + is ~1 GB. Plan for at least 5 GB free on the VM, otherwise + `uv sync` will fail mid-run and leave you with a half-installed venv. diff --git a/tests/claude_code/cron_vm/build_matrix.py b/tests/claude_code/cron_vm/build_matrix.py new file mode 100644 index 00000000000..3df0d574ea8 --- /dev/null +++ b/tests/claude_code/cron_vm/build_matrix.py @@ -0,0 +1,48 @@ +"""Tiny CLI wrapper around `tests.claude_code.matrix_builder.build_from_paths`. + +Exists only so `run_daily.sh` can hand the version metadata + paths into +the matrix builder without re-implementing it in bash. All real logic +lives in `matrix_builder.py`, which has its own unit tests under +`_builder_unit_tests/`. + +Invoked from the cron worktree (where `uv sync` has installed pyyaml), +not the dev checkout — the bash script `cd`s into the worktree before +`uv run python`-ing this file. +""" + +from __future__ import annotations + +import argparse +import datetime +import sys +from pathlib import Path + +from tests.claude_code.matrix_builder import build_from_paths + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--results", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--litellm-version", required=True) + parser.add_argument("--claude-code-version", required=True) + args = parser.parse_args() + + generated_at = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + build_from_paths( + manifest_path=args.manifest, + results_path=args.results, + litellm_version=args.litellm_version, + claude_code_version=args.claude_code_version, + generated_at=generated_at, + output_path=args.output, + ) + print(f"wrote {args.output}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/claude_code/cron_vm/litellm-compat-matrix.service b/tests/claude_code/cron_vm/litellm-compat-matrix.service index 43d8a8326bf..abbd694c17e 100644 --- a/tests/claude_code/cron_vm/litellm-compat-matrix.service +++ b/tests/claude_code/cron_vm/litellm-compat-matrix.service @@ -41,14 +41,9 @@ Group=mateo # is the standard `KEY=value` one line per env var. EnvironmentFile=-/etc/litellm-compat-matrix.env -# Run from the dev checkout so `python -m tests.claude_code.publisher` -# resolves; the publisher itself manages the separate worktree it -# mutates per run. WorkingDirectory=%h/litellm/litellm -# `uv run --frozen` reuses the dev checkout's venv. The publisher then -# bootstraps its own worktree + venv for the proxy. -ExecStart=/usr/bin/env -S uv run --frozen python -m tests.claude_code.publisher +ExecStart=%h/litellm/litellm/tests/claude_code/cron_vm/run_daily.sh # 90 minutes is generous: cold runs do `git clone` + `uv sync` of a new # tag's lockfile, which can take a couple of minutes on a 2-vCPU VM, diff --git a/tests/claude_code/cron_vm/run_daily.sh b/tests/claude_code/cron_vm/run_daily.sh new file mode 100755 index 00000000000..3b53507ae7f --- /dev/null +++ b/tests/claude_code/cron_vm/run_daily.sh @@ -0,0 +1,297 @@ +#!/usr/bin/env bash +# Daily Claude Code compatibility-matrix populator. +# +# Runs from the GCP VM `litellm-compatibility-matrix-populator` via the +# systemd timer in this directory. The flow is: +# +# 1. Resolve the latest LiteLLM v*-stable tag from the GitHub Releases API. +# 2. Update a long-lived worktree at $WORKTREE to that tag and `uv sync` it. +# 3. Boot the proxy as a background subprocess on $PROXY_PORT (default +# 4100; a separate port from the human-tended :4000 proxy). +# 4. Run `pytest tests/claude_code/` against the proxy. Test failures +# become `fail` cells in the JSON, not script errors. +# 5. Hand the per-test results artifact + manifest to a small Python +# CLI (`build_matrix.py`) that wraps the existing +# `matrix_builder.build_from_paths` to produce the published +# compatibility-matrix.json. +# 6. `gh repo clone` litellm-docs, write the JSON to a deterministic +# branch (`compat-matrix/--`), commit, +# `git push --force-with-lease`, and `gh pr create`. +# +# Same-day reruns land on the same branch so they update the existing PR +# rather than spawning a new one. If the JSON is byte-identical to the +# docs branch, we skip the push entirely. +# +# Required commands on $PATH: git, uv, gh, jq, curl, claude, npm. +# Required state: ~/litellm/litellm checked out (this file lives in it), +# $WORKTREE is created on first run, gh is already authenticated. +# +# Override any default by setting the matching env var; see the systemd +# unit for the production wiring. + +set -Eeuo pipefail + +LITELLM_REPO="${LITELLM_REPO:-${HOME}/litellm/litellm}" +WORKTREE="${LITELLM_WORKTREE:-${HOME}/litellm-cron-worktree}" +PROXY_PORT="${PROXY_PORT:-4100}" +PROXY_API_KEY="${PROXY_API_KEY:-sk-cron-matrix}" +DOCS_REPO="${DOCS_REPO:-BerriAI/litellm-docs}" +DOCS_BRANCH="${DOCS_BRANCH:-main}" +DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}" +SKIP_PUBLISH="${SKIP_PUBLISH:-0}" +PYTEST_K="${PYTEST_K:-}" + +POPULATOR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKDIR="$(mktemp -d -t litellm-compat-matrix.XXXXXX)" +PROXY_PID="" + +cleanup() { + local rc=$? + set +e + if [[ -n "${PROXY_PID}" ]] && kill -0 "${PROXY_PID}" 2>/dev/null; then + # Negative pid = process group; the proxy spawns workers that don't + # forward signals from a parent. + kill -TERM "-${PROXY_PID}" 2>/dev/null || true + for _ in 1 2 3 4 5; do + kill -0 "${PROXY_PID}" 2>/dev/null || break + sleep 1 + done + kill -KILL "-${PROXY_PID}" 2>/dev/null || true + fi + rm -rf "${WORKDIR}" + exit "${rc}" +} +trap cleanup EXIT + +log() { printf '==> %s\n' "$*" >&2; } +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +for cmd in git uv gh jq curl claude; do + command -v "${cmd}" >/dev/null 2>&1 || die "missing required command: ${cmd}" +done + +# --------------------------------------------------------------------------- +# 1. Resolve versions +# --------------------------------------------------------------------------- + +# Newest v*-stable release on BerriAI/litellm. The `select(...)` filter +# drops drafts/non-stable, the version_key sort handles 1.10 > 1.9. +GH_AUTH_HEADER=() +if [[ -n "${GITHUB_TOKEN:-}" ]]; then + GH_AUTH_HEADER=(-H "Authorization: Bearer ${GITHUB_TOKEN}") +fi +LITELLM_VERSION="$( + curl -fsS \ + -H 'Accept: application/vnd.github+json' \ + -H 'User-Agent: litellm-compat-matrix' \ + "${GH_AUTH_HEADER[@]}" \ + https://api.github.com/repos/BerriAI/litellm/releases \ + | jq -r ' + [ .[] | .tag_name // empty + | select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+-stable$")) + ] + | sort_by( + capture("^v(?[0-9]+)\\.(?[0-9]+)\\.(?[0-9]+)-stable$") + | [(.a|tonumber), (.b|tonumber), (.c|tonumber)] + ) + | last // empty + ' +)" +[[ -n "${LITELLM_VERSION}" ]] || die "could not resolve latest v*-stable tag" +log "resolved litellm: ${LITELLM_VERSION}" + +CLAUDE_CODE_VERSION="$(claude --version 2>/dev/null | awk '{print $1}')" +[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "could not read 'claude --version'" +log "local claude code: ${CLAUDE_CODE_VERSION}" + +# --------------------------------------------------------------------------- +# 2. Update the worktree to that tag +# --------------------------------------------------------------------------- + +if [[ ! -d "${WORKTREE}/.git" ]]; then + log "first run: cloning litellm into ${WORKTREE}" + mkdir -p "$(dirname "${WORKTREE}")" + git clone https://github.com/BerriAI/litellm.git "${WORKTREE}" +fi + +log "updating worktree to ${LITELLM_VERSION}" +git -C "${WORKTREE}" fetch --tags --force +git -C "${WORKTREE}" reset --hard +# Keep the venv around — uv sync will reconcile it. Drop everything else +# (compat-results.json, __pycache__, etc.) so each run starts clean. +git -C "${WORKTREE}" clean -fdx -e .venv +git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}" + +log "uv sync --frozen" +(cd "${WORKTREE}" && uv sync --frozen) + +PROXY_CONFIG="${WORKTREE}/tests/claude_code/test_config.yaml" +[[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (does ${LITELLM_VERSION} predate the compat matrix work?)" + +# --------------------------------------------------------------------------- +# 3. Boot the proxy +# --------------------------------------------------------------------------- + +log "starting proxy on :${PROXY_PORT}" +( + cd "${WORKTREE}" \ + && setsid uv run litellm --config "${PROXY_CONFIG}" --port "${PROXY_PORT}" \ + >"${WORKDIR}/proxy.log" 2>&1 +) & +PROXY_PID=$! + +HEALTH_URL="http://127.0.0.1:${PROXY_PORT}/health/liveliness" +for _ in $(seq 1 45); do + if curl -fsS "${HEALTH_URL}" >/dev/null 2>&1; then + break + fi + sleep 2 +done +curl -fsS "${HEALTH_URL}" >/dev/null \ + || { tail -50 "${WORKDIR}/proxy.log" >&2; die "proxy did not become healthy"; } + +# --------------------------------------------------------------------------- +# 4. Run pytest +# --------------------------------------------------------------------------- + +RESULTS_JSON="${WORKDIR}/compat-results.json" +PYTEST_ARGS=( + tests/claude_code/ + --ignore=tests/claude_code/_driver_unit_tests + --ignore=tests/claude_code/_builder_unit_tests + --ignore=tests/claude_code/_publisher_unit_tests + --ignore=tests/claude_code/_pr_gate_unit_tests +) +if [[ -n "${PYTEST_K}" ]]; then + log "PYTEST_K set; narrowing to: ${PYTEST_K}" + PYTEST_ARGS+=(-k "${PYTEST_K}") +fi + +log "running pytest" +set +e +( + cd "${WORKTREE}" \ + && ANTHROPIC_BASE_URL="http://127.0.0.1:${PROXY_PORT}" \ + ANTHROPIC_AUTH_TOKEN="${PROXY_API_KEY}" \ + COMPAT_RESULTS_PATH="${RESULTS_JSON}" \ + uv run pytest "${PYTEST_ARGS[@]}" +) +PYTEST_EXIT=$? +set -e +log "pytest exit code: ${PYTEST_EXIT} (failures become 'fail' cells, not script errors)" +[[ -f "${RESULTS_JSON}" ]] || die "pytest did not produce ${RESULTS_JSON}" + +# --------------------------------------------------------------------------- +# 5. Build the matrix JSON +# --------------------------------------------------------------------------- + +MATRIX_JSON="${WORKDIR}/compatibility-matrix.json" +log "building ${MATRIX_JSON}" +( + cd "${WORKTREE}" \ + && uv run python "${POPULATOR_DIR}/build_matrix.py" \ + --manifest "${WORKTREE}/tests/claude_code/manifest.yaml" \ + --results "${RESULTS_JSON}" \ + --output "${MATRIX_JSON}" \ + --litellm-version "${LITELLM_VERSION}" \ + --claude-code-version "${CLAUDE_CODE_VERSION}" +) + +# --------------------------------------------------------------------------- +# 6. Open a docs-repo PR +# --------------------------------------------------------------------------- + +if [[ "${SKIP_PUBLISH}" == "1" ]]; then + cp "${MATRIX_JSON}" "${LITELLM_REPO}/compatibility-matrix.json" + log "SKIP_PUBLISH=1; matrix written to ${LITELLM_REPO}/compatibility-matrix.json" + exit 0 +fi + +DATE_UTC="$(date -u +%Y-%m-%d)" +BRANCH_NAME="compat-matrix/${LITELLM_VERSION}-${CLAUDE_CODE_VERSION}-${DATE_UTC}" +DOCS_CLONE="${WORKDIR}/litellm-docs" + +log "cloning ${DOCS_REPO}@${DOCS_BRANCH}" +gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}" + +cd "${DOCS_CLONE}" +git config user.email "litellm-bot@berri.ai" +git config user.name "litellm-compat-matrix-bot" +git checkout -b "${BRANCH_NAME}" + +mkdir -p "$(dirname "${DOCS_TARGET_PATH}")" +cp "${MATRIX_JSON}" "${DOCS_TARGET_PATH}" +git add "${DOCS_TARGET_PATH}" + +if git diff --cached --quiet; then + log "matrix JSON unchanged from ${DOCS_BRANCH}; skipping PR" + exit 0 +fi + +GENERATED_AT="$(jq -r '.generated_at' "${MATRIX_JSON}")" +COMMIT_MSG="$(cat <&1 +)" +PR_EXIT=$? +set -e +echo "${PR_OUT}" + +if [[ ${PR_EXIT} -ne 0 ]]; then + if grep -q "a pull request for branch.*already exists" <<<"${PR_OUT}"; then + log "PR already exists for ${BRANCH_NAME}; updated branch in place" + else + die "gh pr create failed (exit ${PR_EXIT})" + fi +fi + +log "done" diff --git a/tests/claude_code/publisher.py b/tests/claude_code/publisher.py deleted file mode 100644 index 492d91c6965..00000000000 --- a/tests/claude_code/publisher.py +++ /dev/null @@ -1,709 +0,0 @@ -"""Daily-cron matrix publisher (GCP VM edition). - -End-to-end orchestrator that runs on the dedicated cron VM -`litellm-compatibility-matrix-populator`. The flow is: - - 1. Resolve the latest LiteLLM ``v*-stable`` tag via ``resolver.py``. - 2. Update a long-lived git worktree of ``BerriAI/litellm`` to that tag, - run ``uv sync --frozen`` against it, and boot the proxy as a - subprocess on a non-conflicting local port. Reusing the same worktree - across runs (rather than a fresh tempdir) keeps disk footprint - bounded — ``uv sync`` removes packages no longer pinned and ``git - checkout`` mutates the same files in place. - 3. Run ``pytest tests/claude_code/`` against the proxy. The locally - installed Claude Code CLI is exercised as-is — there is no - ``npm install`` step, so the operator controls when the CLI is - upgraded by running ``npm install -g @anthropic-ai/claude-code@latest`` - out-of-band (typically baked into the VM image or a separate cron). - 4. Build ``compatibility-matrix.json`` from the per-test results - artifact using the Matrix JSON Builder (``matrix_builder.py``). - 5. Open (or update) a pull request against the docs repo with the JSON - change, using a ``gh`` CLI that has been pre-authenticated on the VM - against an account with ``pull-requests: write`` on - ``BerriAI/litellm-docs``. - -Why no Docker -------------- - -The original design pulled ``ghcr.io/berriai/litellm:`` per run on -GitHub-hosted runners. The cron VM does not run docker — installing it -would just trade one set of moving parts (docker daemon, image pulls, -networking) for the simpler "one git checkout + one ``uv sync``" we -already use to start the proxy interactively. Removing the docker code -also halves the publisher's surface area. - -Idempotency ------------ - -Re-runs on the same UTC day with the same resolved versions land on the -same head branch (``compat-matrix/--``). If the JSON is byte-identical to the docs repo's -target branch, the script exits before pushing. If a PR is already open -for the branch, ``gh pr create`` no-ops with a non-fatal message; we -treat that as success. - -The "only ``compatibility-matrix.json`` is ever committed" guarantee is -enforced by ``select_files_to_commit`` rather than by token scope, since -GitHub does not expose file-path-scoped tokens. -""" - -from __future__ import annotations - -import argparse -import datetime -import os -import shutil -import signal -import subprocess -import sys -import tempfile -import time -import urllib.error -import urllib.request -from pathlib import Path -from typing import Any, List, Mapping, Optional, Sequence - -from tests.claude_code.matrix_builder import build_from_paths -from tests.claude_code.resolver import latest_stable_litellm_tag - -DOCS_REPO_DEFAULT = "BerriAI/litellm-docs" -DOCS_TARGET_BASENAME = "compatibility-matrix.json" -DOCS_TARGET_PATH_DEFAULT = f"static/data/{DOCS_TARGET_BASENAME}" -PR_BRANCH_PREFIX = "compat-matrix" - -# Sourced from the same config the human-tended dev proxy uses, but on a -# different port so a developer running the proxy on :4000 doesn't -# collide with the cron run. -DEFAULT_PROXY_PORT = 4100 -DEFAULT_PROXY_API_KEY = "sk-cron-matrix" # the proxy never sees real auth -DEFAULT_PROXY_HEALTH_TIMEOUT_SECONDS = 90 - -# Location of a persistent litellm checkout that the cron mutates each -# run (``git checkout `` + ``uv sync``). Persisting across runs -# keeps disk usage bounded. Override via the ``LITELLM_WORKTREE`` env var -# or ``--worktree`` so the cron can target a deliberate path on the VM. -DEFAULT_WORKTREE = Path.home() / "litellm-cron-worktree" - -REPO_ROOT = Path(__file__).resolve().parents[2] -DEFAULT_MANIFEST = REPO_ROOT / "tests" / "claude_code" / "manifest.yaml" -DEFAULT_RESULTS = REPO_ROOT / "compat-results.json" - - -# --------------------------------------------------------------------------- -# Pure helpers — unit-tested under _publisher_unit_tests/. -# --------------------------------------------------------------------------- - - -def commit_message_for_matrix(matrix: Mapping[str, Any]) -> str: - """Build a deterministic commit message for the docs-repo push. - - Surfaces the three pieces of provenance the docs banner shows - (LiteLLM version, Claude Code version, generated_at) so the docs - repo's git log is self-describing without opening the JSON. - """ - litellm_version = matrix.get("litellm_version", "") - claude_code_version = matrix.get("claude_code_version", "") - generated_at = matrix.get("generated_at", "") - headline = "Update Claude Code compatibility matrix" - body_lines = [ - f"litellm_version: {litellm_version}", - f"claude_code_version: {claude_code_version}", - f"generated_at: {generated_at}", - ] - return headline + "\n\n" + "\n".join(body_lines) + "\n" - - -def pr_branch_name( - *, litellm_version: str, claude_code_version: str, date_utc: str -) -> str: - """Deterministic branch name for the docs-repo PR. - - Two cron runs that resolve to the same (litellm_version, - claude_code_version, UTC date) land on the same branch and therefore - the same PR — the second push is a force-with-lease update of the - existing branch and ``gh pr create`` no-ops. - - Each component is required because: - - ``litellm_version`` distinguishes consecutive stable tags - - ``claude_code_version`` distinguishes a Claude-Code-only refresh - - ``date_utc`` lets us still produce a fresh branch when neither - upstream version moved but a maintainer manually re-ran on a - later day to recover from a transient failure - """ - if not litellm_version: - raise ValueError("litellm_version must be a non-empty string") - if not claude_code_version: - raise ValueError("claude_code_version must be a non-empty string") - if not date_utc: - raise ValueError("date_utc must be a non-empty string") - return f"{PR_BRANCH_PREFIX}/{litellm_version}-{claude_code_version}-{date_utc}" - - -def pr_title_for_matrix(matrix: Mapping[str, Any]) -> str: - """Single-line PR title with the resolved versions inline. - - Mirrors the commit headline so notification subject lines and the - docs repo's PR list are immediately self-describing — maintainers - can triage from the inbox without opening the diff. - """ - litellm_version = matrix.get("litellm_version", "") - claude_code_version = matrix.get("claude_code_version", "") - return ( - "chore(compat-matrix): refresh for " - f"{litellm_version} + claude-code {claude_code_version}" - ) - - -def pr_body_for_matrix(matrix: Mapping[str, Any]) -> str: - """Markdown body summarising the matrix at a glance. - - Renders one line per feature with the per-provider statuses inline, - so reviewers don't have to diff the JSON to see what changed since - the last refresh. The status column ordering follows the manifest - (which the matrix already reflects in ``providers``), keeping the - table stable across days. - """ - litellm_version = matrix.get("litellm_version", "") - claude_code_version = matrix.get("claude_code_version", "") - generated_at = matrix.get("generated_at", "") - providers = list(matrix.get("providers", [])) - features = list(matrix.get("features", [])) - - lines: List[str] = [ - "Automated daily refresh of the Claude Code compatibility matrix.", - "", - "| Field | Value |", - "| --- | --- |", - f"| litellm_version | `{litellm_version}` |", - f"| claude_code_version | `{claude_code_version}` |", - f"| generated_at | `{generated_at}` |", - "", - "## Per-feature results", - "", - ] - for feature in features: - if not isinstance(feature, Mapping): - continue - name = feature.get("name") or feature.get("id") or "" - cells = feature.get("providers") or {} - per_provider = ", ".join( - f"{provider}={(cells.get(provider) or {}).get('status', 'not_tested')}" - for provider in providers - ) - lines.append(f"- **{name}**: {per_provider}") - - lines.extend( - [ - "", - "---", - "", - "Generated by `tests/claude_code/publisher.py`. " - "Close without merging if the diff looks wrong; the next " - "cron run will reopen with fresh results.", - "", - ] - ) - return "\n".join(lines) - - -def select_files_to_commit( - staged_paths: Sequence[str], allowed_basename: str -) -> List[str]: - """Return only the paths whose basename matches the allowlist. - - Even though the docs-repo PR is reviewable, the publisher still - enforces a one-file allowlist as defence in depth — a stray file in - the working tree from a future feature must not be smuggled into the - PR by accident. - """ - return [p for p in staged_paths if os.path.basename(p) == allowed_basename] - - -# --------------------------------------------------------------------------- -# Subprocess + filesystem glue. Side-effectful, deliberately not unit-tested -# (the daily cron's failure surface is the test). -# --------------------------------------------------------------------------- - - -def _now_utc_iso() -> str: - """ISO-8601 UTC timestamp with ``Z`` suffix, matching the v1 schema.""" - return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def _today_utc_date() -> str: - """Fallback UTC date string used when ``generated_at`` is missing.""" - return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") - - -def _run(cmd: Sequence[str], **kwargs: Any) -> subprocess.CompletedProcess: - """Print + run subprocess; raise on nonzero exit unless caller opts out.""" - print("+ " + " ".join(str(part) for part in cmd), flush=True) - return subprocess.run(cmd, check=True, **kwargs) - - -def _get_claude_code_version() -> str: - """Return the version string printed by ``claude --version``.""" - completed = subprocess.run( - ["claude", "--version"], capture_output=True, text=True, check=True - ) - # ``claude --version`` prints e.g. "2.1.120 (Claude Code)"; the first - # whitespace-delimited token is the version. - out = (completed.stdout or "").strip() - return out.split()[0] if out else "" - - -def _ensure_worktree(worktree: Path) -> None: - """Make sure ``worktree`` is a working litellm checkout. - - On first run we ``git clone`` ``BerriAI/litellm`` into the worktree - path. On subsequent runs we reuse what's already there — the run - just needs ``git fetch && git checkout ``. - - Why a clone instead of a ``git worktree`` of the dev checkout: the - dev checkout (``~/litellm/litellm``) is where humans iterate and may - sit on uncommitted changes or arbitrary feature branches. A separate - clone keeps the cron's ``git checkout `` from disturbing that. - """ - if (worktree / ".git").exists(): - return - worktree.parent.mkdir(parents=True, exist_ok=True) - _run( - [ - "git", - "clone", - "https://github.com/BerriAI/litellm.git", - str(worktree), - ] - ) - - -def _checkout_tag_in_worktree(worktree: Path, tag: str) -> None: - """Fetch and ``git checkout `` inside ``worktree``. - - Uses ``git checkout --force`` so any cruft left behind by a previous - run (e.g. ``compat-results.json``, ``__pycache__``) is wiped before - a fresh ``uv sync``. The cron has no use for that state — every run - starts from the published tag. - """ - _run(["git", "fetch", "--tags", "--force"], cwd=worktree) - _run(["git", "reset", "--hard"], cwd=worktree) - _run(["git", "clean", "-fdx", "-e", ".venv"], cwd=worktree) - _run(["git", "checkout", "--force", tag], cwd=worktree) - - -def _uv_sync(worktree: Path) -> None: - """Bring the worktree's ``.venv`` in line with the checked-out tag. - - ``uv sync --frozen`` is deterministic: it installs exactly what the - lockfile says and removes anything no longer referenced. That's the - "doesn't blow up storage" property the operator requires — the venv - can never grow unboundedly across runs. - """ - _run(["uv", "sync", "--frozen"], cwd=worktree) - - -def _start_proxy(worktree: Path, port: int, config_path: Path) -> subprocess.Popen: - """Start the LiteLLM proxy as a subprocess; returns the Popen handle. - - Started in its own process group so we can SIGTERM the whole tree - on shutdown — the proxy itself spawns worker subprocesses that - don't otherwise propagate signals from a parent. - """ - cmd = [ - "uv", - "run", - "litellm", - "--config", - str(config_path), - "--port", - str(port), - ] - print("+ " + " ".join(cmd), flush=True) - return subprocess.Popen( - cmd, - cwd=worktree, - stdout=sys.stdout, - stderr=sys.stderr, - start_new_session=True, - ) - - -def _stop_proxy(proc: subprocess.Popen) -> None: - """SIGTERM the proxy's process group; SIGKILL if it doesn't exit.""" - if proc.poll() is not None: - return - try: - os.killpg(proc.pid, signal.SIGTERM) - except ProcessLookupError: - return - try: - proc.wait(timeout=15) - except subprocess.TimeoutExpired: - try: - os.killpg(proc.pid, signal.SIGKILL) - except ProcessLookupError: - pass - proc.wait(timeout=5) - - -def _wait_for_proxy( - port: int, timeout_seconds: int = DEFAULT_PROXY_HEALTH_TIMEOUT_SECONDS -) -> None: - """Poll ``/health/liveliness`` until it returns 200 or we time out.""" - url = f"http://127.0.0.1:{port}/health/liveliness" - deadline = time.time() + timeout_seconds - last_err: Optional[BaseException] = None - while time.time() < deadline: - try: - with urllib.request.urlopen(url, timeout=5) as resp: # noqa: S310 - if resp.status == 200: - return - except (urllib.error.URLError, OSError) as exc: - last_err = exc - time.sleep(2) - raise RuntimeError( - f"proxy did not become healthy within {timeout_seconds}s: {last_err!r}" - ) - - -# --------------------------------------------------------------------------- -# PR creation against the docs repo. -# --------------------------------------------------------------------------- - - -def publish( - *, - docs_repo: str, - docs_branch: str, - docs_target_path: str, - manifest_path: Path, - results_path: Path, - matrix_output_path: Path, - litellm_version: str, - claude_code_version: str, - generated_at: str, -) -> None: - """Build the matrix JSON and open a PR against the docs repo. - - Auth uses whatever ``gh auth login`` has stashed on the cron VM — - the GCP edition does not pass an explicit token. The account ``gh`` - is logged in as must be a collaborator on ``docs_repo`` with - ``pull-requests: write`` (the same permission ``gh pr create`` - needs interactively). - """ - matrix = build_from_paths( - manifest_path=manifest_path, - results_path=results_path, - litellm_version=litellm_version, - claude_code_version=claude_code_version, - generated_at=generated_at, - output_path=matrix_output_path, - ) - - branch_name = pr_branch_name( - litellm_version=litellm_version, - claude_code_version=claude_code_version, - date_utc=generated_at[:10] if generated_at else _today_utc_date(), - ) - - with tempfile.TemporaryDirectory(prefix="docs-repo-") as workdir: - workdir_path = Path(workdir) - # ``gh repo clone`` reuses the VM's ``gh auth`` state, so we - # don't need to construct an authenticated URL by hand. - _run( - [ - "gh", - "repo", - "clone", - docs_repo, - str(workdir_path), - "--", - "--depth", - "1", - "--branch", - docs_branch, - ] - ) - _run( - ["git", "config", "user.email", "litellm-bot@berri.ai"], - cwd=workdir_path, - ) - _run( - ["git", "config", "user.name", "litellm-compat-matrix-bot"], - cwd=workdir_path, - ) - # Branch always starts from the freshly-cloned base. If the - # remote branch already exists from an earlier run on the same - # day, the later ``git push --force-with-lease`` reconciles — - # we'd rather present the latest matrix JSON than preserve a - # stale intermediate state. - _run(["git", "checkout", "-b", branch_name], cwd=workdir_path) - - target_in_docs = workdir_path / docs_target_path - target_in_docs.parent.mkdir(parents=True, exist_ok=True) - target_in_docs.write_text(matrix_output_path.read_text()) - - # Defence in depth: even if some other tool dropped a file in - # the working tree, only the matrix JSON is staged. - staged = [docs_target_path] - keep = select_files_to_commit(staged, DOCS_TARGET_BASENAME) - if not keep: - raise RuntimeError( - "no allowed files to commit; expected " - f"{DOCS_TARGET_BASENAME!r} but got {staged!r}" - ) - for path in keep: - _run(["git", "add", path], cwd=workdir_path) - - # Skip the push entirely if the JSON is byte-identical to what's - # already on ``docs_branch`` — keeps the docs-repo PR list clean - # during idempotent reruns of the cron. - diff = subprocess.run( - ["git", "diff", "--cached", "--quiet"], - cwd=workdir_path, - check=False, - ) - if diff.returncode == 0: - print("matrix JSON unchanged; skipping PR", flush=True) - return - - _run( - ["git", "commit", "-m", commit_message_for_matrix(matrix)], - cwd=workdir_path, - ) - # ``--force-with-lease`` so a same-day rerun updates the existing - # branch (and therefore the existing PR) safely; the lease check - # ensures we never overwrite a docs-maintainer's manual fixup - # commit on the same branch. - _run( - [ - "git", - "push", - "--force-with-lease", - "--set-upstream", - "origin", - branch_name, - ], - cwd=workdir_path, - ) - - _open_or_update_pr( - docs_repo=docs_repo, - docs_branch=docs_branch, - head_branch=branch_name, - title=pr_title_for_matrix(matrix), - body=pr_body_for_matrix(matrix), - cwd=workdir_path, - ) - - -def _open_or_update_pr( - *, - docs_repo: str, - docs_branch: str, - head_branch: str, - title: str, - body: str, - cwd: Path, -) -> None: - """Open a PR via the ``gh`` CLI; treat 'already exists' as success.""" - completed = subprocess.run( - [ - "gh", - "pr", - "create", - "--repo", - docs_repo, - "--base", - docs_branch, - "--head", - head_branch, - "--title", - title, - "--body", - body, - ], - cwd=cwd, - capture_output=True, - text=True, - check=False, - ) - if completed.returncode == 0: - print(completed.stdout.strip(), flush=True) - return - stderr = completed.stderr or "" - # ``gh pr create`` exits non-zero when a PR already exists for the - # head branch. That's the idempotent re-run path and not an error: - # the branch was already force-pushed above, so the existing PR now - # carries the freshest matrix JSON. - if "a pull request for branch" in stderr and "already exists" in stderr: - print( - f"PR already exists for {head_branch}; updated branch in place", - flush=True, - ) - return - sys.stderr.write(stderr) - raise RuntimeError(f"gh pr create failed with exit code {completed.returncode}") - - -# --------------------------------------------------------------------------- -# Top-level orchestrator. -# --------------------------------------------------------------------------- - - -def main(argv: Optional[Sequence[str]] = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--docs-repo", - default=os.environ.get("DOCS_REPO", DOCS_REPO_DEFAULT), - help="`owner/name` of the docs repo to publish into.", - ) - parser.add_argument( - "--docs-branch", - default=os.environ.get("DOCS_BRANCH", "main"), - help="Base branch on the docs repo to PR against.", - ) - parser.add_argument( - "--docs-target-path", - default=os.environ.get("DOCS_TARGET_PATH", DOCS_TARGET_PATH_DEFAULT), - help="Path inside the docs repo where the matrix JSON lives.", - ) - parser.add_argument( - "--worktree", - type=Path, - default=Path(os.environ.get("LITELLM_WORKTREE", str(DEFAULT_WORKTREE))), - help=( - "Persistent litellm checkout the cron mutates each run " - f"(default: {DEFAULT_WORKTREE})." - ), - ) - parser.add_argument( - "--proxy-port", - type=int, - default=int(os.environ.get("PROXY_PORT", DEFAULT_PROXY_PORT)), - ) - parser.add_argument( - "--manifest", - type=Path, - default=DEFAULT_MANIFEST, - help="Manifest the matrix JSON is built against.", - ) - parser.add_argument( - "--results", - type=Path, - default=DEFAULT_RESULTS, - help="Path where pytest will write the compat-results.json artifact.", - ) - parser.add_argument( - "--matrix-output", - type=Path, - default=REPO_ROOT / DOCS_TARGET_BASENAME, - ) - parser.add_argument( - "--skip-publish", - action="store_true", - help="Run the test pipeline but do not open a PR.", - ) - args = parser.parse_args(argv) - - proxy_proc: Optional[subprocess.Popen] = None - try: - litellm_version = latest_stable_litellm_tag( - token=os.environ.get("GITHUB_TOKEN") - ) - print(f"resolved latest stable litellm: {litellm_version}", flush=True) - - claude_code_version = _get_claude_code_version() - if not claude_code_version: - print( - "could not read 'claude --version'; is the CLI installed?", - file=sys.stderr, - ) - return 2 - print(f"local claude code cli: {claude_code_version}", flush=True) - - _ensure_worktree(args.worktree) - _checkout_tag_in_worktree(args.worktree, litellm_version) - _uv_sync(args.worktree) - - config_path = args.worktree / "tests" / "claude_code" / "test_config.yaml" - if not config_path.exists(): - raise RuntimeError( - f"proxy config not found at {config_path}; the resolved tag " - f"{litellm_version} may predate the compat matrix work" - ) - - proxy_proc = _start_proxy(args.worktree, args.proxy_port, config_path) - _wait_for_proxy(args.proxy_port) - - env = { - **os.environ, - "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{args.proxy_port}", - "ANTHROPIC_AUTH_TOKEN": DEFAULT_PROXY_API_KEY, - "COMPAT_RESULTS_PATH": str(args.results), - } - pytest_cmd = [ - "uv", - "run", - "pytest", - "tests/claude_code/", - "--ignore=tests/claude_code/_driver_unit_tests", - "--ignore=tests/claude_code/_builder_unit_tests", - "--ignore=tests/claude_code/_publisher_unit_tests", - "--ignore=tests/claude_code/_pr_gate_unit_tests", - ] - # Operator escape hatch: PYTEST_K narrows the run to a single - # cell or feature for first-time validation after a CLI/proxy - # upgrade. The matrix builder fills cells we didn't touch with - # `not_tested`, so a PYTEST_K-narrowed run is safe to publish — - # though in practice it's used with --skip-publish. - pytest_k = os.environ.get("PYTEST_K", "").strip() - if pytest_k: - pytest_cmd.extend(["-k", pytest_k]) - print(f"PYTEST_K set; narrowing pytest to: {pytest_k}", flush=True) - # Run pytest from inside the worktree so it picks up the - # checked-out tag's test code (and its conftest hook), not the - # current process's working directory. - subprocess.run(pytest_cmd, env=env, cwd=args.worktree, check=False) - - if args.skip_publish: - print("skip-publish: not opening a PR", flush=True) - return 0 - - publish( - docs_repo=args.docs_repo, - docs_branch=args.docs_branch, - docs_target_path=args.docs_target_path, - manifest_path=args.manifest, - results_path=args.results, - matrix_output_path=args.matrix_output, - litellm_version=litellm_version, - claude_code_version=claude_code_version, - generated_at=_now_utc_iso(), - ) - return 0 - finally: - if proxy_proc is not None: - _stop_proxy(proxy_proc) - - -if __name__ == "__main__": - sys.exit(main()) - - -__all__ = [ - "DOCS_REPO_DEFAULT", - "DOCS_TARGET_BASENAME", - "DOCS_TARGET_PATH_DEFAULT", - "DEFAULT_PROXY_PORT", - "DEFAULT_PROXY_API_KEY", - "DEFAULT_WORKTREE", - "PR_BRANCH_PREFIX", - "commit_message_for_matrix", - "select_files_to_commit", - "pr_branch_name", - "pr_title_for_matrix", - "pr_body_for_matrix", - "publish", - "main", -] diff --git a/tests/claude_code/resolver.py b/tests/claude_code/resolver.py deleted file mode 100644 index dc42e202302..00000000000 --- a/tests/claude_code/resolver.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Latest Stable LiteLLM Resolver. - -Queries the GitHub Releases API for `BerriAI/litellm` and returns the -newest tag matching `v*-stable`. Used by the daily-cron publisher to -decide which Docker image to pull for the matrix run. - -The resolver is intentionally tiny — its only state is the GitHub API -URL constant — and exposes a single public function so unit tests can -inject a fake HTTP getter and run offline. -""" - -from __future__ import annotations - -import json -import re -import urllib.request -from typing import Callable, List, Optional - -GITHUB_RELEASES_URL = "https://api.github.com/repos/BerriAI/litellm/releases" -USER_AGENT = "litellm-compat-matrix-resolver" -REQUEST_TIMEOUT_SECONDS = 30 - -STABLE_TAG_RE = re.compile(r"^v(\d+)\.(\d+)\.(\d+)-stable$") - - -class ResolverError(RuntimeError): - """Raised when the resolver cannot determine a latest-stable tag.""" - - -def _default_http_get(url: str, *, token: Optional[str] = None) -> str: - """Minimal urllib-based GET that the cron VM can call without extra deps. - - Forwards `token` as a Bearer header when set so the cron job can lift - the unauthenticated GitHub rate limit by passing the GitHub App - installation token. - """ - request = urllib.request.Request(url) - request.add_header("User-Agent", USER_AGENT) - request.add_header("Accept", "application/vnd.github+json") - if token: - request.add_header("Authorization", f"Bearer {token}") - with urllib.request.urlopen( # noqa: S310 - URL is hardcoded above - request, timeout=REQUEST_TIMEOUT_SECONDS - ) as response: - return response.read().decode("utf-8") - - -def latest_stable_litellm_tag( - *, - http_get: Optional[Callable[..., str]] = None, - token: Optional[str] = None, -) -> str: - """Return the newest `v*-stable` tag published on BerriAI/litellm. - - Sort is numeric on the (major, minor, patch) triple so v1.10.0-stable - correctly outranks v1.9.5-stable. Releases with no `tag_name` (drafts) - or that don't match the `v*-stable` shape are skipped. - """ - fetch = http_get or _default_http_get - payload = fetch(GITHUB_RELEASES_URL, token=token) - releases = json.loads(payload) - if not isinstance(releases, list): - raise ResolverError( - "github releases response is not a list; got " f"{type(releases).__name__}" - ) - - matched: List[tuple] = [] - for release in releases: - if not isinstance(release, dict): - continue - tag = release.get("tag_name") - if not isinstance(tag, str): - continue - m = STABLE_TAG_RE.match(tag) - if not m: - continue - version_key = tuple(int(x) for x in m.groups()) - matched.append((version_key, tag)) - - if not matched: - raise ResolverError("no v*-stable tags found in github releases response") - - matched.sort(key=lambda pair: pair[0], reverse=True) - return matched[0][1] - - -__all__ = [ - "GITHUB_RELEASES_URL", - "ResolverError", - "latest_stable_litellm_tag", -]