diff --git a/.github/workflows/claude_code_compat_matrix.yml b/.github/workflows/claude_code_compat_matrix.yml new file mode 100644 index 00000000000..9520f321365 --- /dev/null +++ b/.github/workflows/claude_code_compat_matrix.yml @@ -0,0 +1,127 @@ +name: Claude Code Compatibility Matrix (daily cron) + +# Slice 4 of the Claude Code Compatibility Matrix (PRD #26476, issue #26480). +# +# Three triggers per the PRD's "Daily Cron" section: +# - Daily cron (06:00 UTC) — picks up newly-published Claude Code releases. +# - `release` of a `v*-stable` tag on this repo — re-runs the matrix the +# moment a new stable LiteLLM ships. +# - Manual dispatch — operators can re-run the publisher on demand. +# +# The job runs on a GitHub-hosted ubuntu-latest runner, which gives us a +# fresh VM per run and is "isolated from the main CI environment" in the +# sense that nothing else on this runner survives the run. Since the +# always-latest Claude Code CLI is only installed inside this ephemeral +# VM, a malicious or broken Claude Code release cannot affect the trusted +# build infrastructure used by the PR gate (which lives in CircleCI and +# uses a `latest minus 3 days` Claude Code pin). +# +# Cross-repo authentication (per "Cross-repo authentication" in the PRD): +# A GitHub App installed on `BerriAI/litellm-docs` only, scoped to +# `contents: write`, mints an installation token at job-start. The token +# is only ever used by the publisher, which only ever writes +# `compatibility-matrix.json` (enforced by `select_files_to_commit`). + +on: + schedule: + - cron: "0 6 * * *" # daily at 06:00 UTC + release: + types: [published] + workflow_dispatch: + inputs: + skip_publish: + description: "Run the test pipeline but skip the docs-repo push." + required: false + type: boolean + default: false + +permissions: + contents: read + +jobs: + publish-matrix: + # Skip release runs that aren't tagged `v*-stable`. Plain `v1.84.0-rc1` + # or `v1.84.0` releases must NOT republish the matrix — only the + # latest *stable* tag is reflected on the docs page. + if: | + github.repository == 'BerriAI/litellm' && ( + github.event_name != 'release' || + endsWith(github.event.release.tag_name, '-stable') + ) + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - name: Checkout litellm + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + enable-cache: false + + - name: Set up Node (for the Claude Code CLI) + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + + - name: Mint docs-repo installation token from GitHub App + id: docs-token + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 + with: + app-id: ${{ secrets.COMPAT_MATRIX_APP_ID }} + private-key: ${{ secrets.COMPAT_MATRIX_APP_PRIVATE_KEY }} + owner: BerriAI + repositories: litellm-docs + + - name: Install LiteLLM dev deps + run: uv sync --frozen + + - name: Run matrix publisher + env: + # Token used to direct-push compatibility-matrix.json to the docs + # repo's main branch. Comes from the GitHub App installation token + # minted above; scoped to litellm-docs only. + DOCS_REPO_TOKEN: ${{ steps.docs-token.outputs.token }} + # Token used by the resolver to lift the unauthenticated GitHub + # rate limit on the Releases API. The default GITHUB_TOKEN is + # sufficient for read-only access to public release metadata. + GITHUB_TOKEN: ${{ github.token }} + # Real provider credentials needed by the per-cell tests. These + # are the same secrets the LLM-translation workflow uses. + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }} + VERTEXAI_PROJECT: ${{ secrets.VERTEXAI_PROJECT }} + VERTEXAI_LOCATION: ${{ secrets.VERTEXAI_LOCATION }} + GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }} + AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} + AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} + SKIP_PUBLISH: ${{ inputs.skip_publish }} + run: | + set -euo pipefail + if [ "${SKIP_PUBLISH:-false}" = "true" ]; then + uv run python -m tests.claude_code.publisher --skip-publish + else + uv run python -m tests.claude_code.publisher + fi + + - name: Upload compat-results.json artifact (debugging) + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: compat-results-${{ github.run_id }} + path: | + compat-results.json + compatibility-matrix.json + if-no-files-found: ignore + retention-days: 30 diff --git a/.gitignore b/.gitignore index 1a282869434..458bcba0dca 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,5 @@ test-config # Claude Code compatibility-matrix pytest artifact (CI-only output). compat-results.json +# Matrix JSON produced by the daily-cron publisher (pushed to litellm-docs). +compatibility-matrix.json diff --git a/tests/claude_code/_publisher_unit_tests/__init__.py b/tests/claude_code/_publisher_unit_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/claude_code/_publisher_unit_tests/test_publisher.py b/tests/claude_code/_publisher_unit_tests/test_publisher.py new file mode 100644 index 00000000000..faaf5881630 --- /dev/null +++ b/tests/claude_code/_publisher_unit_tests/test_publisher.py @@ -0,0 +1,99 @@ +"""Unit tests for the daily-cron matrix publisher. + +The publisher orchestrates Docker, git, npm and pytest — per the PRD's +"Testing Decisions" section, the orchestration itself is too thin to +warrant heavy mocking. These tests cover the small pure helpers that +do warrant test coverage: + +- `commit_message_for_matrix`: deterministic commit message containing + the LiteLLM and Claude Code versions plus `generated_at`, so the docs + repo's git log shows what produced each push. +- `docker_image_for_tag`: maps a `v*-stable` tag to its `ghcr.io` image. +- `select_files_to_commit`: enforces the "only `compatibility-matrix.json` + is pushed" guarantee that the GitHub App's broad `contents: write` scope + doesn't enforce on its own (per PRD: "File-level restriction is enforced + by script correctness"). +""" + +from __future__ import annotations + +import pytest + +from tests.claude_code.publisher import ( + DOCS_TARGET_BASENAME, + commit_message_for_matrix, + docker_image_for_tag, + select_files_to_commit, +) + + +def test_commit_message_includes_versions_and_timestamp(): + matrix = { + "schema_version": "1", + "generated_at": "2026-04-25T06:00:00Z", + "litellm_version": "v1.83.0-stable", + "claude_code_version": "2.1.120", + "providers": ["anthropic"], + "features": [], + } + message = commit_message_for_matrix(matrix) + # Headline is short and identifies the artifact. + headline, _, body = message.partition("\n") + assert "compatibility matrix" in headline.lower() + # Body must surface the three pieces of provenance the docs banner shows. + assert "v1.83.0-stable" in body + assert "2.1.120" in body + assert "2026-04-25T06:00:00Z" in body + + +def test_commit_message_is_deterministic(): + """Same matrix in → same message out (no clock, no randomness).""" + matrix = { + "litellm_version": "v1.83.0-stable", + "claude_code_version": "2.1.120", + "generated_at": "2026-04-25T06:00:00Z", + } + assert commit_message_for_matrix(matrix) == commit_message_for_matrix(matrix) + + +def test_docker_image_for_tag_targets_berriai_ghcr(): + assert ( + docker_image_for_tag("v1.83.0-stable") + == "ghcr.io/berriai/litellm:v1.83.0-stable" + ) + + +def test_docker_image_for_tag_rejects_empty_tag(): + with pytest.raises(ValueError, match="tag must be a non-empty string"): + docker_image_for_tag("") + + +def test_select_files_to_commit_drops_anything_other_than_matrix(): + """The script's safety net: even if pytest leaves stray artifacts in + the docs-repo checkout, only `compatibility-matrix.json` ever ships. + + Acceptance criterion: "the script does not write any other files". + """ + staged = [ + "static/data/compatibility-matrix.json", + "static/data/notes.txt", + ".github/workflows/secret.yml", + "compat-results.json", + ] + assert select_files_to_commit(staged, DOCS_TARGET_BASENAME) == [ + "static/data/compatibility-matrix.json" + ] + + +def test_select_files_to_commit_returns_empty_when_nothing_matches(): + assert select_files_to_commit(["other.json"], DOCS_TARGET_BASENAME) == [] + + +def test_select_files_to_commit_basename_match_not_substring(): + """`compatibility-matrix.json.bak` must not be treated as the allowed file.""" + assert ( + select_files_to_commit( + ["static/data/compatibility-matrix.json.bak"], DOCS_TARGET_BASENAME + ) + == [] + ) diff --git a/tests/claude_code/_publisher_unit_tests/test_resolver.py b/tests/claude_code/_publisher_unit_tests/test_resolver.py new file mode 100644 index 00000000000..91a89c9b271 --- /dev/null +++ b/tests/claude_code/_publisher_unit_tests/test_resolver.py @@ -0,0 +1,112 @@ +"""Unit tests for the Latest Stable LiteLLM Resolver. + +The resolver is the smallest piece of cron-pipeline glue: it asks the GitHub +Releases API for `BerriAI/litellm` and returns the newest tag matching the +`v*-stable` pattern. Tests inject a fake HTTP getter so they run offline. + +Per the PRD's "Testing Decisions" section, version resolvers were officially +deferred from v0 — but the resolver is also small enough that a few cheap +unit tests are worth more than the daily-cron failing loudly. +""" + +from __future__ import annotations + +import json +from typing import List + +import pytest + +from tests.claude_code.resolver import ( + GITHUB_RELEASES_URL, + ResolverError, + latest_stable_litellm_tag, +) + + +def _fake_getter(payload): + """Return a callable that records calls and returns the given JSON payload. + + Mirrors the `runner=` injection seam used by `cli_driver` tests so the + unit tests don't depend on `urllib`. + """ + + captured: List[dict] = [] + + def getter(url, *, token=None): + captured.append({"url": url, "token": token}) + return json.dumps(payload) + + getter.captured = captured # type: ignore[attr-defined] + return getter + + +def test_resolver_returns_newest_stable_tag(): + """Given a list of releases, the newest `v*-stable` wins.""" + getter = _fake_getter( + [ + {"tag_name": "v1.81.0-stable"}, + {"tag_name": "v1.83.0-stable"}, + {"tag_name": "v1.82.5-stable"}, + ] + ) + assert latest_stable_litellm_tag(http_get=getter) == "v1.83.0-stable" + + +def test_resolver_ignores_non_stable_tags(): + """RC, alpha, beta, plain `v1.83.0`, and noise tags are filtered out.""" + getter = _fake_getter( + [ + {"tag_name": "v1.84.0-rc1"}, + {"tag_name": "v1.84.0-stable.draft"}, + {"tag_name": "v1.84.0"}, + {"tag_name": "v1.83.0-stable"}, + {"tag_name": "stable-2026-04-25"}, + {"tag_name": "v1.85.0-alpha"}, + ] + ) + assert latest_stable_litellm_tag(http_get=getter) == "v1.83.0-stable" + + +def test_resolver_uses_numeric_version_sort_not_lexicographic(): + """v1.10.0-stable must be newer than v1.9.0-stable (numeric, not string).""" + getter = _fake_getter( + [ + {"tag_name": "v1.9.0-stable"}, + {"tag_name": "v1.10.0-stable"}, + {"tag_name": "v1.9.5-stable"}, + ] + ) + assert latest_stable_litellm_tag(http_get=getter) == "v1.10.0-stable" + + +def test_resolver_raises_when_no_stable_tags(): + getter = _fake_getter([{"tag_name": "v1.84.0-rc1"}, {"tag_name": "v1.83.0"}]) + with pytest.raises(ResolverError, match="no v\\*-stable tags found"): + latest_stable_litellm_tag(http_get=getter) + + +def test_resolver_raises_when_response_is_not_a_list(): + getter = _fake_getter({"message": "API rate limit exceeded"}) + with pytest.raises(ResolverError, match="not a list"): + latest_stable_litellm_tag(http_get=getter) + + +def test_resolver_calls_github_releases_endpoint_with_optional_token(): + """The resolver must call the BerriAI/litellm releases API and forward + the auth token (if provided) to the http getter so callers can lift + the unauthenticated rate limit when running on the cron VM.""" + getter = _fake_getter([{"tag_name": "v1.83.0-stable"}]) + latest_stable_litellm_tag(http_get=getter, token="ghs_xxx") + assert getter.captured == [{"url": GITHUB_RELEASES_URL, "token": "ghs_xxx"}] + + +def test_resolver_skips_releases_with_missing_tag_name(): + """Defensive: GitHub draft releases can omit `tag_name`; don't crash.""" + getter = _fake_getter( + [ + {"name": "untagged draft"}, + {"tag_name": None}, + {"tag_name": "v1.83.0-stable"}, + ] + ) + assert latest_stable_litellm_tag(http_get=getter) == "v1.83.0-stable" diff --git a/tests/claude_code/publisher.py b/tests/claude_code/publisher.py new file mode 100644 index 00000000000..1dbf0255350 --- /dev/null +++ b/tests/claude_code/publisher.py @@ -0,0 +1,388 @@ +"""Daily-cron matrix publisher. + +End-to-end orchestrator that runs on the isolated cron VM (per the PRD's +"Two CI environments / Daily Cron" section). The flow is: + + 1. Resolve the latest LiteLLM `v*-stable` tag via `resolver.py`. + 2. Pull the corresponding Docker image and start it as the proxy. + 3. Install the absolute latest Claude Code CLI from npm. + 4. Run `pytest tests/claude_code/` against the proxy. + 5. Build `compatibility-matrix.json` from the per-test results artifact + using the Matrix JSON Builder (`matrix_builder.py`). + 6. Direct-push the JSON to the docs repo's main branch using the + GitHub App installation token mounted as `DOCS_REPO_TOKEN`. + +The orchestration is thin glue over Docker, git, npm and subprocess — per +the PRD's "Testing Decisions" section, it intentionally ships without a +unit-test harness; the daily-cron failure surface is itself the test. The +pure helpers below (commit message, image-name builder, file allowlist) +are unit-tested under `_publisher_unit_tests/`. + +The "only `compatibility-matrix.json` is ever committed" guarantee is +enforced by `select_files_to_commit` rather than by token scope, since +GitHub Apps cannot scope `contents: write` to a single file path. +""" + +from __future__ import annotations + +import argparse +import datetime +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, List, Mapping, Optional, Sequence + +from tests.claude_code.matrix_builder import build_from_paths +from tests.claude_code.resolver import latest_stable_litellm_tag + +DOCS_REPO_DEFAULT = "BerriAI/litellm-docs" +DOCS_TARGET_BASENAME = "compatibility-matrix.json" +DOCS_TARGET_PATH_DEFAULT = f"static/data/{DOCS_TARGET_BASENAME}" +DOCKER_IMAGE_BASE = "ghcr.io/berriai/litellm" +DEFAULT_PROXY_PORT = 4000 +DEFAULT_PROXY_API_KEY = "sk-cron-matrix" # only used inside the ephemeral VM + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_MANIFEST = REPO_ROOT / "tests" / "claude_code" / "manifest.yaml" +DEFAULT_RESULTS = REPO_ROOT / "compat-results.json" + + +def commit_message_for_matrix(matrix: Mapping[str, Any]) -> str: + """Build a deterministic commit message for the docs-repo push. + + Surfaces the three pieces of provenance the docs banner shows + (LiteLLM version, Claude Code version, generated_at) so the docs + repo's git log is self-describing without opening the JSON. + """ + litellm_version = matrix.get("litellm_version", "") + claude_code_version = matrix.get("claude_code_version", "") + generated_at = matrix.get("generated_at", "") + headline = "Update Claude Code compatibility matrix" + body_lines = [ + f"litellm_version: {litellm_version}", + f"claude_code_version: {claude_code_version}", + f"generated_at: {generated_at}", + ] + return headline + "\n\n" + "\n".join(body_lines) + "\n" + + +def docker_image_for_tag(tag: str) -> str: + """Return the ghcr.io image reference for a `v*-stable` tag.""" + if not tag: + raise ValueError("tag must be a non-empty string") + return f"{DOCKER_IMAGE_BASE}:{tag}" + + +def select_files_to_commit( + staged_paths: Sequence[str], allowed_basename: str +) -> List[str]: + """Return only the paths whose basename matches the allowlist. + + The cron VM's GitHub App holds `contents: write` on the entire docs + repo (GitHub does not support file-path-scoped tokens), so the + "only ship the matrix JSON" property is enforced here instead. Any + stray file in the working tree is dropped before the commit step. + """ + return [p for p in staged_paths if os.path.basename(p) == allowed_basename] + + +def _now_utc_iso() -> str: + """ISO-8601 UTC timestamp with `Z` suffix, matching the v1 schema.""" + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _run(cmd: Sequence[str], **kwargs: Any) -> subprocess.CompletedProcess: + """Print + run subprocess; raise on nonzero exit unless caller opts out.""" + print("+ " + " ".join(cmd), flush=True) + return subprocess.run(cmd, check=True, **kwargs) + + +def _get_claude_code_version() -> str: + """Return the version string printed by `claude --version`.""" + completed = subprocess.run( + ["claude", "--version"], capture_output=True, text=True, check=True + ) + # `claude --version` prints e.g. "2.1.120 (Claude Code)"; we keep the + # raw first whitespace-delimited token, which is the version. + out = (completed.stdout or "").strip() + return out.split()[0] if out else "" + + +def _start_proxy(image: str, port: int) -> str: + """Start the LiteLLM proxy via `docker run -d`; returns the container id.""" + completed = subprocess.run( + [ + "docker", + "run", + "-d", + "-p", + f"{port}:4000", + "--name", + "litellm-compat-matrix-proxy", + image, + "--port", + "4000", + ], + capture_output=True, + text=True, + check=True, + ) + container_id = (completed.stdout or "").strip() + if not container_id: + raise RuntimeError("docker run did not return a container id") + return container_id + + +def _stop_proxy(container_id: str) -> None: + subprocess.run(["docker", "rm", "-f", container_id], check=False) + + +def _wait_for_proxy(port: int, timeout_seconds: int = 60) -> None: + """Poll the proxy's /health endpoint until it returns 200 or we time out.""" + import urllib.error + import urllib.request + + url = f"http://127.0.0.1:{port}/health/liveliness" + deadline = time.time() + timeout_seconds + last_err: Optional[BaseException] = None + while time.time() < deadline: + try: + with urllib.request.urlopen(url, timeout=5) as resp: # noqa: S310 + if resp.status == 200: + return + except (urllib.error.URLError, OSError) as exc: + last_err = exc + time.sleep(2) + raise RuntimeError( + f"proxy did not become healthy within {timeout_seconds}s: {last_err!r}" + ) + + +def publish( + *, + docs_repo: str, + docs_branch: str, + docs_target_path: str, + docs_token: str, + manifest_path: Path, + results_path: Path, + matrix_output_path: Path, + litellm_version: str, + claude_code_version: str, + generated_at: str, +) -> None: + """Build the matrix JSON and direct-push it to the docs repo's branch. + + Only `docs_target_path` is staged from the docs-repo working tree — + any other file produced by the build is dropped via + `select_files_to_commit`. + """ + matrix = build_from_paths( + manifest_path=manifest_path, + results_path=results_path, + litellm_version=litellm_version, + claude_code_version=claude_code_version, + generated_at=generated_at, + output_path=matrix_output_path, + ) + + with tempfile.TemporaryDirectory(prefix="docs-repo-") as workdir: + workdir_path = Path(workdir) + clone_url = f"https://x-access-token:{docs_token}@github.com/{docs_repo}.git" + _run( + [ + "git", + "clone", + "--depth", + "1", + "--branch", + docs_branch, + clone_url, + str(workdir_path), + ] + ) + _run( + ["git", "config", "user.email", "litellm-bot@berri.ai"], + cwd=workdir_path, + ) + _run( + ["git", "config", "user.name", "litellm-compat-matrix-bot"], + cwd=workdir_path, + ) + + target_in_docs = workdir_path / docs_target_path + target_in_docs.parent.mkdir(parents=True, exist_ok=True) + target_in_docs.write_text(matrix_output_path.read_text()) + + # Defense in depth: even if some other tool dropped a file in the + # working tree, only the matrix JSON is staged. + staged = [docs_target_path] + keep = select_files_to_commit(staged, DOCS_TARGET_BASENAME) + if not keep: + raise RuntimeError( + "no allowed files to commit; expected " + f"{DOCS_TARGET_BASENAME!r} but got {staged!r}" + ) + for path in keep: + _run(["git", "add", path], cwd=workdir_path) + + # Skip the push entirely if the JSON is byte-identical to what's + # already on main — keeps the docs-repo git log clean during + # idempotent reruns of the cron. + diff = subprocess.run( + ["git", "diff", "--cached", "--quiet"], + cwd=workdir_path, + check=False, + ) + if diff.returncode == 0: + print("matrix JSON unchanged; skipping push", flush=True) + return + + _run( + ["git", "commit", "-m", commit_message_for_matrix(matrix)], + cwd=workdir_path, + ) + _run(["git", "push", "origin", docs_branch], cwd=workdir_path) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--docs-repo", + default=os.environ.get("DOCS_REPO", DOCS_REPO_DEFAULT), + help="`owner/name` of the docs repo to publish into.", + ) + parser.add_argument( + "--docs-branch", + default=os.environ.get("DOCS_BRANCH", "main"), + help="Branch on the docs repo to push to.", + ) + parser.add_argument( + "--docs-target-path", + default=os.environ.get("DOCS_TARGET_PATH", DOCS_TARGET_PATH_DEFAULT), + help="Path inside the docs repo where the matrix JSON lives.", + ) + parser.add_argument( + "--proxy-port", + type=int, + default=int(os.environ.get("PROXY_PORT", DEFAULT_PROXY_PORT)), + ) + parser.add_argument( + "--manifest", + type=Path, + default=DEFAULT_MANIFEST, + ) + parser.add_argument( + "--results", + type=Path, + default=DEFAULT_RESULTS, + help="Path where pytest will write the compat-results.json artifact.", + ) + parser.add_argument( + "--matrix-output", + type=Path, + default=REPO_ROOT / DOCS_TARGET_BASENAME, + ) + parser.add_argument( + "--skip-proxy", + action="store_true", + help=( + "Skip Docker/proxy/CLI/pytest steps and go straight to publish — " + "useful when the workflow runs those steps in separate jobs." + ), + ) + parser.add_argument( + "--skip-publish", + action="store_true", + help="Run the test pipeline but do not push to the docs repo.", + ) + args = parser.parse_args(argv) + + docs_token = os.environ.get("DOCS_REPO_TOKEN", "") + if not args.skip_publish and not docs_token: + print( + "DOCS_REPO_TOKEN is required to push to the docs repo " + "(GitHub App installation token)", + file=sys.stderr, + ) + return 2 + + container_id: Optional[str] = None + litellm_version: str + claude_code_version: str + try: + litellm_version = latest_stable_litellm_tag( + token=os.environ.get("GITHUB_TOKEN") + ) + print(f"resolved latest stable litellm: {litellm_version}", flush=True) + + if not args.skip_proxy: + image = docker_image_for_tag(litellm_version) + _run(["docker", "pull", image]) + container_id = _start_proxy(image, args.proxy_port) + _wait_for_proxy(args.proxy_port) + + _run(["npm", "install", "-g", "@anthropic-ai/claude-code@latest"]) + claude_code_version = _get_claude_code_version() + print(f"installed claude code cli: {claude_code_version}", flush=True) + + env = { + **os.environ, + "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{args.proxy_port}", + "ANTHROPIC_AUTH_TOKEN": DEFAULT_PROXY_API_KEY, + "COMPAT_RESULTS_PATH": str(args.results), + } + _run( + [ + "pytest", + "tests/claude_code/", + "--ignore=tests/claude_code/_driver_unit_tests", + "--ignore=tests/claude_code/_builder_unit_tests", + "--ignore=tests/claude_code/_publisher_unit_tests", + ], + env=env, + check=False, + ) + else: + claude_code_version = os.environ.get("CLAUDE_CODE_VERSION", "") + + if args.skip_publish: + print("skip-publish: not pushing to docs repo", flush=True) + return 0 + + publish( + docs_repo=args.docs_repo, + docs_branch=args.docs_branch, + docs_target_path=args.docs_target_path, + docs_token=docs_token, + manifest_path=args.manifest, + results_path=args.results, + matrix_output_path=args.matrix_output, + litellm_version=litellm_version, + claude_code_version=claude_code_version, + generated_at=_now_utc_iso(), + ) + return 0 + finally: + if container_id is not None: + _stop_proxy(container_id) + + +if __name__ == "__main__": + sys.exit(main()) + + +__all__ = [ + "DOCS_REPO_DEFAULT", + "DOCS_TARGET_BASENAME", + "DOCS_TARGET_PATH_DEFAULT", + "DOCKER_IMAGE_BASE", + "commit_message_for_matrix", + "docker_image_for_tag", + "select_files_to_commit", + "publish", + "main", +] diff --git a/tests/claude_code/resolver.py b/tests/claude_code/resolver.py new file mode 100644 index 00000000000..dc42e202302 --- /dev/null +++ b/tests/claude_code/resolver.py @@ -0,0 +1,91 @@ +"""Latest Stable LiteLLM Resolver. + +Queries the GitHub Releases API for `BerriAI/litellm` and returns the +newest tag matching `v*-stable`. Used by the daily-cron publisher to +decide which Docker image to pull for the matrix run. + +The resolver is intentionally tiny — its only state is the GitHub API +URL constant — and exposes a single public function so unit tests can +inject a fake HTTP getter and run offline. +""" + +from __future__ import annotations + +import json +import re +import urllib.request +from typing import Callable, List, Optional + +GITHUB_RELEASES_URL = "https://api.github.com/repos/BerriAI/litellm/releases" +USER_AGENT = "litellm-compat-matrix-resolver" +REQUEST_TIMEOUT_SECONDS = 30 + +STABLE_TAG_RE = re.compile(r"^v(\d+)\.(\d+)\.(\d+)-stable$") + + +class ResolverError(RuntimeError): + """Raised when the resolver cannot determine a latest-stable tag.""" + + +def _default_http_get(url: str, *, token: Optional[str] = None) -> str: + """Minimal urllib-based GET that the cron VM can call without extra deps. + + Forwards `token` as a Bearer header when set so the cron job can lift + the unauthenticated GitHub rate limit by passing the GitHub App + installation token. + """ + request = urllib.request.Request(url) + request.add_header("User-Agent", USER_AGENT) + request.add_header("Accept", "application/vnd.github+json") + if token: + request.add_header("Authorization", f"Bearer {token}") + with urllib.request.urlopen( # noqa: S310 - URL is hardcoded above + request, timeout=REQUEST_TIMEOUT_SECONDS + ) as response: + return response.read().decode("utf-8") + + +def latest_stable_litellm_tag( + *, + http_get: Optional[Callable[..., str]] = None, + token: Optional[str] = None, +) -> str: + """Return the newest `v*-stable` tag published on BerriAI/litellm. + + Sort is numeric on the (major, minor, patch) triple so v1.10.0-stable + correctly outranks v1.9.5-stable. Releases with no `tag_name` (drafts) + or that don't match the `v*-stable` shape are skipped. + """ + fetch = http_get or _default_http_get + payload = fetch(GITHUB_RELEASES_URL, token=token) + releases = json.loads(payload) + if not isinstance(releases, list): + raise ResolverError( + "github releases response is not a list; got " f"{type(releases).__name__}" + ) + + matched: List[tuple] = [] + for release in releases: + if not isinstance(release, dict): + continue + tag = release.get("tag_name") + if not isinstance(tag, str): + continue + m = STABLE_TAG_RE.match(tag) + if not m: + continue + version_key = tuple(int(x) for x in m.groups()) + matched.append((version_key, tag)) + + if not matched: + raise ResolverError("no v*-stable tags found in github releases response") + + matched.sort(key=lambda pair: pair[0], reverse=True) + return matched[0][1] + + +__all__ = [ + "GITHUB_RELEASES_URL", + "ResolverError", + "latest_stable_litellm_tag", +]