litellm/tests/claude_code/publisher.py

"""Daily-cron matrix publisher.

End-to-end orchestrator that runs on the isolated cron VM (per the PRD's
"Two CI environments / Daily Cron" section). The flow is:

  1. Resolve the latest LiteLLM `v*-stable` tag via `resolver.py`.
  2. Pull the corresponding Docker image and start it as the proxy.
  3. Install the absolute latest Claude Code CLI from npm.
  4. Run `pytest tests/claude_code/` against the proxy.
  5. Build `compatibility-matrix.json` from the per-test results artifact
     using the Matrix JSON Builder (`matrix_builder.py`).
  6. Direct-push the JSON to the docs repo's main branch using the
     GitHub App installation token mounted as `DOCS_REPO_TOKEN`.

The orchestration is thin glue over Docker, git, npm and subprocess — per
the PRD's "Testing Decisions" section, it intentionally ships without a
unit-test harness; the daily-cron failure surface is itself the test. The
pure helpers below (commit message, image-name builder, file allowlist)
are unit-tested under `_publisher_unit_tests/`.

The "only `compatibility-matrix.json` is ever committed" guarantee is
enforced by `select_files_to_commit` rather than by token scope, since
GitHub Apps cannot scope `contents: write` to a single file path.
"""

from __future__ import annotations

import argparse
import datetime
import os
import subprocess
import sys
import tempfile
import time
from pathlib import Path
from typing import Any, List, Mapping, Optional, Sequence

from tests.claude_code.matrix_builder import build_from_paths
from tests.claude_code.resolver import latest_stable_litellm_tag

DOCS_REPO_DEFAULT = "BerriAI/litellm-docs"
DOCS_TARGET_BASENAME = "compatibility-matrix.json"
DOCS_TARGET_PATH_DEFAULT = f"static/data/{DOCS_TARGET_BASENAME}"
DOCKER_IMAGE_BASE = "ghcr.io/berriai/litellm"
DEFAULT_PROXY_PORT = 4000
DEFAULT_PROXY_API_KEY = "sk-cron-matrix"  # only used inside the ephemeral VM

REPO_ROOT = Path(__file__).resolve().parents[2]
DEFAULT_MANIFEST = REPO_ROOT / "tests" / "claude_code" / "manifest.yaml"
DEFAULT_RESULTS = REPO_ROOT / "compat-results.json"


def commit_message_for_matrix(matrix: Mapping[str, Any]) -> str:
    """Build a deterministic commit message for the docs-repo push.

    Surfaces the three pieces of provenance the docs banner shows
    (LiteLLM version, Claude Code version, generated_at) so the docs
    repo's git log is self-describing without opening the JSON.
    """
    litellm_version = matrix.get("litellm_version", "")
    claude_code_version = matrix.get("claude_code_version", "")
    generated_at = matrix.get("generated_at", "")
    headline = "Update Claude Code compatibility matrix"
    body_lines = [
        f"litellm_version: {litellm_version}",
        f"claude_code_version: {claude_code_version}",
        f"generated_at: {generated_at}",
    ]
    return headline + "\n\n" + "\n".join(body_lines) + "\n"


def docker_image_for_tag(tag: str) -> str:
    """Return the ghcr.io image reference for a `v*-stable` tag."""
    if not tag:
        raise ValueError("tag must be a non-empty string")
    return f"{DOCKER_IMAGE_BASE}:{tag}"


def select_files_to_commit(
    staged_paths: Sequence[str], allowed_basename: str
) -> List[str]:
    """Return only the paths whose basename matches the allowlist.

    The cron VM's GitHub App holds `contents: write` on the entire docs
    repo (GitHub does not support file-path-scoped tokens), so the
    "only ship the matrix JSON" property is enforced here instead. Any
    stray file in the working tree is dropped before the commit step.
    """
    return [p for p in staged_paths if os.path.basename(p) == allowed_basename]


def _now_utc_iso() -> str:
    """ISO-8601 UTC timestamp with `Z` suffix, matching the v1 schema."""
    return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")


def _run(cmd: Sequence[str], **kwargs: Any) -> subprocess.CompletedProcess:
    """Print + run subprocess; raise on nonzero exit unless caller opts out."""
    print("+ " + " ".join(cmd), flush=True)
    return subprocess.run(cmd, check=True, **kwargs)


def _get_claude_code_version() -> str:
    """Return the version string printed by `claude --version`."""
    completed = subprocess.run(
        ["claude", "--version"], capture_output=True, text=True, check=True
    )
    # `claude --version` prints e.g. "2.1.120 (Claude Code)"; we keep the
    # raw first whitespace-delimited token, which is the version.
    out = (completed.stdout or "").strip()
    return out.split()[0] if out else ""


def _start_proxy(image: str, port: int) -> str:
    """Start the LiteLLM proxy via `docker run -d`; returns the container id."""
    completed = subprocess.run(
        [
            "docker",
            "run",
            "-d",
            "-p",
            f"{port}:4000",
            "--name",
            "litellm-compat-matrix-proxy",
            image,
            "--port",
            "4000",
        ],
        capture_output=True,
        text=True,
        check=True,
    )
    container_id = (completed.stdout or "").strip()
    if not container_id:
        raise RuntimeError("docker run did not return a container id")
    return container_id


def _stop_proxy(container_id: str) -> None:
    subprocess.run(["docker", "rm", "-f", container_id], check=False)


def _wait_for_proxy(port: int, timeout_seconds: int = 60) -> None:
    """Poll the proxy's /health endpoint until it returns 200 or we time out."""
    import urllib.error
    import urllib.request

    url = f"http://127.0.0.1:{port}/health/liveliness"
    deadline = time.time() + timeout_seconds
    last_err: Optional[BaseException] = None
    while time.time() < deadline:
        try:
            with urllib.request.urlopen(url, timeout=5) as resp:  # noqa: S310
                if resp.status == 200:
                    return
        except (urllib.error.URLError, OSError) as exc:
            last_err = exc
        time.sleep(2)
    raise RuntimeError(
        f"proxy did not become healthy within {timeout_seconds}s: {last_err!r}"
    )


def publish(
    *,
    docs_repo: str,
    docs_branch: str,
    docs_target_path: str,
    docs_token: str,
    manifest_path: Path,
    results_path: Path,
    matrix_output_path: Path,
    litellm_version: str,
    claude_code_version: str,
    generated_at: str,
) -> None:
    """Build the matrix JSON and direct-push it to the docs repo's branch.

    Only `docs_target_path` is staged from the docs-repo working tree —
    any other file produced by the build is dropped via
    `select_files_to_commit`.
    """
    matrix = build_from_paths(
        manifest_path=manifest_path,
        results_path=results_path,
        litellm_version=litellm_version,
        claude_code_version=claude_code_version,
        generated_at=generated_at,
        output_path=matrix_output_path,
    )

    with tempfile.TemporaryDirectory(prefix="docs-repo-") as workdir:
        workdir_path = Path(workdir)
        clone_url = f"https://x-access-token:{docs_token}@github.com/{docs_repo}.git"
        _run(
            [
                "git",
                "clone",
                "--depth",
                "1",
                "--branch",
                docs_branch,
                clone_url,
                str(workdir_path),
            ]
        )
        _run(
            ["git", "config", "user.email", "litellm-bot@berri.ai"],
            cwd=workdir_path,
        )
        _run(
            ["git", "config", "user.name", "litellm-compat-matrix-bot"],
            cwd=workdir_path,
        )

        target_in_docs = workdir_path / docs_target_path
        target_in_docs.parent.mkdir(parents=True, exist_ok=True)
        target_in_docs.write_text(matrix_output_path.read_text())

        # Defense in depth: even if some other tool dropped a file in the
        # working tree, only the matrix JSON is staged.
        staged = [docs_target_path]
        keep = select_files_to_commit(staged, DOCS_TARGET_BASENAME)
        if not keep:
            raise RuntimeError(
                "no allowed files to commit; expected "
                f"{DOCS_TARGET_BASENAME!r} but got {staged!r}"
            )
        for path in keep:
            _run(["git", "add", path], cwd=workdir_path)

        # Skip the push entirely if the JSON is byte-identical to what's
        # already on main — keeps the docs-repo git log clean during
        # idempotent reruns of the cron.
        diff = subprocess.run(
            ["git", "diff", "--cached", "--quiet"],
            cwd=workdir_path,
            check=False,
        )
        if diff.returncode == 0:
            print("matrix JSON unchanged; skipping push", flush=True)
            return

        _run(
            ["git", "commit", "-m", commit_message_for_matrix(matrix)],
            cwd=workdir_path,
        )
        _run(["git", "push", "origin", docs_branch], cwd=workdir_path)


def main(argv: Optional[Sequence[str]] = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--docs-repo",
        default=os.environ.get("DOCS_REPO", DOCS_REPO_DEFAULT),
        help="`owner/name` of the docs repo to publish into.",
    )
    parser.add_argument(
        "--docs-branch",
        default=os.environ.get("DOCS_BRANCH", "main"),
        help="Branch on the docs repo to push to.",
    )
    parser.add_argument(
        "--docs-target-path",
        default=os.environ.get("DOCS_TARGET_PATH", DOCS_TARGET_PATH_DEFAULT),
        help="Path inside the docs repo where the matrix JSON lives.",
    )
    parser.add_argument(
        "--proxy-port",
        type=int,
        default=int(os.environ.get("PROXY_PORT", DEFAULT_PROXY_PORT)),
    )
    parser.add_argument(
        "--manifest",
        type=Path,
        default=DEFAULT_MANIFEST,
    )
    parser.add_argument(
        "--results",
        type=Path,
        default=DEFAULT_RESULTS,
        help="Path where pytest will write the compat-results.json artifact.",
    )
    parser.add_argument(
        "--matrix-output",
        type=Path,
        default=REPO_ROOT / DOCS_TARGET_BASENAME,
    )
    parser.add_argument(
        "--skip-proxy",
        action="store_true",
        help=(
            "Skip Docker/proxy/CLI/pytest steps and go straight to publish — "
            "useful when the workflow runs those steps in separate jobs."
        ),
    )
    parser.add_argument(
        "--skip-publish",
        action="store_true",
        help="Run the test pipeline but do not push to the docs repo.",
    )
    args = parser.parse_args(argv)

    docs_token = os.environ.get("DOCS_REPO_TOKEN", "")
    if not args.skip_publish and not docs_token:
        print(
            "DOCS_REPO_TOKEN is required to push to the docs repo "
            "(GitHub App installation token)",
            file=sys.stderr,
        )
        return 2

    container_id: Optional[str] = None
    litellm_version: str
    claude_code_version: str
    try:
        litellm_version = latest_stable_litellm_tag(
            token=os.environ.get("GITHUB_TOKEN")
        )
        print(f"resolved latest stable litellm: {litellm_version}", flush=True)

        if not args.skip_proxy:
            image = docker_image_for_tag(litellm_version)
            _run(["docker", "pull", image])
            container_id = _start_proxy(image, args.proxy_port)
            _wait_for_proxy(args.proxy_port)

            _run(["npm", "install", "-g", "@anthropic-ai/claude-code@latest"])
            claude_code_version = _get_claude_code_version()
            print(f"installed claude code cli: {claude_code_version}", flush=True)

            env = {
                **os.environ,
                "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{args.proxy_port}",
                "ANTHROPIC_AUTH_TOKEN": DEFAULT_PROXY_API_KEY,
                "COMPAT_RESULTS_PATH": str(args.results),
            }
            _run(
                [
                    "pytest",
                    "tests/claude_code/",
                    "--ignore=tests/claude_code/_driver_unit_tests",
                    "--ignore=tests/claude_code/_builder_unit_tests",
                    "--ignore=tests/claude_code/_publisher_unit_tests",
                ],
                env=env,
                check=False,
            )
        else:
            claude_code_version = os.environ.get("CLAUDE_CODE_VERSION", "")

        if args.skip_publish:
            print("skip-publish: not pushing to docs repo", flush=True)
            return 0

        publish(
            docs_repo=args.docs_repo,
            docs_branch=args.docs_branch,
            docs_target_path=args.docs_target_path,
            docs_token=docs_token,
            manifest_path=args.manifest,
            results_path=args.results,
            matrix_output_path=args.matrix_output,
            litellm_version=litellm_version,
            claude_code_version=claude_code_version,
            generated_at=_now_utc_iso(),
        )
        return 0
    finally:
        if container_id is not None:
            _stop_proxy(container_id)


if __name__ == "__main__":
    sys.exit(main())


__all__ = [
    "DOCS_REPO_DEFAULT",
    "DOCS_TARGET_BASENAME",
    "DOCS_TARGET_PATH_DEFAULT",
    "DOCKER_IMAGE_BASE",
    "commit_message_for_matrix",
    "docker_image_for_tag",
    "select_files_to_commit",
    "publish",
    "main",
]