litellm/tests/unit/test_circleci_rust_toolchain.py

"""Static guardrails for how CircleCI provisions Rust.

The root package builds `litellm-rust` through maturin, so any job that runs
`uv sync` or `uv build` compiles the bridge. The `cimg/python` images ship no
Rust toolchain, and when cargo is missing maturin's `puccinialin` helper
quietly provisions one itself: it fetches `rustup-init` from the unversioned
`https://static.rust-lang.org/rustup/dist/<triple>/` path with no checksum and
installs a floating `stable` toolchain. uv suppresses build-backend output on a
successful sync, so that happens with nothing in the job log to show for it,
and the compiler a job builds with changes whenever upstream publishes.

Two invariants are pinned here:

  1. No step list (job or reusable command) reaches a `uv sync` / `uv build`
     without a Rust toolchain already provisioned ahead of it. That is the
     `install_rust` command on Linux and `install_windows_toolchain` on Windows,
     so the check accepts any command or step that installs a pinned rustup. A new job that syncs without one
     falls back to the unpinned path, which is exactly the regression a static
     check catches at PR time and a green CI run does not.
  2. Both installers pin what they download: an explicit rustup version, a
     verified SHA-256, and the exact toolchain in `rust-toolchain.toml`.
"""

from __future__ import annotations

import re
from pathlib import Path
from typing import Final

import pytest
import yaml

REPO_ROOT = Path(__file__).resolve().parents[2]
CONFIG = REPO_ROOT / ".circleci" / "config.yml"
TOOLCHAIN: Final = REPO_ROOT / "rust-toolchain.toml"

BUILDS_WORKSPACE = re.compile(r"\buv\s+(?:sync|build)\b")
RUSTUP_ARCHIVE_URL = re.compile(r"https://static\.rust-lang\.org/rustup/archive/\d+\.\d+\.\d+/")
EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?(\d+\.\d+\.\d+)\"?")
TOOLCHAIN_CHANNEL: Final = re.compile(r'^channel = "(\d+\.\d+\.\d+)"$', re.MULTILINE)


def _config() -> dict[str, object]:
    return yaml.safe_load(CONFIG.read_text())


def _step_text(step: object) -> str:
    """Flatten one step into the shell text it runs, or '' for a command reference."""
    if isinstance(step, dict):
        run = step.get("run")
        if isinstance(run, str):
            return run
        if isinstance(run, dict):
            command = run.get("command")
            return command if isinstance(command, str) else ""
    return ""


def _pinned_toolchain() -> str:
    match: Final = TOOLCHAIN_CHANNEL.search(TOOLCHAIN.read_text())
    assert match is not None, "rust-toolchain.toml must pin an exact channel"
    return match.group(1)


def _without_comments(text: str) -> str:
    return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#"))


def _installs_pinned_rustup(step: object) -> bool:
    text = _step_text(step)
    return "rustup-init" in text and ("sha256sum" in text or "SHA256" in text)


def _provisioning_commands() -> frozenset[str]:
    return frozenset(
        name.removeprefix("command ")
        for name, steps in _step_lists().items()
        if name.startswith("command ") and any(_installs_pinned_rustup(step) for step in steps)
    )


def _provisions_rust(step: object, provisioning_commands: frozenset[str]) -> bool:
    return (isinstance(step, str) and step in provisioning_commands) or _installs_pinned_rustup(step)


def _step_lists() -> dict[str, list[object]]:
    config = _config()
    lists: dict[str, list[object]] = {}
    for kind in ("jobs", "commands"):
        section = config.get(kind)
        if not isinstance(section, dict):
            continue
        for name, body in section.items():
            steps = body.get("steps") if isinstance(body, dict) else None
            if isinstance(steps, list):
                lists[f"{kind[:-1]} {name}"] = steps
    return lists


def _first_unprovisioned_build(steps: list[object], provisioning_commands: frozenset[str]) -> str | None:
    """Return the shell text of the first workspace build reached without Rust, if any."""
    rust_ready = False
    for step in steps:
        if _provisions_rust(step, provisioning_commands):
            rust_ready = True
        text = _step_text(step)
        if BUILDS_WORKSPACE.search(_without_comments(text)) and not rust_ready:
            return text
    return None


def test_step_lists_exist() -> None:
    lists = _step_lists()
    assert "command install_rust" in lists
    building = {
        name
        for name, steps in lists.items()
        if any(BUILDS_WORKSPACE.search(_without_comments(_step_text(s))) for s in steps)
    }
    assert "command setup_litellm_test_deps" in building, f"found {sorted(building)}"


def test_no_workspace_build_without_a_provisioned_rust_toolchain() -> None:
    provisioning_commands: Final = _provisioning_commands()
    assert {"install_rust", "install_windows_toolchain"} <= provisioning_commands
    offenders = {
        name: build
        for name, steps in _step_lists().items()
        if (build := _first_unprovisioned_build(steps, provisioning_commands)) is not None
    }
    assert not offenders, (
        "these CircleCI step lists run `uv sync`/`uv build` with no Rust toolchain provisioned first, "
        "so maturin will download an unpinned rustup and a floating toolchain instead: "
        f"{ {name: build.strip().splitlines()[0] for name, build in offenders.items()} }"
    )


@pytest.fixture(name="install_rust_command")
def _install_rust_command() -> str:
    steps = _step_lists()["command install_rust"]
    return "\n".join(_step_text(step) for step in steps)


def test_install_rust_pins_the_rustup_version_in_the_url(install_rust_command: str) -> None:
    assert RUSTUP_ARCHIVE_URL.search(install_rust_command), (
        "install_rust must download rustup-init from a version-pinned /rustup/archive/<x.y.z>/ URL; "
        "the /rustup/dist/ path always serves whatever rustup is current"
    )
    assert "/rustup/dist/" not in install_rust_command


def test_install_rust_verifies_the_installer_checksum(install_rust_command: str) -> None:
    assert "sha256sum -c" in install_rust_command
    assert re.search(r"RUSTUP_SHA256=[0-9a-f]{64}\b", install_rust_command), (
        "install_rust must compare the downloaded installer against a hardcoded SHA-256 "
        "taken from rust-lang's published .sha256 sidecar"
    )
    checksum_index = install_rust_command.index("sha256sum -c")
    execute_index = install_rust_command.index("/tmp/rustup-init -y")
    assert checksum_index < execute_index, "the checksum must be verified before the installer is executed"


def test_install_rust_pins_an_exact_toolchain_version(install_rust_command: str) -> None:
    match: Final = EXACT_TOOLCHAIN.search(install_rust_command)
    assert match is not None, (
        "install_rust must pin an exact toolchain version (e.g. 1.98.0); a channel name like "
        "stable/beta/nightly makes the compiler drift with whatever upstream published that day"
    )
    assert match.group(1) == _pinned_toolchain()


def test_windows_installer_matches_the_repo_toolchain() -> None:
    windows_steps: Final = _step_lists()["command install_windows_toolchain"]
    windows_command: Final = "\n".join(_step_text(step) for step in windows_steps)
    match: Final = EXACT_TOOLCHAIN.search(windows_command)
    assert match is not None
    assert match.group(1) == _pinned_toolchain()