diff --git a/.circleci/config.yml b/.circleci/config.yml index 9063ba1d4a6..790cd6c7010 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,6 +88,36 @@ commands: rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" + install_rust: + description: "Install pinned rustup (1.28.2) and Rust toolchain (1.97.1) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself." + steps: + - run: + name: Install Rust (rustup 1.28.2, toolchain 1.97.1) + command: | + case "$(uname -m)" in + x86_64) + RUSTUP_TRIPLE=x86_64-unknown-linux-gnu + RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c + ;; + aarch64) + RUSTUP_TRIPLE=aarch64-unknown-linux-gnu + RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c + ;; + *) + echo "install_rust: unsupported architecture $(uname -m)" >&2 + exit 1 + ;; + esac + curl -sSLf -o /tmp/rustup-init \ + "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init" + echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - + chmod +x /tmp/rustup-init + /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.97.1 + rm -f /tmp/rustup-init + echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.cargo/bin:$PATH" + rustc --version + cargo --version start_postgres: description: "Start a postgres-db container on port 5432 and wait until it accepts connections." parameters: @@ -198,6 +228,7 @@ commands: - checkout - setup_google_dns - install_uv + - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} @@ -312,6 +343,7 @@ jobs: - checkout - setup_google_dns - install_uv + - install_rust - run: name: Build the wheel environment: @@ -344,6 +376,7 @@ jobs: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -417,6 +450,7 @@ jobs: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -491,6 +525,7 @@ jobs: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -542,6 +577,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -608,6 +644,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -648,6 +685,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -689,6 +727,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -722,6 +761,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} @@ -772,6 +812,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} @@ -823,6 +864,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -856,6 +898,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} @@ -902,6 +945,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -948,6 +992,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -990,6 +1035,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1036,6 +1082,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1083,6 +1130,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} @@ -1123,6 +1171,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1168,6 +1217,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1212,6 +1262,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1244,6 +1295,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1287,6 +1339,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1331,6 +1384,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1375,6 +1429,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1406,6 +1461,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1452,6 +1508,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1497,6 +1554,7 @@ jobs: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1547,6 +1605,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1571,6 +1630,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1597,6 +1657,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1698,6 +1759,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1793,6 +1855,7 @@ jobs: at: ~/project - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1881,6 +1944,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -1964,6 +2028,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -2096,6 +2161,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -2182,6 +2248,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -2278,6 +2345,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -2356,6 +2424,7 @@ jobs: - setup_google_dns # Remove Docker CLI installation since it's already available in machine executor - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -2437,6 +2506,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -2576,6 +2646,7 @@ jobs: - skip_if_unrelated_changes - setup_google_dns - install_uv + - install_rust - run: name: Install Dependencies command: | @@ -2766,6 +2837,7 @@ jobs: category: client - setup_google_dns - install_uv + - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} @@ -2908,6 +2980,7 @@ jobs: category: client - setup_google_dns - install_uv + - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} diff --git a/tests/test_litellm/test_circleci_rust_toolchain.py b/tests/test_litellm/test_circleci_rust_toolchain.py new file mode 100644 index 00000000000..c35ced51e16 --- /dev/null +++ b/tests/test_litellm/test_circleci_rust_toolchain.py @@ -0,0 +1,148 @@ +"""Static guardrails for how CircleCI provisions Rust. + +The root package builds `litellm-rust` through maturin, so any job that runs +`uv sync` or `uv build` compiles the bridge. The `cimg/python` images ship no +Rust toolchain, and when cargo is missing maturin's `puccinialin` helper +quietly provisions one itself: it fetches `rustup-init` from the unversioned +`https://static.rust-lang.org/rustup/dist//` path with no checksum and +installs a floating `stable` toolchain. uv suppresses build-backend output on a +successful sync, so that happens with nothing in the job log to show for it, +and the compiler a job builds with changes whenever upstream publishes. + +Two invariants are pinned here: + + 1. No step list (job or reusable command) reaches a `uv sync` / `uv build` + without a Rust toolchain already provisioned ahead of it. That is the + `install_rust` command on Linux and an inline pinned rustup install in the + Windows job, so the check accepts either. A new job that syncs without one + falls back to the unpinned path, which is exactly the regression a static + check catches at PR time and a green CI run does not. + 2. `install_rust` itself pins what it downloads: an explicit rustup version in + the URL, a verified SHA-256, and an exact toolchain version rather than a + channel name. + +The Windows job predates `install_rust` and provisions its toolchain inline, so +invariant 2 is scoped to `install_rust`; invariant 1 covers both. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +CONFIG = REPO_ROOT / ".circleci" / "config.yml" + +BUILDS_WORKSPACE = re.compile(r"\buv\s+(?:sync|build)\b") +RUSTUP_ARCHIVE_URL = re.compile(r"https://static\.rust-lang\.org/rustup/archive/\d+\.\d+\.\d+/") +EXACT_TOOLCHAIN = re.compile(r"--default-toolchain\s+\"?\d+\.\d+\.\d+\"?") + + +def _config() -> dict[str, object]: + return yaml.safe_load(CONFIG.read_text()) + + +def _step_text(step: object) -> str: + """Flatten one step into the shell text it runs, or '' for a command reference.""" + if isinstance(step, dict): + run = step.get("run") + if isinstance(run, str): + return run + if isinstance(run, dict): + command = run.get("command") + return command if isinstance(command, str) else "" + return "" + + +def _without_comments(text: str) -> str: + return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) + + +def _provisions_rust(step: object) -> bool: + if step == "install_rust": + return True + text = _step_text(step) + return "rustup-init" in text and ("sha256sum" in text or "SHA256" in text) + + +def _step_lists() -> dict[str, list[object]]: + config = _config() + lists: dict[str, list[object]] = {} + for kind in ("jobs", "commands"): + section = config.get(kind) + if not isinstance(section, dict): + continue + for name, body in section.items(): + steps = body.get("steps") if isinstance(body, dict) else None + if isinstance(steps, list): + lists[f"{kind[:-1]} {name}"] = steps + return lists + + +def _first_unprovisioned_build(steps: list[object]) -> str | None: + """Return the shell text of the first workspace build reached without Rust, if any.""" + rust_ready = False + for step in steps: + if _provisions_rust(step): + rust_ready = True + text = _step_text(step) + if BUILDS_WORKSPACE.search(_without_comments(text)) and not rust_ready: + return text + return None + + +def test_step_lists_exist() -> None: + lists = _step_lists() + assert "command install_rust" in lists + building = { + name + for name, steps in lists.items() + if any(BUILDS_WORKSPACE.search(_without_comments(_step_text(s))) for s in steps) + } + assert len(building) > 10, f"expected many workspace-building step lists, found {sorted(building)}" + + +def test_no_workspace_build_without_a_provisioned_rust_toolchain() -> None: + offenders = { + name: build for name, steps in _step_lists().items() if (build := _first_unprovisioned_build(steps)) is not None + } + assert not offenders, ( + "these CircleCI step lists run `uv sync`/`uv build` with no Rust toolchain provisioned first, " + "so maturin will download an unpinned rustup and a floating toolchain instead: " + f"{ {name: build.strip().splitlines()[0] for name, build in offenders.items()} }" + ) + + +@pytest.fixture(name="install_rust_command") +def _install_rust_command() -> str: + steps = _step_lists()["command install_rust"] + return "\n".join(_step_text(step) for step in steps) + + +def test_install_rust_pins_the_rustup_version_in_the_url(install_rust_command: str) -> None: + assert RUSTUP_ARCHIVE_URL.search(install_rust_command), ( + "install_rust must download rustup-init from a version-pinned /rustup/archive// URL; " + "the /rustup/dist/ path always serves whatever rustup is current" + ) + assert "/rustup/dist/" not in install_rust_command + + +def test_install_rust_verifies_the_installer_checksum(install_rust_command: str) -> None: + assert "sha256sum -c" in install_rust_command + assert re.search(r"RUSTUP_SHA256=[0-9a-f]{64}\b", install_rust_command), ( + "install_rust must compare the downloaded installer against a hardcoded SHA-256 " + "taken from rust-lang's published .sha256 sidecar" + ) + checksum_index = install_rust_command.index("sha256sum -c") + execute_index = install_rust_command.index("/tmp/rustup-init -y") + assert checksum_index < execute_index, "the checksum must be verified before the installer is executed" + + +def test_install_rust_pins_an_exact_toolchain_version(install_rust_command: str) -> None: + assert EXACT_TOOLCHAIN.search(install_rust_command), ( + "install_rust must pin an exact toolchain version (e.g. 1.97.1); a channel name like " + "stable/beta/nightly makes the compiler drift with whatever upstream published that day" + )