diff --git a/.circleci/AGENTS.md b/.circleci/AGENTS.md new file mode 100644 index 00000000000..f2821001a9d --- /dev/null +++ b/.circleci/AGENTS.md @@ -0,0 +1,5 @@ +# .circleci + +The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values + +Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because the project's SSH checkout key no longer works and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ce5ea8b6fe..09e3c9a8bd3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,18 +21,29 @@ orbs: win: circleci/windows@5.0 # Add Windows orb commands: - checkout_migration_source: + checkout_over_https: + parameters: + revision: + type: string + default: "" + shell: + type: string + default: /bin/bash -eo pipefail steps: - run: - name: Select the requested migration test revision + name: Check out the public repository over HTTPS + shell: << parameters.shell >> environment: - MIGRATION_SOURCE_SHA: << pipeline.parameters.migration_source_sha >> + REQUESTED_REVISION: << parameters.revision >> command: | - revision="${MIGRATION_SOURCE_SHA:-$CIRCLE_SHA1}" + revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}" [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1 git init git remote add origin https://github.com/BerriAI/litellm.git - git fetch --depth 1 origin "$revision" + for _ in 1 2 3; do + git fetch --filter=blob:none origin "$revision" && break + sleep 5 + done git checkout --detach FETCH_HEAD skip_if_unrelated_changes: parameters: @@ -347,7 +358,7 @@ commands: - when: condition: << parameters.checkout >> steps: - - checkout + - checkout_over_https - setup_google_dns - install_uv - install_rust @@ -377,7 +388,8 @@ jobs: CARGO_HTTP_MULTIPLEXING: "false" CARGO_NET_RETRY: "5" steps: - - checkout + - checkout_over_https: + shell: bash.exe - run: name: Skip job when no windows-release-relevant files changed shell: bash.exe @@ -451,7 +463,7 @@ jobs: password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - - checkout + - checkout_over_https - setup_google_dns - install_uv - install_rust @@ -482,7 +494,7 @@ jobs: working_directory: ~/project steps: - - checkout + - checkout_over_https - skip_if_unrelated_changes - attach_workspace: at: ~/project @@ -559,7 +571,7 @@ jobs: resource_class: medium working_directory: ~/project steps: - - checkout + - checkout_over_https - run: name: Generate LiteLLM master key command: | @@ -659,7 +671,7 @@ jobs: resource_class: large working_directory: ~/project steps: - - checkout + - checkout_over_https - run: name: Generate LiteLLM master key command: | @@ -740,7 +752,7 @@ jobs: resource_class: large working_directory: ~/project steps: - - checkout + - checkout_over_https - skip_if_unrelated_changes: category: client - setup_google_dns @@ -794,7 +806,7 @@ jobs: # the rest of the suite: nothing else performs a logout. PROXY_LOGOUT_URL: "https://www.example.com" steps: - - checkout + - checkout_over_https - run: name: Generate LiteLLM master key command: | @@ -940,7 +952,7 @@ jobs: # and by migration.serverRootPath.config.ts, which refuses to run without it. SERVER_ROOT_PATH: "/litellm" steps: - - checkout + - checkout_over_https - run: name: Generate LiteLLM master key command: | @@ -1066,11 +1078,12 @@ jobs: - when: condition: << parameters.migration_qualification >> steps: - - checkout_migration_source + - checkout_over_https: + revision: << pipeline.parameters.migration_source_sha >> - unless: condition: << parameters.migration_qualification >> steps: - - checkout + - checkout_over_https - skip_if_unrelated_changes - run: @@ -1122,7 +1135,8 @@ jobs: MIGRATION_TEST_OUTPUT: /tmp/migration-results PYTHONPATH: tests/e2e steps: - - checkout_migration_source + - checkout_over_https: + revision: << pipeline.parameters.migration_source_sha >> - run: name: Generate LiteLLM master key command: | @@ -1197,7 +1211,7 @@ jobs: resource_class: medium working_directory: ~/project steps: - - checkout + - checkout_over_https - skip_if_unrelated_changes - attach_workspace: at: ~/project @@ -1371,7 +1385,7 @@ jobs: environment: UV_PYTHON_DOWNLOADS: never steps: - - checkout + - checkout_over_https - install_uv - run: name: Install the locked shard verifier and coverage dependencies @@ -1494,7 +1508,7 @@ jobs: environment: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test steps: - - checkout + - checkout_over_https - skip_if_unrelated_changes - setup_litellm_test_deps: checkout: false diff --git a/tests/unit/test_circleci_checkout.py b/tests/unit/test_circleci_checkout.py new file mode 100644 index 00000000000..469d51fdd22 --- /dev/null +++ b/tests/unit/test_circleci_checkout.py @@ -0,0 +1,220 @@ +import os +import shutil +import subprocess +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +import yaml +from pydantic import BaseModel, ConfigDict + +CIRCLECI: Final = Path(__file__).resolve().parents[2] / ".circleci" +CONFIG: Final = CIRCLECI / "config.yml" +PUBLIC_REPO: Final = "https://github.com/BerriAI/litellm.git" +LINUX_SHELL: Final = ("bash", "-eo", "pipefail", "-c") +WINDOWS_SHELL: Final = ("bash", "-c") + + +class _Run(BaseModel): + model_config = ConfigDict(frozen=True) + command: str + + +class _Step(BaseModel): + model_config = ConfigDict(frozen=True) + run: _Run + + +class _Command(BaseModel): + model_config = ConfigDict(frozen=True) + steps: tuple[_Step] + + +class _Commands(BaseModel): + model_config = ConfigDict(frozen=True) + checkout_over_https: _Command + + +class _Config(BaseModel): + model_config = ConfigDict(frozen=True) + commands: _Commands + + +@dataclass(frozen=True, slots=True) +class _Upstream: + url: str + older: str + newer: str + + +def _step_name(step: object) -> str: + if isinstance(step, str): + return step + if isinstance(step, dict) and len(step) == 1: + return str(next(iter(step))) + return "" + + +def _step_names(node: object) -> Iterator[str]: + if isinstance(node, dict): + for key, value in node.items(): + if key in ("steps", "pre-steps", "post-steps") and isinstance(value, list): + yield from map(_step_name, value) + yield from _step_names(value) + if isinstance(node, list): + for item in node: + yield from _step_names(item) + + +def _git(cwd: Path, *args: str) -> str: + return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout.strip() + + +def _commit(work: Path, files: Mapping[str, str], message: str) -> str: + for name, content in files.items(): + (work / name).write_text(content) + _git(work, "add", "-A") + _git(work, "commit", "-q", "--allow-empty", "-m", message) + return _git(work, "rev-parse", "HEAD") + + +def _new_upstream(tmp_path: Path) -> tuple[Path, Path]: + remote: Final = tmp_path / "remote.git" + work: Final = tmp_path / "work" + work.mkdir() + _git(tmp_path, "init", "-q", "--bare", str(remote)) + _git(remote, "config", "uploadpack.allowFilter", "true") + _git(work, "init", "-q", "-b", "main") + _git(work, "config", "user.email", "t@t") + _git(work, "config", "user.name", "t") + return remote, work + + +def _publish_two_commits(tmp_path: Path) -> _Upstream: + remote, work = _new_upstream(tmp_path) + older: Final = _commit(work, {"app.py": "x = 1\n"}, "pipeline commit") + newer: Final = _commit(work, {"app.py": "x = 2\n"}, "main moved on") + _git(work, "push", "-q", str(remote), "main") + return _Upstream(url=remote.as_uri(), older=older, newer=newer) + + +def _publish_docs_only_pr(tmp_path: Path) -> _Upstream: + remote, work = _new_upstream(tmp_path) + main: Final = _commit(work, {"litellm_inference.py": "x\n"}, "base") + _git(work, "checkout", "-q", "-b", "litellm_feature") + head: Final = _commit(work, {"README.md": "# docs\n"}, "docs") + _git(work, "push", "-q", str(remote), "main", "litellm_feature") + return _Upstream(url=remote.as_uri(), older=main, newer=head) + + +def _stub(bin_dir: Path, name: str, script: str) -> None: + bin_dir.mkdir(exist_ok=True) + (bin_dir / name).write_text(f"#!/usr/bin/env bash\n{script}\n") + (bin_dir / name).chmod(0o755) + + +def _job_env(tmp_path: Path, upstream: _Upstream) -> dict[str, str]: + return { + "PATH": f"{tmp_path / 'bin'}{os.pathsep}{os.environ['PATH']}", + "HOME": str(tmp_path), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "GIT_SSH_COMMAND": "false", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": f"url.{upstream.url}.insteadOf", + "GIT_CONFIG_VALUE_0": PUBLIC_REPO, + } + + +def _run_checkout( + tmp_path: Path, + upstream: _Upstream, + revisions: Mapping[str, str], + shell: tuple[str, ...] = LINUX_SHELL, +) -> subprocess.CompletedProcess[str]: + job: Final = tmp_path / "job" + job.mkdir() + command: Final = _Config.model_validate(yaml.safe_load(CONFIG.read_text())).commands.checkout_over_https + return subprocess.run( + [*shell, command.steps[0].run.command], + cwd=job, + env={**_job_env(tmp_path, upstream), **revisions}, + capture_output=True, + text=True, + ) + + +def test_every_clone_goes_over_https_so_no_job_needs_the_ssh_checkout_key() -> None: + names: Final = tuple(_step_names(yaml.safe_load(CONFIG.read_text()))) + assert "checkout_over_https" in names + assert "checkout" not in names + + +@pytest.mark.parametrize( + "revisions", + ( + {"CIRCLE_SHA1": "older"}, + {"CIRCLE_SHA1": "older", "REQUESTED_REVISION": ""}, + {"CIRCLE_SHA1": "newer", "REQUESTED_REVISION": "older"}, + ), +) +def test_a_job_checks_out_its_own_commit_from_the_public_https_url_after_main_moves( + tmp_path: Path, revisions: Mapping[str, str] +) -> None: + upstream: Final = _publish_two_commits(tmp_path) + shas: Final = {"older": upstream.older, "newer": upstream.newer, "": ""} + result: Final = _run_checkout(tmp_path, upstream, {name: shas[value] for name, value in revisions.items()}) + assert result.returncode == 0, result.stderr + assert _git(tmp_path / "job", "rev-parse", "HEAD") == upstream.older + + +@pytest.mark.parametrize("shell", (LINUX_SHELL, WINDOWS_SHELL), ids=("linux", "windows-bash-exe")) +def test_a_revision_that_is_not_a_full_sha_is_refused_before_git_runs(tmp_path: Path, shell: tuple[str, ...]) -> None: + upstream: Final = _publish_two_commits(tmp_path) + result: Final = _run_checkout( + tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer, "REQUESTED_REVISION": "main"}, shell + ) + assert result.returncode != 0 + assert not (tmp_path / "job" / ".git").exists() + + +def test_a_docs_only_pr_job_still_skips_backend_work_after_the_checkout(tmp_path: Path) -> None: + upstream: Final = _publish_docs_only_pr(tmp_path) + _stub(tmp_path / "bin", "circleci-agent", 'echo "circleci-agent $*"') + checkout: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer}) + assert checkout.returncode == 0, checkout.stderr + path_filter: Final = subprocess.run( + ["bash", str(CIRCLECI / "scripts" / "path_filter.sh"), "backend"], + cwd=tmp_path / "job", + env={**_job_env(tmp_path, upstream), "CIRCLE_PULL_REQUEST": "https://github.com/BerriAI/litellm/pull/1"}, + capture_output=True, + text=True, + ) + assert path_filter.returncode == 0, path_filter.stderr + assert "circleci-agent step halt" in path_filter.stdout + + +@pytest.mark.parametrize("shell", (LINUX_SHELL, WINDOWS_SHELL), ids=("linux", "windows-bash-exe")) +@pytest.mark.parametrize(("failing_fetches", "checked_out"), ((2, True), (3, False))) +def test_a_fetch_that_fails_is_retried_and_the_step_fails_once_retries_run_out( + tmp_path: Path, shell: tuple[str, ...], failing_fetches: int, checked_out: bool +) -> None: + upstream: Final = _publish_two_commits(tmp_path) + fetches: Final = tmp_path / "fetches" + _stub(tmp_path / "bin", "sleep", "exit 0") + _stub( + tmp_path / "bin", + "git", + f'''if [ "$1" = fetch ]; then + echo x >> "{fetches}" + [ "$(wc -l < "{fetches}")" -le {failing_fetches} ] && exit 128 +fi +exec "{shutil.which("git")}" "$@"''', + ) + result: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.older}, shell) + assert (result.returncode == 0) is checked_out, result.stderr + assert len(fetches.read_text().splitlines()) == min(failing_fetches + 1, 3) + head: Final = subprocess.run(["git", "rev-parse", "HEAD"], cwd=tmp_path / "job", capture_output=True, text=True) + assert (head.stdout.strip() == upstream.older) is checked_out