Merge 92c69e6155 into a392a39627

.circleci/AGENTS.md

@@ -0,0 +1,5 @@
+# .circleci
+
+The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values
+
+Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because the project's SSH checkout key no longer works and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base

.circleci/config.yml

@@ -21,18 +21,29 @@ orbs:
   win: circleci/windows@5.0 # Add Windows orb
 
 commands:
-  checkout_migration_source:
+  checkout_over_https:
+    parameters:
+      revision:
+        type: string
+        default: ""
+      shell:
+        type: string
+        default: /bin/bash -eo pipefail
     steps:
       - run:
-          name: Select the requested migration test revision
+          name: Check out the public repository over HTTPS
+          shell: << parameters.shell >>
           environment:
-            MIGRATION_SOURCE_SHA: << pipeline.parameters.migration_source_sha >>
+            REQUESTED_REVISION: << parameters.revision >>
           command: |
-            revision="${MIGRATION_SOURCE_SHA:-$CIRCLE_SHA1}"
+            revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}"
             [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1
             git init
             git remote add origin https://github.com/BerriAI/litellm.git
-            git fetch --depth 1 origin "$revision"
+            for _ in 1 2 3; do
+              git fetch --filter=blob:none origin "$revision" && break
+              sleep 5
+            done
             git checkout --detach FETCH_HEAD
   skip_if_unrelated_changes:
     parameters:
@@ -347,7 +358,7 @@ commands:
       - when:
           condition: << parameters.checkout >>
           steps:
-            - checkout
+            - checkout_over_https
       - setup_google_dns
       - install_uv
       - install_rust
@@ -377,7 +388,8 @@ jobs:
       CARGO_HTTP_MULTIPLEXING: "false"
       CARGO_NET_RETRY: "5"
     steps:
-      - checkout
+      - checkout_over_https:
+          shell: bash.exe
       - run:
           name: Skip job when no windows-release-relevant files changed
           shell: bash.exe
@@ -451,7 +463,7 @@ jobs:
           password: ${DOCKERHUB_PASSWORD}
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - setup_google_dns
       - install_uv
       - install_rust
@@ -482,7 +494,7 @@ jobs:
     working_directory: ~/project
 
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes
       - attach_workspace:
           at: ~/project
@@ -559,7 +571,7 @@ jobs:
     resource_class: medium
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -659,7 +671,7 @@ jobs:
     resource_class: large
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -740,7 +752,7 @@ jobs:
     resource_class: large
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes:
           category: client
       - setup_google_dns
@@ -794,7 +806,7 @@ jobs:
       # the rest of the suite: nothing else performs a logout.
       PROXY_LOGOUT_URL: "https://www.example.com"
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -940,7 +952,7 @@ jobs:
       # and by migration.serverRootPath.config.ts, which refuses to run without it.
       SERVER_ROOT_PATH: "/litellm"
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -1066,11 +1078,12 @@ jobs:
       - when:
           condition: << parameters.migration_qualification >>
           steps:
-            - checkout_migration_source
+            - checkout_over_https:
+                revision: << pipeline.parameters.migration_source_sha >>
       - unless:
           condition: << parameters.migration_qualification >>
           steps:
-            - checkout
+            - checkout_over_https
             - skip_if_unrelated_changes
 
       - run:
@@ -1122,7 +1135,8 @@ jobs:
       MIGRATION_TEST_OUTPUT: /tmp/migration-results
       PYTHONPATH: tests/e2e
     steps:
-      - checkout_migration_source
+      - checkout_over_https:
+          revision: << pipeline.parameters.migration_source_sha >>
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -1197,7 +1211,7 @@ jobs:
     resource_class: medium
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes
       - attach_workspace:
           at: ~/project
@@ -1371,7 +1385,7 @@ jobs:
     environment:
       UV_PYTHON_DOWNLOADS: never
     steps:
-      - checkout
+      - checkout_over_https
       - install_uv
       - run:
           name: Install the locked shard verifier and coverage dependencies
@@ -1494,7 +1508,7 @@ jobs:
     environment:
       DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes
       - setup_litellm_test_deps:
           checkout: false

tests/unit/test_circleci_checkout.py

@@ -0,0 +1,220 @@
+import os
+import shutil
+import subprocess
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+import pytest
+import yaml
+from pydantic import BaseModel, ConfigDict
+
+CIRCLECI: Final = Path(__file__).resolve().parents[2] / ".circleci"
+CONFIG: Final = CIRCLECI / "config.yml"
+PUBLIC_REPO: Final = "https://github.com/BerriAI/litellm.git"
+LINUX_SHELL: Final = ("bash", "-eo", "pipefail", "-c")
+WINDOWS_SHELL: Final = ("bash", "-c")
+
+
+class _Run(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    command: str
+
+
+class _Step(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    run: _Run
+
+
+class _Command(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    steps: tuple[_Step]
+
+
+class _Commands(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    checkout_over_https: _Command
+
+
+class _Config(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    commands: _Commands
+
+
+@dataclass(frozen=True, slots=True)
+class _Upstream:
+    url: str
+    older: str
+    newer: str
+
+
+def _step_name(step: object) -> str:
+    if isinstance(step, str):
+        return step
+    if isinstance(step, dict) and len(step) == 1:
+        return str(next(iter(step)))
+    return ""
+
+
+def _step_names(node: object) -> Iterator[str]:
+    if isinstance(node, dict):
+        for key, value in node.items():
+            if key in ("steps", "pre-steps", "post-steps") and isinstance(value, list):
+                yield from map(_step_name, value)
+            yield from _step_names(value)
+    if isinstance(node, list):
+        for item in node:
+            yield from _step_names(item)
+
+
+def _git(cwd: Path, *args: str) -> str:
+    return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout.strip()
+
+
+def _commit(work: Path, files: Mapping[str, str], message: str) -> str:
+    for name, content in files.items():
+        (work / name).write_text(content)
+    _git(work, "add", "-A")
+    _git(work, "commit", "-q", "--allow-empty", "-m", message)
+    return _git(work, "rev-parse", "HEAD")
+
+
+def _new_upstream(tmp_path: Path) -> tuple[Path, Path]:
+    remote: Final = tmp_path / "remote.git"
+    work: Final = tmp_path / "work"
+    work.mkdir()
+    _git(tmp_path, "init", "-q", "--bare", str(remote))
+    _git(remote, "config", "uploadpack.allowFilter", "true")
+    _git(work, "init", "-q", "-b", "main")
+    _git(work, "config", "user.email", "t@t")
+    _git(work, "config", "user.name", "t")
+    return remote, work
+
+
+def _publish_two_commits(tmp_path: Path) -> _Upstream:
+    remote, work = _new_upstream(tmp_path)
+    older: Final = _commit(work, {"app.py": "x = 1\n"}, "pipeline commit")
+    newer: Final = _commit(work, {"app.py": "x = 2\n"}, "main moved on")
+    _git(work, "push", "-q", str(remote), "main")
+    return _Upstream(url=remote.as_uri(), older=older, newer=newer)
+
+
+def _publish_docs_only_pr(tmp_path: Path) -> _Upstream:
+    remote, work = _new_upstream(tmp_path)
+    main: Final = _commit(work, {"litellm_inference.py": "x\n"}, "base")
+    _git(work, "checkout", "-q", "-b", "litellm_feature")
+    head: Final = _commit(work, {"README.md": "# docs\n"}, "docs")
+    _git(work, "push", "-q", str(remote), "main", "litellm_feature")
+    return _Upstream(url=remote.as_uri(), older=main, newer=head)
+
+
+def _stub(bin_dir: Path, name: str, script: str) -> None:
+    bin_dir.mkdir(exist_ok=True)
+    (bin_dir / name).write_text(f"#!/usr/bin/env bash\n{script}\n")
+    (bin_dir / name).chmod(0o755)
+
+
+def _job_env(tmp_path: Path, upstream: _Upstream) -> dict[str, str]:
+    return {
+        "PATH": f"{tmp_path / 'bin'}{os.pathsep}{os.environ['PATH']}",
+        "HOME": str(tmp_path),
+        "GIT_CONFIG_NOSYSTEM": "1",
+        "GIT_TERMINAL_PROMPT": "0",
+        "GIT_SSH_COMMAND": "false",
+        "GIT_CONFIG_COUNT": "1",
+        "GIT_CONFIG_KEY_0": f"url.{upstream.url}.insteadOf",
+        "GIT_CONFIG_VALUE_0": PUBLIC_REPO,
+    }
+
+
+def _run_checkout(
+    tmp_path: Path,
+    upstream: _Upstream,
+    revisions: Mapping[str, str],
+    shell: tuple[str, ...] = LINUX_SHELL,
+) -> subprocess.CompletedProcess[str]:
+    job: Final = tmp_path / "job"
+    job.mkdir()
+    command: Final = _Config.model_validate(yaml.safe_load(CONFIG.read_text())).commands.checkout_over_https
+    return subprocess.run(
+        [*shell, command.steps[0].run.command],
+        cwd=job,
+        env={**_job_env(tmp_path, upstream), **revisions},
+        capture_output=True,
+        text=True,
+    )
+
+
+def test_every_clone_goes_over_https_so_no_job_needs_the_ssh_checkout_key() -> None:
+    names: Final = tuple(_step_names(yaml.safe_load(CONFIG.read_text())))
+    assert "checkout_over_https" in names
+    assert "checkout" not in names
+
+
+@pytest.mark.parametrize(
+    "revisions",
+    (
+        {"CIRCLE_SHA1": "older"},
+        {"CIRCLE_SHA1": "older", "REQUESTED_REVISION": ""},
+        {"CIRCLE_SHA1": "newer", "REQUESTED_REVISION": "older"},
+    ),
+)
+def test_a_job_checks_out_its_own_commit_from_the_public_https_url_after_main_moves(
+    tmp_path: Path, revisions: Mapping[str, str]
+) -> None:
+    upstream: Final = _publish_two_commits(tmp_path)
+    shas: Final = {"older": upstream.older, "newer": upstream.newer, "": ""}
+    result: Final = _run_checkout(tmp_path, upstream, {name: shas[value] for name, value in revisions.items()})
+    assert result.returncode == 0, result.stderr
+    assert _git(tmp_path / "job", "rev-parse", "HEAD") == upstream.older
+
+
+@pytest.mark.parametrize("shell", (LINUX_SHELL, WINDOWS_SHELL), ids=("linux", "windows-bash-exe"))
+def test_a_revision_that_is_not_a_full_sha_is_refused_before_git_runs(tmp_path: Path, shell: tuple[str, ...]) -> None:
+    upstream: Final = _publish_two_commits(tmp_path)
+    result: Final = _run_checkout(
+        tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer, "REQUESTED_REVISION": "main"}, shell
+    )
+    assert result.returncode != 0
+    assert not (tmp_path / "job" / ".git").exists()
+
+
+def test_a_docs_only_pr_job_still_skips_backend_work_after_the_checkout(tmp_path: Path) -> None:
+    upstream: Final = _publish_docs_only_pr(tmp_path)
+    _stub(tmp_path / "bin", "circleci-agent", 'echo "circleci-agent $*"')
+    checkout: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer})
+    assert checkout.returncode == 0, checkout.stderr
+    path_filter: Final = subprocess.run(
+        ["bash", str(CIRCLECI / "scripts" / "path_filter.sh"), "backend"],
+        cwd=tmp_path / "job",
+        env={**_job_env(tmp_path, upstream), "CIRCLE_PULL_REQUEST": "https://github.com/BerriAI/litellm/pull/1"},
+        capture_output=True,
+        text=True,
+    )
+    assert path_filter.returncode == 0, path_filter.stderr
+    assert "circleci-agent step halt" in path_filter.stdout
+
+
+@pytest.mark.parametrize("shell", (LINUX_SHELL, WINDOWS_SHELL), ids=("linux", "windows-bash-exe"))
+@pytest.mark.parametrize(("failing_fetches", "checked_out"), ((2, True), (3, False)))
+def test_a_fetch_that_fails_is_retried_and_the_step_fails_once_retries_run_out(
+    tmp_path: Path, shell: tuple[str, ...], failing_fetches: int, checked_out: bool
+) -> None:
+    upstream: Final = _publish_two_commits(tmp_path)
+    fetches: Final = tmp_path / "fetches"
+    _stub(tmp_path / "bin", "sleep", "exit 0")
+    _stub(
+        tmp_path / "bin",
+        "git",
+        f'''if [ "$1" = fetch ]; then
+  echo x >> "{fetches}"
+  [ "$(wc -l < "{fetches}")" -le {failing_fetches} ] && exit 128
+fi
+exec "{shutil.which("git")}" "$@"''',
+    )
+    result: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.older}, shell)
+    assert (result.returncode == 0) is checked_out, result.stderr
+    assert len(fetches.read_text().splitlines()) == min(failing_fetches + 1, 3)
+    head: Final = subprocess.run(["git", "rev-parse", "HEAD"], cwd=tmp_path / "job", capture_output=True, text=True)
+    assert (head.stdout.strip() == upstream.older) is checked_out