From 81f81234f1b4be197707343bc7f460a51ae36438 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:34:35 -0700 Subject: [PATCH 1/3] ci(circleci): clone the public repo over HTTPS so main jobs stop failing at checkout The project's SSH checkout key stopped authenticating on Oct 27, so every job's built-in checkout step failed with Permission denied (publickey). The repo is public, so each job now fetches its pipeline's commit anonymously over HTTPS, the way the migration jobs already did --- .circleci/config.yml | 49 +++++++++++++++++----------- tests/unit/test_circleci_checkout.py | 32 ++++++++++++++++++ 2 files changed, 62 insertions(+), 19 deletions(-) create mode 100644 tests/unit/test_circleci_checkout.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ce5ea8b6fe..cf0c99fb1d3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,14 +21,22 @@ orbs: win: circleci/windows@5.0 # Add Windows orb commands: - checkout_migration_source: + checkout_over_https: + parameters: + revision: + type: string + default: "" + shell: + type: string + default: /bin/bash -eo pipefail steps: - run: - name: Select the requested migration test revision + name: Check out the public repository over HTTPS + shell: << parameters.shell >> environment: - MIGRATION_SOURCE_SHA: << pipeline.parameters.migration_source_sha >> + REQUESTED_REVISION: << parameters.revision >> command: | - revision="${MIGRATION_SOURCE_SHA:-$CIRCLE_SHA1}" + revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}" [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1 git init git remote add origin https://github.com/BerriAI/litellm.git @@ -347,7 +355,7 @@ commands: - when: condition: << parameters.checkout >> steps: - - checkout + - checkout_over_https - setup_google_dns - install_uv - install_rust @@ -377,7 +385,8 @@ jobs: CARGO_HTTP_MULTIPLEXING: "false" CARGO_NET_RETRY: "5" steps: - - checkout + - checkout_over_https: + shell: bash.exe - run: name: Skip job when no windows-release-relevant files changed shell: bash.exe @@ -451,7 +460,7 @@ jobs: password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - - checkout + - checkout_over_https - setup_google_dns - install_uv - install_rust @@ -482,7 +491,7 @@ jobs: working_directory: ~/project steps: - - checkout + - checkout_over_https - skip_if_unrelated_changes - attach_workspace: at: ~/project @@ -559,7 +568,7 @@ jobs: resource_class: medium working_directory: ~/project steps: - - checkout + - checkout_over_https - run: name: Generate LiteLLM master key command: | @@ -659,7 +668,7 @@ jobs: resource_class: large working_directory: ~/project steps: - - checkout + - checkout_over_https - run: name: Generate LiteLLM master key command: | @@ -740,7 +749,7 @@ jobs: resource_class: large working_directory: ~/project steps: - - checkout + - checkout_over_https - skip_if_unrelated_changes: category: client - setup_google_dns @@ -794,7 +803,7 @@ jobs: # the rest of the suite: nothing else performs a logout. PROXY_LOGOUT_URL: "https://www.example.com" steps: - - checkout + - checkout_over_https - run: name: Generate LiteLLM master key command: | @@ -940,7 +949,7 @@ jobs: # and by migration.serverRootPath.config.ts, which refuses to run without it. SERVER_ROOT_PATH: "/litellm" steps: - - checkout + - checkout_over_https - run: name: Generate LiteLLM master key command: | @@ -1066,11 +1075,12 @@ jobs: - when: condition: << parameters.migration_qualification >> steps: - - checkout_migration_source + - checkout_over_https: + revision: << pipeline.parameters.migration_source_sha >> - unless: condition: << parameters.migration_qualification >> steps: - - checkout + - checkout_over_https - skip_if_unrelated_changes - run: @@ -1122,7 +1132,8 @@ jobs: MIGRATION_TEST_OUTPUT: /tmp/migration-results PYTHONPATH: tests/e2e steps: - - checkout_migration_source + - checkout_over_https: + revision: << pipeline.parameters.migration_source_sha >> - run: name: Generate LiteLLM master key command: | @@ -1197,7 +1208,7 @@ jobs: resource_class: medium working_directory: ~/project steps: - - checkout + - checkout_over_https - skip_if_unrelated_changes - attach_workspace: at: ~/project @@ -1371,7 +1382,7 @@ jobs: environment: UV_PYTHON_DOWNLOADS: never steps: - - checkout + - checkout_over_https - install_uv - run: name: Install the locked shard verifier and coverage dependencies @@ -1494,7 +1505,7 @@ jobs: environment: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test steps: - - checkout + - checkout_over_https - skip_if_unrelated_changes - setup_litellm_test_deps: checkout: false diff --git a/tests/unit/test_circleci_checkout.py b/tests/unit/test_circleci_checkout.py new file mode 100644 index 00000000000..056f2aea56a --- /dev/null +++ b/tests/unit/test_circleci_checkout.py @@ -0,0 +1,32 @@ +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import yaml + +CONFIG: Final = Path(__file__).resolve().parents[2] / ".circleci" / "config.yml" + + +def _step_name(step: object) -> str: + if isinstance(step, str): + return step + if isinstance(step, dict) and len(step) == 1: + return str(next(iter(step))) + return "" + + +def _step_names(node: object) -> Iterator[str]: + if isinstance(node, dict): + for key, value in node.items(): + if key == "steps" and isinstance(value, list): + yield from map(_step_name, value) + yield from _step_names(value) + if isinstance(node, list): + for item in node: + yield from _step_names(item) + + +def test_every_clone_goes_over_https_because_the_project_has_no_ssh_checkout_key() -> None: + names: Final = tuple(_step_names(yaml.safe_load(CONFIG.read_text()))) + assert "checkout_over_https" in names + assert "checkout" not in names From dadec59f41e6953972b47c75824e05cb79407fd9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:00:31 -0700 Subject: [PATCH 2/3] fix(circleci): let the PR path filter read history after the shallow clone and test the clone step itself --- .circleci/scripts/path_filter.sh | 4 +- tests/unit/test_circleci_checkout.py | 113 +++++++++++++++++++++++- tests/unit/test_circleci_path_filter.py | 11 +++ 3 files changed, 125 insertions(+), 3 deletions(-) diff --git a/.circleci/scripts/path_filter.sh b/.circleci/scripts/path_filter.sh index c3cf6e566ff..73171dce75c 100755 --- a/.circleci/scripts/path_filter.sh +++ b/.circleci/scripts/path_filter.sh @@ -14,7 +14,9 @@ run_full() { candidate_bases="${PATH_FILTER_BASE_BRANCH:-main}" merge_base="" for base in $candidate_bases; do - git fetch --quiet origin "$base" 2>/dev/null || continue + unshallow="" + [ "$(git rev-parse --is-shallow-repository)" = true ] && unshallow="--unshallow" + git fetch --quiet $unshallow origin "$base" 2>/dev/null || continue candidate="$(git merge-base HEAD FETCH_HEAD 2>/dev/null)" || continue [ -n "$candidate" ] || continue if [ -z "$merge_base" ] || git merge-base --is-ancestor "$merge_base" "$candidate" 2>/dev/null; then diff --git a/tests/unit/test_circleci_checkout.py b/tests/unit/test_circleci_checkout.py index 056f2aea56a..8d45b0ba9b8 100644 --- a/tests/unit/test_circleci_checkout.py +++ b/tests/unit/test_circleci_checkout.py @@ -1,10 +1,48 @@ -from collections.abc import Iterator +import os +import subprocess +from collections.abc import Iterator, Mapping +from dataclasses import dataclass from pathlib import Path from typing import Final +import pytest import yaml +from pydantic import BaseModel, ConfigDict CONFIG: Final = Path(__file__).resolve().parents[2] / ".circleci" / "config.yml" +PUBLIC_REPO: Final = "https://github.com/BerriAI/litellm.git" + + +class _Run(BaseModel): + model_config = ConfigDict(frozen=True) + command: str + + +class _Step(BaseModel): + model_config = ConfigDict(frozen=True) + run: _Run + + +class _Command(BaseModel): + model_config = ConfigDict(frozen=True) + steps: tuple[_Step] + + +class _Commands(BaseModel): + model_config = ConfigDict(frozen=True) + checkout_over_https: _Command + + +class _Config(BaseModel): + model_config = ConfigDict(frozen=True) + commands: _Commands + + +@dataclass(frozen=True, slots=True) +class _Upstream: + url: str + older: str + newer: str def _step_name(step: object) -> str: @@ -18,7 +56,7 @@ def _step_name(step: object) -> str: def _step_names(node: object) -> Iterator[str]: if isinstance(node, dict): for key, value in node.items(): - if key == "steps" and isinstance(value, list): + if key in ("steps", "pre-steps", "post-steps") and isinstance(value, list): yield from map(_step_name, value) yield from _step_names(value) if isinstance(node, list): @@ -26,7 +64,78 @@ def _step_names(node: object) -> Iterator[str]: yield from _step_names(item) +def _git(cwd: Path, *args: str) -> str: + return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout.strip() + + +def _commit(work: Path, message: str) -> str: + _git(work, "commit", "-q", "--allow-empty", "-m", message) + return _git(work, "rev-parse", "HEAD") + + +def _publish_two_commits(tmp_path: Path) -> _Upstream: + remote: Final = tmp_path / "remote.git" + work: Final = tmp_path / "work" + work.mkdir() + _git(tmp_path, "init", "-q", "--bare", str(remote)) + _git(work, "init", "-q", "-b", "main") + _git(work, "config", "user.email", "t@t") + _git(work, "config", "user.name", "t") + older: Final = _commit(work, "pipeline commit") + newer: Final = _commit(work, "main moved on") + _git(work, "push", "-q", str(remote), "main") + return _Upstream(url=remote.as_uri(), older=older, newer=newer) + + +def _run_checkout(tmp_path: Path, upstream: _Upstream, revisions: Mapping[str, str]) -> subprocess.CompletedProcess[str]: + job: Final = tmp_path / "job" + job.mkdir() + command: Final = _Config.model_validate(yaml.safe_load(CONFIG.read_text())).commands.checkout_over_https + return subprocess.run( + ["bash", "-eo", "pipefail", "-c", command.steps[0].run.command], + cwd=job, + env={ + "PATH": os.environ["PATH"], + "HOME": str(tmp_path), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "GIT_SSH_COMMAND": "false", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": f"url.{upstream.url}.insteadOf", + "GIT_CONFIG_VALUE_0": PUBLIC_REPO, + **revisions, + }, + capture_output=True, + text=True, + ) + + def test_every_clone_goes_over_https_because_the_project_has_no_ssh_checkout_key() -> None: names: Final = tuple(_step_names(yaml.safe_load(CONFIG.read_text()))) assert "checkout_over_https" in names assert "checkout" not in names + + +@pytest.mark.parametrize( + "revisions", + ( + {"CIRCLE_SHA1": "older"}, + {"CIRCLE_SHA1": "older", "REQUESTED_REVISION": ""}, + {"CIRCLE_SHA1": "newer", "REQUESTED_REVISION": "older"}, + ), +) +def test_a_job_checks_out_its_own_commit_from_the_public_https_url_after_main_moves( + tmp_path: Path, revisions: Mapping[str, str] +) -> None: + upstream: Final = _publish_two_commits(tmp_path) + shas: Final = {"older": upstream.older, "newer": upstream.newer, "": ""} + result: Final = _run_checkout(tmp_path, upstream, {name: shas[value] for name, value in revisions.items()}) + assert result.returncode == 0, result.stderr + assert _git(tmp_path / "job", "rev-parse", "HEAD") == upstream.older + + +def test_a_revision_that_is_not_a_full_sha_is_refused_before_git_runs(tmp_path: Path) -> None: + upstream: Final = _publish_two_commits(tmp_path) + result: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer, "REQUESTED_REVISION": "main"}) + assert result.returncode != 0 + assert not (tmp_path / "job" / ".git").exists() diff --git a/tests/unit/test_circleci_path_filter.py b/tests/unit/test_circleci_path_filter.py index 9f695960517..1e1e641ab24 100644 --- a/tests/unit/test_circleci_path_filter.py +++ b/tests/unit/test_circleci_path_filter.py @@ -236,6 +236,17 @@ def test_path_filter_runs_backend_pr(tmp_path: Path) -> None: assert "halt" not in result.stdout +def test_path_filter_halts_docs_only_pr_from_a_depth_one_checkout(tmp_path: Path) -> None: + work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) + _git(work, "push", "-q", "origin", "litellm_feature") + shallow = tmp_path / "shallow" + remote = (tmp_path / "remote.git").as_uri() + subprocess.run(["git", "clone", "-q", "--depth", "1", "-b", "litellm_feature", remote, str(shallow)], check=True) + result = _run_path_filter(shallow, tmp_path, "backend", SCRIPTS_DIR) + assert result.returncode == 0 + assert "circleci-agent step halt" in result.stdout + + def test_path_filter_fails_open_when_not_a_pr(tmp_path: Path) -> None: work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) result = _run_path_filter(work, tmp_path, "backend", SCRIPTS_DIR, is_pr=False) From 92c69e6155006b7e6868931975e8e57933e51f8a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:37:13 -0700 Subject: [PATCH 3/3] fix(circleci): clone full history without file contents, retry the fetch, and record the integration schedule The depth-1 clone left the PR path filter without a merge base, and the unshallow that fixed it pulled about 1.4 GB per filtered job. A blobless fetch gives every job the history the built-in checkout step gave, so the path filter change goes back to main's version. The fetch now gets three tries, the tests run the step under the Windows job's bare bash too, and .circleci/AGENTS.md records the integration schedule's cron --- .circleci/AGENTS.md | 5 + .circleci/config.yml | 5 +- .circleci/scripts/path_filter.sh | 4 +- tests/unit/test_circleci_checkout.py | 121 ++++++++++++++++++++---- tests/unit/test_circleci_path_filter.py | 11 --- 5 files changed, 110 insertions(+), 36 deletions(-) create mode 100644 .circleci/AGENTS.md diff --git a/.circleci/AGENTS.md b/.circleci/AGENTS.md new file mode 100644 index 00000000000..f2821001a9d --- /dev/null +++ b/.circleci/AGENTS.md @@ -0,0 +1,5 @@ +# .circleci + +The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values + +Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because the project's SSH checkout key no longer works and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base diff --git a/.circleci/config.yml b/.circleci/config.yml index cf0c99fb1d3..09e3c9a8bd3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,7 +40,10 @@ commands: [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1 git init git remote add origin https://github.com/BerriAI/litellm.git - git fetch --depth 1 origin "$revision" + for _ in 1 2 3; do + git fetch --filter=blob:none origin "$revision" && break + sleep 5 + done git checkout --detach FETCH_HEAD skip_if_unrelated_changes: parameters: diff --git a/.circleci/scripts/path_filter.sh b/.circleci/scripts/path_filter.sh index 73171dce75c..c3cf6e566ff 100755 --- a/.circleci/scripts/path_filter.sh +++ b/.circleci/scripts/path_filter.sh @@ -14,9 +14,7 @@ run_full() { candidate_bases="${PATH_FILTER_BASE_BRANCH:-main}" merge_base="" for base in $candidate_bases; do - unshallow="" - [ "$(git rev-parse --is-shallow-repository)" = true ] && unshallow="--unshallow" - git fetch --quiet $unshallow origin "$base" 2>/dev/null || continue + git fetch --quiet origin "$base" 2>/dev/null || continue candidate="$(git merge-base HEAD FETCH_HEAD 2>/dev/null)" || continue [ -n "$candidate" ] || continue if [ -z "$merge_base" ] || git merge-base --is-ancestor "$merge_base" "$candidate" 2>/dev/null; then diff --git a/tests/unit/test_circleci_checkout.py b/tests/unit/test_circleci_checkout.py index 8d45b0ba9b8..469d51fdd22 100644 --- a/tests/unit/test_circleci_checkout.py +++ b/tests/unit/test_circleci_checkout.py @@ -1,4 +1,5 @@ import os +import shutil import subprocess from collections.abc import Iterator, Mapping from dataclasses import dataclass @@ -9,8 +10,11 @@ import pytest import yaml from pydantic import BaseModel, ConfigDict -CONFIG: Final = Path(__file__).resolve().parents[2] / ".circleci" / "config.yml" +CIRCLECI: Final = Path(__file__).resolve().parents[2] / ".circleci" +CONFIG: Final = CIRCLECI / "config.yml" PUBLIC_REPO: Final = "https://github.com/BerriAI/litellm.git" +LINUX_SHELL: Final = ("bash", "-eo", "pipefail", "-c") +WINDOWS_SHELL: Final = ("bash", "-c") class _Run(BaseModel): @@ -68,49 +72,81 @@ def _git(cwd: Path, *args: str) -> str: return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout.strip() -def _commit(work: Path, message: str) -> str: +def _commit(work: Path, files: Mapping[str, str], message: str) -> str: + for name, content in files.items(): + (work / name).write_text(content) + _git(work, "add", "-A") _git(work, "commit", "-q", "--allow-empty", "-m", message) return _git(work, "rev-parse", "HEAD") -def _publish_two_commits(tmp_path: Path) -> _Upstream: +def _new_upstream(tmp_path: Path) -> tuple[Path, Path]: remote: Final = tmp_path / "remote.git" work: Final = tmp_path / "work" work.mkdir() _git(tmp_path, "init", "-q", "--bare", str(remote)) + _git(remote, "config", "uploadpack.allowFilter", "true") _git(work, "init", "-q", "-b", "main") _git(work, "config", "user.email", "t@t") _git(work, "config", "user.name", "t") - older: Final = _commit(work, "pipeline commit") - newer: Final = _commit(work, "main moved on") + return remote, work + + +def _publish_two_commits(tmp_path: Path) -> _Upstream: + remote, work = _new_upstream(tmp_path) + older: Final = _commit(work, {"app.py": "x = 1\n"}, "pipeline commit") + newer: Final = _commit(work, {"app.py": "x = 2\n"}, "main moved on") _git(work, "push", "-q", str(remote), "main") return _Upstream(url=remote.as_uri(), older=older, newer=newer) -def _run_checkout(tmp_path: Path, upstream: _Upstream, revisions: Mapping[str, str]) -> subprocess.CompletedProcess[str]: +def _publish_docs_only_pr(tmp_path: Path) -> _Upstream: + remote, work = _new_upstream(tmp_path) + main: Final = _commit(work, {"litellm_inference.py": "x\n"}, "base") + _git(work, "checkout", "-q", "-b", "litellm_feature") + head: Final = _commit(work, {"README.md": "# docs\n"}, "docs") + _git(work, "push", "-q", str(remote), "main", "litellm_feature") + return _Upstream(url=remote.as_uri(), older=main, newer=head) + + +def _stub(bin_dir: Path, name: str, script: str) -> None: + bin_dir.mkdir(exist_ok=True) + (bin_dir / name).write_text(f"#!/usr/bin/env bash\n{script}\n") + (bin_dir / name).chmod(0o755) + + +def _job_env(tmp_path: Path, upstream: _Upstream) -> dict[str, str]: + return { + "PATH": f"{tmp_path / 'bin'}{os.pathsep}{os.environ['PATH']}", + "HOME": str(tmp_path), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "GIT_SSH_COMMAND": "false", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": f"url.{upstream.url}.insteadOf", + "GIT_CONFIG_VALUE_0": PUBLIC_REPO, + } + + +def _run_checkout( + tmp_path: Path, + upstream: _Upstream, + revisions: Mapping[str, str], + shell: tuple[str, ...] = LINUX_SHELL, +) -> subprocess.CompletedProcess[str]: job: Final = tmp_path / "job" job.mkdir() command: Final = _Config.model_validate(yaml.safe_load(CONFIG.read_text())).commands.checkout_over_https return subprocess.run( - ["bash", "-eo", "pipefail", "-c", command.steps[0].run.command], + [*shell, command.steps[0].run.command], cwd=job, - env={ - "PATH": os.environ["PATH"], - "HOME": str(tmp_path), - "GIT_CONFIG_NOSYSTEM": "1", - "GIT_TERMINAL_PROMPT": "0", - "GIT_SSH_COMMAND": "false", - "GIT_CONFIG_COUNT": "1", - "GIT_CONFIG_KEY_0": f"url.{upstream.url}.insteadOf", - "GIT_CONFIG_VALUE_0": PUBLIC_REPO, - **revisions, - }, + env={**_job_env(tmp_path, upstream), **revisions}, capture_output=True, text=True, ) -def test_every_clone_goes_over_https_because_the_project_has_no_ssh_checkout_key() -> None: +def test_every_clone_goes_over_https_so_no_job_needs_the_ssh_checkout_key() -> None: names: Final = tuple(_step_names(yaml.safe_load(CONFIG.read_text()))) assert "checkout_over_https" in names assert "checkout" not in names @@ -134,8 +170,51 @@ def test_a_job_checks_out_its_own_commit_from_the_public_https_url_after_main_mo assert _git(tmp_path / "job", "rev-parse", "HEAD") == upstream.older -def test_a_revision_that_is_not_a_full_sha_is_refused_before_git_runs(tmp_path: Path) -> None: +@pytest.mark.parametrize("shell", (LINUX_SHELL, WINDOWS_SHELL), ids=("linux", "windows-bash-exe")) +def test_a_revision_that_is_not_a_full_sha_is_refused_before_git_runs(tmp_path: Path, shell: tuple[str, ...]) -> None: upstream: Final = _publish_two_commits(tmp_path) - result: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer, "REQUESTED_REVISION": "main"}) + result: Final = _run_checkout( + tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer, "REQUESTED_REVISION": "main"}, shell + ) assert result.returncode != 0 assert not (tmp_path / "job" / ".git").exists() + + +def test_a_docs_only_pr_job_still_skips_backend_work_after_the_checkout(tmp_path: Path) -> None: + upstream: Final = _publish_docs_only_pr(tmp_path) + _stub(tmp_path / "bin", "circleci-agent", 'echo "circleci-agent $*"') + checkout: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.newer}) + assert checkout.returncode == 0, checkout.stderr + path_filter: Final = subprocess.run( + ["bash", str(CIRCLECI / "scripts" / "path_filter.sh"), "backend"], + cwd=tmp_path / "job", + env={**_job_env(tmp_path, upstream), "CIRCLE_PULL_REQUEST": "https://github.com/BerriAI/litellm/pull/1"}, + capture_output=True, + text=True, + ) + assert path_filter.returncode == 0, path_filter.stderr + assert "circleci-agent step halt" in path_filter.stdout + + +@pytest.mark.parametrize("shell", (LINUX_SHELL, WINDOWS_SHELL), ids=("linux", "windows-bash-exe")) +@pytest.mark.parametrize(("failing_fetches", "checked_out"), ((2, True), (3, False))) +def test_a_fetch_that_fails_is_retried_and_the_step_fails_once_retries_run_out( + tmp_path: Path, shell: tuple[str, ...], failing_fetches: int, checked_out: bool +) -> None: + upstream: Final = _publish_two_commits(tmp_path) + fetches: Final = tmp_path / "fetches" + _stub(tmp_path / "bin", "sleep", "exit 0") + _stub( + tmp_path / "bin", + "git", + f'''if [ "$1" = fetch ]; then + echo x >> "{fetches}" + [ "$(wc -l < "{fetches}")" -le {failing_fetches} ] && exit 128 +fi +exec "{shutil.which("git")}" "$@"''', + ) + result: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.older}, shell) + assert (result.returncode == 0) is checked_out, result.stderr + assert len(fetches.read_text().splitlines()) == min(failing_fetches + 1, 3) + head: Final = subprocess.run(["git", "rev-parse", "HEAD"], cwd=tmp_path / "job", capture_output=True, text=True) + assert (head.stdout.strip() == upstream.older) is checked_out diff --git a/tests/unit/test_circleci_path_filter.py b/tests/unit/test_circleci_path_filter.py index 1e1e641ab24..9f695960517 100644 --- a/tests/unit/test_circleci_path_filter.py +++ b/tests/unit/test_circleci_path_filter.py @@ -236,17 +236,6 @@ def test_path_filter_runs_backend_pr(tmp_path: Path) -> None: assert "halt" not in result.stdout -def test_path_filter_halts_docs_only_pr_from_a_depth_one_checkout(tmp_path: Path) -> None: - work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) - _git(work, "push", "-q", "origin", "litellm_feature") - shallow = tmp_path / "shallow" - remote = (tmp_path / "remote.git").as_uri() - subprocess.run(["git", "clone", "-q", "--depth", "1", "-b", "litellm_feature", remote, str(shallow)], check=True) - result = _run_path_filter(shallow, tmp_path, "backend", SCRIPTS_DIR) - assert result.returncode == 0 - assert "circleci-agent step halt" in result.stdout - - def test_path_filter_fails_open_when_not_a_pr(tmp_path: Path) -> None: work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) result = _run_path_filter(work, tmp_path, "backend", SCRIPTS_DIR, is_pr=False)