From eb5ba4875714394f0af26798a1987d37883d4c2f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:17:48 -0700 Subject: [PATCH] fix(circleci): stop on the fetch error once the three tries run out, and name the GitHub App fallback as the cause --- .circleci/AGENTS.md | 2 +- .circleci/config.yml | 3 ++- tests/unit/test_circleci_checkout.py | 17 +++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.circleci/AGENTS.md b/.circleci/AGENTS.md index f2821001a9d..d4a94646287 100644 --- a/.circleci/AGENTS.md +++ b/.circleci/AGENTS.md @@ -2,4 +2,4 @@ The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values -Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because the project's SSH checkout key no longer works and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base +Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because CircleCI falls back to the project's SSH checkout key without its GitHub App access, GitHub rejects that key, and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base diff --git a/.circleci/config.yml b/.circleci/config.yml index 09e3c9a8bd3..ac057aaa7dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,8 +40,9 @@ commands: [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1 git init git remote add origin https://github.com/BerriAI/litellm.git - for _ in 1 2 3; do + for attempt in 1 2 3; do git fetch --filter=blob:none origin "$revision" && break + [[ $attempt -lt 3 ]] || exit 1 sleep 5 done git checkout --detach FETCH_HEAD diff --git a/tests/unit/test_circleci_checkout.py b/tests/unit/test_circleci_checkout.py index 469d51fdd22..ee474d99fc6 100644 --- a/tests/unit/test_circleci_checkout.py +++ b/tests/unit/test_circleci_checkout.py @@ -198,23 +198,24 @@ def test_a_docs_only_pr_job_still_skips_backend_work_after_the_checkout(tmp_path @pytest.mark.parametrize("shell", (LINUX_SHELL, WINDOWS_SHELL), ids=("linux", "windows-bash-exe")) @pytest.mark.parametrize(("failing_fetches", "checked_out"), ((2, True), (3, False))) -def test_a_fetch_that_fails_is_retried_and_the_step_fails_once_retries_run_out( +def test_a_fetch_that_fails_is_retried_and_the_step_stops_at_the_fetch_once_retries_run_out( tmp_path: Path, shell: tuple[str, ...], failing_fetches: int, checked_out: bool ) -> None: upstream: Final = _publish_two_commits(tmp_path) - fetches: Final = tmp_path / "fetches" - _stub(tmp_path / "bin", "sleep", "exit 0") + calls: Final = tmp_path / "calls" + _stub(tmp_path / "bin", "sleep", f'echo sleep >> "{calls}"') _stub( tmp_path / "bin", "git", - f'''if [ "$1" = fetch ]; then - echo x >> "{fetches}" - [ "$(wc -l < "{fetches}")" -le {failing_fetches} ] && exit 128 -fi + f'''echo "$1" >> "{calls}" +if [ "$1" = fetch ] && [ "$(grep -c '^fetch$' "{calls}")" -le {failing_fetches} ]; then exit 128; fi exec "{shutil.which("git")}" "$@"''', ) result: Final = _run_checkout(tmp_path, upstream, {"CIRCLE_SHA1": upstream.older}, shell) assert (result.returncode == 0) is checked_out, result.stderr - assert len(fetches.read_text().splitlines()) == min(failing_fetches + 1, 3) + steps: Final = calls.read_text().splitlines() + assert steps.count("fetch") == min(failing_fetches + 1, 3) + assert steps.count("sleep") == steps.count("fetch") - 1 + assert ("checkout" in steps) is checked_out head: Final = subprocess.run(["git", "rev-parse", "HEAD"], cwd=tmp_path / "job", capture_output=True, text=True) assert (head.stdout.strip() == upstream.older) is checked_out