From f5438d121aeca3c5b10bbb6942a820028aa2a6a2 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 4 Jul 2026 19:15:08 -0700 Subject: [PATCH] ci: gate CircleCI jobs on changed paths (#32080) * ci: gate CircleCI jobs on changed paths Every CircleCI job used to run on every PR. Now each job starts with a lightweight `skip_if_unrelated_changes` step that inspects the PR diff and halts the job as successful when nothing relevant changed. Docs-only PRs (*.md, *.mdx, docs/) run nothing, UI-only PRs (ui/) run just the frontend jobs, and any backend change still runs both the backend and frontend jobs. The decision logic lives in .circleci/scripts/classify_changes.sh (pure, reads the changed-file list on stdin) so it can be unit tested, while path_filter.sh handles the git plumbing and fails open (runs the job) on any uncertainty such as a missing merge base or a non-PR pipeline. Halting via `circleci-agent step halt` keeps the job green, so required status checks are never left pending. The Windows smoke job is intentionally left ungated to avoid cross-platform shell fragility * fix(ci): keep path filter fail-open when classifier errors Guard the classify_changes.sh invocation with `|| run_full` so a broken or non-zero classifier runs the job instead of falling through to a silent halt, and mark the advisory logging pipe best-effort with `|| true`. Add path_filter.sh regression tests covering the docs-only halt, backend run, non-PR fail-open, and classifier-failure fail-open paths --- .circleci/config.yml | 61 +++++++ .circleci/scripts/classify_changes.sh | 27 +++ .circleci/scripts/path_filter.sh | 40 +++++ .../test_litellm/test_circleci_path_filter.py | 167 ++++++++++++++++++ 4 files changed, 295 insertions(+) create mode 100755 .circleci/scripts/classify_changes.sh create mode 100755 .circleci/scripts/path_filter.sh create mode 100644 tests/test_litellm/test_circleci_path_filter.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 909331a1f34..f95802128b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,6 +5,16 @@ orbs: win: circleci/windows@5.0 # Add Windows orb commands: + skip_if_unrelated_changes: + parameters: + category: + type: enum + enum: ["backend", "client"] + default: "backend" + steps: + - run: + name: "Skip job when no << parameters.category >>-relevant files changed" + command: bash .circleci/scripts/path_filter.sh << parameters.category >> setup_google_dns: steps: - run: @@ -282,6 +292,7 @@ jobs: parallelism: 4 steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: @@ -354,6 +365,7 @@ jobs: parallelism: 4 steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: @@ -427,6 +439,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: @@ -480,6 +493,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -545,6 +559,7 @@ jobs: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -584,6 +599,7 @@ jobs: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -624,6 +640,7 @@ jobs: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -656,6 +673,7 @@ jobs: FAKE_OPENAI_API_BASE: http://127.0.0.1:8190 steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - restore_cache: @@ -705,6 +723,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - restore_cache: @@ -755,6 +774,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -787,6 +807,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - restore_cache: @@ -832,6 +853,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -877,6 +899,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -918,6 +941,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -963,6 +987,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1007,6 +1032,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - restore_cache: @@ -1045,6 +1071,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1089,6 +1116,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1132,6 +1160,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1163,6 +1192,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1205,6 +1235,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1248,6 +1279,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1291,6 +1323,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1321,6 +1354,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1366,6 +1400,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1407,6 +1442,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: @@ -1459,6 +1495,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1482,6 +1519,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1507,6 +1545,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1531,6 +1570,7 @@ jobs: steps: - checkout + - skip_if_unrelated_changes - attach_workspace: at: ~/project - setup_google_dns @@ -1606,6 +1646,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1698,6 +1739,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - attach_workspace: at: ~/project - setup_google_dns @@ -1787,6 +1829,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -1869,6 +1912,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -2000,6 +2044,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -2085,6 +2130,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -2180,6 +2226,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -2252,6 +2299,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns # Remove Docker CLI installation since it's already available in machine executor - install_uv @@ -2333,6 +2381,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -2471,6 +2520,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - setup_google_dns - install_uv - run: @@ -2537,6 +2587,7 @@ jobs: - *python312_image steps: - checkout + - skip_if_unrelated_changes - attach_workspace: at: . # Check file locations @@ -2567,6 +2618,8 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes: + category: client - setup_google_dns - restore_cache: keys: @@ -2609,6 +2662,8 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes: + category: client - setup_google_dns - restore_cache: keys: @@ -2654,6 +2709,8 @@ jobs: PROXY_LOGOUT_URL: "https://www.example.com" steps: - checkout + - skip_if_unrelated_changes: + category: client - setup_google_dns - install_uv - restore_cache: @@ -2791,6 +2848,8 @@ jobs: SERVER_ROOT_PATH: "/litellm" steps: - checkout + - skip_if_unrelated_changes: + category: client - setup_google_dns - install_uv - restore_cache: @@ -2892,6 +2951,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - run: name: Build Docker image @@ -2917,6 +2977,7 @@ jobs: working_directory: ~/project steps: - checkout + - skip_if_unrelated_changes - attach_workspace: at: ~/project - setup_google_dns diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh new file mode 100755 index 00000000000..2c15428be6a --- /dev/null +++ b/.circleci/scripts/classify_changes.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -uo pipefail + +category="${1:?usage: classify_changes.sh }" + +has_client=false +has_backend=false +while IFS= read -r file || [ -n "$file" ]; do + [ -n "$file" ] || continue + case "$file" in + ui/*) has_client=true ;; + docs/* | *.md | *.mdx) : ;; + *) has_backend=true ;; + esac +done + +case "$category" in + backend) + [ "$has_backend" = true ] && echo run || echo skip + ;; + client) + { [ "$has_client" = true ] || [ "$has_backend" = true ]; } && echo run || echo skip + ;; + *) + echo run + ;; +esac diff --git a/.circleci/scripts/path_filter.sh b/.circleci/scripts/path_filter.sh new file mode 100755 index 00000000000..dcf64a24399 --- /dev/null +++ b/.circleci/scripts/path_filter.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -uo pipefail + +category="${1:?usage: path_filter.sh }" +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +run_full() { + echo "path-filter[$category]: running job ($1)" + exit 0 +} + +[ -n "${CIRCLE_PULL_REQUEST:-}" ] || run_full "not a pull request" + +candidate_bases="main litellm_internal_staging litellm_oss_staging" +merge_base="" +for base in $candidate_bases; do + git fetch --quiet origin "$base" 2>/dev/null || continue + candidate="$(git merge-base HEAD FETCH_HEAD 2>/dev/null)" || continue + [ -n "$candidate" ] || continue + if [ -z "$merge_base" ] || git merge-base --is-ancestor "$merge_base" "$candidate" 2>/dev/null; then + merge_base="$candidate" + fi +done + +[ -n "$merge_base" ] || run_full "could not resolve a merge base against $candidate_bases" + +changed="$(git diff --name-only "$merge_base" HEAD 2>/dev/null)" || run_full "git diff failed" +[ -n "$changed" ] || run_full "no files changed vs $merge_base" + +echo "path-filter[$category]: changed files vs ${merge_base}:" +printf '%s\n' "$changed" | sed 's/^/ /' || true + +decision="$(printf '%s\n' "$changed" | bash "$here/classify_changes.sh" "$category")" || run_full "classify_changes.sh failed" + +if [ "$decision" = run ]; then + run_full "$category-relevant changes detected" +fi + +echo "path-filter[$category]: only unrelated (docs/client) changes detected; halting job as successful" +circleci-agent step halt diff --git a/tests/test_litellm/test_circleci_path_filter.py b/tests/test_litellm/test_circleci_path_filter.py new file mode 100644 index 00000000000..b8e763b4979 --- /dev/null +++ b/tests/test_litellm/test_circleci_path_filter.py @@ -0,0 +1,167 @@ +"""Regression tests for CircleCI change-based job gating. + +`.circleci/scripts/classify_changes.sh` is the pure decision function behind +`path_filter.sh`: given the list of files a PR changed (on stdin) and a job +category, it prints `run` or `skip`. The gating contract we lock in here: + + * docs-only changes (``*.md``, ``*.mdx``, ``docs/``) run nothing + * client-only changes (``ui/``) run client jobs but skip backend jobs + * any backend change runs both client and backend jobs + +If this logic silently regresses, real test jobs get skipped, so these cases +are the guardrail against that. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +SCRIPTS_DIR = Path(__file__).resolve().parents[2] / ".circleci" / "scripts" +SCRIPT = SCRIPTS_DIR / "classify_changes.sh" +PATH_FILTER = SCRIPTS_DIR / "path_filter.sh" + + +def classify(category: str, changed: list[str]) -> str: + result = subprocess.run( + ["bash", str(SCRIPT), category], + input="\n".join(changed), + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +DOCS = ["README.md", "docs/my_website/index.mdx", "litellm/anywhere.md"] +CLIENT = ["ui/litellm-dashboard/src/App.tsx"] +BACKEND = ["litellm/main.py"] + + +@pytest.mark.parametrize( + "category,changed,expected", + [ + # docs-only: skip everything + ("backend", DOCS, "skip"), + ("client", DOCS, "skip"), + ("backend", [], "skip"), + ("client", [], "skip"), + # client-only: backend skips, client runs + ("backend", CLIENT, "skip"), + ("client", CLIENT, "run"), + ("backend", CLIENT + DOCS, "skip"), + ("client", CLIENT + DOCS, "run"), + # any backend change: both run ("backend runs both") + ("backend", BACKEND, "run"), + ("client", BACKEND, "run"), + ("backend", BACKEND + DOCS, "run"), + ("client", BACKEND + DOCS, "run"), + ("backend", BACKEND + CLIENT, "run"), + ("client", BACKEND + CLIENT, "run"), + ], +) +def test_classify_decisions(category: str, changed: list[str], expected: str) -> None: + assert classify(category, changed) == expected + + +def test_markdown_under_ui_counts_as_client_not_docs() -> None: + assert classify("client", ["ui/litellm-dashboard/README.md"]) == "run" + assert classify("backend", ["ui/litellm-dashboard/README.md"]) == "skip" + + +def test_non_docs_directory_with_docs_in_name_is_backend() -> None: + assert classify("backend", ["documentation_tests/foo.py"]) == "run" + + +def test_unknown_category_fails_open_to_run() -> None: + assert classify("mystery", DOCS) == "run" + + +def _git(cwd: Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True) + + +def _pr_repo(tmp_path: Path, feature_files: dict[str, str]) -> Path: + """A repo whose HEAD is a feature branch off `main` with `feature_files` changed.""" + remote = tmp_path / "remote.git" + subprocess.run(["git", "init", "-q", "--bare", str(remote)], check=True) + work = tmp_path / "work" + work.mkdir() + _git(work, "init", "-q", "-b", "main") + _git(work, "config", "user.email", "t@t") + _git(work, "config", "user.name", "t") + _git(work, "remote", "add", "origin", str(remote)) + (work / "litellm_core.py").write_text("x\n") + _git(work, "add", ".") + _git(work, "commit", "-qm", "base") + _git(work, "push", "-q", "origin", "main") + _git(work, "checkout", "-q", "-b", "litellm_feature") + for rel, content in feature_files.items(): + target = work / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content) + _git(work, "add", "-A") + _git(work, "commit", "-qm", "feature") + return work + + +def _run_path_filter(work: Path, tmp_path: Path, category: str, scripts_dir: Path, is_pr: bool = True): + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + stub = bin_dir / "circleci-agent" + stub.write_text("#!/usr/bin/env bash\necho \"[stub] circleci-agent $*\"\nexit 0\n") + stub.chmod(0o755) + env = dict(os.environ) + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + env.pop("CIRCLE_PULL_REQUEST", None) + if is_pr: + env["CIRCLE_PULL_REQUEST"] = "https://github.com/x/y/pull/1" + return subprocess.run( + ["bash", str(scripts_dir / "path_filter.sh"), category], + cwd=work, + capture_output=True, + text=True, + env=env, + ) + + +def test_path_filter_halts_docs_only_pr(tmp_path: Path) -> None: + work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) + result = _run_path_filter(work, tmp_path, "backend", SCRIPTS_DIR) + assert result.returncode == 0 + assert "circleci-agent step halt" in result.stdout + + +def test_path_filter_runs_backend_pr(tmp_path: Path) -> None: + work = _pr_repo(tmp_path, {"litellm/new.py": "y\n"}) + result = _run_path_filter(work, tmp_path, "backend", SCRIPTS_DIR) + assert result.returncode == 0 + assert "running job" in result.stdout + assert "halt" not in result.stdout + + +def test_path_filter_fails_open_when_not_a_pr(tmp_path: Path) -> None: + work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) + result = _run_path_filter(work, tmp_path, "backend", SCRIPTS_DIR, is_pr=False) + assert result.returncode == 0 + assert "not a pull request" in result.stdout + assert "halt" not in result.stdout + + +def test_path_filter_fails_open_when_classifier_errors(tmp_path: Path) -> None: + """Regression: a broken classifier must run the job, never silently halt it.""" + broken_scripts = tmp_path / "broken_scripts" + broken_scripts.mkdir() + shutil.copy(PATH_FILTER, broken_scripts / "path_filter.sh") + (broken_scripts / "classify_changes.sh").write_text("#!/usr/bin/env bash\nexit 1\n") + (broken_scripts / "classify_changes.sh").chmod(0o755) + + work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) + result = _run_path_filter(work, tmp_path, "backend", broken_scripts) + assert result.returncode == 0 + assert "classify_changes.sh failed" in result.stdout + assert "halt" not in result.stdout