From 7b6e16cfd315d940d8b34cebd39ef16f3d2f0c26 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 19 Aug 2026 18:32:21 -0700 Subject: [PATCH] perf(ci): gate the lint, MCP and dashboard jobs on the pull request's file list (#37559) PR #37550 taught the backend unit-test shards to read the pull request's own file list, but four required jobs were never wired to that gate and ran in full on every pull request regardless of what it touched. A UI-only pull request still paid roughly 17 runner-minutes of Python work it could not have affected, and a backend-only one still installed and built the dashboard. Lint and the MCP suite now take the existing backend decision. The dashboard build and unit tests take a new ui decision, which tracks ui/ rather than reusing client: client deliberately runs whenever the backend changes, because it gates CircleCI's end-to-end jobs that drive a real proxy, while the build and the unit tests cannot see the backend at all. CI config counts as ui-relevant too, so a pull request that rewrites the dashboard workflows still exercises them instead of shipping unvalidated. The gate stays inside the job rather than moving to on.paths or to a job-level condition on the shard callers. A workflow filtered out by on.paths never starts and never reports, so a required check waits forever, and a skipped caller job publishes its own name instead of the nested " / Run tests" the ruleset requires. Both were measured before settling on this shape. Three setup steps in the shard base and in the documentation job also leaked past the gate, so a skipped shard still spent about twelve seconds installing uv and restoring its cache. They now carry the same condition, and the documentation job stops cloning litellm-docs when it has nothing to validate. --- .circleci/scripts/classify_changes.sh | 7 ++- .../actions/detect-backend-changes/action.yml | 34 ----------- .github/actions/detect-changes/action.yml | 41 ++++++++++++++ ...t_backend_changes.sh => detect_changes.sh} | 9 +-- .github/workflows/_test-unit-base.yml | 7 ++- .github/workflows/test-linting.yml | 23 ++++++++ .github/workflows/test-litellm-ui-build.yml | 10 ++++ .github/workflows/test-litellm-ui-unit.yml | 11 ++++ .github/workflows/test-mcp.yml | 9 +++ .github/workflows/test-unit-documentation.yml | 12 ++-- .../test_litellm/test_circleci_path_filter.py | 36 ++++++++++++ ...kend_changes.py => test_detect_changes.py} | 56 +++++++++++++++++-- 12 files changed, 205 insertions(+), 50 deletions(-) delete mode 100644 .github/actions/detect-backend-changes/action.yml create mode 100644 .github/actions/detect-changes/action.yml rename .github/scripts/{detect_backend_changes.sh => detect_changes.sh} (80%) rename tests/test_litellm/{test_detect_backend_changes.py => test_detect_changes.py} (73%) diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index 2ca2654a207..7aa0c3544ee 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -1,15 +1,17 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: classify_changes.sh }" +category="${1:?usage: classify_changes.sh }" has_client=false has_backend=false +has_ci=false while IFS= read -r file || [ -n "$file" ]; do [ -n "$file" ] || continue case "$file" in ui/* | tests/e2e/ui/*) has_client=true ;; docs/* | *.md | *.mdx) : ;; + .github/* | .circleci/*) has_ci=true; has_backend=true ;; *) has_backend=true ;; esac done @@ -21,6 +23,9 @@ case "$category" in client) { [ "$has_client" = true ] || [ "$has_backend" = true ]; } && echo run || echo skip ;; + ui) + { [ "$has_client" = true ] || [ "$has_ci" = true ]; } && echo run || echo skip + ;; *) echo run ;; diff --git a/.github/actions/detect-backend-changes/action.yml b/.github/actions/detect-backend-changes/action.yml deleted file mode 100644 index 6e6019e4af9..00000000000 --- a/.github/actions/detect-backend-changes/action.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: "Detect backend-relevant changes" -description: >- - Classify the pull request's changed files with .circleci/scripts/classify_changes.sh - and expose decision=run|skip. decision=skip means only ui/**, **.md or **.mdx files - changed, so callers can short-circuit expensive steps while the job still completes - successfully and satisfies its required status check. The file list comes from the - pull request itself rather than from a git diff, because the checked-out merge ref is - recomputed as the base branch advances and would otherwise attribute the base - branch's own commits to the pull request. The decision defaults to run for any non - pull_request event or whenever the changed set cannot be resolved, so tests are never - skipped when the classification is uncertain. - -inputs: - github-token: - description: "Token used to list the pull request's files; needs pull-requests: read" - required: false - default: ${{ github.token }} - -outputs: - decision: - description: "run when backend-relevant files changed, otherwise skip" - value: ${{ steps.classify.outputs.decision }} - -runs: - using: composite - steps: - - id: classify - shell: bash - env: - GH_TOKEN: ${{ inputs.github-token }} - REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - CHANGED_FILE_COUNT: ${{ github.event.pull_request.changed_files }} - run: bash "${GITHUB_ACTION_PATH}/../../scripts/detect_backend_changes.sh" diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml new file mode 100644 index 00000000000..9b22d2c23a8 --- /dev/null +++ b/.github/actions/detect-changes/action.yml @@ -0,0 +1,41 @@ +name: "Detect relevant changes" +description: >- + Classify the pull request's changed files with .circleci/scripts/classify_changes.sh + and expose decision=run|skip for one category. backend means anything outside ui/, + docs/ and markdown; ui means the dashboard sources alone. decision=skip lets callers + short-circuit expensive steps while the job still completes successfully and satisfies + its required status check, which a paths: filter cannot do because a workflow that + never starts never reports. The file list comes from the pull request itself rather + than from a git diff, because the checked-out merge ref is recomputed as the base + branch advances and would otherwise attribute the base branch's own commits to the + pull request. The decision defaults to run for any non pull_request event or whenever + the changed set cannot be resolved, so jobs are never skipped when the classification + is uncertain. + +inputs: + category: + description: "Which classification to apply: backend, client or ui" + required: false + default: backend + github-token: + description: "Token used to list the pull request's files; needs pull-requests: read" + required: false + default: ${{ github.token }} + +outputs: + decision: + description: "run when category-relevant files changed, otherwise skip" + value: ${{ steps.classify.outputs.decision }} + +runs: + using: composite + steps: + - id: classify + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + CATEGORY: ${{ inputs.category }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + CHANGED_FILE_COUNT: ${{ github.event.pull_request.changed_files }} + run: bash "${GITHUB_ACTION_PATH}/../../scripts/detect_changes.sh" diff --git a/.github/scripts/detect_backend_changes.sh b/.github/scripts/detect_changes.sh similarity index 80% rename from .github/scripts/detect_backend_changes.sh rename to .github/scripts/detect_changes.sh index 7be49992b37..2d427c92fb5 100755 --- a/.github/scripts/detect_backend_changes.sh +++ b/.github/scripts/detect_changes.sh @@ -2,15 +2,16 @@ set -uo pipefail readonly API_FILE_CEILING=3000 +readonly CATEGORY="${CATEGORY:-backend}" decide() { - echo "detect-backend-changes: decision=$1" + echo "detect-changes[${CATEGORY}]: decision=$1" [ -z "${GITHUB_OUTPUT:-}" ] || echo "decision=$1" >>"${GITHUB_OUTPUT}" exit 0 } run_full() { - echo "detect-backend-changes: $1; running job" + echo "detect-changes[${CATEGORY}]: $1; running job" decide run } @@ -30,10 +31,10 @@ changed="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[]. run_full "could not list the files on PR #${PR_NUMBER}" [ -n "${changed}" ] || run_full "the API listed no files on PR #${PR_NUMBER}" -echo "detect-backend-changes: files changed by PR #${PR_NUMBER}:" +echo "detect-changes[${CATEGORY}]: files changed by PR #${PR_NUMBER}:" printf '%s\n' "${changed}" | sed 's/^/ /' -decision="$(printf '%s\n' "${changed}" | bash "${classify}" backend)" || +decision="$(printf '%s\n' "${changed}" | bash "${classify}" "${CATEGORY}")" || run_full "classify_changes.sh failed" case "${decision}" in run | skip) decide "${decision}" ;; diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 6ad240717e8..4f4339a360a 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -72,24 +72,27 @@ jobs: with: persist-credentials: false - - name: Detect backend-relevant changes + - name: Detect relevant changes id: changes timeout-minutes: 2 - uses: ./.github/actions/detect-backend-changes + uses: ./.github/actions/detect-changes - name: Set up Python + if: steps.changes.outputs.decision != 'skip' timeout-minutes: 3 uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' timeout-minutes: 3 uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Cache uv dependencies + if: steps.changes.outputs.decision != 'skip' timeout-minutes: 5 uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 69495cff896..7a0ae0faaa0 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -24,6 +24,7 @@ jobs: # re-running basedpyright over the merge-base tree. permissions: contents: read + pull-requests: read actions: read steps: @@ -37,7 +38,12 @@ jobs: clean: true persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + - name: Fetch gate base (merge-base with target branch) + if: steps.changes.outputs.decision != 'skip' env: GH_TOKEN: ${{ github.token }} BASE_SHA: ${{ github.event.pull_request.base.sha }} @@ -50,39 +56,47 @@ jobs: echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV" - name: Set up Python + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Clean Python cache + if: steps.changes.outputs.decision != 'skip' run: | find . -type d -name "__pycache__" -exec rm -rf {} + || true find . -name "*.pyc" -delete || true - name: Check uv.lock is up to date + if: steps.changes.outputs.decision != 'skip' run: | uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1) - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: | uv sync --frozen --group proxy-dev --group e2e-dev - name: Cache Prisma binaries + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/cache-prisma-binaries # basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma) # only after `prisma generate` writes prisma/client.py et al. Without this the # DB wrappers typed against the generated client would degrade to Unknown. - name: Generate Prisma client + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Check ruff format + if: steps.changes.outputs.decision != 'skip' run: | git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then @@ -92,6 +106,7 @@ jobs: xargs uv run --no-sync ruff format --check --exclude '/enterprise/' < "$RUNNER_TEMP/ruff_format_files.txt" - name: Debug - Check file state + if: steps.changes.outputs.decision != 'skip' run: | echo "Current branch:" git branch --show-current @@ -101,30 +116,36 @@ jobs: head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 - name: Run Ruff linting + if: steps.changes.outputs.decision != 'skip' run: | cd litellm uv run --no-sync ruff check . cd .. - name: Check strict-rule budget (delta vs base) + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA" - name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base) + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA" - name: Print OpenAI version + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')" - name: Check basedpyright budget (delta vs base) + if: steps.changes.outputs.decision != 'skip' env: GH_TOKEN: ${{ github.token }} run: | uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA" - name: Check tests/e2e basedpyright (zero errors) + if: steps.changes.outputs.decision != 'skip' run: | if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then uv run --no-sync basedpyright tests/e2e @@ -133,12 +154,14 @@ jobs: fi - name: Check for circular imports + if: steps.changes.outputs.decision != 'skip' run: | cd litellm uv run --no-sync python ../tests/documentation_tests/test_circular_imports.py cd .. - name: Check import safety + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index 618b0195b5a..b3a07a6e0ff 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -1,6 +1,7 @@ name: UI Build Check permissions: contents: read + pull-requests: read on: pull_request: @@ -28,7 +29,14 @@ jobs: with: persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + with: + category: ui + - name: Setup Node.js + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version-file: ui/litellm-dashboard/.nvmrc @@ -36,7 +44,9 @@ jobs: cache-dependency-path: ui/litellm-dashboard/package-lock.json - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: npm ci - name: Build + if: steps.changes.outputs.decision != 'skip' run: npm run build diff --git a/.github/workflows/test-litellm-ui-unit.yml b/.github/workflows/test-litellm-ui-unit.yml index 69cbc082d98..a7329432be5 100644 --- a/.github/workflows/test-litellm-ui-unit.yml +++ b/.github/workflows/test-litellm-ui-unit.yml @@ -1,6 +1,7 @@ name: UI Unit Tests permissions: contents: read + pull-requests: read on: pull_request: @@ -32,7 +33,14 @@ jobs: fetch-depth: 1 persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + with: + category: ui + - name: Setup Node.js + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version-file: ui/litellm-dashboard/.nvmrc @@ -40,14 +48,17 @@ jobs: cache-dependency-path: ui/litellm-dashboard/package-lock.json - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: npm ci - name: Run UI type tests (Vitest) + if: steps.changes.outputs.decision != 'skip' env: CI: "true" run: npm run test:types - name: Run UI unit tests (Vitest) + if: steps.changes.outputs.decision != 'skip' env: CI: "true" GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index 05cc13d0af2..95187ef2835 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -10,6 +10,7 @@ on: permissions: contents: read + pull-requests: read concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -25,26 +26,34 @@ jobs: with: persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + - name: Thank You Message run: | echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - name: Set up Python + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: | uv lock --check .github/scripts/uv_sync_with_retries.sh --frozen --group proxy-dev --extra proxy --extra semantic-router - name: Run MCP tests + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5 diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml index 00566e13f53..cb8035aafa1 100644 --- a/.github/workflows/test-unit-documentation.yml +++ b/.github/workflows/test-unit-documentation.yml @@ -32,28 +32,32 @@ jobs: with: persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + - name: Checkout litellm-docs into docs/my-website (for documentation_tests) + if: steps.changes.outputs.decision != 'skip' uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: repository: BerriAI/litellm-docs path: docs/my-website persist-credentials: false - - name: Detect backend-relevant changes - id: changes - uses: ./.github/actions/detect-backend-changes - - name: Set up Python + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Cache uv dependencies + if: steps.changes.outputs.decision != 'skip' uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | diff --git a/tests/test_litellm/test_circleci_path_filter.py b/tests/test_litellm/test_circleci_path_filter.py index b8e763b4979..b427a1a3bd8 100644 --- a/tests/test_litellm/test_circleci_path_filter.py +++ b/tests/test_litellm/test_circleci_path_filter.py @@ -7,6 +7,9 @@ category, it prints `run` or `skip`. The gating contract we lock in here: * docs-only changes (``*.md``, ``*.mdx``, ``docs/``) run nothing * client-only changes (``ui/``) run client jobs but skip backend jobs * any backend change runs both client and backend jobs + * ``ui`` tracks ``ui/`` plus CI config, so a backend-only change skips it + where ``client`` would still run, while a change to the workflows that + define the dashboard jobs still exercises them If this logic silently regresses, real test jobs get skipped, so these cases are the guardrail against that. @@ -40,6 +43,7 @@ def classify(category: str, changed: list[str]) -> str: DOCS = ["README.md", "docs/my_website/index.mdx", "litellm/anywhere.md"] CLIENT = ["ui/litellm-dashboard/src/App.tsx"] BACKEND = ["litellm/main.py"] +CI = [".github/workflows/test-litellm-ui-unit.yml"] @pytest.mark.parametrize( @@ -48,20 +52,27 @@ BACKEND = ["litellm/main.py"] # docs-only: skip everything ("backend", DOCS, "skip"), ("client", DOCS, "skip"), + ("ui", DOCS, "skip"), ("backend", [], "skip"), ("client", [], "skip"), + ("ui", [], "skip"), # client-only: backend skips, client runs ("backend", CLIENT, "skip"), ("client", CLIENT, "run"), + ("ui", CLIENT, "run"), ("backend", CLIENT + DOCS, "skip"), ("client", CLIENT + DOCS, "run"), + ("ui", CLIENT + DOCS, "run"), # any backend change: both run ("backend runs both") ("backend", BACKEND, "run"), ("client", BACKEND, "run"), + ("ui", BACKEND, "skip"), ("backend", BACKEND + DOCS, "run"), ("client", BACKEND + DOCS, "run"), + ("ui", BACKEND + DOCS, "skip"), ("backend", BACKEND + CLIENT, "run"), ("client", BACKEND + CLIENT, "run"), + ("ui", BACKEND + CLIENT, "run"), ], ) def test_classify_decisions(category: str, changed: list[str], expected: str) -> None: @@ -71,6 +82,31 @@ def test_classify_decisions(category: str, changed: list[str], expected: str) -> def test_markdown_under_ui_counts_as_client_not_docs() -> None: assert classify("client", ["ui/litellm-dashboard/README.md"]) == "run" assert classify("backend", ["ui/litellm-dashboard/README.md"]) == "skip" + assert classify("ui", ["ui/litellm-dashboard/README.md"]) == "run" + + +def test_ci_config_changes_reach_every_category() -> None: + """A workflow edit has to exercise the jobs it defines, otherwise the change + ships unvalidated: the dashboard jobs would skip on the very pull request + that rewrites them.""" + assert classify("ui", CI) == "run" + assert classify("backend", CI) == "run" + assert classify("client", CI) == "run" + + +def test_markdown_under_dot_github_is_still_docs() -> None: + """`.github/**` counting as CI config must not drag the pull request template + and other markdown back into running the full suite.""" + assert classify("ui", [".github/pull_request_template.md"]) == "skip" + assert classify("backend", [".github/pull_request_template.md"]) == "skip" + + +def test_ui_and_client_diverge_on_a_backend_only_change() -> None: + """`client` gates CircleCI's dashboard end-to-end jobs, which drive a real + proxy and so must run on backend changes. `ui` gates the dashboard build and + its unit tests, which cannot see the backend at all.""" + assert classify("client", BACKEND) == "run" + assert classify("ui", BACKEND) == "skip" def test_non_docs_directory_with_docs_in_name_is_backend() -> None: diff --git a/tests/test_litellm/test_detect_backend_changes.py b/tests/test_litellm/test_detect_changes.py similarity index 73% rename from tests/test_litellm/test_detect_backend_changes.py rename to tests/test_litellm/test_detect_changes.py index 527d4143a9c..d8feb2371a3 100644 --- a/tests/test_litellm/test_detect_backend_changes.py +++ b/tests/test_litellm/test_detect_changes.py @@ -1,12 +1,13 @@ """Regression tests for the GitHub Actions change-based job gating. -`.github/scripts/detect_backend_changes.sh` decides whether a pull request's -backend unit-test jobs do real work. It asks the API which files the pull -request touches and hands them to `classify_changes.sh`. The contract locked in -here: +`.github/scripts/detect_changes.sh` decides whether a pull request's jobs do +real work. It asks the API which files the pull request touches and hands them +to `classify_changes.sh` under one category. The contract locked in here: * a UI-only pull request skips backend jobs even when the checked-out merge ref carries backend commits from the base branch + * the ui category is the mirror image: it skips when only backend files + changed, so a backend-only PR stops building and unit-testing the dashboard * anything the classification cannot resolve (no pull request, an API failure, a truncated file list, a broken classifier) runs the job """ @@ -19,7 +20,7 @@ import subprocess from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] -SCRIPT = REPO_ROOT / ".github" / "scripts" / "detect_backend_changes.sh" +SCRIPT = REPO_ROOT / ".github" / "scripts" / "detect_changes.sh" CLASSIFIER = REPO_ROOT / ".circleci" / "scripts" / "classify_changes.sh" UI_FILE = "ui/litellm-dashboard/src/components/Teams.tsx" @@ -84,6 +85,7 @@ def _run( changed_file_count: str | None = None, gh_exit_code: int = 0, classifier_body: str | None = None, + category: str | None = None, ) -> tuple[str, str]: """Run the script against a stubbed `gh`; returns (decision, stdout).""" bin_dir = tmp_path / "bin" @@ -102,6 +104,10 @@ def _run( env["REPO"] = "BerriAI/litellm" env["PR_NUMBER"] = pr_number env["CHANGED_FILE_COUNT"] = changed_file_count if changed_file_count is not None else str(len(files)) + if category is not None: + env["CATEGORY"] = category + else: + env.pop("CATEGORY", None) tree = _scripts_tree(tmp_path, classifier_body) result = subprocess.run( @@ -187,3 +193,43 @@ def test_unexpected_classifier_output_runs(tmp_path: Path) -> None: ) assert decision == "decision=run" assert "unexpected decision: maybe" in stdout + + +def test_ui_category_skips_a_backend_only_pr(tmp_path: Path) -> None: + """The dashboard build and its unit tests cannot be affected by a pull request + that touches no `ui/` file, and the `client` category cannot express that + because it deliberately runs whenever the backend changes.""" + decision, _ = _run(tmp_path, files=[BACKEND_FILE], category="ui") + assert decision == "decision=skip" + + +def test_ui_category_runs_a_ui_only_pr(tmp_path: Path) -> None: + decision, _ = _run(tmp_path, files=[UI_FILE], category="ui") + assert decision == "decision=run" + + +def test_ui_category_runs_a_mixed_pr(tmp_path: Path) -> None: + decision, _ = _run(tmp_path, files=[UI_FILE, BACKEND_FILE], category="ui") + assert decision == "decision=run" + + +def test_absent_category_still_runs_a_backend_pr(tmp_path: Path) -> None: + """Callers that pass no category keep the pre-existing backend behaviour.""" + assert _run(tmp_path, files=[BACKEND_FILE])[0] == "decision=run" + + +def test_absent_category_still_skips_a_ui_pr(tmp_path: Path) -> None: + assert _run(tmp_path, files=[UI_FILE])[0] == "decision=skip" + + +def test_ui_category_fails_open_when_the_api_fails(tmp_path: Path) -> None: + decision, stdout = _run(tmp_path, files=[], gh_exit_code=1, category="ui") + assert decision == "decision=run" + assert "detect-changes[ui]" in stdout + + +def test_ui_category_runs_when_the_ui_workflows_themselves_change(tmp_path: Path) -> None: + """Without this the dashboard jobs would skip on the pull request that edits + them, shipping a workflow change nothing ever exercised.""" + decision, _ = _run(tmp_path, files=[".github/workflows/test-litellm-ui-unit.yml"], category="ui") + assert decision == "decision=run"