diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index 2ca2654a207..7aa0c3544ee 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -1,15 +1,17 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: classify_changes.sh }" +category="${1:?usage: classify_changes.sh }" has_client=false has_backend=false +has_ci=false while IFS= read -r file || [ -n "$file" ]; do [ -n "$file" ] || continue case "$file" in ui/* | tests/e2e/ui/*) has_client=true ;; docs/* | *.md | *.mdx) : ;; + .github/* | .circleci/*) has_ci=true; has_backend=true ;; *) has_backend=true ;; esac done @@ -21,6 +23,9 @@ case "$category" in client) { [ "$has_client" = true ] || [ "$has_backend" = true ]; } && echo run || echo skip ;; + ui) + { [ "$has_client" = true ] || [ "$has_ci" = true ]; } && echo run || echo skip + ;; *) echo run ;; diff --git a/.github/actions/detect-backend-changes/action.yml b/.github/actions/detect-backend-changes/action.yml deleted file mode 100644 index 6e6019e4af9..00000000000 --- a/.github/actions/detect-backend-changes/action.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: "Detect backend-relevant changes" -description: >- - Classify the pull request's changed files with .circleci/scripts/classify_changes.sh - and expose decision=run|skip. decision=skip means only ui/**, **.md or **.mdx files - changed, so callers can short-circuit expensive steps while the job still completes - successfully and satisfies its required status check. The file list comes from the - pull request itself rather than from a git diff, because the checked-out merge ref is - recomputed as the base branch advances and would otherwise attribute the base - branch's own commits to the pull request. The decision defaults to run for any non - pull_request event or whenever the changed set cannot be resolved, so tests are never - skipped when the classification is uncertain. - -inputs: - github-token: - description: "Token used to list the pull request's files; needs pull-requests: read" - required: false - default: ${{ github.token }} - -outputs: - decision: - description: "run when backend-relevant files changed, otherwise skip" - value: ${{ steps.classify.outputs.decision }} - -runs: - using: composite - steps: - - id: classify - shell: bash - env: - GH_TOKEN: ${{ inputs.github-token }} - REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - CHANGED_FILE_COUNT: ${{ github.event.pull_request.changed_files }} - run: bash "${GITHUB_ACTION_PATH}/../../scripts/detect_backend_changes.sh" diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml new file mode 100644 index 00000000000..9b22d2c23a8 --- /dev/null +++ b/.github/actions/detect-changes/action.yml @@ -0,0 +1,41 @@ +name: "Detect relevant changes" +description: >- + Classify the pull request's changed files with .circleci/scripts/classify_changes.sh + and expose decision=run|skip for one category. backend means anything outside ui/, + docs/ and markdown; ui means the dashboard sources alone. decision=skip lets callers + short-circuit expensive steps while the job still completes successfully and satisfies + its required status check, which a paths: filter cannot do because a workflow that + never starts never reports. The file list comes from the pull request itself rather + than from a git diff, because the checked-out merge ref is recomputed as the base + branch advances and would otherwise attribute the base branch's own commits to the + pull request. The decision defaults to run for any non pull_request event or whenever + the changed set cannot be resolved, so jobs are never skipped when the classification + is uncertain. + +inputs: + category: + description: "Which classification to apply: backend, client or ui" + required: false + default: backend + github-token: + description: "Token used to list the pull request's files; needs pull-requests: read" + required: false + default: ${{ github.token }} + +outputs: + decision: + description: "run when category-relevant files changed, otherwise skip" + value: ${{ steps.classify.outputs.decision }} + +runs: + using: composite + steps: + - id: classify + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + CATEGORY: ${{ inputs.category }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + CHANGED_FILE_COUNT: ${{ github.event.pull_request.changed_files }} + run: bash "${GITHUB_ACTION_PATH}/../../scripts/detect_changes.sh" diff --git a/.github/scripts/detect_backend_changes.sh b/.github/scripts/detect_changes.sh similarity index 80% rename from .github/scripts/detect_backend_changes.sh rename to .github/scripts/detect_changes.sh index 7be49992b37..2d427c92fb5 100755 --- a/.github/scripts/detect_backend_changes.sh +++ b/.github/scripts/detect_changes.sh @@ -2,15 +2,16 @@ set -uo pipefail readonly API_FILE_CEILING=3000 +readonly CATEGORY="${CATEGORY:-backend}" decide() { - echo "detect-backend-changes: decision=$1" + echo "detect-changes[${CATEGORY}]: decision=$1" [ -z "${GITHUB_OUTPUT:-}" ] || echo "decision=$1" >>"${GITHUB_OUTPUT}" exit 0 } run_full() { - echo "detect-backend-changes: $1; running job" + echo "detect-changes[${CATEGORY}]: $1; running job" decide run } @@ -30,10 +31,10 @@ changed="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[]. run_full "could not list the files on PR #${PR_NUMBER}" [ -n "${changed}" ] || run_full "the API listed no files on PR #${PR_NUMBER}" -echo "detect-backend-changes: files changed by PR #${PR_NUMBER}:" +echo "detect-changes[${CATEGORY}]: files changed by PR #${PR_NUMBER}:" printf '%s\n' "${changed}" | sed 's/^/ /' -decision="$(printf '%s\n' "${changed}" | bash "${classify}" backend)" || +decision="$(printf '%s\n' "${changed}" | bash "${classify}" "${CATEGORY}")" || run_full "classify_changes.sh failed" case "${decision}" in run | skip) decide "${decision}" ;; diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 6ad240717e8..4f4339a360a 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -72,24 +72,27 @@ jobs: with: persist-credentials: false - - name: Detect backend-relevant changes + - name: Detect relevant changes id: changes timeout-minutes: 2 - uses: ./.github/actions/detect-backend-changes + uses: ./.github/actions/detect-changes - name: Set up Python + if: steps.changes.outputs.decision != 'skip' timeout-minutes: 3 uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' timeout-minutes: 3 uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Cache uv dependencies + if: steps.changes.outputs.decision != 'skip' timeout-minutes: 5 uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 69495cff896..7a0ae0faaa0 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -24,6 +24,7 @@ jobs: # re-running basedpyright over the merge-base tree. permissions: contents: read + pull-requests: read actions: read steps: @@ -37,7 +38,12 @@ jobs: clean: true persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + - name: Fetch gate base (merge-base with target branch) + if: steps.changes.outputs.decision != 'skip' env: GH_TOKEN: ${{ github.token }} BASE_SHA: ${{ github.event.pull_request.base.sha }} @@ -50,39 +56,47 @@ jobs: echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV" - name: Set up Python + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Clean Python cache + if: steps.changes.outputs.decision != 'skip' run: | find . -type d -name "__pycache__" -exec rm -rf {} + || true find . -name "*.pyc" -delete || true - name: Check uv.lock is up to date + if: steps.changes.outputs.decision != 'skip' run: | uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1) - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: | uv sync --frozen --group proxy-dev --group e2e-dev - name: Cache Prisma binaries + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/cache-prisma-binaries # basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma) # only after `prisma generate` writes prisma/client.py et al. Without this the # DB wrappers typed against the generated client would degrade to Unknown. - name: Generate Prisma client + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - name: Check ruff format + if: steps.changes.outputs.decision != 'skip' run: | git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then @@ -92,6 +106,7 @@ jobs: xargs uv run --no-sync ruff format --check --exclude '/enterprise/' < "$RUNNER_TEMP/ruff_format_files.txt" - name: Debug - Check file state + if: steps.changes.outputs.decision != 'skip' run: | echo "Current branch:" git branch --show-current @@ -101,30 +116,36 @@ jobs: head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 - name: Run Ruff linting + if: steps.changes.outputs.decision != 'skip' run: | cd litellm uv run --no-sync ruff check . cd .. - name: Check strict-rule budget (delta vs base) + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA" - name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base) + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA" - name: Print OpenAI version + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')" - name: Check basedpyright budget (delta vs base) + if: steps.changes.outputs.decision != 'skip' env: GH_TOKEN: ${{ github.token }} run: | uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA" - name: Check tests/e2e basedpyright (zero errors) + if: steps.changes.outputs.decision != 'skip' run: | if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then uv run --no-sync basedpyright tests/e2e @@ -133,12 +154,14 @@ jobs: fi - name: Check for circular imports + if: steps.changes.outputs.decision != 'skip' run: | cd litellm uv run --no-sync python ../tests/documentation_tests/test_circular_imports.py cd .. - name: Check import safety + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index 618b0195b5a..b3a07a6e0ff 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -1,6 +1,7 @@ name: UI Build Check permissions: contents: read + pull-requests: read on: pull_request: @@ -28,7 +29,14 @@ jobs: with: persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + with: + category: ui + - name: Setup Node.js + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version-file: ui/litellm-dashboard/.nvmrc @@ -36,7 +44,9 @@ jobs: cache-dependency-path: ui/litellm-dashboard/package-lock.json - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: npm ci - name: Build + if: steps.changes.outputs.decision != 'skip' run: npm run build diff --git a/.github/workflows/test-litellm-ui-unit.yml b/.github/workflows/test-litellm-ui-unit.yml index 69cbc082d98..a7329432be5 100644 --- a/.github/workflows/test-litellm-ui-unit.yml +++ b/.github/workflows/test-litellm-ui-unit.yml @@ -1,6 +1,7 @@ name: UI Unit Tests permissions: contents: read + pull-requests: read on: pull_request: @@ -32,7 +33,14 @@ jobs: fetch-depth: 1 persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + with: + category: ui + - name: Setup Node.js + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version-file: ui/litellm-dashboard/.nvmrc @@ -40,14 +48,17 @@ jobs: cache-dependency-path: ui/litellm-dashboard/package-lock.json - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: npm ci - name: Run UI type tests (Vitest) + if: steps.changes.outputs.decision != 'skip' env: CI: "true" run: npm run test:types - name: Run UI unit tests (Vitest) + if: steps.changes.outputs.decision != 'skip' env: CI: "true" GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml index 05cc13d0af2..95187ef2835 100644 --- a/.github/workflows/test-mcp.yml +++ b/.github/workflows/test-mcp.yml @@ -10,6 +10,7 @@ on: permissions: contents: read + pull-requests: read concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -25,26 +26,34 @@ jobs: with: persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + - name: Thank You Message run: | echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY - name: Set up Python + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Install dependencies + if: steps.changes.outputs.decision != 'skip' run: | uv lock --check .github/scripts/uv_sync_with_retries.sh --frozen --group proxy-dev --extra proxy --extra semantic-router - name: Run MCP tests + if: steps.changes.outputs.decision != 'skip' run: | uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5 diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml index 00566e13f53..cb8035aafa1 100644 --- a/.github/workflows/test-unit-documentation.yml +++ b/.github/workflows/test-unit-documentation.yml @@ -32,28 +32,32 @@ jobs: with: persist-credentials: false + - name: Detect relevant changes + id: changes + uses: ./.github/actions/detect-changes + - name: Checkout litellm-docs into docs/my-website (for documentation_tests) + if: steps.changes.outputs.decision != 'skip' uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: repository: BerriAI/litellm-docs path: docs/my-website persist-credentials: false - - name: Detect backend-relevant changes - id: changes - uses: ./.github/actions/detect-backend-changes - - name: Set up Python + if: steps.changes.outputs.decision != 'skip' uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Set up uv + if: steps.changes.outputs.decision != 'skip' uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - name: Cache uv dependencies + if: steps.changes.outputs.decision != 'skip' uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | diff --git a/tests/test_litellm/test_circleci_path_filter.py b/tests/test_litellm/test_circleci_path_filter.py index b8e763b4979..b427a1a3bd8 100644 --- a/tests/test_litellm/test_circleci_path_filter.py +++ b/tests/test_litellm/test_circleci_path_filter.py @@ -7,6 +7,9 @@ category, it prints `run` or `skip`. The gating contract we lock in here: * docs-only changes (``*.md``, ``*.mdx``, ``docs/``) run nothing * client-only changes (``ui/``) run client jobs but skip backend jobs * any backend change runs both client and backend jobs + * ``ui`` tracks ``ui/`` plus CI config, so a backend-only change skips it + where ``client`` would still run, while a change to the workflows that + define the dashboard jobs still exercises them If this logic silently regresses, real test jobs get skipped, so these cases are the guardrail against that. @@ -40,6 +43,7 @@ def classify(category: str, changed: list[str]) -> str: DOCS = ["README.md", "docs/my_website/index.mdx", "litellm/anywhere.md"] CLIENT = ["ui/litellm-dashboard/src/App.tsx"] BACKEND = ["litellm/main.py"] +CI = [".github/workflows/test-litellm-ui-unit.yml"] @pytest.mark.parametrize( @@ -48,20 +52,27 @@ BACKEND = ["litellm/main.py"] # docs-only: skip everything ("backend", DOCS, "skip"), ("client", DOCS, "skip"), + ("ui", DOCS, "skip"), ("backend", [], "skip"), ("client", [], "skip"), + ("ui", [], "skip"), # client-only: backend skips, client runs ("backend", CLIENT, "skip"), ("client", CLIENT, "run"), + ("ui", CLIENT, "run"), ("backend", CLIENT + DOCS, "skip"), ("client", CLIENT + DOCS, "run"), + ("ui", CLIENT + DOCS, "run"), # any backend change: both run ("backend runs both") ("backend", BACKEND, "run"), ("client", BACKEND, "run"), + ("ui", BACKEND, "skip"), ("backend", BACKEND + DOCS, "run"), ("client", BACKEND + DOCS, "run"), + ("ui", BACKEND + DOCS, "skip"), ("backend", BACKEND + CLIENT, "run"), ("client", BACKEND + CLIENT, "run"), + ("ui", BACKEND + CLIENT, "run"), ], ) def test_classify_decisions(category: str, changed: list[str], expected: str) -> None: @@ -71,6 +82,31 @@ def test_classify_decisions(category: str, changed: list[str], expected: str) -> def test_markdown_under_ui_counts_as_client_not_docs() -> None: assert classify("client", ["ui/litellm-dashboard/README.md"]) == "run" assert classify("backend", ["ui/litellm-dashboard/README.md"]) == "skip" + assert classify("ui", ["ui/litellm-dashboard/README.md"]) == "run" + + +def test_ci_config_changes_reach_every_category() -> None: + """A workflow edit has to exercise the jobs it defines, otherwise the change + ships unvalidated: the dashboard jobs would skip on the very pull request + that rewrites them.""" + assert classify("ui", CI) == "run" + assert classify("backend", CI) == "run" + assert classify("client", CI) == "run" + + +def test_markdown_under_dot_github_is_still_docs() -> None: + """`.github/**` counting as CI config must not drag the pull request template + and other markdown back into running the full suite.""" + assert classify("ui", [".github/pull_request_template.md"]) == "skip" + assert classify("backend", [".github/pull_request_template.md"]) == "skip" + + +def test_ui_and_client_diverge_on_a_backend_only_change() -> None: + """`client` gates CircleCI's dashboard end-to-end jobs, which drive a real + proxy and so must run on backend changes. `ui` gates the dashboard build and + its unit tests, which cannot see the backend at all.""" + assert classify("client", BACKEND) == "run" + assert classify("ui", BACKEND) == "skip" def test_non_docs_directory_with_docs_in_name_is_backend() -> None: diff --git a/tests/test_litellm/test_detect_backend_changes.py b/tests/test_litellm/test_detect_changes.py similarity index 73% rename from tests/test_litellm/test_detect_backend_changes.py rename to tests/test_litellm/test_detect_changes.py index 527d4143a9c..d8feb2371a3 100644 --- a/tests/test_litellm/test_detect_backend_changes.py +++ b/tests/test_litellm/test_detect_changes.py @@ -1,12 +1,13 @@ """Regression tests for the GitHub Actions change-based job gating. -`.github/scripts/detect_backend_changes.sh` decides whether a pull request's -backend unit-test jobs do real work. It asks the API which files the pull -request touches and hands them to `classify_changes.sh`. The contract locked in -here: +`.github/scripts/detect_changes.sh` decides whether a pull request's jobs do +real work. It asks the API which files the pull request touches and hands them +to `classify_changes.sh` under one category. The contract locked in here: * a UI-only pull request skips backend jobs even when the checked-out merge ref carries backend commits from the base branch + * the ui category is the mirror image: it skips when only backend files + changed, so a backend-only PR stops building and unit-testing the dashboard * anything the classification cannot resolve (no pull request, an API failure, a truncated file list, a broken classifier) runs the job """ @@ -19,7 +20,7 @@ import subprocess from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] -SCRIPT = REPO_ROOT / ".github" / "scripts" / "detect_backend_changes.sh" +SCRIPT = REPO_ROOT / ".github" / "scripts" / "detect_changes.sh" CLASSIFIER = REPO_ROOT / ".circleci" / "scripts" / "classify_changes.sh" UI_FILE = "ui/litellm-dashboard/src/components/Teams.tsx" @@ -84,6 +85,7 @@ def _run( changed_file_count: str | None = None, gh_exit_code: int = 0, classifier_body: str | None = None, + category: str | None = None, ) -> tuple[str, str]: """Run the script against a stubbed `gh`; returns (decision, stdout).""" bin_dir = tmp_path / "bin" @@ -102,6 +104,10 @@ def _run( env["REPO"] = "BerriAI/litellm" env["PR_NUMBER"] = pr_number env["CHANGED_FILE_COUNT"] = changed_file_count if changed_file_count is not None else str(len(files)) + if category is not None: + env["CATEGORY"] = category + else: + env.pop("CATEGORY", None) tree = _scripts_tree(tmp_path, classifier_body) result = subprocess.run( @@ -187,3 +193,43 @@ def test_unexpected_classifier_output_runs(tmp_path: Path) -> None: ) assert decision == "decision=run" assert "unexpected decision: maybe" in stdout + + +def test_ui_category_skips_a_backend_only_pr(tmp_path: Path) -> None: + """The dashboard build and its unit tests cannot be affected by a pull request + that touches no `ui/` file, and the `client` category cannot express that + because it deliberately runs whenever the backend changes.""" + decision, _ = _run(tmp_path, files=[BACKEND_FILE], category="ui") + assert decision == "decision=skip" + + +def test_ui_category_runs_a_ui_only_pr(tmp_path: Path) -> None: + decision, _ = _run(tmp_path, files=[UI_FILE], category="ui") + assert decision == "decision=run" + + +def test_ui_category_runs_a_mixed_pr(tmp_path: Path) -> None: + decision, _ = _run(tmp_path, files=[UI_FILE, BACKEND_FILE], category="ui") + assert decision == "decision=run" + + +def test_absent_category_still_runs_a_backend_pr(tmp_path: Path) -> None: + """Callers that pass no category keep the pre-existing backend behaviour.""" + assert _run(tmp_path, files=[BACKEND_FILE])[0] == "decision=run" + + +def test_absent_category_still_skips_a_ui_pr(tmp_path: Path) -> None: + assert _run(tmp_path, files=[UI_FILE])[0] == "decision=skip" + + +def test_ui_category_fails_open_when_the_api_fails(tmp_path: Path) -> None: + decision, stdout = _run(tmp_path, files=[], gh_exit_code=1, category="ui") + assert decision == "decision=run" + assert "detect-changes[ui]" in stdout + + +def test_ui_category_runs_when_the_ui_workflows_themselves_change(tmp_path: Path) -> None: + """Without this the dashboard jobs would skip on the pull request that edits + them, shipping a workflow change nothing ever exercised.""" + decision, _ = _run(tmp_path, files=[".github/workflows/test-litellm-ui-unit.yml"], category="ui") + assert decision == "decision=run"