litellm/tests/test_litellm/test_detect_changes.py

"""Regression tests for the GitHub Actions change-based job gating.

`.github/scripts/detect_changes.sh` decides whether a pull request's jobs do
real work. It asks the API which files the pull request touches and hands them
to `classify_changes.sh` under one category. The contract locked in here:

  * a UI-only pull request skips backend jobs even when the checked-out merge
    ref carries backend commits from the base branch
  * the ui category is the mirror image: it skips when only backend files
    changed, so a backend-only PR stops building and unit-testing the dashboard
  * anything the classification cannot resolve (no pull request, an API
    failure, a truncated file list, a broken classifier) runs the job
"""

from __future__ import annotations

import os
import shutil
import subprocess
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parents[2]
SCRIPT = REPO_ROOT / ".github" / "scripts" / "detect_changes.sh"
CLASSIFIER = REPO_ROOT / ".circleci" / "scripts" / "classify_changes.sh"

UI_FILE = "ui/litellm-dashboard/src/components/Teams.tsx"
BACKEND_FILE = "litellm/proxy/proxy_server.py"


def _git(cwd: Path, *args: str) -> None:
    subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True)


def _merge_ref_checkout(tmp_path: Path) -> Path:
    """A checkout shaped like `refs/pull/N/merge`: a UI-only branch merged into a
    base tip that has moved ahead by a backend commit since the branch was cut."""
    work = tmp_path / "work"
    work.mkdir()
    _git(work, "init", "-q", "-b", "main")
    _git(work, "config", "user.email", "t@t")
    _git(work, "config", "user.name", "t")
    (work / "seed.txt").write_text("seed\n")
    _git(work, "add", "-A")
    _git(work, "commit", "-qm", "base")

    _git(work, "checkout", "-q", "-b", "feature")
    ui = work / UI_FILE
    ui.parent.mkdir(parents=True, exist_ok=True)
    ui.write_text("export const Teams = () => null\n")
    _git(work, "add", "-A")
    _git(work, "commit", "-qm", "ui change")

    _git(work, "checkout", "-q", "main")
    backend = work / BACKEND_FILE
    backend.parent.mkdir(parents=True, exist_ok=True)
    backend.write_text("x = 1\n")
    _git(work, "add", "-A")
    _git(work, "commit", "-qm", "someone else's backend change")

    _git(work, "merge", "-q", "--no-ff", "-m", "Merge feature into main", "feature")
    return work


def _scripts_tree(tmp_path: Path, classifier_body: str | None = None) -> Path:
    """Copy the scripts into a throwaway tree, preserving their relative layout."""
    root = tmp_path / "tree"
    (root / ".github" / "scripts").mkdir(parents=True)
    (root / ".circleci" / "scripts").mkdir(parents=True)
    shutil.copy(SCRIPT, root / ".github" / "scripts" / SCRIPT.name)
    target = root / ".circleci" / "scripts" / CLASSIFIER.name
    if classifier_body is None:
        shutil.copy(CLASSIFIER, target)
    else:
        target.write_text(classifier_body)
    target.chmod(0o755)
    return root


def _run(
    tmp_path: Path,
    *,
    files: list[str],
    cwd: Path | None = None,
    pr_number: str = "37540",
    changed_file_count: str | None = None,
    gh_exit_code: int = 0,
    classifier_body: str | None = None,
    category: str | None = None,
) -> tuple[str, str]:
    """Run the script against a stubbed `gh`; returns (decision, stdout)."""
    bin_dir = tmp_path / "bin"
    bin_dir.mkdir(exist_ok=True)
    listing = "".join(f"echo {f}\n" for f in files)
    stub = bin_dir / "gh"
    stub.write_text(f"#!/usr/bin/env bash\n{listing}exit {gh_exit_code}\n")
    stub.chmod(0o755)

    output_file = tmp_path / "github_output"
    output_file.write_text("")

    env = {k: v for k, v in os.environ.items() if k not in {"GH_TOKEN", "GITHUB_TOKEN"}}
    env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
    env["GITHUB_OUTPUT"] = str(output_file)
    env["REPO"] = "BerriAI/litellm"
    env["PR_NUMBER"] = pr_number
    env["CHANGED_FILE_COUNT"] = changed_file_count if changed_file_count is not None else str(len(files))
    if category is not None:
        env["CATEGORY"] = category
    else:
        env.pop("CATEGORY", None)

    tree = _scripts_tree(tmp_path, classifier_body)
    result = subprocess.run(
        ["bash", str(tree / ".github" / "scripts" / SCRIPT.name)],
        cwd=cwd or tmp_path,
        capture_output=True,
        text=True,
        env=env,
        check=True,
    )
    return output_file.read_text().strip(), result.stdout


def test_ui_only_pr_skips_even_when_the_merge_ref_carries_backend_commits(tmp_path: Path) -> None:
    """The bug this replaces: diffing the checked-out merge ref against the event's
    base sha attributed the base branch's own backend commits to the pull request,
    so every UI-only PR ran the full backend suite."""
    work = _merge_ref_checkout(tmp_path)
    tracked = subprocess.run(
        ["git", "diff", "--name-only", "HEAD~2", "HEAD"],
        cwd=work,
        capture_output=True,
        text=True,
        check=True,
    ).stdout.split()
    assert BACKEND_FILE in tracked, "the checkout must contain the base branch's backend commit"

    decision, _ = _run(tmp_path, files=[UI_FILE], cwd=work)
    assert decision == "decision=skip"


def test_backend_file_in_the_pr_runs(tmp_path: Path) -> None:
    decision, _ = _run(tmp_path, files=[UI_FILE, BACKEND_FILE])
    assert decision == "decision=run"


def test_docs_only_pr_skips(tmp_path: Path) -> None:
    decision, _ = _run(tmp_path, files=["README.md", "docs/my-website/index.mdx"])
    assert decision == "decision=skip"


def test_non_pull_request_event_runs(tmp_path: Path) -> None:
    decision, stdout = _run(tmp_path, files=[UI_FILE], pr_number="")
    assert decision == "decision=run"
    assert "not a pull_request event" in stdout


def test_api_failure_runs(tmp_path: Path) -> None:
    decision, stdout = _run(tmp_path, files=[], gh_exit_code=1)
    assert decision == "decision=run"
    assert "could not list the files" in stdout


def test_empty_file_list_runs(tmp_path: Path) -> None:
    decision, stdout = _run(tmp_path, files=[], changed_file_count="0")
    assert decision == "decision=run"
    assert "listed no files" in stdout


def test_pr_past_the_listing_ceiling_runs(tmp_path: Path) -> None:
    """The API caps its file listing, so a larger PR would be classified from a
    truncated set and could skip backend jobs it needs."""
    decision, stdout = _run(tmp_path, files=[UI_FILE], changed_file_count="3001")
    assert decision == "decision=run"
    assert "past the 3000-file listing ceiling" in stdout


def test_broken_classifier_runs(tmp_path: Path) -> None:
    decision, stdout = _run(
        tmp_path,
        files=[UI_FILE],
        classifier_body="#!/usr/bin/env bash\nexit 1\n",
    )
    assert decision == "decision=run"
    assert "classify_changes.sh failed" in stdout


def test_unexpected_classifier_output_runs(tmp_path: Path) -> None:
    decision, stdout = _run(
        tmp_path,
        files=[UI_FILE],
        classifier_body="#!/usr/bin/env bash\ncat >/dev/null\necho maybe\n",
    )
    assert decision == "decision=run"
    assert "unexpected decision: maybe" in stdout


def test_ui_category_skips_a_backend_only_pr(tmp_path: Path) -> None:
    """The dashboard build and its unit tests cannot be affected by a pull request
    that touches no `ui/` file, and the `client` category cannot express that
    because it deliberately runs whenever the backend changes."""
    decision, _ = _run(tmp_path, files=[BACKEND_FILE], category="ui")
    assert decision == "decision=skip"


def test_ui_category_runs_a_ui_only_pr(tmp_path: Path) -> None:
    decision, _ = _run(tmp_path, files=[UI_FILE], category="ui")
    assert decision == "decision=run"


def test_ui_category_runs_a_mixed_pr(tmp_path: Path) -> None:
    decision, _ = _run(tmp_path, files=[UI_FILE, BACKEND_FILE], category="ui")
    assert decision == "decision=run"


def test_absent_category_still_runs_a_backend_pr(tmp_path: Path) -> None:
    """Callers that pass no category keep the pre-existing backend behaviour."""
    assert _run(tmp_path, files=[BACKEND_FILE])[0] == "decision=run"


def test_absent_category_still_skips_a_ui_pr(tmp_path: Path) -> None:
    assert _run(tmp_path, files=[UI_FILE])[0] == "decision=skip"


def test_ui_category_fails_open_when_the_api_fails(tmp_path: Path) -> None:
    decision, stdout = _run(tmp_path, files=[], gh_exit_code=1, category="ui")
    assert decision == "decision=run"
    assert "detect-changes[ui]" in stdout


def test_ui_category_runs_when_the_ui_workflows_themselves_change(tmp_path: Path) -> None:
    """Without this the dashboard jobs would skip on the pull request that edits
    them, shipping a workflow change nothing ever exercised."""
    decision, _ = _run(tmp_path, files=[".github/workflows/test-litellm-ui-unit.yml"], category="ui")
    assert decision == "decision=run"