diff --git a/.circleci/config.yml b/.circleci/config.yml index aaa2edd9e0e..f626ea1a58d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,7 +38,7 @@ commands: parameters: category: type: enum - enum: ["backend", "client", "provider-harness"] + enum: ["backend", "client", "provider-harness", "mcp-integration"] default: "backend" steps: - run: @@ -3162,6 +3162,13 @@ jobs: environment: MCP_CONFORMANCE_CODECOV_FLAG: mcp-conformance steps: + - when: + condition: + equal: [mcp, << parameters.suite >>] + steps: + - checkout + - skip_if_unrelated_changes: + category: mcp-integration - setup_litellm_test_deps - when: condition: diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index 387197b65d7..d82f8196823 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: classify_changes.sh }" +category="${1:?usage: classify_changes.sh }" has_client=false has_backend=false @@ -9,10 +9,24 @@ has_ci=false has_provider_harness=false has_cost_map=false has_mcp_dependencies=false +has_mcp_integration=false has_windows_release=false outside_cost_map_set=false while IFS= read -r file || [ -n "$file" ]; do [ -n "$file" ] || continue + case "$file" in + docs/* | *.md | *.mdx) : ;; + *[mM][cC][pP]* | tests/conftest.py | tests/*/conftest.py | tests/integration/* | tests/*/_support/* | tests/unit/integration_support/* | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py) + has_mcp_integration=true ;; + ui/*) : ;; + tests/*) + case "${file##*/}" in + test_*.py) : ;; + *) has_mcp_integration=true ;; + esac + ;; + *) has_mcp_integration=true ;; + esac case "$file" in *.md | *.mdx) : ;; pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py) @@ -42,6 +56,9 @@ while IFS= read -r file || [ -n "$file" ]; do done case "$category" in + mcp-integration) + [ "$has_mcp_integration" = true ] && echo run || echo skip + ;; mcp-dependencies) [ "$has_mcp_dependencies" = true ] && echo run || echo skip ;; diff --git a/.circleci/scripts/path_filter.sh b/.circleci/scripts/path_filter.sh index 3050674f562..ca95062f59d 100755 --- a/.circleci/scripts/path_filter.sh +++ b/.circleci/scripts/path_filter.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: path_filter.sh }" +category="${1:?usage: path_filter.sh }" here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" run_full() { @@ -24,7 +24,7 @@ done [ -n "$merge_base" ] || run_full "could not resolve a merge base against $candidate_bases" -changed="$(git diff --name-only "$merge_base" HEAD 2>/dev/null)" || run_full "git diff failed" +changed="$(git diff --name-only --no-renames "$merge_base" HEAD 2>/dev/null)" || run_full "git diff failed" [ -n "$changed" ] || run_full "no files changed vs $merge_base" echo "path-filter[$category]: changed files vs ${merge_base}:" @@ -32,9 +32,11 @@ printf '%s\n' "$changed" | sed 's/^/ /' || true decision="$(printf '%s\n' "$changed" | bash "$here/classify_changes.sh" "$category")" || run_full "classify_changes.sh failed" -if [ "$decision" = run ]; then - run_full "$category-relevant changes detected" -fi +case "$decision" in + run) run_full "$category-relevant changes detected" ;; + skip) : ;; + *) run_full "classify_changes.sh printed an unexpected decision: $decision" ;; +esac -echo "path-filter[$category]: only unrelated changes detected; halting job as successful" +echo "path-filter[$category]: not applicable, only unrelated changes detected; halting job as successful" circleci-agent step halt diff --git a/tests/integration/README.md b/tests/integration/README.md index 3770c85813f..b469a662f7a 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -128,3 +128,12 @@ status in the existing main ruleset. Close LIT-7744 after this baseline is merge required. LIT-7745 can be developed alongside gate completion; its shared-path changes must pass the baseline and their added legacy/modern tests before merging. Later coverage extensions belong to their implementation tickets rather than keeping LIT-7744 open indefinitely + +The `ci/circleci: integration-mcp` status is published for every normal PR pipeline. Before installing +dependencies or starting services, its `mcp-integration` path filter completes the job successfully +as "not applicable" for changes confined to documentation, frontend files unrelated to MCP, or +unrelated tests. MCP files, shared test fixtures and integration helpers always run the full gate. +Runtime code, dependencies, CI configuration and unrecognized paths also run it conservatively. +Non-PR pipelines, unavailable merge bases, empty diffs and failed or invalid classification run the +suite. Renames include both old and new paths. Filtering selects whether to run the entire job; +it never reduces the 181 required cases for an applicable run diff --git a/tests/unit/test_circleci_path_filter.py b/tests/unit/test_circleci_path_filter.py index 3776aea28e4..3775ae32d89 100644 --- a/tests/unit/test_circleci_path_filter.py +++ b/tests/unit/test_circleci_path_filter.py @@ -21,6 +21,7 @@ import os import shutil import subprocess from pathlib import Path +from typing import Final import pytest @@ -49,6 +50,32 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"] @pytest.mark.parametrize( "category,changed,expected", [ + ("mcp-integration", ["litellm/proxy/_experimental/mcp_server/operations.py"], "run"), + ("mcp-integration", ["litellm/experimental_mcp_client/client.py"], "run"), + ("mcp-integration", ["litellm/proxy/auth/auth_checks.py"], "run"), + ("mcp-integration", ["litellm/caching/caching.py"], "run"), + ("mcp-integration", ["enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py"], "run"), + ("mcp-integration", ["pyproject.toml", "uv.lock"], "run"), + ("mcp-integration", [".circleci/config.yml"], "run"), + ("mcp-integration", [".github/workflows/test-unit.yml"], "run"), + ("mcp-integration", ["tests/integration/_support/process.py"], "run"), + ("mcp-integration", ["tests/integration/conformance_coverage.toml"], "run"), + ("mcp-integration", ["tests/conftest.py"], "run"), + ("mcp-integration", ["tests/unit/conftest.py"], "run"), + ("mcp-integration", ["tests/unit/integration_support/test_process.py"], "run"), + ("mcp-integration", ["tests/unit/test_circleci_path_filter.py"], "run"), + ("mcp-integration", ["tests/shared_auth.py"], "run"), + ("mcp-integration", ["tests/test_litellm/helpers.py"], "run"), + ("mcp-integration", ["tests/__init__.py"], "run"), + ("mcp-integration", ["tests/e2e/fixtures/policy.json"], "run"), + ("mcp-integration", ["tests/e2e/mcp/test_authorization.py"], "run"), + ("mcp-integration", ["ui/litellm-dashboard/src/components/mcp_servers.tsx"], "run"), + ("mcp-integration", ["new_runtime/adapter.py"], "run"), + ("mcp-integration", DOCS, "skip"), + ("mcp-integration", CLIENT, "skip"), + ("mcp-integration", ["tests/test_litellm/llms/anthropic/test_chat.py"], "skip"), + ("mcp-integration", DOCS + ["tests/unit/test_router.py"], "skip"), + ("mcp-integration", CLIENT + ["litellm/proxy/proxy_server.py"], "run"), ("mcp-dependencies", ["pyproject.toml"], "run"), ("mcp-dependencies", ["uv.lock"], "run"), ("mcp-dependencies", ["litellm/experimental_mcp_client/client.py"], "run"), @@ -207,7 +234,7 @@ def _run_path_filter(work: Path, tmp_path: Path, category: str, scripts_dir: Pat bin_dir = tmp_path / "bin" bin_dir.mkdir(exist_ok=True) stub = bin_dir / "circleci-agent" - stub.write_text("#!/usr/bin/env bash\necho \"[stub] circleci-agent $*\"\nexit 0\n") + stub.write_text('#!/usr/bin/env bash\necho "[stub] circleci-agent $*"\nexit 0\n') stub.chmod(0o755) env = dict(os.environ) env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" @@ -223,30 +250,34 @@ def _run_path_filter(work: Path, tmp_path: Path, category: str, scripts_dir: Pat ) -def test_path_filter_halts_docs_only_pr(tmp_path: Path) -> None: +@pytest.mark.parametrize("category", ("backend", "mcp-integration")) +def test_path_filter_halts_docs_only_pr(tmp_path: Path, category: str) -> None: work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) - result = _run_path_filter(work, tmp_path, "backend", SCRIPTS_DIR) + result = _run_path_filter(work, tmp_path, category, SCRIPTS_DIR) assert result.returncode == 0 assert "circleci-agent step halt" in result.stdout -def test_path_filter_runs_backend_pr(tmp_path: Path) -> None: +@pytest.mark.parametrize("category", ("backend", "mcp-integration")) +def test_path_filter_runs_backend_pr(tmp_path: Path, category: str) -> None: work = _pr_repo(tmp_path, {"litellm/new.py": "y\n"}) - result = _run_path_filter(work, tmp_path, "backend", SCRIPTS_DIR) + result = _run_path_filter(work, tmp_path, category, SCRIPTS_DIR) assert result.returncode == 0 assert "running job" in result.stdout assert "halt" not in result.stdout -def test_path_filter_fails_open_when_not_a_pr(tmp_path: Path) -> None: +@pytest.mark.parametrize("category", ("backend", "mcp-integration")) +def test_path_filter_fails_open_when_not_a_pr(tmp_path: Path, category: str) -> None: work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) - result = _run_path_filter(work, tmp_path, "backend", SCRIPTS_DIR, is_pr=False) + result = _run_path_filter(work, tmp_path, category, SCRIPTS_DIR, is_pr=False) assert result.returncode == 0 assert "not a pull request" in result.stdout assert "halt" not in result.stdout -def test_path_filter_fails_open_when_classifier_errors(tmp_path: Path) -> None: +@pytest.mark.parametrize("category", ("backend", "mcp-integration")) +def test_path_filter_fails_open_when_classifier_errors(tmp_path: Path, category: str) -> None: """Regression: a broken classifier must run the job, never silently halt it.""" broken_scripts = tmp_path / "broken_scripts" broken_scripts.mkdir() @@ -255,7 +286,44 @@ def test_path_filter_fails_open_when_classifier_errors(tmp_path: Path) -> None: (broken_scripts / "classify_changes.sh").chmod(0o755) work = _pr_repo(tmp_path, {"README.md": "# docs\n"}) - result = _run_path_filter(work, tmp_path, "backend", broken_scripts) + result = _run_path_filter(work, tmp_path, category, broken_scripts) assert result.returncode == 0 assert "classify_changes.sh failed" in result.stdout assert "halt" not in result.stdout + + +@pytest.mark.parametrize("decision", ("", "unexpected")) +def test_path_filter_runs_when_classifier_output_is_invalid(tmp_path: Path, decision: str) -> None: + scripts: Final = tmp_path / "scripts" + scripts.mkdir() + shutil.copy(PATH_FILTER, scripts / "path_filter.sh") + (scripts / "classify_changes.sh").write_text(f"#!/usr/bin/env bash\nprintf '%s' '{decision}'\n") + work: Final = _pr_repo(tmp_path, {"README.md": "# docs\n"}) + result: Final = _run_path_filter(work, tmp_path, "mcp-integration", scripts) + assert result.returncode == 0 + assert "running job" in result.stdout + assert "halt" not in result.stdout + + +@pytest.mark.parametrize("uncertainty", ("missing_remote", "empty_diff")) +def test_mcp_path_filter_runs_when_changed_files_are_unavailable(tmp_path: Path, uncertainty: str) -> None: + work: Final = _pr_repo(tmp_path, {"README.md": "# docs\n"}) + if uncertainty == "missing_remote": + _git(work, "remote", "remove", "origin") + else: + _git(work, "push", "-q", "origin", "HEAD:main") + result: Final = _run_path_filter(work, tmp_path, "mcp-integration", SCRIPTS_DIR) + assert result.returncode == 0 + assert "running job" in result.stdout + assert "halt" not in result.stdout + + +def test_mcp_path_filter_includes_deleted_side_of_rename(tmp_path: Path) -> None: + work: Final = _pr_repo(tmp_path, {"litellm/experimental_mcp_client/client.py": "original client\n"}) + _git(work, "push", "-q", "origin", "HEAD:main") + _git(work, "mv", "litellm/experimental_mcp_client/client.py", "README.md") + _git(work, "commit", "-qm", "move client out of runtime") + result: Final = _run_path_filter(work, tmp_path, "mcp-integration", SCRIPTS_DIR) + assert result.returncode == 0 + assert "running job" in result.stdout + assert "halt" not in result.stdout