diff --git a/.circleci/config.yml b/.circleci/config.yml index ceeb4b5fbf8..ad9dcfd559c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,7 +38,7 @@ commands: parameters: category: type: enum - enum: ["backend", "client", "redis-compat"] + enum: ["backend", "client"] default: "backend" steps: - run: @@ -339,8 +339,15 @@ commands: Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`"" } setup_litellm_test_deps: + parameters: + checkout: + type: boolean + default: true steps: - - checkout + - when: + condition: << parameters.checkout >> + steps: + - checkout - setup_google_dns - install_uv - install_rust @@ -376,13 +383,28 @@ jobs: shell: bash.exe command: bash .circleci/scripts/path_filter.sh windows-release - run: - name: Install Python + name: Install Python 3.11.9 from python.org + no_output_timeout: 6m command: | - choco install python --version=3.11.0 -y --no-progress --force - refreshenv - python --version - environment: - CHOCOLATEY_CONFIRM_ALL: "true" + $ProgressPreference = "SilentlyContinue" + $installer = Join-Path $env:TEMP "python-3.11.9-amd64.exe" + Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe -OutFile $installer -TimeoutSec 120 + $expected = "5ee42c4eee1e6b4464bb23722f90b45303f79442df63083f05322f1785f5fdde" + $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() + if ($actual -ne $expected) { + throw "Python installer hash mismatch: expected $expected got $actual" + } + $arguments = "/quiet InstallAllUsers=1 PrependPath=1 Include_test=0 Include_doc=0 Include_launcher=0 TargetDir=C:\Python311" + $process = Start-Process -FilePath $installer -ArgumentList $arguments -PassThru + if (-not $process.WaitForExit(300000)) { + $process.Kill() + throw "Python installer did not finish within 5 minutes" + } + if ($process.ExitCode -ne 0) { + throw "Python installer exited with $($process.ExitCode)" + } + Remove-Item $installer + & C:\Python311\python.exe --version - install_windows_toolchain - run: name: Record the Rust build environment for the release cargo cache key @@ -405,7 +427,7 @@ jobs: environment: UV_HTTP_TIMEOUT: "300" command: | - $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path" + $env:Path = "C:\Python311;C:\Python311\Scripts;$HOME\.cargo\bin;$HOME\.local\bin;$env:Path" uv build --wheel --out-dir dist if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE @@ -512,11 +534,12 @@ jobs: --set image.repository=litellm-ci \ --set image.tag=${IMAGE_TAG} \ --set image.pullPolicy=Never - # Wait for pod to be ready - echo "Waiting 30 seconds for pod to be ready..." - sleep 30 + if ! kubectl rollout status deployment/litellm --timeout=180s; then + kubectl describe pods -l app.kubernetes.io/name=litellm + kubectl logs -l app.kubernetes.io/name=litellm --tail=200 + exit 1 + fi - # Print pod logs before running tests echo "Printing pod logs..." kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") @@ -1446,7 +1469,8 @@ jobs: steps: - checkout - skip_if_unrelated_changes - - setup_litellm_test_deps + - setup_litellm_test_deps: + checkout: false - start_postgres: db_name: litellm_test image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 @@ -1491,108 +1515,6 @@ jobs: - store_artifacts: path: test-results - mcp_integration: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - environment: - COVERAGE_CORE: sysmon - LITELLM_LOCAL_MODEL_COST_MAP: "True" - steps: - - checkout - - skip_if_unrelated_changes - - setup_litellm_test_deps - - run: - name: Generate Prisma client - command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - run: - name: Install MCP SDK1 peer - command: | - uv venv --python 3.12 .venv-mcp-peer - uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' - echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV" - - run: - name: Run MCP integration tests - command: | - mkdir -p test-results - env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY \ - timeout --signal=TERM 20m uv run --no-sync pytest \ - tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \ - tests/unit/responses/mcp --tb=short -vv --maxfail=10 -n 2 --dist=loadscope --reruns 0 \ - --reruns-delay 1 --timeout=120 --rerun-except "from pytest-timeout" --durations=20 \ - --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml \ - --cov-config=pyproject.toml -o junit_family=xunit1 --junitxml=test-results/junit.xml - - install_codecov_cli - - run: - name: Upload MCP integration coverage - when: always - command: | - [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; } - codecov upload-process --disable-search --fail-on-error -f coverage.xml \ - -F mcp-integration -C "$CIRCLE_SHA1" \ - -n "mcp-integration-${CIRCLE_BUILD_NUM}" --git-service github - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - - redis_compat: - parameters: - redis_py: - type: string - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - skip_if_unrelated_changes: - category: redis-compat - - setup_litellm_test_deps - - run: - name: Pin redis-py version - command: | - uv pip install "redis==<< parameters.redis_py >>" - uv run --no-sync python -c "import redis; assert redis.__version__ == '<< parameters.redis_py >>', redis.__version__; print('redis-py', redis.__version__)" - - run: - name: Install redis-server 7.2.16 - command: | - mkdir -p "$HOME/.local/bin" - cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)" - docker cp "${cid}":/usr/local/bin/redis-server "$HOME/.local/bin/redis-server" - docker rm "$cid" - redis-server --version | grep -q 'v=7.2.16' - - run: - name: Run Redis compatibility tests - command: | - mkdir -p test-results - env -u CASSETTE_REDIS_URL -u AZURE_CLIENT_ID -u AZURE_CLIENT_SECRET -u AZURE_TENANT_ID \ - timeout --signal=TERM 15m uv run --no-sync pytest \ - tests/unit/test_redis.py tests/unit/caching/test_redis_connection_pool.py \ - tests/unit/caching/test_redis_cluster_cache.py tests/unit/caching/test_evicted_client_closer.py \ - tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_azure_credentials \ - tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_gcp_credentials \ - --tb=short -vv --reruns 2 --reruns-delay 1 --durations=20 --cov=./litellm \ - --cov-report=xml:coverage.xml -o junit_family=xunit1 --junitxml=test-results/junit.xml - - when: - condition: - equal: ["5.3.1", << parameters.redis_py >>] - steps: - - install_codecov_cli - - run: - name: Upload Redis compatibility coverage - when: always - command: | - [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; } - codecov upload-process --disable-search --fail-on-error -f coverage.xml \ - -F redis-compat -C "$CIRCLE_SHA1" \ - -n "redis-compat-${CIRCLE_BUILD_NUM}" --git-service github - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - workflows: migration_startup_scheduled: triggers: @@ -1749,19 +1671,6 @@ workflows: coverage_flag: roi-postgres timeout_minutes: 10 filters: *integration_default_run - - mcp_integration: - name: mcp-integration - filters: *integration_default_run - - redis_compat: - name: redis-compat-<< matrix.redis_py >> - matrix: - parameters: - redis_py: - - "5.3.1" - - "6.4.0" - - "7.4.1" - - "8.0.1" - filters: *integration_default_run - windows_release_wheel: filters: *integration_default_run - base_sdk_install: diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index cca7f19a02a..d2ab3478397 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -14,7 +14,7 @@ outside_cost_map_set=false while IFS= read -r file || [ -n "$file" ]; do [ -n "$file" ] || continue case "$file" in - litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock) + litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .circleci/scripts/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock) has_redis_compat=true ;; esac case "$file" in diff --git a/.circleci/scripts/path_filter.sh b/.circleci/scripts/path_filter.sh index 111fa2b57ca..c3cf6e566ff 100755 --- a/.circleci/scripts/path_filter.sh +++ b/.circleci/scripts/path_filter.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: path_filter.sh }" +category="${1:?usage: path_filter.sh }" here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" run_full() { diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 23a340cd436..aa107a42e8e 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -802,6 +802,28 @@ jobs: job-timeout-minutes: 60 dist: loadscope + - shard: mcp-server + test-path: |- + tests/unit/experimental_mcp_client + tests/unit/proxy/_experimental/mcp_server + tests/unit/responses/mcp + --ignore=tests/unit/experimental_mcp_client/test_mcp_client.py + --ignore=tests/unit/proxy/_experimental/mcp_server/test_capabilities.py + --ignore=tests/unit/proxy/_experimental/mcp_server/test_interactions.py + --ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py + --ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py + --ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py + --ignore=tests/unit/proxy/_experimental/mcp_server/test_operations.py + --ignore=tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py + --ignore=tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py + --ignore=tests/unit/proxy/_experimental/mcp_server/outbound_credentials + python-version: "3.12" + workers: 4 + reruns: 0 + timeout-minutes: 20 + job-timeout-minutes: 60 + dist: loadscope + - shard: custom-logging test-path: |- tests/unit/proxy/test_proxy_server_custom_logger.py @@ -1324,6 +1346,115 @@ jobs: - name: Check consumer types with the installed SDK if: steps.changes.outputs.decision != 'skip' run: uv run --no-sync basedpyright --pythonpath .venv/bin/python --project tests/typing/pyrightconfig.json + redis-compat: + name: redis-py ${{ matrix.redis-py }} compatibility + needs: rust-bridge + if: ${{ !cancelled() }} + permissions: + contents: read + id-token: write + pull-requests: read + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + redis-py: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"] + env: + UV_PYTHON: "3.12" + LITELLM_LOCAL_MODEL_COST_MAP: "True" + REDIS_PY_VERSION: ${{ matrix.redis-py }} + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + timeout-minutes: 3 + with: + persist-credentials: false + + - name: Detect relevant changes + id: changes + timeout-minutes: 2 + uses: ./.github/actions/detect-changes + with: + category: redis-compat + + - name: Set up Python + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 3 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ env.UV_PYTHON }} + + - name: Set up uv + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 3 + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache uv dependencies + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 5 + uses: ./.github/actions/cache-uv-downloads + + - name: Set up the Rust build + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 5 + uses: ./.github/actions/rust-bridge + with: + artifact: ${{ needs.rust-bridge.outputs.artifact }} + + - name: Install dependencies with redis-py ${{ matrix.redis-py }} + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 8 + env: + RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }} + run: | + if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils + else + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project + uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl + cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so + fi + uv pip install --python .venv/bin/python "redis==${REDIS_PY_VERSION:?}" + uv run --no-sync python -c 'import os, redis; assert redis.__version__ == os.environ["REDIS_PY_VERSION"], redis.__version__' + + - name: Install redis-server 7.2.16 + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 5 + run: | + mkdir -p "$RUNNER_TEMP/redis-bin" + cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)" + docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server" + docker rm "$cid" + "$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16' + echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH" + + - name: Run Redis compatibility tests + if: steps.changes.outputs.decision != 'skip' + timeout-minutes: 15 + env: + COVERAGE_CORE: sysmon + run: | + uv run --no-sync pytest \ + tests/unit/test_redis.py \ + tests/unit/caching/test_redis_connection_pool.py \ + tests/unit/caching/test_redis_cluster_cache.py \ + tests/unit/caching/test_evicted_client_closer.py \ + tests/integration/sdk/test_redis_cluster_iam_auth.py \ + --tb=short -vv -rs -p no:rerunfailures -p no:pytest-retry --timeout=120 --durations=20 \ + --cov=./litellm --cov-report=xml:coverage.xml + + - name: Upload redis-py ${{ matrix.redis-py }} coverage to Codecov + if: always() && steps.changes.outputs.decision != 'skip' && matrix.redis-py == '5.3.1' + continue-on-error: true + uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5 + with: + use_oidc: true + files: coverage.xml + root_dir: ${{ github.workspace }} + flags: redis-compat + fail_ci_if_error: false coverage: name: coverage needs: unit @@ -1369,10 +1500,20 @@ jobs: root_dir: ${{ github.workspace }} flags: unit fail_ci_if_error: false + + - name: Upload MCP server coverage under its own flag + continue-on-error: true + uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5 + with: + use_oidc: true + files: coverage-reports/coverage-mcp-server-${{ github.run_id }}-${{ github.run_attempt }}/coverage.xml + root_dir: ${{ github.workspace }} + flags: mcp-integration + fail_ci_if_error: false unit-passed: name: unit passed permissions: {} - needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs, helm, openai-sdk-compat] + needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs, helm, openai-sdk-compat, redis-compat] if: always() runs-on: ubuntu-latest timeout-minutes: 2 diff --git a/tests/integration/README.md b/tests/integration/README.md index a90faa0823d..8f818225505 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -4,6 +4,8 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport +The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so they run in the GitHub Actions `redis-compat` matrix against every supported redis-py major rather than in `run_integration.sh` + The `cost` group is driven by the `CostTrackingTestCase` literals under `cost_calculation/cases/.py` (gathered by `cost_calculation/catalog.py`), with the scripted cost map in `cost_calculation/cost_map.py`. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL` Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist. Management, accounting, providers and extensions give each worker a private database, Redis, upstream, fake provider and proxy, with a peer and read-only database role where required. The pytest controller owns these services and closes them after worker failures. Their cases use pytest-xdist load groups inside the files selected by CircleCI timing splits. Module and class fixtures are grouped automatically, so authors do not need scheduling markers. Function fixtures remain independently schedulable. Tests that own an independent mutable sink for every case should use function fixtures, so a shared module fixture does not serialize them or share pending observations. Nested harness probes discard the parent run's shard, worker and result settings, then supply their own selections. MCP keeps its existing resource ownership and explicit load groups. Automatic group suffixes are removed from JUnit and execution records to preserve case identities. Owned proxy helpers wait for a healthy response from every configured, live worker before yielding. Serial runs keep the supplied deployment, including routing parity runs. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. When that deployment runs more than one proxy worker, set `INTEGRATION_PROXY_WORKERS` to the count so a test that writes a model and then calls it waits out the config reload interval, the only cross-worker convergence bound the wire exposes. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate diff --git a/tests/integration/run.py b/tests/integration/run.py index 97c79d1845d..1f9a64bd1ea 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -32,6 +32,7 @@ GITHUB_FILES: Final = frozenset( "tests/integration/database/image/test_offline_image_migration.py", "tests/integration/database/image/test_ui_image_serves_offline.py", "tests/integration/mcp/test_interactions.py", + "tests/integration/sdk/test_redis_cluster_iam_auth.py", } ) diff --git a/tests/unit/test_circleci_path_filter.py b/tests/unit/test_circleci_path_filter.py index b432f358f85..0856115e39b 100644 --- a/tests/unit/test_circleci_path_filter.py +++ b/tests/unit/test_circleci_path_filter.py @@ -124,8 +124,10 @@ def test_classify_decisions(category: str, changed: list[str], expected: str) -> ("litellm/caching/redis_cache.py", "run"), ("tests/unit/caching/test_redis_cluster_cache.py", "run"), ("tests/integration/sdk/test_redis_cluster_iam_auth.py", "run"), - (".circleci/config.yml", "run"), + (".github/workflows/test-unit.yml", "run"), + ("tests/integration/run.py", "run"), ("uv.lock", "run"), + (".circleci/config.yml", "skip"), ("litellm/router.py", "skip"), ("docs/my-website/redis.md", "skip"), ),