ci: move redis-compat and MCP unit suites from CircleCI to GitHub Actions (#46026)

The redis-compat matrix and mcp-integration ran unit tests on five large
CircleCI machines per integration run. redis-compat now runs as a redis-py
matrix in the unit workflow with the same pinned redis-server 7.2.16 binary,
no reruns, and the redis-compat Codecov flag on the 5.3.1 leg. The MCP unit
directories not already claimed by mcp-elicitation and mcp-oauth run in a new
mcp-server shard whose report is also uploaded under the mcp-integration flag.

The Windows release wheel job installs Python from a checksum-verified
python.org installer with a bounded wait instead of Chocolatey, helm chart
testing waits on the rollout instead of sleeping, and roi-database no longer
checks out the repository twice.

Co-authored-by: Yuneng Jiang <yuneng@berri.ai>
This commit is contained in:
devin-ai-integration[bot] 2026-10-12 02:24:31 -07:00 • committed by GitHub
parent daa3d36c4d
commit 4fd0051346
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 188 additions and 133 deletions

View file

@ -38,7 +38,7 @@ commands:
parameters:
category:
type: enum
enum: ["backend", "client", "redis-compat"]
enum: ["backend", "client"]
default: "backend"
steps:
- run:
@ -339,8 +339,15 @@ commands:
Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`""
}
setup_litellm_test_deps:
parameters:
checkout:
type: boolean
default: true
steps:
- checkout
- when:
condition: << parameters.checkout >>
steps:
- checkout
- setup_google_dns
- install_uv
- install_rust
@ -376,13 +383,28 @@ jobs:
shell: bash.exe
command: bash .circleci/scripts/path_filter.sh windows-release
- run:
name: Install Python
name: Install Python 3.11.9 from python.org
no_output_timeout: 6m
command: |
choco install python --version=3.11.0 -y --no-progress --force
refreshenv
python --version
environment:
CHOCOLATEY_CONFIRM_ALL: "true"
$ProgressPreference = "SilentlyContinue"
$installer = Join-Path $env:TEMP "python-3.11.9-amd64.exe"
Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe -OutFile $installer -TimeoutSec 120
$expected = "5ee42c4eee1e6b4464bb23722f90b45303f79442df63083f05322f1785f5fdde"
$actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower()
if ($actual -ne $expected) {
throw "Python installer hash mismatch: expected $expected got $actual"
}
$arguments = "/quiet InstallAllUsers=1 PrependPath=1 Include_test=0 Include_doc=0 Include_launcher=0 TargetDir=C:\Python311"
$process = Start-Process -FilePath $installer -ArgumentList $arguments -PassThru
if (-not $process.WaitForExit(300000)) {
$process.Kill()
throw "Python installer did not finish within 5 minutes"
}
if ($process.ExitCode -ne 0) {
throw "Python installer exited with $($process.ExitCode)"
}
Remove-Item $installer
& C:\Python311\python.exe --version
- install_windows_toolchain
- run:
name: Record the Rust build environment for the release cargo cache key
@ -405,7 +427,7 @@ jobs:
environment:
UV_HTTP_TIMEOUT: "300"
command: |
$env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
$env:Path = "C:\Python311;C:\Python311\Scripts;$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
uv build --wheel --out-dir dist
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
@ -512,11 +534,12 @@ jobs:
--set image.repository=litellm-ci \
--set image.tag=${IMAGE_TAG} \
--set image.pullPolicy=Never
# Wait for pod to be ready
echo "Waiting 30 seconds for pod to be ready..."
sleep 30
if ! kubectl rollout status deployment/litellm --timeout=180s; then
kubectl describe pods -l app.kubernetes.io/name=litellm
kubectl logs -l app.kubernetes.io/name=litellm --tail=200
exit 1
fi
# Print pod logs before running tests
echo "Printing pod logs..."
kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
@ -1446,7 +1469,8 @@ jobs:
steps:
- checkout
- skip_if_unrelated_changes
- setup_litellm_test_deps
- setup_litellm_test_deps:
checkout: false
- start_postgres:
db_name: litellm_test
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
@ -1491,108 +1515,6 @@ jobs:
- store_artifacts:
path: test-results
mcp_integration:
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
environment:
COVERAGE_CORE: sysmon
LITELLM_LOCAL_MODEL_COST_MAP: "True"
steps:
- checkout
- skip_if_unrelated_changes
- setup_litellm_test_deps
- run:
name: Generate Prisma client
command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- run:
name: Install MCP SDK1 peer
command: |
uv venv --python 3.12 .venv-mcp-peer
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
- run:
name: Run MCP integration tests
command: |
mkdir -p test-results
env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY \
timeout --signal=TERM 20m uv run --no-sync pytest \
tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \
tests/unit/responses/mcp --tb=short -vv --maxfail=10 -n 2 --dist=loadscope --reruns 0 \
--reruns-delay 1 --timeout=120 --rerun-except "from pytest-timeout" --durations=20 \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml \
--cov-config=pyproject.toml -o junit_family=xunit1 --junitxml=test-results/junit.xml
- install_codecov_cli
- run:
name: Upload MCP integration coverage
when: always
command: |
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
-F mcp-integration -C "$CIRCLE_SHA1" \
-n "mcp-integration-${CIRCLE_BUILD_NUM}" --git-service github
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
redis_compat:
parameters:
redis_py:
type: string
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes:
category: redis-compat
- setup_litellm_test_deps
- run:
name: Pin redis-py version
command: |
uv pip install "redis==<< parameters.redis_py >>"
uv run --no-sync python -c "import redis; assert redis.__version__ == '<< parameters.redis_py >>', redis.__version__; print('redis-py', redis.__version__)"
- run:
name: Install redis-server 7.2.16
command: |
mkdir -p "$HOME/.local/bin"
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
docker cp "${cid}":/usr/local/bin/redis-server "$HOME/.local/bin/redis-server"
docker rm "$cid"
redis-server --version | grep -q 'v=7.2.16'
- run:
name: Run Redis compatibility tests
command: |
mkdir -p test-results
env -u CASSETTE_REDIS_URL -u AZURE_CLIENT_ID -u AZURE_CLIENT_SECRET -u AZURE_TENANT_ID \
timeout --signal=TERM 15m uv run --no-sync pytest \
tests/unit/test_redis.py tests/unit/caching/test_redis_connection_pool.py \
tests/unit/caching/test_redis_cluster_cache.py tests/unit/caching/test_evicted_client_closer.py \
tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_azure_credentials \
tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_gcp_credentials \
--tb=short -vv --reruns 2 --reruns-delay 1 --durations=20 --cov=./litellm \
--cov-report=xml:coverage.xml -o junit_family=xunit1 --junitxml=test-results/junit.xml
- when:
condition:
equal: ["5.3.1", << parameters.redis_py >>]
steps:
- install_codecov_cli
- run:
name: Upload Redis compatibility coverage
when: always
command: |
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
-F redis-compat -C "$CIRCLE_SHA1" \
-n "redis-compat-${CIRCLE_BUILD_NUM}" --git-service github
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
workflows:
migration_startup_scheduled:
triggers:
@ -1749,19 +1671,6 @@ workflows:
coverage_flag: roi-postgres
timeout_minutes: 10
filters: *integration_default_run
- mcp_integration:
name: mcp-integration
filters: *integration_default_run
- redis_compat:
name: redis-compat-<< matrix.redis_py >>
matrix:
parameters:
redis_py:
- "5.3.1"
- "6.4.0"
- "7.4.1"
- "8.0.1"
filters: *integration_default_run
- windows_release_wheel:
filters: *integration_default_run
- base_sdk_install:

View file

@ -14,7 +14,7 @@ outside_cost_map_set=false
while IFS= read -r file || [ -n "$file" ]; do
[ -n "$file" ] || continue
case "$file" in
litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock)
litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .circleci/scripts/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock)
has_redis_compat=true ;;
esac
case "$file" in

View file

@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -uo pipefail
category="${1:?usage: path_filter.sh <backend|client|redis-compat>}"
category="${1:?usage: path_filter.sh <backend|client|windows-release>}"
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
run_full() {

View file

@ -802,6 +802,28 @@ jobs:
job-timeout-minutes: 60
dist: loadscope
- shard: mcp-server
test-path: |-
tests/unit/experimental_mcp_client
tests/unit/proxy/_experimental/mcp_server
tests/unit/responses/mcp
--ignore=tests/unit/experimental_mcp_client/test_mcp_client.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_capabilities.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_interactions.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_operations.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
--ignore=tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
--ignore=tests/unit/proxy/_experimental/mcp_server/outbound_credentials
python-version: "3.12"
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
dist: loadscope
- shard: custom-logging
test-path: |-
tests/unit/proxy/test_proxy_server_custom_logger.py
@ -1324,6 +1346,115 @@ jobs:
- name: Check consumer types with the installed SDK
if: steps.changes.outputs.decision != 'skip'
run: uv run --no-sync basedpyright --pythonpath .venv/bin/python --project tests/typing/pyrightconfig.json
redis-compat:
name: redis-py ${{ matrix.redis-py }} compatibility
needs: rust-bridge
if: ${{ !cancelled() }}
permissions:
contents: read
id-token: write
pull-requests: read
runs-on: ubuntu-latest
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
redis-py: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"]
env:
UV_PYTHON: "3.12"
LITELLM_LOCAL_MODEL_COST_MAP: "True"
REDIS_PY_VERSION: ${{ matrix.redis-py }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
timeout-minutes: 3
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
timeout-minutes: 2
uses: ./.github/actions/detect-changes
with:
category: redis-compat
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ env.UV_PYTHON }}
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/cache-uv-downloads
- name: Set up the Rust build
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/rust-bridge
with:
artifact: ${{ needs.rust-bridge.outputs.artifact }}
- name: Install dependencies with redis-py ${{ matrix.redis-py }}
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 8
env:
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
run: |
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
fi
uv pip install --python .venv/bin/python "redis==${REDIS_PY_VERSION:?}"
uv run --no-sync python -c 'import os, redis; assert redis.__version__ == os.environ["REDIS_PY_VERSION"], redis.__version__'
- name: Install redis-server 7.2.16
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
run: |
mkdir -p "$RUNNER_TEMP/redis-bin"
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
docker rm "$cid"
"$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
- name: Run Redis compatibility tests
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 15
env:
COVERAGE_CORE: sysmon
run: |
uv run --no-sync pytest \
tests/unit/test_redis.py \
tests/unit/caching/test_redis_connection_pool.py \
tests/unit/caching/test_redis_cluster_cache.py \
tests/unit/caching/test_evicted_client_closer.py \
tests/integration/sdk/test_redis_cluster_iam_auth.py \
--tb=short -vv -rs -p no:rerunfailures -p no:pytest-retry --timeout=120 --durations=20 \
--cov=./litellm --cov-report=xml:coverage.xml
- name: Upload redis-py ${{ matrix.redis-py }} coverage to Codecov
if: always() && steps.changes.outputs.decision != 'skip' && matrix.redis-py == '5.3.1'
continue-on-error: true
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
files: coverage.xml
root_dir: ${{ github.workspace }}
flags: redis-compat
fail_ci_if_error: false
coverage:
name: coverage
needs: unit
@ -1369,10 +1500,20 @@ jobs:
root_dir: ${{ github.workspace }}
flags: unit
fail_ci_if_error: false
- name: Upload MCP server coverage under its own flag
continue-on-error: true
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
files: coverage-reports/coverage-mcp-server-${{ github.run_id }}-${{ github.run_attempt }}/coverage.xml
root_dir: ${{ github.workspace }}
flags: mcp-integration
fail_ci_if_error: false
unit-passed:
name: unit passed
permissions: {}
needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs, helm, openai-sdk-compat]
needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs, helm, openai-sdk-compat, redis-compat]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 2

View file

@ -4,6 +4,8 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local
The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport
The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so they run in the GitHub Actions `redis-compat` matrix against every supported redis-py major rather than in `run_integration.sh`
The `cost` group is driven by the `CostTrackingTestCase` literals under `cost_calculation/cases/<provider>.py` (gathered by `cost_calculation/catalog.py`), with the scripted cost map in `cost_calculation/cost_map.py`. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/<scenario_id>`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL`
Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist. Management, accounting, providers and extensions give each worker a private database, Redis, upstream, fake provider and proxy, with a peer and read-only database role where required. The pytest controller owns these services and closes them after worker failures. Their cases use pytest-xdist load groups inside the files selected by CircleCI timing splits. Module and class fixtures are grouped automatically, so authors do not need scheduling markers. Function fixtures remain independently schedulable. Tests that own an independent mutable sink for every case should use function fixtures, so a shared module fixture does not serialize them or share pending observations. Nested harness probes discard the parent run's shard, worker and result settings, then supply their own selections. MCP keeps its existing resource ownership and explicit load groups. Automatic group suffixes are removed from JUnit and execution records to preserve case identities. Owned proxy helpers wait for a healthy response from every configured, live worker before yielding. Serial runs keep the supplied deployment, including routing parity runs. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. When that deployment runs more than one proxy worker, set `INTEGRATION_PROXY_WORKERS` to the count so a test that writes a model and then calls it waits out the config reload interval, the only cross-worker convergence bound the wire exposes. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate

View file

@ -32,6 +32,7 @@ GITHUB_FILES: Final = frozenset(
"tests/integration/database/image/test_offline_image_migration.py",
"tests/integration/database/image/test_ui_image_serves_offline.py",
"tests/integration/mcp/test_interactions.py",
"tests/integration/sdk/test_redis_cluster_iam_auth.py",
}
)

View file

@ -124,8 +124,10 @@ def test_classify_decisions(category: str, changed: list[str], expected: str) ->
("litellm/caching/redis_cache.py", "run"),
("tests/unit/caching/test_redis_cluster_cache.py", "run"),
("tests/integration/sdk/test_redis_cluster_iam_auth.py", "run"),
(".circleci/config.yml", "run"),
(".github/workflows/test-unit.yml", "run"),
("tests/integration/run.py", "run"),
("uv.lock", "run"),
(".circleci/config.yml", "skip"),
("litellm/router.py", "skip"),
("docs/my-website/redis.md", "skip"),
),