mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-14 04:09:09 +00:00
ci: move redis-compat and MCP unit suites from CircleCI to GitHub Actions (#46026)
The redis-compat matrix and mcp-integration ran unit tests on five large CircleCI machines per integration run. redis-compat now runs as a redis-py matrix in the unit workflow with the same pinned redis-server 7.2.16 binary, no reruns, and the redis-compat Codecov flag on the 5.3.1 leg. The MCP unit directories not already claimed by mcp-elicitation and mcp-oauth run in a new mcp-server shard whose report is also uploaded under the mcp-integration flag. The Windows release wheel job installs Python from a checksum-verified python.org installer with a bounded wait instead of Chocolatey, helm chart testing waits on the rollout instead of sleeping, and roi-database no longer checks out the repository twice. Co-authored-by: Yuneng Jiang <yuneng@berri.ai>
This commit is contained in:
parent
daa3d36c4d
commit
4fd0051346
7 changed files with 188 additions and 133 deletions
|
|
@ -38,7 +38,7 @@ commands:
|
|||
parameters:
|
||||
category:
|
||||
type: enum
|
||||
enum: ["backend", "client", "redis-compat"]
|
||||
enum: ["backend", "client"]
|
||||
default: "backend"
|
||||
steps:
|
||||
- run:
|
||||
|
|
@ -339,8 +339,15 @@ commands:
|
|||
Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`""
|
||||
}
|
||||
setup_litellm_test_deps:
|
||||
parameters:
|
||||
checkout:
|
||||
type: boolean
|
||||
default: true
|
||||
steps:
|
||||
- checkout
|
||||
- when:
|
||||
condition: << parameters.checkout >>
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- install_rust
|
||||
|
|
@ -376,13 +383,28 @@ jobs:
|
|||
shell: bash.exe
|
||||
command: bash .circleci/scripts/path_filter.sh windows-release
|
||||
- run:
|
||||
name: Install Python
|
||||
name: Install Python 3.11.9 from python.org
|
||||
no_output_timeout: 6m
|
||||
command: |
|
||||
choco install python --version=3.11.0 -y --no-progress --force
|
||||
refreshenv
|
||||
python --version
|
||||
environment:
|
||||
CHOCOLATEY_CONFIRM_ALL: "true"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
$installer = Join-Path $env:TEMP "python-3.11.9-amd64.exe"
|
||||
Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe -OutFile $installer -TimeoutSec 120
|
||||
$expected = "5ee42c4eee1e6b4464bb23722f90b45303f79442df63083f05322f1785f5fdde"
|
||||
$actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower()
|
||||
if ($actual -ne $expected) {
|
||||
throw "Python installer hash mismatch: expected $expected got $actual"
|
||||
}
|
||||
$arguments = "/quiet InstallAllUsers=1 PrependPath=1 Include_test=0 Include_doc=0 Include_launcher=0 TargetDir=C:\Python311"
|
||||
$process = Start-Process -FilePath $installer -ArgumentList $arguments -PassThru
|
||||
if (-not $process.WaitForExit(300000)) {
|
||||
$process.Kill()
|
||||
throw "Python installer did not finish within 5 minutes"
|
||||
}
|
||||
if ($process.ExitCode -ne 0) {
|
||||
throw "Python installer exited with $($process.ExitCode)"
|
||||
}
|
||||
Remove-Item $installer
|
||||
& C:\Python311\python.exe --version
|
||||
- install_windows_toolchain
|
||||
- run:
|
||||
name: Record the Rust build environment for the release cargo cache key
|
||||
|
|
@ -405,7 +427,7 @@ jobs:
|
|||
environment:
|
||||
UV_HTTP_TIMEOUT: "300"
|
||||
command: |
|
||||
$env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
|
||||
$env:Path = "C:\Python311;C:\Python311\Scripts;$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
|
||||
uv build --wheel --out-dir dist
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
exit $LASTEXITCODE
|
||||
|
|
@ -512,11 +534,12 @@ jobs:
|
|||
--set image.repository=litellm-ci \
|
||||
--set image.tag=${IMAGE_TAG} \
|
||||
--set image.pullPolicy=Never
|
||||
# Wait for pod to be ready
|
||||
echo "Waiting 30 seconds for pod to be ready..."
|
||||
sleep 30
|
||||
if ! kubectl rollout status deployment/litellm --timeout=180s; then
|
||||
kubectl describe pods -l app.kubernetes.io/name=litellm
|
||||
kubectl logs -l app.kubernetes.io/name=litellm --tail=200
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Print pod logs before running tests
|
||||
echo "Printing pod logs..."
|
||||
kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
|
||||
|
||||
|
|
@ -1446,7 +1469,8 @@ jobs:
|
|||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_litellm_test_deps
|
||||
- setup_litellm_test_deps:
|
||||
checkout: false
|
||||
- start_postgres:
|
||||
db_name: litellm_test
|
||||
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
||||
|
|
@ -1491,108 +1515,6 @@ jobs:
|
|||
- store_artifacts:
|
||||
path: test-results
|
||||
|
||||
mcp_integration:
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
working_directory: ~/project
|
||||
environment:
|
||||
COVERAGE_CORE: sysmon
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_litellm_test_deps
|
||||
- run:
|
||||
name: Generate Prisma client
|
||||
command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
- run:
|
||||
name: Install MCP SDK1 peer
|
||||
command: |
|
||||
uv venv --python 3.12 .venv-mcp-peer
|
||||
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
|
||||
- run:
|
||||
name: Run MCP integration tests
|
||||
command: |
|
||||
mkdir -p test-results
|
||||
env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY \
|
||||
timeout --signal=TERM 20m uv run --no-sync pytest \
|
||||
tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \
|
||||
tests/unit/responses/mcp --tb=short -vv --maxfail=10 -n 2 --dist=loadscope --reruns 0 \
|
||||
--reruns-delay 1 --timeout=120 --rerun-except "from pytest-timeout" --durations=20 \
|
||||
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml \
|
||||
--cov-config=pyproject.toml -o junit_family=xunit1 --junitxml=test-results/junit.xml
|
||||
- install_codecov_cli
|
||||
- run:
|
||||
name: Upload MCP integration coverage
|
||||
when: always
|
||||
command: |
|
||||
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
|
||||
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
|
||||
-F mcp-integration -C "$CIRCLE_SHA1" \
|
||||
-n "mcp-integration-${CIRCLE_BUILD_NUM}" --git-service github
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- store_artifacts:
|
||||
path: test-results
|
||||
|
||||
redis_compat:
|
||||
parameters:
|
||||
redis_py:
|
||||
type: string
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes:
|
||||
category: redis-compat
|
||||
- setup_litellm_test_deps
|
||||
- run:
|
||||
name: Pin redis-py version
|
||||
command: |
|
||||
uv pip install "redis==<< parameters.redis_py >>"
|
||||
uv run --no-sync python -c "import redis; assert redis.__version__ == '<< parameters.redis_py >>', redis.__version__; print('redis-py', redis.__version__)"
|
||||
- run:
|
||||
name: Install redis-server 7.2.16
|
||||
command: |
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
|
||||
docker cp "${cid}":/usr/local/bin/redis-server "$HOME/.local/bin/redis-server"
|
||||
docker rm "$cid"
|
||||
redis-server --version | grep -q 'v=7.2.16'
|
||||
- run:
|
||||
name: Run Redis compatibility tests
|
||||
command: |
|
||||
mkdir -p test-results
|
||||
env -u CASSETTE_REDIS_URL -u AZURE_CLIENT_ID -u AZURE_CLIENT_SECRET -u AZURE_TENANT_ID \
|
||||
timeout --signal=TERM 15m uv run --no-sync pytest \
|
||||
tests/unit/test_redis.py tests/unit/caching/test_redis_connection_pool.py \
|
||||
tests/unit/caching/test_redis_cluster_cache.py tests/unit/caching/test_evicted_client_closer.py \
|
||||
tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_azure_credentials \
|
||||
tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_gcp_credentials \
|
||||
--tb=short -vv --reruns 2 --reruns-delay 1 --durations=20 --cov=./litellm \
|
||||
--cov-report=xml:coverage.xml -o junit_family=xunit1 --junitxml=test-results/junit.xml
|
||||
- when:
|
||||
condition:
|
||||
equal: ["5.3.1", << parameters.redis_py >>]
|
||||
steps:
|
||||
- install_codecov_cli
|
||||
- run:
|
||||
name: Upload Redis compatibility coverage
|
||||
when: always
|
||||
command: |
|
||||
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
|
||||
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
|
||||
-F redis-compat -C "$CIRCLE_SHA1" \
|
||||
-n "redis-compat-${CIRCLE_BUILD_NUM}" --git-service github
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- store_artifacts:
|
||||
path: test-results
|
||||
|
||||
workflows:
|
||||
migration_startup_scheduled:
|
||||
triggers:
|
||||
|
|
@ -1749,19 +1671,6 @@ workflows:
|
|||
coverage_flag: roi-postgres
|
||||
timeout_minutes: 10
|
||||
filters: *integration_default_run
|
||||
- mcp_integration:
|
||||
name: mcp-integration
|
||||
filters: *integration_default_run
|
||||
- redis_compat:
|
||||
name: redis-compat-<< matrix.redis_py >>
|
||||
matrix:
|
||||
parameters:
|
||||
redis_py:
|
||||
- "5.3.1"
|
||||
- "6.4.0"
|
||||
- "7.4.1"
|
||||
- "8.0.1"
|
||||
filters: *integration_default_run
|
||||
- windows_release_wheel:
|
||||
filters: *integration_default_run
|
||||
- base_sdk_install:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ outside_cost_map_set=false
|
|||
while IFS= read -r file || [ -n "$file" ]; do
|
||||
[ -n "$file" ] || continue
|
||||
case "$file" in
|
||||
litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock)
|
||||
litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .circleci/scripts/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock)
|
||||
has_redis_compat=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
category="${1:?usage: path_filter.sh <backend|client|redis-compat>}"
|
||||
category="${1:?usage: path_filter.sh <backend|client|windows-release>}"
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
run_full() {
|
||||
|
|
|
|||
143
.github/workflows/test-unit.yml
vendored
143
.github/workflows/test-unit.yml
vendored
|
|
@ -802,6 +802,28 @@ jobs:
|
|||
job-timeout-minutes: 60
|
||||
dist: loadscope
|
||||
|
||||
- shard: mcp-server
|
||||
test-path: |-
|
||||
tests/unit/experimental_mcp_client
|
||||
tests/unit/proxy/_experimental/mcp_server
|
||||
tests/unit/responses/mcp
|
||||
--ignore=tests/unit/experimental_mcp_client/test_mcp_client.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_capabilities.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_interactions.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_operations.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py
|
||||
--ignore=tests/unit/proxy/_experimental/mcp_server/outbound_credentials
|
||||
python-version: "3.12"
|
||||
workers: 4
|
||||
reruns: 0
|
||||
timeout-minutes: 20
|
||||
job-timeout-minutes: 60
|
||||
dist: loadscope
|
||||
|
||||
- shard: custom-logging
|
||||
test-path: |-
|
||||
tests/unit/proxy/test_proxy_server_custom_logger.py
|
||||
|
|
@ -1324,6 +1346,115 @@ jobs:
|
|||
- name: Check consumer types with the installed SDK
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: uv run --no-sync basedpyright --pythonpath .venv/bin/python --project tests/typing/pyrightconfig.json
|
||||
redis-compat:
|
||||
name: redis-py ${{ matrix.redis-py }} compatibility
|
||||
needs: rust-bridge
|
||||
if: ${{ !cancelled() }}
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
pull-requests: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
redis-py: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"]
|
||||
env:
|
||||
UV_PYTHON: "3.12"
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
REDIS_PY_VERSION: ${{ matrix.redis-py }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
timeout-minutes: 3
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Detect relevant changes
|
||||
id: changes
|
||||
timeout-minutes: 2
|
||||
uses: ./.github/actions/detect-changes
|
||||
with:
|
||||
category: redis-compat
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 3
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
|
||||
- name: Set up uv
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 3
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache uv dependencies
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/cache-uv-downloads
|
||||
|
||||
- name: Set up the Rust build
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/rust-bridge
|
||||
with:
|
||||
artifact: ${{ needs.rust-bridge.outputs.artifact }}
|
||||
|
||||
- name: Install dependencies with redis-py ${{ matrix.redis-py }}
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 8
|
||||
env:
|
||||
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
|
||||
run: |
|
||||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
fi
|
||||
uv pip install --python .venv/bin/python "redis==${REDIS_PY_VERSION:?}"
|
||||
uv run --no-sync python -c 'import os, redis; assert redis.__version__ == os.environ["REDIS_PY_VERSION"], redis.__version__'
|
||||
|
||||
- name: Install redis-server 7.2.16
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/redis-bin"
|
||||
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
|
||||
docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
|
||||
docker rm "$cid"
|
||||
"$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
|
||||
echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Run Redis compatibility tests
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
COVERAGE_CORE: sysmon
|
||||
run: |
|
||||
uv run --no-sync pytest \
|
||||
tests/unit/test_redis.py \
|
||||
tests/unit/caching/test_redis_connection_pool.py \
|
||||
tests/unit/caching/test_redis_cluster_cache.py \
|
||||
tests/unit/caching/test_evicted_client_closer.py \
|
||||
tests/integration/sdk/test_redis_cluster_iam_auth.py \
|
||||
--tb=short -vv -rs -p no:rerunfailures -p no:pytest-retry --timeout=120 --durations=20 \
|
||||
--cov=./litellm --cov-report=xml:coverage.xml
|
||||
|
||||
- name: Upload redis-py ${{ matrix.redis-py }} coverage to Codecov
|
||||
if: always() && steps.changes.outputs.decision != 'skip' && matrix.redis-py == '5.3.1'
|
||||
continue-on-error: true
|
||||
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
|
||||
with:
|
||||
use_oidc: true
|
||||
files: coverage.xml
|
||||
root_dir: ${{ github.workspace }}
|
||||
flags: redis-compat
|
||||
fail_ci_if_error: false
|
||||
coverage:
|
||||
name: coverage
|
||||
needs: unit
|
||||
|
|
@ -1369,10 +1500,20 @@ jobs:
|
|||
root_dir: ${{ github.workspace }}
|
||||
flags: unit
|
||||
fail_ci_if_error: false
|
||||
|
||||
- name: Upload MCP server coverage under its own flag
|
||||
continue-on-error: true
|
||||
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
|
||||
with:
|
||||
use_oidc: true
|
||||
files: coverage-reports/coverage-mcp-server-${{ github.run_id }}-${{ github.run_attempt }}/coverage.xml
|
||||
root_dir: ${{ github.workspace }}
|
||||
flags: mcp-integration
|
||||
fail_ci_if_error: false
|
||||
unit-passed:
|
||||
name: unit passed
|
||||
permissions: {}
|
||||
needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs, helm, openai-sdk-compat]
|
||||
needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs, helm, openai-sdk-compat, redis-compat]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local
|
|||
|
||||
The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport
|
||||
|
||||
The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so they run in the GitHub Actions `redis-compat` matrix against every supported redis-py major rather than in `run_integration.sh`
|
||||
|
||||
The `cost` group is driven by the `CostTrackingTestCase` literals under `cost_calculation/cases/<provider>.py` (gathered by `cost_calculation/catalog.py`), with the scripted cost map in `cost_calculation/cost_map.py`. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/<scenario_id>`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL`
|
||||
|
||||
Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist. Management, accounting, providers and extensions give each worker a private database, Redis, upstream, fake provider and proxy, with a peer and read-only database role where required. The pytest controller owns these services and closes them after worker failures. Their cases use pytest-xdist load groups inside the files selected by CircleCI timing splits. Module and class fixtures are grouped automatically, so authors do not need scheduling markers. Function fixtures remain independently schedulable. Tests that own an independent mutable sink for every case should use function fixtures, so a shared module fixture does not serialize them or share pending observations. Nested harness probes discard the parent run's shard, worker and result settings, then supply their own selections. MCP keeps its existing resource ownership and explicit load groups. Automatic group suffixes are removed from JUnit and execution records to preserve case identities. Owned proxy helpers wait for a healthy response from every configured, live worker before yielding. Serial runs keep the supplied deployment, including routing parity runs. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. When that deployment runs more than one proxy worker, set `INTEGRATION_PROXY_WORKERS` to the count so a test that writes a model and then calls it waits out the config reload interval, the only cross-worker convergence bound the wire exposes. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ GITHUB_FILES: Final = frozenset(
|
|||
"tests/integration/database/image/test_offline_image_migration.py",
|
||||
"tests/integration/database/image/test_ui_image_serves_offline.py",
|
||||
"tests/integration/mcp/test_interactions.py",
|
||||
"tests/integration/sdk/test_redis_cluster_iam_auth.py",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -124,8 +124,10 @@ def test_classify_decisions(category: str, changed: list[str], expected: str) ->
|
|||
("litellm/caching/redis_cache.py", "run"),
|
||||
("tests/unit/caching/test_redis_cluster_cache.py", "run"),
|
||||
("tests/integration/sdk/test_redis_cluster_iam_auth.py", "run"),
|
||||
(".circleci/config.yml", "run"),
|
||||
(".github/workflows/test-unit.yml", "run"),
|
||||
("tests/integration/run.py", "run"),
|
||||
("uv.lock", "run"),
|
||||
(".circleci/config.yml", "skip"),
|
||||
("litellm/router.py", "skip"),
|
||||
("docs/my-website/redis.md", "skip"),
|
||||
),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue