version: 2.1 parameters: run_migration_tests: type: boolean default: false migration_candidate_image: type: string default: "" migration_baseline_image: type: string default: "ghcr.io/berriai/litellm-database:v1.102.0" migration_source_sha: type: string default: "" routing_parity_base: type: string default: "" orbs: codecov: codecov/codecov@4.0.1 node: circleci/node@5.1.0 # Add this line to declare the node orb win: circleci/windows@5.0 # Add Windows orb commands: checkout_over_https: parameters: revision: type: string default: "" shell: type: string default: /bin/bash -eo pipefail steps: - run: name: Check out the public repository over HTTPS shell: << parameters.shell >> environment: REQUESTED_REVISION: << parameters.revision >> command: | revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}" [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1 git init git remote add origin https://github.com/BerriAI/litellm.git for attempt in 1 2 3; do git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break [[ $attempt -lt 3 ]] || exit 1 sleep 5 done skip_if_unrelated_changes: parameters: category: type: enum enum: ["backend", "client"] default: "backend" steps: - run: name: "Skip job when no << parameters.category >>-relevant files changed" command: bash .circleci/scripts/path_filter.sh << parameters.category >> setup_google_dns: steps: - run: name: "Configure Google DNS" command: | # Backup original resolv.conf sudo cp /etc/resolv.conf /etc/resolv.conf.backup # Add both local and Google DNS servers echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf wait_for_service: description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)" parameters: url: type: string timeout: type: string default: "60" steps: - run: name: "Wait for << parameters.url >>" command: | TIMEOUT=<< parameters.timeout >> URL="<< parameters.url >>" ELAPSED=0 echo "Waiting up to ${TIMEOUT}s for ${URL} ..." if echo "$URL" | grep -q '^tcp://'; then HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1) PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2) while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do sleep 2; ELAPSED=$((ELAPSED+2)) if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi done else while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do sleep 2; ELAPSED=$((ELAPSED+2)) if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi done fi echo "Service ready after ${ELAPSED}s" install_helm: steps: - run: name: Install Helm v3.17.3 command: | curl -sSLf -o /tmp/helm.tar.gz \ https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c - sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm rm -f /tmp/helm.tar.gz install_kind: steps: - run: name: Install Kind v0.20.0 command: | curl -sSLf -o /tmp/kind \ https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c - chmod +x /tmp/kind sudo mv /tmp/kind /usr/local/bin/kind install_uv: description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH." steps: - run: name: Install uv (pinned 0.10.9) command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" install_node: description: "Install the Node.js version pinned in ui/litellm-dashboard/.nvmrc (24.19.0, which bundles npm 11.17.0) with checksum verification, and prepend it to PATH. Run this on any executor whose image does not already ship that version, or `npm ci` in ui/litellm-dashboard fails EBADENGINE against the engines floor. Installs into /opt/node rather than over /usr/local on purpose: cimg/python:*-browsers ships its own node there, and unpacking the tarball on top of it leaves npm 11.17 files merged with the image's npm 11.9 tree, which reports the new version and then exits 1 on `npm ci` with no error text at all. Requires checkout, which the .nvmrc drift check reads." steps: - run: name: Install Node.js 24.19.0 command: | NODE_VERSION="24.19.0" NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz" NODE_EXPECTED_SHA="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647" NVMRC_VERSION="$(tr -d '[:space:]' < ui/litellm-dashboard/.nvmrc)" if [ "$NVMRC_VERSION" != "$NODE_VERSION" ]; then echo "install_node: ui/litellm-dashboard/.nvmrc pins ${NVMRC_VERSION} but this command pins ${NODE_VERSION}; update NODE_VERSION and NODE_EXPECTED_SHA together" >&2 exit 1 fi curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c - sudo mkdir -p /opt/node sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /opt/node --strip-components=1 rm -f "/tmp/${NODE_TARBALL}" echo 'export PATH="/opt/node/bin:$PATH"' >> "$BASH_ENV" export PATH="/opt/node/bin:$PATH" node --version npm --version install_rust: description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself. Also restores the dev-profile cargo cache that save_cargo_target writes on main, minus the workspace crates' fingerprints so those always rebuild from the checked-out source." steps: - run: name: Install Rust (rustup 1.28.2, toolchain 1.98.0) command: | case "$(uname -m)" in x86_64) RUSTUP_TRIPLE=x86_64-unknown-linux-gnu RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c ;; aarch64) RUSTUP_TRIPLE=aarch64-unknown-linux-gnu RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c ;; *) echo "install_rust: unsupported architecture $(uname -m)" >&2 exit 1 ;; esac curl -sSLf --retry 3 --retry-all-errors --retry-max-time 120 --connect-timeout 15 --max-time 60 -o /tmp/rustup-init \ "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init" echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - chmod +x /tmp/rustup-init /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0 rm -f /tmp/rustup-init echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" echo 'export CARGO_INCREMENTAL=0' >> "$BASH_ENV" export PATH="$HOME/.cargo/bin:$PATH" rustc --version cargo --version { rustc -vV; cc --version; cat /etc/os-release; } > /tmp/cargo-build-env - restore_cache: keys: - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}- - run: name: Force a rebuild of the workspace crates restored from the cargo cache command: rm -rf litellm-rust/target/debug/.fingerprint/litellm-* save_cargo_target: steps: - when: condition: equal: [main, << pipeline.git.branch >>] steps: - save_cache: key: v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} paths: - ~/.cargo/registry - ~/project/litellm-rust/target/debug start_postgres: description: "Start a postgres-db container on port 5432 and wait until it accepts connections." parameters: db_name: type: string default: circle_test image: type: string default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26 server_args: type: string default: "" steps: - run: name: Start PostgreSQL command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=<< parameters.db_name >> \ -p 5432:5432 \ << parameters.image >> << parameters.server_args >> - wait_for_service: url: tcp://localhost:5432 timeout: "60" start_redis: description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys." steps: - run: name: Start Redis command: | docker run -d \ --name redis-cache \ -p 6379:6379 \ redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 - wait_for_service: url: tcp://localhost:6379 timeout: "60" install_codecov_cli: steps: - run: name: Install Codecov CLI (pinned v11.3.1) when: always command: | curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM [ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ] (cd /tmp && sha256sum -c codecov.SHA256SUM) chmod +x /tmp/codecov mkdir -p "$HOME/.local/bin" mv /tmp/codecov "$HOME/.local/bin/codecov" start_openai_record_replay_proxy: description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay Redis cache hits. Cache misses call the live provider and store its response, so these legacy E2E jobs still require provider availability and may incur provider charges. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." steps: - run: name: Start record/replay proxy background: true command: | CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \ RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \ uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090 - run: name: Wait for record/replay proxy command: | for i in $(seq 1 30); do if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then echo "record/replay proxy is up" exit 0 fi sleep 1 done echo "record/replay proxy did not become ready" >&2 exit 1 start_fake_openai_endpoint: description: "Start the canned OpenAI mock (tests/_fake_openai_endpoint_server.py) on host port 8190 and wait until healthy. Models whose api_base points here (via FAKE_OPENAI_API_BASE) get well-formed chat/text/embedding responses with realistic usage, so the E2E run neither pays for nor depends on the live provider. A request whose model is '429' returns HTTP 429 for rate-limit/cooldown tests. Run after uv deps are synced." steps: - run: name: Start fake OpenAI endpoint background: true command: | uv run --no-sync python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 - run: name: Wait for fake OpenAI endpoint command: | for i in $(seq 1 30); do if curl -sf http://localhost:8190/health >/dev/null 2>&1; then echo "fake OpenAI endpoint is up" exit 0 fi sleep 1 done echo "fake OpenAI endpoint did not become ready" >&2 exit 1 setup_litellm_enterprise_pip: steps: - run: name: "Install local version of litellm-enterprise" command: | # litellm-enterprise is a uv workspace member and is already installed # by the main `uv sync --all-groups --all-extras`. Do NOT run # `uv sync --package litellm-enterprise` here — that overwrites the # shared .venv and strips out dev/test deps (pytest, prisma, etc.). uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" install_windows_toolchain: steps: - run: name: Install Rust and uv no_output_timeout: 30m command: | $rustupInit = Join-Path $env:TEMP "rustup-init.exe" $rustupVersion = "1.28.2" $rustupUrl = "https://static.rust-lang.org/rustup/archive/$rustupVersion/x86_64-pc-windows-msvc/rustup-init.exe" Invoke-WebRequest -Uri $rustupUrl -OutFile $rustupInit $rustupExpected = "88d8258dcf6ae4f7a80c7d1088e1f36fa7025a1cfd1343731b4ee6f385121fc0" $rustupActual = (Get-FileHash -Path $rustupInit -Algorithm SHA256).Hash.ToLower() if ($rustupActual -ne $rustupExpected) { throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual" } & $rustupInit -y --profile minimal --default-toolchain 1.98.0 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Remove-Item $rustupInit $cargoBin = Join-Path $HOME ".cargo\bin" $env:Path = "$cargoBin;$env:Path" rustc --version cargo --version $installer = Join-Path $env:TEMP "uv-install.ps1" Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer $expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d" $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() if ($actual -ne $expected) { throw "uv installer hash mismatch: expected $expected got $actual" } & $installer Remove-Item $installer $uvBin = Join-Path $HOME ".local\bin" $env:Path = "$uvBin;$env:Path" if (!(Test-Path $PROFILE)) { New-Item -ItemType File -Force -Path $PROFILE | Out-Null } if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) { Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`"" } if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) { Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`"" } setup_litellm_test_deps: parameters: checkout: type: boolean default: true steps: - when: condition: << parameters.checkout >> steps: - checkout_over_https - setup_google_dns - install_uv - install_rust - restore_cache: keys: - v3-integration-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - ~/.cache/uv key: v3-integration-uv-cache-{{ checksum "uv.lock" }} - save_cargo_target jobs: windows_release_wheel: executor: name: win/default shell: powershell.exe size: xlarge working_directory: ~/project environment: UV_PYTHON: "3.11" CARGO_HTTP_MULTIPLEXING: "false" CARGO_NET_RETRY: "5" steps: - checkout_over_https: shell: bash.exe - run: name: Skip job when no windows-release-relevant files changed shell: bash.exe command: bash .circleci/scripts/path_filter.sh windows-release - run: name: Install Python 3.11.9 from python.org no_output_timeout: 6m command: | $ProgressPreference = "SilentlyContinue" $installer = Join-Path $env:TEMP "python-3.11.9-amd64.exe" Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe -OutFile $installer -TimeoutSec 120 $expected = "5ee42c4eee1e6b4464bb23722f90b45303f79442df63083f05322f1785f5fdde" $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() if ($actual -ne $expected) { throw "Python installer hash mismatch: expected $expected got $actual" } $arguments = "/quiet InstallAllUsers=1 PrependPath=1 Include_test=0 Include_doc=0 Include_launcher=0 TargetDir=C:\Python311" $process = Start-Process -FilePath $installer -ArgumentList $arguments -PassThru if (-not $process.WaitForExit(300000)) { $process.Kill() throw "Python installer did not finish within 5 minutes" } if ($process.ExitCode -ne 0) { throw "Python installer exited with $($process.ExitCode)" } Remove-Item $installer & C:\Python311\python.exe --version - install_windows_toolchain - run: name: Record the Rust build environment for the release cargo cache key command: | & "$HOME\.cargo\bin\rustc.exe" -vV | Out-File -Encoding ascii .cargo-build-env - restore_cache: keys: - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}- - run: name: Force a rebuild of the workspace crates restored from the cargo cache command: | $fingerprints = "litellm-rust/target/release/.fingerprint" if (Test-Path $fingerprints) { Get-ChildItem -Path $fingerprints -Filter "litellm-*" | Remove-Item -Recurse -Force } - run: name: Build the release wheel and install it under a worst-case MAX_PATH prefix no_output_timeout: 30m environment: UV_HTTP_TIMEOUT: "300" command: | $env:Path = "C:\Python311;C:\Python311\Scripts;$HOME\.cargo\bin;$HOME\.local\bin;$env:Path" uv build --wheel --out-dir dist if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python tests/windows_tests/check_windows_wheel_install.py - when: condition: equal: [main, << pipeline.git.branch >>] steps: - save_cache: key: v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} paths: - ~/.cargo/registry - ~/project/litellm-rust/target/release base_sdk_install: docker: - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout_over_https - setup_google_dns - install_uv - install_rust - run: name: Build the wheel no_output_timeout: 30m environment: UV_HTTP_TIMEOUT: "300" command: | uv build --wheel --out-dir dist - run: name: Install the wheel with no extras and smoke-check it environment: UV_HTTP_TIMEOUT: "300" command: | uv venv /tmp/base-sdk --python 3.12 VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl /tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py - run: name: Guard against MAX_PATH-busting packaged wheel paths command: | python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only helm_chart_testing: machine: image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker resource_class: medium working_directory: ~/project steps: - checkout_over_https - skip_if_unrelated_changes - attach_workspace: at: ~/project - setup_google_dns - install_helm - install_kind # Install kubectl (pinned version with hardcoded checksum) - run: name: Install kubectl v1.31.4 command: | curl -sSLf -o /tmp/kubectl \ https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c - chmod +x /tmp/kubectl sudo mv /tmp/kubectl /usr/local/bin/ # Create kind cluster - run: name: Create Kind Cluster command: | kind create cluster --name litellm-test - run: name: Load Docker Database Image for helm tests command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load IMAGE_TAG=${CIRCLE_SHA1:-ci} docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG} - run: name: Load Docker image into Kind command: | IMAGE_TAG=${CIRCLE_SHA1:-ci} kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test # Run helm lint - run: name: Run helm lint command: | helm lint ./helm/litellm-helm # Run helm tests - run: name: Run helm tests command: | IMAGE_TAG=${CIRCLE_SHA1:-ci} helm install litellm ./helm/litellm-helm -f ./helm/litellm-helm/ci/test-values.yaml \ --set image.repository=litellm-ci \ --set image.tag=${IMAGE_TAG} \ --set image.pullPolicy=Never if ! kubectl rollout status deployment/litellm --timeout=180s; then kubectl describe pods -l app.kubernetes.io/name=litellm kubectl logs -l app.kubernetes.io/name=litellm --tail=200 exit 1 fi echo "Printing pod logs..." kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") # Run the helm tests helm test litellm --logs # Cleanup - run: name: Cleanup command: | kind delete cluster --name litellm-test when: always # This ensures cleanup runs even if previous steps fail db_migration_disable_update_check: machine: image: ubuntu-2204:2024.04.1 resource_class: medium working_directory: ~/project steps: - checkout_over_https - run: name: Generate LiteLLM master key command: | key="$(openssl rand -hex 16)" printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_fake_openai_endpoint - start_postgres: db_name: litellm_test - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Seed database with real schema command: | docker run -d \ -p 4001:4000 \ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \ --name schema-seed \ --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --use_prisma_db_push - wait_for_service: url: http://localhost:4001 timeout: "300" - run: name: Stop schema seed container command: docker stop schema-seed && docker rm schema-seed - run: name: Run Docker container with bad schema and disabled updates command: | docker run -d \ -p 4000:4000 \ -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DISABLE_SCHEMA_UPDATE="True" \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ --name my-app \ --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - wait_for_service: url: http://localhost:4000 timeout: "60" - run: name: Check container logs for expected message command: | echo "=== Printing Full Container Startup Logs ===" LOG_OUTPUT="$(docker logs my-app 2>&1)" printf '%s\n' "$LOG_OUTPUT" echo "=== End of Full Container Startup Logs ===" if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then echo "Expected message found in logs. Test passed." else echo "Expected message not found in logs. Test failed." exit 1 fi - run: name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ -v \ --junitxml=test-results/junit-2.xml \ --durations=5" no_output_timeout: 15m - store_test_results: path: test-results proxy_logging_guardrails_model_info_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout_over_https - run: name: Generate LiteLLM master key command: | key="$(openssl rand -hex 16)" printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - start_openai_record_replay_proxy - start_fake_openai_endpoint # Second Docker Container Run with Different Config # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license - run: name: Run Second Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE="bad-license" \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app-3 \ -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - run: name: Start outputting logs for second container command: docker logs -f my-app-3 background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run second round of tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ -v \ --junitxml=test-results/junit-2.xml \ --durations=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results e2e_ui_build: docker: - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} resource_class: large working_directory: ~/project steps: - checkout_over_https - skip_if_unrelated_changes: category: client - setup_google_dns - install_node - restore_cache: keys: - ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - run: name: Install dashboard dependencies command: | cd ui/litellm-dashboard npm ci - save_cache: key: ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} paths: - ui/litellm-dashboard/node_modules - run: name: Build UI bundle command: | cd ui/litellm-dashboard npm run build mkdir -p ../../ui-bundle mv out ../../ui-bundle/out find ../../ui-bundle/out -name '*.html' ! -name 'index.html' | while read -r f; do d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" done - persist_to_workspace: root: ~/project paths: - ui-bundle/out e2e_ui_testing: docker: - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: e2euser POSTGRES_PASSWORD: e2epassword POSTGRES_DB: litellm_e2e resource_class: xlarge working_directory: ~/project environment: DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" CI: "true" # Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can # assert the redirect. Set at job level so both the proxy boot step and the # Playwright step (whose skip guard reads this) see the same value. Safe for # the rest of the suite: nothing else performs a logout. PROXY_LOGOUT_URL: "https://www.example.com" steps: - checkout_over_https - run: name: Generate LiteLLM master key command: | key="$(openssl rand -hex 16)" printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - skip_if_unrelated_changes: category: client - attach_workspace: at: ~/project - setup_google_dns - install_node - install_uv - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Python dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - save_cache: key: v1-uv-cache-{{ checksum "uv.lock" }} paths: - ~/.cache/uv - restore_cache: keys: - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - run: name: Install Node dependencies and Playwright # The cimg/python:3.12-browsers image already ships the Chromium system # libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.). # `--with-deps` triggers a redundant apt-get update + install that adds # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors, # so we install just the browser binary. command: | cd tests/e2e/ui npm ci npx playwright install chromium - save_cache: key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} paths: - tests/e2e/ui/node_modules - ~/.cache/ms-playwright - run: name: Use built UI bundle command: | rm -rf litellm/proxy/_experimental/out cp -r ui-bundle/out litellm/proxy/_experimental/out - wait_for_service: url: tcp://localhost:5432 timeout: "30" - run: name: Push Prisma schema command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - run: name: Seed database command: | PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ -f tests/e2e/ui/fixtures/seed.sql - run: name: Start mock LLM server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py background: true - run: name: Start mock Presidio server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py background: true - run: name: Wait for mock Presidio server command: | for i in $(seq 1 30); do if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi sleep 1 done echo "Mock Presidio server never answered /health on port 8091" >&2 exit 1 - run: name: Start LiteLLM proxy environment: MOCK_LLM_URL: "http://127.0.0.1:8090/v1" DISABLE_SCHEMA_UPDATE: "true" SERVER_ROOT_PATH: "" # PROXY_LOGOUT_URL is inherited from the job-level environment so the # proxy and proxyLogoutUrl.spec.ts agree on the logout target. # LITELLM_LICENSE is forwarded from the project env so premium-gated # UI flows can be exercised. license.spec.ts asserts the resulting # JWT carries premium_user=true; if it ever stops being passed, that # test fails loudly rather than silently regressing premium coverage. command: | LITELLM_LICENSE="$LITELLM_LICENSE" \ uv run --no-sync python -m litellm.proxy.proxy_cli \ --config tests/e2e/ui/fixtures/config.yml \ --port 4000 background: true - run: name: Wait for proxy to be ready command: | for i in $(seq 1 60); do HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true) if [ "$HTTP_CODE" = "200" ]; then echo "Proxy is ready" exit 0 fi sleep 2 done echo "Proxy failed to start" exit 1 - run: name: Run Playwright E2E tests # Forward LITELLM_LICENSE so license.spec.ts can detect that the # proxy was launched with a license and assert premium_user=true. command: | cd tests/e2e/ui LITELLM_LICENSE="$LITELLM_LICENSE" \ npx playwright test --config playwright.config.ts --workers=3 --retries=0 no_output_timeout: 10m - store_artifacts: path: tests/e2e/ui/test-results destination: e2e-test-results - store_artifacts: path: tests/e2e/ui/playwright-report destination: e2e-playwright-report e2e_ui_testing_server_root_path: docker: - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: e2euser POSTGRES_PASSWORD: e2epassword POSTGRES_DB: litellm_e2e resource_class: large working_directory: ~/project environment: DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" CI: "true" # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH # is read both by the proxy at boot (to rewrite the built UI bundle in place) # and by migration.serverRootPath.config.ts, which refuses to run without it. SERVER_ROOT_PATH: "/litellm" steps: - checkout_over_https - run: name: Generate LiteLLM master key command: | key="$(openssl rand -hex 16)" printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - skip_if_unrelated_changes: category: client - attach_workspace: at: ~/project - setup_google_dns - install_node - install_uv - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Python dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - save_cache: key: v1-uv-cache-{{ checksum "uv.lock" }} paths: - ~/.cache/uv - restore_cache: keys: - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - run: name: Install Node dependencies and Playwright command: | cd tests/e2e/ui npm ci npx playwright install chromium - save_cache: key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} paths: - tests/e2e/ui/node_modules - ~/.cache/ms-playwright - run: name: Use built UI bundle command: | rm -rf litellm/proxy/_experimental/out cp -r ui-bundle/out litellm/proxy/_experimental/out - wait_for_service: url: tcp://localhost:5432 timeout: "30" - run: name: Push Prisma schema command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - run: name: Seed database command: | PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ -f tests/e2e/ui/fixtures/seed.sql - run: name: Start mock LLM server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py background: true - run: name: Start mock Presidio server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py background: true - run: name: Wait for mock Presidio server command: | for i in $(seq 1 30); do if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi sleep 1 done echo "Mock Presidio server never answered /health on port 8091" >&2 exit 1 - run: name: Start LiteLLM proxy under a server root path environment: MOCK_LLM_URL: "http://127.0.0.1:8090/v1" DISABLE_SCHEMA_UPDATE: "true" # Output flows to this step's own log, so a boot crash is visible here # rather than swallowed by a downstream readiness probe. command: | LITELLM_LICENSE="$LITELLM_LICENSE" \ uv run --no-sync python -m litellm.proxy.proxy_cli \ --config tests/e2e/ui/fixtures/config.yml \ --port 4000 background: true - run: name: Wait for prefixed proxy to be ready command: | for i in $(seq 1 60); do HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer $LITELLM_MASTER_KEY" http://127.0.0.1:4000/litellm/health 2>/dev/null || true) if [ "$HTTP_CODE" = "200" ]; then echo "Prefixed proxy is ready" exit 0 fi sleep 2 done echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log" exit 1 - run: name: Run migration smoke under SERVER_ROOT_PATH command: | cd tests/e2e/ui LITELLM_LICENSE="$LITELLM_LICENSE" \ npx playwright test --config migration.serverRootPath.config.ts no_output_timeout: 10m - store_artifacts: path: tests/e2e/ui/test-results destination: e2e-server-root-path-test-results - store_artifacts: path: tests/e2e/ui/playwright-report destination: e2e-server-root-path-playwright-report build_docker_database_image: parameters: migration_qualification: type: boolean default: false machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - when: condition: << parameters.migration_qualification >> steps: - checkout_over_https: revision: << pipeline.parameters.migration_source_sha >> - unless: condition: << parameters.migration_qualification >> steps: - checkout_over_https - skip_if_unrelated_changes - run: name: Build Docker image environment: MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >> command: | if [ -n "$MIGRATION_CANDIDATE_IMAGE" ]; then [[ "$MIGRATION_CANDIDATE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+@sha256:[0-9a-f]{64}$ ]] || exit 1 docker pull "$MIGRATION_CANDIDATE_IMAGE" docker tag "$MIGRATION_CANDIDATE_IMAGE" litellm-docker-database:ci else docker build \ --label org.opencontainers.image.revision="$(git rev-parse HEAD)" \ -t litellm-docker-database:ci \ -f docker/Dockerfile.database . fi python3 .circleci/scripts/run_migration_tests.py record-image - run: name: Save Docker image to workspace root command: | docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst - persist_to_workspace: root: . paths: - litellm-docker-database.tar.zst - migration-image.json migration_startup_tests: parameters: suite: type: enum enum: [startup, recovery, legacy, upgrade, shaped] baseline: type: boolean default: false machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project environment: LITELLM_MIGRATION_TESTS: "1" LITELLM_MIGRATION_TEST_IMAGE: litellm-docker-database:ci LITELLM_MIGRATION_BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >> MIGRATION_TEST_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres MIGRATION_TEST_CONTAINER_ADMIN_URL: postgresql://postgres:postgres@host.docker.internal:5432/postgres MIGRATION_TEST_OUTPUT: /tmp/migration-results PYTHONPATH: tests/e2e steps: - checkout_over_https: revision: << pipeline.parameters.migration_source_sha >> - run: name: Generate LiteLLM master key command: | key="$(openssl rand -hex 16)" printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - install_uv - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install test dependencies command: uv sync --frozen --all-groups --all-extras --python 3.12 - attach_workspace: at: ~/project - run: name: Load the shared candidate and start PostgreSQL command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker run -d --name migration-postgres \ -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres \ -p 5432:5432 \ postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - wait_for_service: url: tcp://localhost:5432 timeout: "60" - when: condition: << parameters.baseline >> steps: - run: name: Pull the baseline release the upgrade starts from environment: BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >> command: | [[ "$BASELINE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+(@sha256:[0-9a-f]{64}|:v[0-9][0-9a-z.-]*)$ ]] || exit 1 docker pull "$BASELINE_IMAGE" - run: name: Run migration startup regressions environment: MIGRATION_TEST_SUITE: << parameters.suite >> MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >> command: | mkdir -p /tmp/migration-results uv run --no-sync python .circleci/scripts/run_migration_tests.py no_output_timeout: 15m - store_test_results: path: /tmp/migration-results/junit - run: name: Package migration diagnostics when: always command: | mkdir -p /tmp/migration-artifacts if [ -d /tmp/migration-results ]; then tar -czf /tmp/migration-artifacts/diagnostics.tar.gz -C /tmp/migration-results . if [ -f /tmp/migration-results/verdict.json ]; then cp /tmp/migration-results/verdict.json /tmp/migration-artifacts/verdict.json fi fi - store_artifacts: path: /tmp/migration-artifacts destination: migration-results - run: name: Remove migration test containers when: always command: | docker ps -aq --filter label=litellm-migration-test=true | xargs -r docker rm -f docker rm -f migration-postgres || true test_bad_database_url: machine: image: ubuntu-2204:2024.04.1 resource_class: medium working_directory: ~/project steps: - checkout_over_https - skip_if_unrelated_changes - attach_workspace: at: ~/project - setup_google_dns - start_postgres - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker tag litellm-docker-database:ci myapp:latest - run: name: Run Docker container with bad DATABASE_URL command: | set +e docker run --name my-app \ -p 4000:4000 \ -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ myapp:latest \ --port 4000 > docker_output.log 2>&1 echo "$?" > docker_exit_code set -e - run: name: Display Docker logs command: cat docker_output.log - run: name: Proxy must refuse to serve on an unreachable database command: | fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; } exit_code="$(cat docker_exit_code)" [ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database" grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server" ! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state" ! docker exec my-app true 2>/dev/null || fail "container is still running" echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed." integration_contracts: parameters: suite: type: string mode: type: enum enum: [standard, replica] default: standard parallelism: type: integer default: 1 resource_class: type: enum enum: [large, xlarge] default: large machine: image: ubuntu-2204:2024.04.1 resource_class: << parameters.resource_class >> working_directory: ~/project parallelism: << parameters.parallelism >> steps: - setup_litellm_test_deps - when: condition: equal: [browser, << parameters.suite >>] steps: - install_node - restore_cache: keys: - integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} - run: name: Install locked browser dependencies command: | cd ui/litellm-dashboard npm ci cd ../../tests/e2e/ui npm ci sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l \ timeout --signal=TERM --kill-after=20s 6m node node_modules/@playwright/test/cli.js install-deps chromium timeout --signal=TERM --kill-after=20s 3m node node_modules/@playwright/test/cli.js install chromium - save_cache: key: integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} paths: - ~/.npm - ~/.cache/ms-playwright - run: name: Build the candidate dashboard command: cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build - when: condition: equal: [mcp, << parameters.suite >>] steps: - run: name: Install MCP SDK1 peer command: | uv venv --python 3.12 .venv-mcp-peer uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV" - when: condition: equal: [sdk, << parameters.suite >>] steps: - run: name: Install redis-server 7.2.16 command: | mkdir -p /tmp/redis-bin cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)" docker cp "${cid}:/usr/local/bin/redis-server" /tmp/redis-bin/redis-server docker rm "$cid" /tmp/redis-bin/redis-server --version | grep -q 'v=7.2.16' echo 'export PATH="/tmp/redis-bin:$PATH"' >> "$BASH_ENV" - start_postgres: image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 server_args: "-c max_connections=300 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - start_redis - run: name: Run owned integration contracts command: bash .circleci/scripts/run_integration.sh << parameters.suite >> << parameters.mode >> no_output_timeout: 15m - run: name: Stop owned database and Redis when: always command: | mkdir -p test-results/services-<< parameters.suite >>-<< parameters.mode >> docker logs postgres-db > test-results/services-<< parameters.suite >>-<< parameters.mode >>/postgres.log 2>&1 || true docker logs redis-cache > test-results/services-<< parameters.suite >>-<< parameters.mode >>/redis.log 2>&1 || true docker rm -f postgres-db redis-cache test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - when: condition: not: equal: [1, << parameters.parallelism >>] steps: - run: name: Prepare this shard's execution record command: | results="test-results/integration-<< parameters.suite >>" if [ "<< parameters.mode >>" = replica ]; then results="$results-replica" fi mkdir -p "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX" cp "$results/execution.json" "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX/execution.json" if [ "<< parameters.suite >>" = mcp ]; then mkdir -p "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage" cp test-results/integration-mcp/coverage/data "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage/data.$CIRCLE_NODE_INDEX" fi - persist_to_workspace: root: /tmp/integration-shards paths: - << parameters.suite >>-<< parameters.mode >> - store_test_results: path: test-results - store_artifacts: path: test-results integration_shard_coverage: parameters: suite: type: enum enum: [management, accounting, database, extensions, providers, mcp] mode: type: enum enum: [standard, replica] default: standard shards: type: integer docker: - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} resource_class: small working_directory: ~/project environment: UV_PYTHON_DOWNLOADS: never steps: - checkout_over_https - install_uv - run: name: Install the locked shard verifier and coverage dependencies command: | uv export --frozen --all-groups --all-extras --no-emit-project --no-emit-workspace --no-header --no-annotate \ | awk '/^[^ ]/ { keep = $0 ~ /^(annotated-types|coverage|pydantic|pydantic-core|typing-extensions|typing-inspection)==/ } keep' \ > /tmp/fan-in-requirements.txt uv venv .venv --python 3.12 uv pip install --no-config --python .venv --require-hashes -r /tmp/fan-in-requirements.txt - attach_workspace: at: /tmp/integration-shards - run: name: Verify every << parameters.suite >> case ran exactly once command: | .venv/bin/python -m tests.integration._support.sharding /tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >> \ --shards << parameters.shards >> - when: condition: equal: [mcp, << parameters.suite >>] steps: - run: name: Combine coverage from every MCP shard command: | mkdir -p test-results/integration-mcp/coverage export COVERAGE_FILE="$PWD/test-results/integration-mcp/coverage/data" .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml /tmp/integration-shards/mcp-standard/*/coverage .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > test-results/integration-mcp/coverage/coverage.txt .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d test-results/integration-mcp/coverage/html - store_artifacts: path: test-results/integration-mcp/coverage routing_parity: parameters: suite: type: string machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - setup_litellm_test_deps - run: name: Check out base product code environment: ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >> command: | [[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1 git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE" git rm -r -f --quiet litellm enterprise litellm-proxy-extras git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras git reset --quiet test -f litellm/rust_bridge/_native.abi3.so - start_postgres: image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - start_redis - run: name: Run base side command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity base no_output_timeout: 15m - run: name: Stop base database and Redis when: always command: | mkdir -p test-results/services-<< parameters.suite >>-parity-base docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-base/postgres.log 2>&1 || true docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-base/redis.log 2>&1 || true docker rm -f postgres-db redis-cache test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - run: name: Check out head product code command: | git rm -r -f --quiet litellm enterprise litellm-proxy-extras git checkout "$CIRCLE_SHA1" -- litellm enterprise litellm-proxy-extras git reset --quiet test -f litellm/rust_bridge/_native.abi3.so - start_postgres: image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - start_redis - run: name: Run head side command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity head no_output_timeout: 15m - run: name: Stop head database and Redis when: always command: | mkdir -p test-results/services-<< parameters.suite >>-parity-head docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-head/postgres.log 2>&1 || true docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-head/redis.log 2>&1 || true docker rm -f postgres-db redis-cache test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - run: name: Compare routing parity command: PYTHONPATH="$PWD/tests" .venv/bin/python -m integration._support.routing check test-results/parity-<< parameters.suite >>/base test-results/parity-<< parameters.suite >>/head - store_test_results: path: test-results - store_artifacts: path: test-results postgres_suite: parameters: test_path: type: string seed: type: boolean coverage_flag: type: string default: "" timeout_minutes: type: integer workers: type: integer default: 1 machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project environment: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test steps: - checkout_over_https - skip_if_unrelated_changes - setup_litellm_test_deps: checkout: false - start_postgres: db_name: litellm_test image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - run: name: Generate Prisma client command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - when: condition: << parameters.seed >> steps: - run: name: Seed database schema command: uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - run: name: Run << parameters.test_path >> command: | mkdir -p test-results coverage_args=() worker_args=() if [ "<< parameters.workers >>" -gt 1 ]; then worker_args=(-n << parameters.workers >> --dist=worksteal) fi if [ -n "<< parameters.coverage_flag >>" ]; then coverage_args=(--cov=./litellm --cov-report=xml:coverage.xml) fi timeout --signal=TERM << parameters.timeout_minutes >>m uv run --no-sync pytest \ << parameters.test_path >> -vv --tb=short --durations=10 -o junit_family=xunit1 \ --junitxml=test-results/junit.xml "${coverage_args[@]}" "${worker_args[@]}" - when: condition: << parameters.coverage_flag >> steps: - install_codecov_cli - run: name: Upload coverage when: always command: | [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; } codecov upload-process --disable-search --fail-on-error -f coverage.xml \ -F << parameters.coverage_flag >> -C "$CIRCLE_SHA1" \ -n "<< parameters.coverage_flag >>-${CIRCLE_BUILD_NUM}" --git-service github - store_test_results: path: test-results - store_artifacts: path: test-results workflows: migration_startup_scheduled: triggers: - schedule: cron: "17 0,6,12,18 * * *" filters: branches: only: main jobs: - build_docker_database_image: migration_qualification: true - migration_startup_tests: name: migration-startup suite: startup requires: [build_docker_database_image] - migration_startup_tests: name: migration-recovery suite: recovery requires: [build_docker_database_image] - migration_startup_tests: name: migration-legacy-and-pooling suite: legacy requires: [build_docker_database_image] - migration_startup_tests: name: migration-upgrade suite: upgrade baseline: true requires: [build_docker_database_image] - migration_startup_tests: name: migration-upgrade-shaped suite: shaped baseline: true requires: [build_docker_database_image] integration: jobs: - integration_contracts: name: integration-<< matrix.suite >> filters: &integration_default_run not pipeline.parameters.run_migration_tests and pipeline.parameters.routing_parity_base == "" matrix: parameters: suite: [sdk, cost, browser] parallelism: 1 - integration_contracts: name: integration-security suite: security parallelism: 1 resource_class: xlarge filters: *integration_default_run - integration_contracts: name: integration-management suite: management parallelism: 4 resource_class: xlarge filters: *integration_default_run - integration_shard_coverage: name: integration-management-coverage suite: management shards: 4 requires: [integration-management] filters: *integration_default_run - integration_contracts: name: integration-accounting suite: accounting parallelism: 4 resource_class: xlarge filters: *integration_default_run - integration_shard_coverage: name: integration-accounting-coverage suite: accounting shards: 4 requires: [integration-accounting] filters: *integration_default_run - integration_contracts: name: integration-providers suite: providers parallelism: 4 resource_class: xlarge filters: *integration_default_run - integration_shard_coverage: name: integration-providers-coverage suite: providers shards: 4 requires: - integration-providers filters: *integration_default_run - integration_contracts: name: integration-database suite: database parallelism: 2 filters: *integration_default_run - integration_shard_coverage: name: integration-database-coverage suite: database shards: 2 requires: [integration-database] filters: *integration_default_run - integration_contracts: name: integration-mcp suite: mcp parallelism: 2 filters: *integration_default_run - integration_shard_coverage: name: integration-mcp-coverage suite: mcp shards: 2 requires: - integration-mcp filters: *integration_default_run - integration_contracts: name: integration-extensions suite: extensions parallelism: 8 resource_class: xlarge filters: *integration_default_run - integration_shard_coverage: name: integration-extensions-coverage suite: extensions shards: 8 requires: - integration-extensions filters: *integration_default_run - integration_contracts: name: integration-management-replica suite: management mode: replica parallelism: 4 resource_class: xlarge filters: *integration_default_run - integration_shard_coverage: name: integration-management-replica-coverage suite: management mode: replica shards: 4 requires: [integration-management-replica] filters: *integration_default_run - integration_contracts: name: integration-database-replica suite: database mode: replica parallelism: 2 filters: *integration_default_run - integration_shard_coverage: name: integration-database-replica-coverage suite: database mode: replica shards: 2 requires: [integration-database-replica] filters: *integration_default_run - postgres_suite: name: roi-database test_path: tests/integration/database/test_roi_observed.py seed: false coverage_flag: roi-postgres timeout_minutes: 10 filters: *integration_default_run - windows_release_wheel: filters: *integration_default_run - base_sdk_install: filters: *integration_default_run - build_docker_database_image: filters: *integration_default_run - proxy_logging_guardrails_model_info_tests: requires: - build_docker_database_image filters: *integration_default_run - db_migration_disable_update_check: requires: - build_docker_database_image filters: *integration_default_run - test_bad_database_url: requires: - build_docker_database_image filters: *integration_default_run - helm_chart_testing: requires: - build_docker_database_image filters: *integration_default_run - e2e_ui_build: filters: *integration_default_run - e2e_ui_testing: requires: [e2e_ui_build] filters: *integration_default_run - e2e_ui_testing_server_root_path: requires: [e2e_ui_build] filters: *integration_default_run - build_docker_database_image: name: build_docker_database_image-migration migration_qualification: true filters: pipeline.parameters.run_migration_tests - migration_startup_tests: name: migration-startup suite: startup requires: [build_docker_database_image-migration] filters: pipeline.parameters.run_migration_tests - migration_startup_tests: name: migration-recovery suite: recovery requires: [build_docker_database_image-migration] filters: pipeline.parameters.run_migration_tests - migration_startup_tests: name: migration-legacy-and-pooling suite: legacy requires: [build_docker_database_image-migration] filters: pipeline.parameters.run_migration_tests - migration_startup_tests: name: migration-upgrade suite: upgrade baseline: true requires: [build_docker_database_image-migration] filters: pipeline.parameters.run_migration_tests - migration_startup_tests: name: migration-upgrade-shaped suite: shaped baseline: true requires: [build_docker_database_image-migration] filters: pipeline.parameters.run_migration_tests - routing_parity: name: routing-parity-<< matrix.suite >> matrix: parameters: suite: [management, accounting, database, providers, extensions, cost, mcp] filters: pipeline.parameters.routing_parity_base != ""