version: 2.1 orbs: codecov: codecov/codecov@4.0.1 node: circleci/node@5.1.0 # Add this line to declare the node orb win: circleci/windows@5.0 # Add Windows orb commands: skip_if_unrelated_changes: parameters: category: type: enum enum: ["backend", "client"] default: "backend" steps: - run: name: "Skip job when no << parameters.category >>-relevant files changed" command: bash .circleci/scripts/path_filter.sh << parameters.category >> setup_google_dns: steps: - run: name: "Configure Google DNS" command: | # Backup original resolv.conf sudo cp /etc/resolv.conf /etc/resolv.conf.backup # Add both local and Google DNS servers echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf wait_for_service: description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)" parameters: url: type: string timeout: type: string default: "60" steps: - run: name: "Wait for << parameters.url >>" command: | TIMEOUT=<< parameters.timeout >> URL="<< parameters.url >>" ELAPSED=0 echo "Waiting up to ${TIMEOUT}s for ${URL} ..." if echo "$URL" | grep -q '^tcp://'; then HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1) PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2) while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do sleep 2; ELAPSED=$((ELAPSED+2)) if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi done else while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do sleep 2; ELAPSED=$((ELAPSED+2)) if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi done fi echo "Service ready after ${ELAPSED}s" install_helm: steps: - run: name: Install Helm v3.17.3 command: | curl -sSLf -o /tmp/helm.tar.gz \ https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c - sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm rm -f /tmp/helm.tar.gz install_kind: steps: - run: name: Install Kind v0.20.0 command: | curl -sSLf -o /tmp/kind \ https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c - chmod +x /tmp/kind sudo mv /tmp/kind /usr/local/bin/kind install_uv: description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH." steps: - run: name: Install uv (pinned 0.10.9) command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" install_node: description: "Install the Node.js version pinned in ui/litellm-dashboard/.nvmrc (24.19.0, which bundles npm 11.17.0) with checksum verification, and prepend it to PATH. Run this on any executor whose image does not already ship that version, or `npm ci` in ui/litellm-dashboard fails EBADENGINE against the engines floor. Installs into /opt/node rather than over /usr/local on purpose: cimg/python:*-browsers ships its own node there, and unpacking the tarball on top of it leaves npm 11.17 files merged with the image's npm 11.9 tree, which reports the new version and then exits 1 on `npm ci` with no error text at all. Requires checkout, which the .nvmrc drift check reads." steps: - run: name: Install Node.js 24.19.0 command: | NODE_VERSION="24.19.0" NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz" NODE_EXPECTED_SHA="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647" NVMRC_VERSION="$(tr -d '[:space:]' < ui/litellm-dashboard/.nvmrc)" if [ "$NVMRC_VERSION" != "$NODE_VERSION" ]; then echo "install_node: ui/litellm-dashboard/.nvmrc pins ${NVMRC_VERSION} but this command pins ${NODE_VERSION}; update NODE_VERSION and NODE_EXPECTED_SHA together" >&2 exit 1 fi curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c - sudo mkdir -p /opt/node sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /opt/node --strip-components=1 rm -f "/tmp/${NODE_TARBALL}" echo 'export PATH="/opt/node/bin:$PATH"' >> "$BASH_ENV" export PATH="/opt/node/bin:$PATH" node --version npm --version install_rust: description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself." steps: - run: name: Install Rust (rustup 1.28.2, toolchain 1.98.0) command: | case "$(uname -m)" in x86_64) RUSTUP_TRIPLE=x86_64-unknown-linux-gnu RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c ;; aarch64) RUSTUP_TRIPLE=aarch64-unknown-linux-gnu RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c ;; *) echo "install_rust: unsupported architecture $(uname -m)" >&2 exit 1 ;; esac curl -sSLf -o /tmp/rustup-init \ "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init" echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - chmod +x /tmp/rustup-init /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0 rm -f /tmp/rustup-init echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.cargo/bin:$PATH" rustc --version cargo --version start_postgres: description: "Start a postgres-db container on port 5432 and wait until it accepts connections." parameters: db_name: type: string default: circle_test steps: - run: name: Start PostgreSQL command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=<< parameters.db_name >> \ -p 5432:5432 \ postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26 - wait_for_service: url: tcp://localhost:5432 timeout: "60" start_redis: description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys." steps: - run: name: Start Redis command: | docker run -d \ --name redis-cache \ -p 6379:6379 \ redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 - wait_for_service: url: tcp://localhost:6379 timeout: "60" start_openai_record_replay_proxy: description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." steps: - run: name: Start record/replay proxy background: true command: | CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \ RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \ uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090 - run: name: Wait for record/replay proxy command: | for i in $(seq 1 30); do if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then echo "record/replay proxy is up" exit 0 fi sleep 1 done echo "record/replay proxy did not become ready" >&2 exit 1 start_fake_openai_endpoint: description: "Start the canned OpenAI mock (tests/_fake_openai_endpoint_server.py) on host port 8190 and wait until healthy. Models whose api_base points here (via FAKE_OPENAI_API_BASE) get well-formed chat/text/embedding responses with realistic usage, so the E2E run neither pays for nor depends on the live provider. A request whose model is '429' returns HTTP 429 for rate-limit/cooldown tests. Run after uv deps are synced." steps: - run: name: Start fake OpenAI endpoint background: true command: | uv run --no-sync python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 - run: name: Wait for fake OpenAI endpoint command: | for i in $(seq 1 30); do if curl -sf http://localhost:8190/health >/dev/null 2>&1; then echo "fake OpenAI endpoint is up" exit 0 fi sleep 1 done echo "fake OpenAI endpoint did not become ready" >&2 exit 1 start_cost_center_service: description: "Start the stand-in cost center validation service (tests/store_model_in_db_tests/cost_center_service.py) on host port 9414 and wait until healthy. The proxy's team-metadata validator (team_metadata_validator_e2e.py, impl 'http') reaches it via TEAM_METADATA_VALIDATION_SERVICE_URL=http://host.docker.internal:9414/validate. Run after uv deps are synced." steps: - run: name: Start cost center validation service background: true command: | uv run --no-sync python tests/store_model_in_db_tests/cost_center_service.py --host 0.0.0.0 --port 9414 - run: name: Wait for cost center validation service command: | for i in $(seq 1 30); do if curl -sf http://localhost:9414/health >/dev/null 2>&1; then echo "cost center validation service is up" exit 0 fi sleep 1 done echo "cost center validation service did not become ready" >&2 exit 1 setup_litellm_enterprise_pip: steps: - run: name: "Install local version of litellm-enterprise" command: | # litellm-enterprise is a uv workspace member and is already installed # by the main `uv sync --all-groups --all-extras`. Do NOT run # `uv sync --package litellm-enterprise` here — that overwrites the # shared .venv and strips out dev/test deps (pytest, prisma, etc.). uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" setup_litellm_test_deps: steps: - checkout - setup_google_dns - install_uv - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} jobs: # Add Windows testing job using_litellm_on_windows: executor: name: win/default shell: powershell.exe working_directory: ~/project environment: UV_PYTHON: "3.11" CARGO_HTTP_MULTIPLEXING: "false" CARGO_NET_RETRY: "5" steps: - checkout - run: name: Install Python command: | choco install python --version=3.11.0 -y --no-progress --force refreshenv python --version environment: CHOCOLATEY_CONFIRM_ALL: "true" - run: name: Install Dependencies environment: UV_HTTP_TIMEOUT: "300" command: | $rustupInit = Join-Path $env:TEMP "rustup-init.exe" $rustupVersion = "1.28.2" $rustupUrl = "https://static.rust-lang.org/rustup/archive/$rustupVersion/x86_64-pc-windows-msvc/rustup-init.exe" Invoke-WebRequest -Uri $rustupUrl -OutFile $rustupInit $rustupExpected = "88d8258dcf6ae4f7a80c7d1088e1f36fa7025a1cfd1343731b4ee6f385121fc0" $rustupActual = (Get-FileHash -Path $rustupInit -Algorithm SHA256).Hash.ToLower() if ($rustupActual -ne $rustupExpected) { throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual" } & $rustupInit -y --profile minimal --default-toolchain 1.98.0 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Remove-Item $rustupInit $cargoBin = Join-Path $HOME ".cargo\bin" $env:Path = "$cargoBin;$env:Path" rustc --version cargo --version $installer = Join-Path $env:TEMP "uv-install.ps1" Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer $expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d" $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() if ($actual -ne $expected) { throw "uv installer hash mismatch: expected $expected got $actual" } & $installer Remove-Item $installer $uvBin = Join-Path $HOME ".local\bin" $env:Path = "$uvBin;$env:Path" if (!(Test-Path $PROFILE)) { New-Item -ItemType File -Force -Path $PROFILE | Out-Null } if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) { Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`"" } if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) { Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`"" } for ($attempt = 1; $attempt -le 5; $attempt++) { Write-Host "uv sync attempt $attempt/5" uv sync --frozen --group dev --python 3.11 if ($LASTEXITCODE -eq 0) { break } if ($attempt -eq 5) { exit $LASTEXITCODE } Start-Sleep -Seconds 15 } - run: name: Run Windows-specific test command: | uv run --no-sync python -m pytest tests/windows_tests/ -v - run: name: Guard against MAX_PATH-busting packaged wheel paths environment: UV_HTTP_TIMEOUT: "300" command: | $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path" cargo --version Get-ChildItem -Path "litellm\rust_bridge" -Filter "_native*" -File -ErrorAction SilentlyContinue | Remove-Item -Force uv build --wheel --out-dir dist uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py base_sdk_install: docker: - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - install_rust - run: name: Build the wheel environment: UV_HTTP_TIMEOUT: "300" command: | uv build --wheel --out-dir dist - run: name: Install the wheel with no extras and smoke-check it environment: UV_HTTP_TIMEOUT: "300" command: | uv venv /tmp/base-sdk --python 3.12 VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl /tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py local_testing_part1: docker: - &python312_image image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project parallelism: 4 steps: - checkout - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Run prisma ./docker/entrypoint.sh command: | set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e # Run pytest and generate JUnit XML report - run: name: Run tests (Part 1 - A-M) command: | mkdir test-results # Discover test files (A-M) TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov=./enterprise/litellm_enterprise \ --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=20 \ -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ -n 4 \ --timeout=300 \ --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | # When CI reruns only the failed tests, a parallel node can receive # zero tests and pytest never writes coverage. Emit empty placeholders # so persist_to_workspace and the downstream coverage combine stay green. if [ -f coverage.xml ]; then mv coverage.xml local_testing_part1_coverage.xml mv .coverage local_testing_part1_coverage else touch local_testing_part1_coverage.xml local_testing_part1_coverage fi # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - local_testing_part1_coverage.xml - local_testing_part1_coverage local_testing_part2: docker: - *python312_image working_directory: ~/project parallelism: 4 steps: - checkout - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Run prisma ./docker/entrypoint.sh command: | set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e # Run pytest and generate JUnit XML report - run: name: Run tests (Part 2 - N-Z) command: | mkdir test-results # Discover test files (N-Z) TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov=./enterprise/litellm_enterprise \ --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=20 \ -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ -n 4 \ --timeout=300 \ --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | # When CI reruns only the failed tests, a parallel node can receive # zero tests and pytest never writes coverage. Emit empty placeholders # so persist_to_workspace and the downstream coverage combine stay green. if [ -f coverage.xml ]; then mv coverage.xml local_testing_part2_coverage.xml mv .coverage local_testing_part2_coverage else touch local_testing_part2_coverage.xml local_testing_part2_coverage fi # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - local_testing_part2_coverage.xml - local_testing_part2_coverage langfuse_logging_unit_tests: docker: - *python312_image working_directory: ~/project resource_class: medium steps: - checkout - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Run prisma ./docker/entrypoint.sh command: | set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit.xml \ --durations=5 \ -k \"langfuse\"" no_output_timeout: 15m # Store test results - store_test_results: path: test-results auth_ui_unit_tests: docker: - *python312_image - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: litellm_test working_directory: ~/project environment: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} - wait_for_service: url: tcp://localhost:5432 timeout: "60" - run: name: Seed DB schema via prisma db push command: | set +e uv run --no-sync litellm --skip_server_startup --use_prisma_db_push set -e - run: name: Generate Prisma Client command: uv run --no-sync python -m prisma generate # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml auth_ui_unit_tests_coverage.xml mv .coverage auth_ui_unit_tests_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - auth_ui_unit_tests_coverage.xml - auth_ui_unit_tests_coverage litellm_router_testing: # Runs all tests with the "router" keyword docker: - *python312_image working_directory: ~/project resource_class: large parallelism: 4 environment: FAKE_OPENAI_API_BASE: http://127.0.0.1:8190 steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} - start_fake_openai_endpoint # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | # On a "rerun failed tests" build a parallel node can receive no # tests, so the test command never creates test-results. Pre-create it # so store_test_results doesn't fail the node on a missing path. mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ -k 'router' \ -n 4 \ --junitxml=test-results/junit.xml \ --durations=5 \ --timeout=300 --timeout_method=thread" no_output_timeout: 15m # Store test results - store_test_results: path: test-results litellm_router_unit_testing: # Runs all tests with the "router" keyword docker: - *python312_image working_directory: ~/project resource_class: large steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml router_unit_tests_coverage.xml mv .coverage router_unit_tests_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - router_unit_tests_coverage.xml - router_unit_tests_coverage litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - *python312_image working_directory: ~/project resource_class: medium steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit.xml \ --durations=5 \ -k \"assistants\"" no_output_timeout: 15m # Store test results - store_test_results: path: test-results llm_translation_testing: docker: - *python312_image working_directory: ~/project resource_class: xlarge steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging # Subdirectories with dedicated jobs (maintain this list as new jobs are added) mkdir -p test-results # Glob excludes the realtime/ subdirectory since it has its own job TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit.xml \ --durations=20 \ -n 4 \ --timeout=120 --timeout_method=thread \ --retries 2 --retry-delay 5 \ --max-worker-restart=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results realtime_translation_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run realtime tests command: | # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=20 \ -n 4 \ --timeout=120 --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml realtime_translation_coverage.xml mv .coverage realtime_translation_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - realtime_translation_coverage.xml - realtime_translation_coverage agent_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/agent_tests/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml agent_coverage.xml mv .coverage agent_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - agent_coverage.xml - agent_coverage guardrails_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results export LITELLM_LOG=WARNING TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2 \ --timeout=120 --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml guardrails_coverage.xml mv .coverage guardrails_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - guardrails_coverage.xml - guardrails_coverage google_generate_content_endpoint_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ --retries 3 --retry-delay 5" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml google_generate_content_endpoint_coverage.xml mv .coverage google_generate_content_endpoint_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - google_generate_content_endpoint_coverage.xml - google_generate_content_endpoint_coverage llm_responses_api_testing: docker: - *python312_image working_directory: ~/project resource_class: large environment: REQUEST_TIMEOUT: "180" steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 8 \ --reruns 1 --only-rerun Timeout" no_output_timeout: 15m # Store test results - store_test_results: path: test-results ocr_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(printf "%s\n%s\n" \ "$(circleci tests glob "tests/ocr_tests/**/test_*.py")" \ "tests/test_litellm/ocr/test_rust_bridge.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml ocr_coverage.xml mv .coverage ocr_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - ocr_coverage.xml - ocr_coverage search_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml search_coverage.xml mv .coverage search_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - search_coverage.xml - search_coverage batches_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml batches_coverage.xml mv .coverage batches_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - batches_coverage.xml - batches_coverage litellm_utils_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml litellm_utils_coverage.xml mv .coverage litellm_utils_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - litellm_utils_coverage.xml - litellm_utils_coverage pass_through_unit_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml pass_through_unit_tests_coverage.xml mv .coverage pass_through_unit_tests_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - pass_through_unit_tests_coverage.xml - pass_through_unit_tests_coverage image_gen_testing: docker: - *python312_image working_directory: ~/project resource_class: large steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" no_output_timeout: 15m # Store test results - store_test_results: path: test-results logging_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | mkdir -p test-results export LITELLM_LOG=WARNING TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ -n 4 \ --junitxml=test-results/junit.xml \ --durations=5 \ --timeout=120 --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml logging_coverage.xml || true mv .coverage logging_coverage || true # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - logging_coverage.xml - logging_coverage audio_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml audio_coverage.xml mv .coverage audio_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - audio_coverage.xml - audio_coverage redis_caching_unit_tests: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(printf "%s\n" \ tests/local_testing/test_dual_cache.py \ tests/local_testing/test_redis_batch_optimizations.py \ tests/local_testing/test_router_utils.py) echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -s \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 -n 2 \ --reruns 2 --reruns-delay 1" no_output_timeout: 20m - run: name: Rename the coverage files command: | mv coverage.xml redis_caching_coverage.xml mv .coverage redis_caching_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - redis_caching_coverage.xml - redis_caching_coverage installing_litellm_on_python: docker: - *python312_image working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - run: name: Run tests command: | uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver" installing_litellm_on_python_3_13: docker: - image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: medium steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.13 - run: name: Run tests command: | uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver" installing_litellm_on_python_v2_migration_resolver: docker: - *python312_image - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: litellm_test working_directory: ~/project environment: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - wait_for_service: url: tcp://localhost:5432 timeout: "60" - run: name: Run v2 migration resolver proxy smoke test command: | uv run --no-sync python -m pytest -vv \ tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver helm_chart_testing: machine: image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker resource_class: medium working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - attach_workspace: at: ~/project - setup_google_dns - install_helm - install_kind # Install kubectl (pinned version with hardcoded checksum) - run: name: Install kubectl v1.31.4 command: | curl -sSLf -o /tmp/kubectl \ https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c - chmod +x /tmp/kubectl sudo mv /tmp/kubectl /usr/local/bin/ # Create kind cluster - run: name: Create Kind Cluster command: | kind create cluster --name litellm-test - run: name: Load Docker Database Image for helm tests command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load IMAGE_TAG=${CIRCLE_SHA1:-ci} docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG} - run: name: Load Docker image into Kind command: | IMAGE_TAG=${CIRCLE_SHA1:-ci} kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test # Run helm lint - run: name: Run helm lint command: | helm lint ./helm/litellm-helm # Run helm tests - run: name: Run helm tests command: | IMAGE_TAG=${CIRCLE_SHA1:-ci} helm install litellm ./helm/litellm-helm -f ./helm/litellm-helm/ci/test-values.yaml \ --set image.repository=litellm-ci \ --set image.tag=${IMAGE_TAG} \ --set image.pullPolicy=Never # Wait for pod to be ready echo "Waiting 30 seconds for pod to be ready..." sleep 30 # Print pod logs before running tests echo "Printing pod logs..." kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") # Run the helm tests helm test litellm --logs # Cleanup - run: name: Cleanup command: | kind delete cluster --name litellm-test when: always # This ensures cleanup runs even if previous steps fail db_migration_disable_update_check: machine: image: ubuntu-2204:2024.04.1 resource_class: medium working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_fake_openai_endpoint - start_postgres: db_name: litellm_test - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Seed database with real schema command: | docker run -d \ -p 4001:4000 \ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e LITELLM_MASTER_KEY="sk-1234" \ --name schema-seed \ --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --use_prisma_db_push - wait_for_service: url: http://localhost:4001 timeout: "300" - run: name: Stop schema seed container command: docker stop schema-seed && docker rm schema-seed - run: name: Run Docker container with bad schema and disabled updates command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DISABLE_SCHEMA_UPDATE="True" \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ --name my-app \ --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - wait_for_service: url: http://localhost:4000 timeout: "60" - run: name: Check container logs for expected message command: | echo "=== Printing Full Container Startup Logs ===" LOG_OUTPUT="$(docker logs my-app 2>&1)" printf '%s\n' "$LOG_OUTPUT" echo "=== End of Full Container Startup Logs ===" if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then echo "Expected message found in logs. Test passed." else echo "Expected message not found in logs. Test failed." exit 1 fi - run: name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit-2.xml \ --durations=5" no_output_timeout: 15m - store_test_results: path: test-results build_and_test: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - attach_workspace: at: ~/project - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker tag litellm-docker-database:ci my-app:latest - start_openai_record_replay_proxy - start_fake_openai_endpoint - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e USE_PRISMA_MIGRATE=True \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e AZURE_API_KEY=$AZURE_API_KEY \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e GROQ_API_KEY=$GROQ_API_KEY \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e AUTO_INFER_REGION=True \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ -e RECORDER_OPENAI_BASE_URL=http://host.docker.internal:8090/v1 \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ my-app:latest \ --config /app/config.yaml \ --port 4000 - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results # Original used `tests/*.py` (top-level only); the `--ignore=...` # flags were vestigial since shell globbing did not descend into # subdirectories. Replicate by globbing only top-level test files. TEST_FILES=$(circleci tests glob "tests/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -s -v \ --junitxml=test-results/junit.xml \ -n 4 \ --durations=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results e2e_openai_endpoints: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e AZURE_API_KEY=$AZURE_API_KEY \ -e AZURE_API_BASE=$AZURE_API_BASE \ -e AZURE_API_VERSION="2024-05-01-preview" \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e GROQ_API_KEY=$GROQ_API_KEY \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e AUTO_INFER_REGION=True \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -s -vv \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results proxy_logging_guardrails_model_info_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - start_openai_record_replay_proxy - start_fake_openai_endpoint - run: name: Run Docker container # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e RECORDER_COHERE_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.cohere.com \ -e GCS_FLUSH_INTERVAL="1" \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \ -v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m # Clean up first container - run: name: Stop and remove first container command: | docker stop my-app docker rm my-app # Second Docker Container Run with Different Config # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license - run: name: Run Second Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE="bad-license" \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app-3 \ -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - run: name: Start outputting logs for second container command: docker logs -f my-app-3 background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run second round of tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit-2.xml \ --durations=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results proxy_spend_accuracy_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - start_redis - start_fake_openai_endpoint - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container # Point the proxy at the job-local Redis (start_redis) instead of the # shared remote Redis. The Redis transaction buffer uses a single # global pod-lock key (cronjob_lock:db_spend_update_job) and a single # global buffer list (litellm_spend_update_buffer); sharing those # across concurrent CI pipelines causes spend flushes to stall or # land in the wrong DB, which is what makes this test flaky. command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=host.docker.internal \ -e REDIS_PORT=6379 \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e PROXY_BATCH_WRITE_AT=2 \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m - store_test_results: path: test-results - run: name: Stop and remove first container when: always command: | docker stop my-app docker rm my-app docker stop redis-cache docker rm redis-cache proxy_multi_instance_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - start_fake_openai_endpoint - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container 1 # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - run: name: Run Docker container 2 command: | docker run -d \ -p 4001:4001 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app-2 \ -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4001 - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - wait_for_service: url: http://localhost:4001 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m # Clean up first container # Store test results - store_test_results: path: test-results proxy_store_model_in_db_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - start_fake_openai_endpoint - start_cost_center_service - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ --restart on-failure \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e STORE_MODEL_IN_DB="True" \ -e LITELLM_MASTER_KEY="sk-1234" \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e TEAM_METADATA_VALIDATION_SERVICE_URL=http://host.docker.internal:9414/validate \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \ -v $(pwd)/litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py:/app/team_metadata_validator_e2e.py \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m - run: name: Stop and remove containers command: | docker stop my-app || true docker rm my-app || true docker stop postgres-db || true docker rm postgres-db || true when: always - store_test_results: path: test-results proxy_build_from_pip_tests: # Change from docker to machine executor machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns # Remove Docker CLI installation since it's already available in machine executor - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - run: name: Build Docker image command: | docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . - start_postgres - start_fake_openai_endpoint - run: name: Run Docker container # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e GCS_FLUSH_INTERVAL="1" \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ my-app:latest \ --config /app/config.yaml \ --port 4000 - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --junitxml=test-results/junit-2.xml \ --durations=5" no_output_timeout: 15m # Clean up first container - store_test_results: path: test-results - run: name: Stop and remove first container command: | docker stop my-app || true docker rm my-app || true docker stop postgres-db || true docker rm postgres-db || true when: always proxy_pass_through_endpoint_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e GEMINI_API_KEY=$GEMINI_API_KEY \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \ -e AZURE_API_KEY=$AZURE_API_KEY \ -e AZURE_API_BASE=$AZURE_API_BASE \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" # Install Node.js directly from nodejs.org with SHA256 verification, # instead of piping NodeSource's setup_24.x apt-repo installer into # sudo bash (which runs a mutable upstream script unattended). - install_node - run: name: Install Node.js test dependencies command: | cd tests/pass_through_tests npm ci - run: name: Run Vertex AI, Google AI Studio Node.js tests command: | cd tests/pass_through_tests NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose no_output_timeout: 30m - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results proxy_e2e_anthropic_messages_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - setup_google_dns - install_uv - install_rust - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - start_openai_record_replay_proxy - run: name: Run Docker container with test config command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e LITELLM_MASTER_KEY="sk-1234" \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e RECORDER_ANTHROPIC_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.anthropic.com \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME="us-east-1" \ -e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \ -e LITELLM_LOG=ERROR \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run Claude Agent SDK E2E Tests command: | mkdir -p test-results export LITELLM_PROXY_URL="http://localhost:4000" export LITELLM_API_KEY="sk-1234" TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -s \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results upload-coverage: docker: - *python312_image steps: - checkout - skip_if_unrelated_changes - attach_workspace: at: . # Check file locations - run: name: Check coverage file location command: | echo "Current directory:" ls -la echo "\nContents of tests/llm_translation:" ls -la tests/llm_translation - install_uv - run: name: Combine Coverage command: | uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage uv tool run --from 'coverage[toml]==7.10.6' coverage xml - codecov/upload: file: ./coverage.xml flags: circleci e2e_ui_testing: docker: - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: e2euser POSTGRES_PASSWORD: e2epassword POSTGRES_DB: litellm_e2e resource_class: large working_directory: ~/project environment: DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" CI: "true" # Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can # assert the redirect. Set at job level so both the proxy boot step and the # Playwright step (whose skip guard reads this) see the same value. Safe for # the rest of the suite: nothing else performs a logout. PROXY_LOGOUT_URL: "https://www.example.com" steps: - checkout - skip_if_unrelated_changes: category: client - setup_google_dns - install_node - install_uv - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Python dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - save_cache: key: v1-uv-cache-{{ checksum "uv.lock" }} paths: - ~/.cache/uv - restore_cache: keys: - ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} - run: name: Install Node dependencies and Playwright # The cimg/python:3.12-browsers image already ships the Chromium system # libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.). # `--with-deps` triggers a redundant apt-get update + install that adds # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors, # so we install just the browser binary. command: | cd ui/litellm-dashboard npm ci cd ../../tests/e2e/ui npm ci npx playwright install chromium - save_cache: key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} paths: - ui/litellm-dashboard/node_modules - tests/e2e/ui/node_modules - ~/.cache/ms-playwright - run: name: Build UI from source # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`. # GNU cp (used on CircleCI's Ubuntu image) interprets that as "copy the # source directory as a child of the destination" when the destination # already exists — silently creating `_experimental/out/out/` instead of # replacing the served bundle. The proxy continued serving whatever was # checked into `_experimental/out/*`, so this job was effectively testing # the pre-build bundle on every run. Replace-and-move guarantees the # freshly built bundle is what the proxy actually serves. command: | cd ui/litellm-dashboard npm run build rm -rf ../../litellm/proxy/_experimental/out mv out ../../litellm/proxy/_experimental/out # Restructure HTML so extensionless routes work (login.html -> login/index.html) find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" done - wait_for_service: url: tcp://localhost:5432 timeout: "30" - run: name: Push Prisma schema command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - run: name: Seed database command: | PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ -f tests/e2e/ui/fixtures/seed.sql - run: name: Start mock LLM server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py background: true - run: name: Start mock Presidio server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py background: true - run: name: Wait for mock Presidio server command: | for i in $(seq 1 30); do if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi sleep 1 done echo "Mock Presidio server never answered /health on port 8091" >&2 exit 1 - run: name: Start LiteLLM proxy environment: LITELLM_MASTER_KEY: "sk-1234" MOCK_LLM_URL: "http://127.0.0.1:8090/v1" DISABLE_SCHEMA_UPDATE: "true" SERVER_ROOT_PATH: "" # PROXY_LOGOUT_URL is inherited from the job-level environment so the # proxy and proxyLogoutUrl.spec.ts agree on the logout target. # LITELLM_LICENSE is forwarded from the project env so premium-gated # UI flows can be exercised. license.spec.ts asserts the resulting # JWT carries premium_user=true; if it ever stops being passed, that # test fails loudly rather than silently regressing premium coverage. command: | LITELLM_LICENSE="$LITELLM_LICENSE" \ uv run --no-sync python -m litellm.proxy.proxy_cli \ --config tests/e2e/ui/fixtures/config.yml \ --port 4000 background: true - run: name: Wait for proxy to be ready command: | for i in $(seq 1 60); do HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true) if [ "$HTTP_CODE" = "200" ]; then echo "Proxy is ready" exit 0 fi sleep 2 done echo "Proxy failed to start" exit 1 - run: name: Run Playwright E2E tests # Forward LITELLM_LICENSE so license.spec.ts can detect that the # proxy was launched with a license and assert premium_user=true. command: | cd tests/e2e/ui LITELLM_LICENSE="$LITELLM_LICENSE" \ npx playwright test --config playwright.config.ts no_output_timeout: 10m - store_artifacts: path: tests/e2e/ui/test-results destination: e2e-test-results - store_artifacts: path: tests/e2e/ui/playwright-report destination: e2e-playwright-report e2e_ui_testing_server_root_path: docker: - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: e2euser POSTGRES_PASSWORD: e2epassword POSTGRES_DB: litellm_e2e resource_class: large working_directory: ~/project environment: DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" CI: "true" # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH # is read both by the proxy at boot (to rewrite the built UI bundle in place) # and by migration.serverRootPath.config.ts, which refuses to run without it. SERVER_ROOT_PATH: "/litellm" steps: - checkout - skip_if_unrelated_changes: category: client - setup_google_dns - install_node - install_uv - install_rust - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Python dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - save_cache: key: v1-uv-cache-{{ checksum "uv.lock" }} paths: - ~/.cache/uv - restore_cache: keys: - ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} - run: name: Install Node dependencies and Playwright command: | cd ui/litellm-dashboard npm ci cd ../../tests/e2e/ui npm ci npx playwright install chromium - save_cache: key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} paths: - ui/litellm-dashboard/node_modules - tests/e2e/ui/node_modules - ~/.cache/ms-playwright - run: name: Build UI from source command: | cd ui/litellm-dashboard npm run build rm -rf ../../litellm/proxy/_experimental/out mv out ../../litellm/proxy/_experimental/out find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" done - wait_for_service: url: tcp://localhost:5432 timeout: "30" - run: name: Push Prisma schema command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - run: name: Seed database command: | PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ -f tests/e2e/ui/fixtures/seed.sql - run: name: Start mock LLM server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py background: true - run: name: Start mock Presidio server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py background: true - run: name: Wait for mock Presidio server command: | for i in $(seq 1 30); do if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi sleep 1 done echo "Mock Presidio server never answered /health on port 8091" >&2 exit 1 - run: name: Start LiteLLM proxy under a server root path environment: LITELLM_MASTER_KEY: "sk-1234" MOCK_LLM_URL: "http://127.0.0.1:8090/v1" DISABLE_SCHEMA_UPDATE: "true" # Output flows to this step's own log, so a boot crash is visible here # rather than swallowed by a downstream readiness probe. command: | LITELLM_LICENSE="$LITELLM_LICENSE" \ uv run --no-sync python -m litellm.proxy.proxy_cli \ --config tests/e2e/ui/fixtures/config.yml \ --port 4000 background: true - run: name: Wait for prefixed proxy to be ready command: | for i in $(seq 1 60); do HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer sk-1234" http://127.0.0.1:4000/litellm/health 2>/dev/null || true) if [ "$HTTP_CODE" = "200" ]; then echo "Prefixed proxy is ready" exit 0 fi sleep 2 done echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log" exit 1 - run: name: Run migration smoke under SERVER_ROOT_PATH command: | cd tests/e2e/ui LITELLM_LICENSE="$LITELLM_LICENSE" \ npx playwright test --config migration.serverRootPath.config.ts no_output_timeout: 10m - store_artifacts: path: tests/e2e/ui/test-results destination: e2e-server-root-path-test-results - store_artifacts: path: tests/e2e/ui/playwright-report destination: e2e-server-root-path-playwright-report build_docker_database_image: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - run: name: Build Docker image command: | docker build \ -t litellm-docker-database:ci \ -f docker/Dockerfile.database . - run: name: Save Docker image to workspace root command: | docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst - persist_to_workspace: root: . paths: - litellm-docker-database.tar.zst test_bad_database_url: machine: image: ubuntu-2204:2024.04.1 resource_class: medium working_directory: ~/project steps: - checkout - skip_if_unrelated_changes - attach_workspace: at: ~/project - setup_google_dns - start_postgres - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker tag litellm-docker-database:ci myapp:latest - run: name: Run Docker container with bad DATABASE_URL command: | docker run --name my-app \ -p 4000:4000 \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ myapp:latest \ --port 4000 > docker_output.log 2>&1 || true - run: name: Display Docker logs command: cat docker_output.log - run: name: Check for expected error command: | if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \ (grep -q "Database setup failed after multiple retries" docker_output.log || \ grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then echo "Expected error found. Test passed." else echo "Expected error not found. Test failed." cat docker_output.log exit 1 fi workflows: build_and_test: jobs: - using_litellm_on_windows: filters: &main_branches branches: only: - main - /litellm_.*/ - base_sdk_install: filters: *main_branches - local_testing_part1: filters: *main_branches - local_testing_part2: filters: *main_branches - langfuse_logging_unit_tests: filters: *main_branches - litellm_assistants_api_testing: filters: *main_branches - litellm_router_testing: filters: *main_branches - litellm_router_unit_testing: filters: *main_branches - auth_ui_unit_tests: filters: *main_branches - build_docker_database_image: filters: *main_branches - e2e_ui_testing: filters: *main_branches - e2e_ui_testing_server_root_path: filters: *main_branches - build_and_test: requires: - build_docker_database_image filters: *main_branches - e2e_openai_endpoints: requires: - build_docker_database_image filters: *main_branches - proxy_logging_guardrails_model_info_tests: requires: - build_docker_database_image filters: *main_branches - proxy_spend_accuracy_tests: requires: - build_docker_database_image filters: *main_branches - proxy_multi_instance_tests: requires: - build_docker_database_image filters: *main_branches - proxy_store_model_in_db_tests: requires: - build_docker_database_image filters: *main_branches - proxy_build_from_pip_tests: filters: *main_branches - proxy_pass_through_endpoint_tests: requires: - build_docker_database_image filters: *main_branches - proxy_e2e_anthropic_messages_tests: requires: - build_docker_database_image filters: *main_branches - llm_translation_testing: filters: *main_branches - realtime_translation_testing: filters: *main_branches - agent_testing: filters: *main_branches - guardrails_testing: filters: *main_branches - google_generate_content_endpoint_testing: filters: *main_branches - llm_responses_api_testing: filters: *main_branches - ocr_testing: filters: *main_branches - search_testing: filters: *main_branches - batches_testing: filters: *main_branches - litellm_utils_testing: filters: *main_branches - pass_through_unit_testing: filters: *main_branches - image_gen_testing: filters: *main_branches - logging_testing: filters: *main_branches - audio_testing: filters: *main_branches - redis_caching_unit_tests: filters: *main_branches - upload-coverage: requires: - realtime_translation_testing - agent_testing - google_generate_content_endpoint_testing - guardrails_testing - ocr_testing - search_testing - batches_testing - litellm_utils_testing - pass_through_unit_testing - image_gen_testing - logging_testing - audio_testing - redis_caching_unit_tests - langfuse_logging_unit_tests - local_testing_part1 - local_testing_part2 - litellm_assistants_api_testing - litellm_router_unit_testing - auth_ui_unit_tests - db_migration_disable_update_check: requires: - build_docker_database_image filters: *main_branches - installing_litellm_on_python: filters: *main_branches - installing_litellm_on_python_3_13: filters: *main_branches - installing_litellm_on_python_v2_migration_resolver: filters: *main_branches - helm_chart_testing: requires: - build_docker_database_image filters: *main_branches - test_bad_database_url: requires: - build_docker_database_image filters: *main_branches