version: 2.1 orbs: codecov: codecov/codecov@4.0.1 node: circleci/node@5.1.0 # Add this line to declare the node orb win: circleci/windows@5.0 # Add Windows orb commands: setup_google_dns: steps: - run: name: "Configure Google DNS" command: | # Backup original resolv.conf sudo cp /etc/resolv.conf /etc/resolv.conf.backup # Add both local and Google DNS servers echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf wait_for_service: description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)" parameters: url: type: string timeout: type: string default: "60" steps: - run: name: "Wait for << parameters.url >>" command: | TIMEOUT=<< parameters.timeout >> URL="<< parameters.url >>" ELAPSED=0 echo "Waiting up to ${TIMEOUT}s for ${URL} ..." if echo "$URL" | grep -q '^tcp://'; then HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1) PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2) while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do sleep 2; ELAPSED=$((ELAPSED+2)) if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi done else while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do sleep 2; ELAPSED=$((ELAPSED+2)) if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi done fi echo "Service ready after ${ELAPSED}s" install_helm: steps: - run: name: Install Helm v3.17.3 command: | curl -sSLf -o /tmp/helm.tar.gz \ https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c - sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm rm -f /tmp/helm.tar.gz install_kind: steps: - run: name: Install Kind v0.20.0 command: | curl -sSLf -o /tmp/kind \ https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c - chmod +x /tmp/kind sudo mv /tmp/kind /usr/local/bin/kind install_uv: description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH." steps: - run: name: Install uv (pinned 0.10.9) command: | curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh rm -f /tmp/uv-install.sh echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" start_postgres: description: "Start a postgres-db container on port 5432 and wait until it accepts connections." parameters: db_name: type: string default: circle_test steps: - run: name: Start PostgreSQL command: | docker run -d \ --name postgres-db \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_DB=<< parameters.db_name >> \ -p 5432:5432 \ postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26 - wait_for_service: url: tcp://localhost:5432 timeout: "60" start_redis: description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys." steps: - run: name: Start Redis command: | docker run -d \ --name redis-cache \ -p 6379:6379 \ redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 - wait_for_service: url: tcp://localhost:6379 timeout: "60" start_openai_record_replay_proxy: description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." steps: - run: name: Start record/replay proxy background: true command: | CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \ RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \ uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090 - run: name: Wait for record/replay proxy command: | for i in $(seq 1 30); do if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then echo "record/replay proxy is up" exit 0 fi sleep 1 done echo "record/replay proxy did not become ready" >&2 exit 1 start_fake_openai_endpoint: description: "Start the canned OpenAI mock (tests/_fake_openai_endpoint_server.py) on host port 8190 and wait until healthy. Models whose api_base points here (via FAKE_OPENAI_API_BASE) get well-formed chat/text/embedding responses with realistic usage, so the E2E run neither pays for nor depends on the live provider. A request whose model is '429' returns HTTP 429 for rate-limit/cooldown tests. Run after uv deps are synced." steps: - run: name: Start fake OpenAI endpoint background: true command: | uv run --no-sync python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 - run: name: Wait for fake OpenAI endpoint command: | for i in $(seq 1 30); do if curl -sf http://localhost:8190/health >/dev/null 2>&1; then echo "fake OpenAI endpoint is up" exit 0 fi sleep 1 done echo "fake OpenAI endpoint did not become ready" >&2 exit 1 setup_litellm_enterprise_pip: steps: - run: name: "Install local version of litellm-enterprise" command: | # litellm-enterprise is a uv workspace member and is already installed # by the main `uv sync --all-groups --all-extras`. Do NOT run # `uv sync --package litellm-enterprise` here — that overwrites the # shared .venv and strips out dev/test deps (pytest, prisma, etc.). uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" setup_litellm_test_deps: steps: - checkout - setup_google_dns - install_uv - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} jobs: # Add Windows testing job using_litellm_on_windows: executor: name: win/default shell: powershell.exe working_directory: ~/project environment: UV_PYTHON: "3.11" CARGO_HTTP_MULTIPLEXING: "false" CARGO_NET_RETRY: "5" steps: - checkout - run: name: Install Python command: | choco install python --version=3.11.0 -y --no-progress --force refreshenv python --version environment: CHOCOLATEY_CONFIRM_ALL: "true" - run: name: Install Dependencies environment: UV_HTTP_TIMEOUT: "300" command: | $rustupInit = Join-Path $env:TEMP "rustup-init.exe" $rustupVersion = "1.28.2" $rustupUrl = "https://static.rust-lang.org/rustup/archive/$rustupVersion/x86_64-pc-windows-msvc/rustup-init.exe" Invoke-WebRequest -Uri $rustupUrl -OutFile $rustupInit $rustupExpected = "88d8258dcf6ae4f7a80c7d1088e1f36fa7025a1cfd1343731b4ee6f385121fc0" $rustupActual = (Get-FileHash -Path $rustupInit -Algorithm SHA256).Hash.ToLower() if ($rustupActual -ne $rustupExpected) { throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual" } & $rustupInit -y --profile minimal --default-toolchain stable if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Remove-Item $rustupInit $cargoBin = Join-Path $HOME ".cargo\bin" $env:Path = "$cargoBin;$env:Path" rustc --version cargo --version $installer = Join-Path $env:TEMP "uv-install.ps1" Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer $expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d" $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() if ($actual -ne $expected) { throw "uv installer hash mismatch: expected $expected got $actual" } & $installer Remove-Item $installer $uvBin = Join-Path $HOME ".local\bin" $env:Path = "$uvBin;$env:Path" if (!(Test-Path $PROFILE)) { New-Item -ItemType File -Force -Path $PROFILE | Out-Null } if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) { Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`"" } if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) { Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`"" } for ($attempt = 1; $attempt -le 5; $attempt++) { Write-Host "uv sync attempt $attempt/5" uv sync --frozen --group dev --python 3.11 if ($LASTEXITCODE -eq 0) { break } if ($attempt -eq 5) { exit $LASTEXITCODE } Start-Sleep -Seconds 15 } - run: name: Run Windows-specific test command: | uv run --no-sync python -m pytest tests/windows_tests/ -v - run: name: Guard against MAX_PATH-busting packaged wheel paths environment: UV_HTTP_TIMEOUT: "300" command: | $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path" cargo --version Get-ChildItem -Path "litellm\rust_bridge" -Filter "_native*" -File -ErrorAction SilentlyContinue | Remove-Item -Force uv build --wheel --out-dir dist uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py local_testing_part1: docker: - &python312_image image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project parallelism: 4 steps: - checkout - setup_google_dns - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Run prisma ./docker/entrypoint.sh command: | set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e # Run pytest and generate JUnit XML report - run: name: Run tests (Part 1 - A-M) command: | mkdir test-results # Discover test files (A-M) TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm \ --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=20 \ -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ -n 4 \ --timeout=300 \ --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | # When CI reruns only the failed tests, a parallel node can receive # zero tests and pytest never writes coverage. Emit empty placeholders # so persist_to_workspace and the downstream coverage combine stay green. if [ -f coverage.xml ]; then mv coverage.xml local_testing_part1_coverage.xml mv .coverage local_testing_part1_coverage else touch local_testing_part1_coverage.xml local_testing_part1_coverage fi # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - local_testing_part1_coverage.xml - local_testing_part1_coverage local_testing_part2: docker: - *python312_image working_directory: ~/project parallelism: 4 steps: - checkout - setup_google_dns - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Run prisma ./docker/entrypoint.sh command: | set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e # Run pytest and generate JUnit XML report - run: name: Run tests (Part 2 - N-Z) command: | mkdir test-results # Discover test files (N-Z) TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm \ --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=20 \ -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ -n 4 \ --timeout=300 \ --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | # When CI reruns only the failed tests, a parallel node can receive # zero tests and pytest never writes coverage. Emit empty placeholders # so persist_to_workspace and the downstream coverage combine stay green. if [ -f coverage.xml ]; then mv coverage.xml local_testing_part2_coverage.xml mv .coverage local_testing_part2_coverage else touch local_testing_part2_coverage.xml local_testing_part2_coverage fi # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - local_testing_part2_coverage.xml - local_testing_part2_coverage langfuse_logging_unit_tests: docker: - *python312_image working_directory: ~/project resource_class: medium steps: - checkout - setup_google_dns - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Run prisma ./docker/entrypoint.sh command: | set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh set -e # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit.xml \ --durations=5 \ -k \"langfuse\"" no_output_timeout: 15m # Store test results - store_test_results: path: test-results auth_ui_unit_tests: docker: - *python312_image - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: litellm_test working_directory: ~/project environment: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} - wait_for_service: url: tcp://localhost:5432 timeout: "60" - run: name: Seed DB schema via prisma db push command: | set +e uv run --no-sync litellm --skip_server_startup --use_prisma_db_push set -e - run: name: Generate Prisma Client command: uv run --no-sync python -m prisma generate # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml auth_ui_unit_tests_coverage.xml mv .coverage auth_ui_unit_tests_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - auth_ui_unit_tests_coverage.xml - auth_ui_unit_tests_coverage proxy_behavior_tests: docker: - *python312_image - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: litellm_test working_directory: ~/project environment: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - wait_for_service: url: tcp://localhost:5432 timeout: "60" - run: name: Seed DB schema via prisma db push command: | uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - run: name: Generate Prisma Client command: uv run --no-sync python -m prisma generate - run: name: Run proxy management behavior tests command: | mkdir -p test-results uv run --no-sync python -m pytest tests/proxy_behavior \ -v --junitxml=test-results/junit.xml --durations=10 no_output_timeout: 15m - store_test_results: path: test-results proxy_security_tests: docker: - *python312_image - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: litellm_test working_directory: ~/project environment: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - wait_for_service: url: tcp://localhost:5432 timeout: "60" - run: name: Seed DB schema via prisma db push command: | uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - run: name: Generate Prisma Client command: uv run --no-sync python -m prisma generate - run: name: Run proxy security tests command: | mkdir -p test-results uv run --no-sync python -m pytest tests/proxy_security_tests \ -v --junitxml=test-results/junit.xml --durations=10 no_output_timeout: 15m - store_test_results: path: test-results schema_migration_check: docker: - *python312_image - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: litellm_test working_directory: ~/project environment: # An empty database; the test applies every committed migration itself. DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - wait_for_service: url: tcp://localhost:5432 timeout: "60" - run: name: Generate Prisma Client command: uv run --no-sync python -m prisma generate - run: name: Check schema.prisma is in sync with committed migrations command: | mkdir -p test-results uv run --no-sync python -m pytest tests/proxy_migration_tests \ -v --junitxml=test-results/junit.xml --durations=10 no_output_timeout: 15m - store_test_results: path: test-results litellm_router_testing: # Runs all tests with the "router" keyword docker: - *python312_image working_directory: ~/project resource_class: large parallelism: 4 environment: FAKE_OPENAI_API_BASE: http://127.0.0.1:8190 steps: - checkout - setup_google_dns - install_uv - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} - start_fake_openai_endpoint # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | # On a "rerun failed tests" build a parallel node can receive no # tests, so the test command never creates test-results. Pre-create it # so store_test_results doesn't fail the node on a missing path. mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ -k 'router' \ -n 4 \ --junitxml=test-results/junit.xml \ --durations=5 \ --timeout=300 --timeout_method=thread" no_output_timeout: 15m # Store test results - store_test_results: path: test-results litellm_router_unit_testing: # Runs all tests with the "router" keyword docker: - *python312_image working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns - install_uv - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml router_unit_tests_coverage.xml mv .coverage router_unit_tests_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - router_unit_tests_coverage.xml - router_unit_tests_coverage litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - *python312_image working_directory: ~/project resource_class: medium steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit.xml \ --durations=5 \ -k \"assistants\"" no_output_timeout: 15m # Store test results - store_test_results: path: test-results llm_translation_testing: docker: - *python312_image working_directory: ~/project resource_class: xlarge steps: - checkout - setup_google_dns - install_uv - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging # Subdirectories with dedicated jobs (maintain this list as new jobs are added) mkdir -p test-results # Glob excludes the realtime/ subdirectory since it has its own job TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit.xml \ --durations=20 \ -n 4 \ --timeout=120 --timeout_method=thread \ --retries 2 --retry-delay 5 \ --max-worker-restart=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results realtime_translation_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run realtime tests command: | # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=20 \ -n 4 \ --timeout=120 --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml realtime_translation_coverage.xml mv .coverage realtime_translation_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - realtime_translation_coverage.xml - realtime_translation_coverage agent_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml agent_coverage.xml mv .coverage agent_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - agent_coverage.xml - agent_coverage guardrails_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results export LITELLM_LOG=WARNING TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2 \ --timeout=120 --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml guardrails_coverage.xml mv .coverage guardrails_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - guardrails_coverage.xml - guardrails_coverage google_generate_content_endpoint_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ --retries 3 --retry-delay 5" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml google_generate_content_endpoint_coverage.xml mv .coverage google_generate_content_endpoint_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - google_generate_content_endpoint_coverage.xml - google_generate_content_endpoint_coverage llm_responses_api_testing: docker: - *python312_image working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns - install_uv - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 8" no_output_timeout: 15m # Store test results - store_test_results: path: test-results ocr_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(printf "%s\n%s\n" \ "$(circleci tests glob "tests/ocr_tests/**/test_*.py")" \ "tests/test_litellm/ocr/test_rust_bridge.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml ocr_coverage.xml mv .coverage ocr_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - ocr_coverage.xml - ocr_coverage search_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml search_coverage.xml mv .coverage search_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - search_coverage.xml - search_coverage litellm_mapped_enterprise_tests: docker: - *python312_image working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - run: name: Run enterprise tests command: | uv run --no-sync python -m prisma generate mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit-enterprise.xml \ --durations=10 \ -n 4" no_output_timeout: 15m # Store test results - store_test_results: path: test-results batches_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml batches_coverage.xml mv .coverage batches_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - batches_coverage.xml - batches_coverage litellm_utils_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 2" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml litellm_utils_coverage.xml mv .coverage litellm_utils_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - litellm_utils_coverage.xml - litellm_utils_coverage pass_through_unit_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml pass_through_unit_tests_coverage.xml mv .coverage pass_through_unit_tests_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - pass_through_unit_tests_coverage.xml - pass_through_unit_tests_coverage image_gen_testing: docker: - *python312_image working_directory: ~/project resource_class: large steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit.xml \ --durations=5 \ -n 4" no_output_timeout: 15m # Store test results - store_test_results: path: test-results logging_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: name: Run tests command: | mkdir -p test-results export LITELLM_LOG=WARNING TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov-report=xml \ -n 4 \ --junitxml=test-results/junit.xml \ --durations=5 \ --timeout=120 --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml logging_coverage.xml || true mv .coverage logging_coverage || true # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - logging_coverage.xml - logging_coverage audio_testing: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m - run: name: Rename the coverage files command: | mv coverage.xml audio_coverage.xml mv .coverage audio_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - audio_coverage.xml - audio_coverage redis_caching_unit_tests: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - save_cache: paths: - ~/.cache/uv key: v1-uv-cache-{{ checksum "uv.lock" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(printf "%s\n" \ tests/local_testing/test_dual_cache.py \ tests/local_testing/test_redis_batch_optimizations.py \ tests/local_testing/test_router_utils.py) echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ --durations=5 -n 2 \ --reruns 2 --reruns-delay 1" no_output_timeout: 20m - run: name: Rename the coverage files command: | mv coverage.xml redis_caching_coverage.xml mv .coverage redis_caching_coverage # Store test results - store_test_results: path: test-results - persist_to_workspace: root: . paths: - redis_caching_coverage.xml - redis_caching_coverage installing_litellm_on_python: docker: - *python312_image working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - run: name: Run tests command: | uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver" installing_litellm_on_python_3_13: docker: - image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project resource_class: medium steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.13 - run: name: Run tests command: | uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver" installing_litellm_on_python_v2_migration_resolver: docker: - *python312_image - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: litellm_test working_directory: ~/project environment: DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - setup_litellm_enterprise_pip - wait_for_service: url: tcp://localhost:5432 timeout: "60" - run: name: Run v2 migration resolver proxy smoke test command: | uv run --no-sync python -m pytest -vv \ tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver helm_chart_testing: machine: image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker resource_class: medium working_directory: ~/project steps: - checkout - attach_workspace: at: ~/project - setup_google_dns - install_helm - install_kind # Install kubectl (pinned version with hardcoded checksum) - run: name: Install kubectl v1.31.4 command: | curl -sSLf -o /tmp/kubectl \ https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c - chmod +x /tmp/kubectl sudo mv /tmp/kubectl /usr/local/bin/ # Create kind cluster - run: name: Create Kind Cluster command: | kind create cluster --name litellm-test - run: name: Load Docker Database Image for helm tests command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load IMAGE_TAG=${CIRCLE_SHA1:-ci} docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG} - run: name: Load Docker image into Kind command: | IMAGE_TAG=${CIRCLE_SHA1:-ci} kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test # Run helm lint - run: name: Run helm lint command: | helm lint ./deploy/charts/litellm-helm # Run helm tests - run: name: Run helm tests command: | IMAGE_TAG=${CIRCLE_SHA1:-ci} helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \ --set image.repository=litellm-ci \ --set image.tag=${IMAGE_TAG} \ --set image.pullPolicy=Never # Wait for pod to be ready echo "Waiting 30 seconds for pod to be ready..." sleep 30 # Print pod logs before running tests echo "Printing pod logs..." kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") # Run the helm tests helm test litellm --logs # Cleanup - run: name: Cleanup command: | kind delete cluster --name litellm-test when: always # This ensures cleanup runs even if previous steps fail db_migration_disable_update_check: machine: image: ubuntu-2204:2024.04.1 resource_class: medium working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_fake_openai_endpoint - start_postgres: db_name: litellm_test - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Seed database with real schema command: | docker run -d \ -p 4001:4000 \ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e LITELLM_MASTER_KEY="sk-1234" \ --name schema-seed \ --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --use_prisma_db_push - wait_for_service: url: http://localhost:4001 timeout: "300" - run: name: Stop schema seed container command: docker stop schema-seed && docker rm schema-seed - run: name: Run Docker container with bad schema and disabled updates command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DISABLE_SCHEMA_UPDATE="True" \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ --name my-app \ --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - wait_for_service: url: http://localhost:4000 timeout: "60" - run: name: Check container logs for expected message command: | echo "=== Printing Full Container Startup Logs ===" LOG_OUTPUT="$(docker logs my-app 2>&1)" printf '%s\n' "$LOG_OUTPUT" echo "=== End of Full Container Startup Logs ===" if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then echo "Expected message found in logs. Test passed." else echo "Expected message not found in logs. Test failed." exit 1 fi - run: name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit-2.xml \ --durations=5" no_output_timeout: 15m - store_test_results: path: test-results build_and_test: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - attach_workspace: at: ~/project - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker tag litellm-docker-database:ci my-app:latest - start_openai_record_replay_proxy - start_fake_openai_endpoint - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e USE_PRISMA_MIGRATE=True \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e AZURE_API_KEY=$AZURE_API_KEY \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e GROQ_API_KEY=$GROQ_API_KEY \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e AUTO_INFER_REGION=True \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ -e RECORDER_OPENAI_BASE_URL=http://host.docker.internal:8090/v1 \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ my-app:latest \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results # Original used `tests/*.py` (top-level only); the `--ignore=...` # flags were vestigial since shell globbing did not descend into # subdirectories. Replicate by globbing only top-level test files. TEST_FILES=$(circleci tests glob "tests/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -s -v -x \ --junitxml=test-results/junit.xml \ -n 4 \ --durations=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results e2e_openai_endpoints: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e AZURE_API_KEY=$AZURE_API_KEY \ -e AZURE_API_BASE=$AZURE_API_BASE \ -e AZURE_API_VERSION="2024-05-01-preview" \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e GROQ_API_KEY=$GROQ_API_KEY \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e AUTO_INFER_REGION=True \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -s -vv \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results proxy_logging_guardrails_model_info_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - start_openai_record_replay_proxy - start_fake_openai_endpoint - run: name: Run Docker container # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e RECORDER_COHERE_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.cohere.com \ -e GCS_FLUSH_INTERVAL="1" \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \ -v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m # Clean up first container - run: name: Stop and remove first container command: | docker stop my-app docker rm my-app # Second Docker Container Run with Different Config # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license - run: name: Run Second Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE="bad-license" \ --add-host host.docker.internal:host-gateway \ --name my-app-3 \ -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug - run: name: Start outputting logs for second container command: docker logs -f my-app-3 background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run second round of tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit-2.xml \ --durations=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results proxy_spend_accuracy_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - start_redis - start_fake_openai_endpoint - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container # Point the proxy at the job-local Redis (start_redis) instead of the # shared remote Redis. The Redis transaction buffer uses a single # global pod-lock key (cronjob_lock:db_spend_update_job) and a single # global buffer list (litellm_spend_update_buffer); sharing those # across concurrent CI pipelines causes spend flushes to stall or # land in the wrong DB, which is what makes this test flaky. command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=host.docker.internal \ -e REDIS_PORT=6379 \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e PROXY_BATCH_WRITE_AT=2 \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m - store_test_results: path: test-results - run: name: Stop and remove first container when: always command: | docker stop my-app docker rm my-app docker stop redis-cache docker rm redis-cache proxy_multi_instance_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - start_fake_openai_endpoint - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container 1 # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Run Docker container 2 command: | docker run -d \ -p 4001:4001 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ --add-host host.docker.internal:host-gateway \ --name my-app-2 \ -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4001 \ --detailed_debug - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - wait_for_service: url: http://localhost:4001 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m # Clean up first container # Store test results - store_test_results: path: test-results proxy_store_model_in_db_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - start_fake_openai_endpoint - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e STORE_MODEL_IN_DB="True" \ -e LITELLM_MASTER_KEY="sk-1234" \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m - run: name: Stop and remove containers command: | docker stop my-app || true docker rm my-app || true docker stop postgres-db || true docker rm postgres-db || true when: always - store_test_results: path: test-results proxy_build_from_pip_tests: # Change from docker to machine executor machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns # Remove Docker CLI installation since it's already available in machine executor - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - run: name: Build Docker image command: | docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . - start_postgres - start_fake_openai_endpoint - run: name: Run Docker container # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e GCS_FLUSH_INTERVAL="1" \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ my-app:latest \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --junitxml=test-results/junit-2.xml \ --durations=5" no_output_timeout: 15m # Clean up first container - store_test_results: path: test-results - run: name: Stop and remove first container command: | docker stop my-app || true docker rm my-app || true docker stop postgres-db || true docker rm postgres-db || true when: always proxy_pass_through_endpoint_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e GEMINI_API_KEY=$GEMINI_API_KEY \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \ -e AZURE_API_KEY=$AZURE_API_KEY \ -e AZURE_API_BASE=$AZURE_API_BASE \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" # Add Ruby installation and testing before the existing Node.js and Python tests - run: name: Install Ruby and Bundler command: | # Clone RVM at pinned tag and verify the commit SHA matches the # published tag before running its install script. RVM_VERSION="1.29.12" RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81" git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)" if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2 exit 1 fi # Import RVM signing keys (used by `rvm install` to verify Ruby tarballs) gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB # Install RVM from the verified checkout. The install script # sources `scripts/functions/installer` using paths relative to # its own working directory, so it must be run from /tmp/rvm. (cd /tmp/rvm && ./install --path "$HOME/.rvm") source "$HOME/.rvm/scripts/rvm" # Install Ruby 3.2.2 (RVM verifies the tarball PGP signature) rvm install 3.2.2 rvm use 3.2.2 --default # Install latest Bundler gem install bundler - run: name: Run Ruby tests command: | source $HOME/.rvm/scripts/rvm cd tests/pass_through_tests/ruby_passthrough_tests bundle install bundle exec rspec no_output_timeout: 30m # Install Node.js directly from nodejs.org with SHA256 verification, # instead of piping NodeSource's setup_18.x apt-repo installer into # sudo bash (which runs a mutable upstream script unattended). - run: name: Install Node.js 18.20.8 command: | NODE_VERSION="18.20.8" NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz" NODE_EXPECTED_SHA="5467ee62d6af1411d46b6a10e3fb5cacc92734dbcef465fea14e7b90993001c9" curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c - sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /usr/local --strip-components=1 rm -f "/tmp/${NODE_TARBALL}" node --version npm --version - run: name: Install Node.js test dependencies command: | cd tests/pass_through_tests npm ci - run: name: Run Vertex AI, Google AI Studio Node.js tests command: | cd tests/pass_through_tests NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose no_output_timeout: 30m - run: name: Run tests command: | mkdir -p test-results TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results proxy_e2e_anthropic_messages_tests: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - setup_google_dns - install_uv - run: name: Install Dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 - start_postgres - attach_workspace: at: ~/project - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - start_openai_record_replay_proxy - run: name: Run Docker container with test config command: | docker run -d \ -p 4000:4000 \ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e LITELLM_MASTER_KEY="sk-1234" \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e RECORDER_ANTHROPIC_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.anthropic.com \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME="us-east-1" \ -e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug - run: name: Start outputting logs command: docker logs -f my-app background: true - wait_for_service: url: http://localhost:4000 timeout: "300" - run: name: Run Claude Agent SDK E2E Tests command: | mkdir -p test-results export LITELLM_PROXY_URL="http://localhost:4000" export LITELLM_API_KEY="sk-1234" TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --junitxml=test-results/junit.xml \ --durations=5" no_output_timeout: 15m # Store test results - store_test_results: path: test-results upload-coverage: docker: - *python312_image steps: - checkout - attach_workspace: at: . # Check file locations - run: name: Check coverage file location command: | echo "Current directory:" ls -la echo "\nContents of tests/llm_translation:" ls -la tests/llm_translation - install_uv - run: name: Combine Coverage command: | uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage uv tool run --from 'coverage[toml]==7.10.6' coverage xml - codecov/upload: file: ./coverage.xml flags: circleci ui_build: docker: - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} resource_class: medium+ working_directory: ~/project steps: - checkout - setup_google_dns - restore_cache: keys: - ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - ui-build-deps-v1- - restore_cache: keys: - ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - ui-nextjs-cache-v1- - run: name: Install dependencies command: | cd ui/litellm-dashboard npm ci - save_cache: key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} paths: - ui/litellm-dashboard/node_modules - run: name: Build UI command: | cd ui/litellm-dashboard source ./build_ui.sh - save_cache: key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} paths: - ui/litellm-dashboard/.next/cache - persist_to_workspace: root: . paths: - litellm/proxy/_experimental/out ui_unit_tests: docker: - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} resource_class: xlarge working_directory: ~/project steps: - checkout - setup_google_dns - restore_cache: keys: - ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - ui-unit-deps-v1- - run: name: Install dependencies command: | cd ui/litellm-dashboard npm ci - save_cache: key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} paths: - ui/litellm-dashboard/node_modules - run: name: Run UI unit tests (Vitest) command: | cd ui/litellm-dashboard CI=true npm run test -- --run \ --pool forks --poolOptions.forks.maxForks=8 e2e_ui_testing: docker: - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: e2euser POSTGRES_PASSWORD: e2epassword POSTGRES_DB: litellm_e2e resource_class: large working_directory: ~/project environment: DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" CI: "true" # Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can # assert the redirect. Set at job level so both the proxy boot step and the # Playwright step (whose skip guard reads this) see the same value. Safe for # the rest of the suite: nothing else performs a logout. PROXY_LOGOUT_URL: "https://www.example.com" steps: - checkout - setup_google_dns - install_uv - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Python dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - save_cache: key: v1-uv-cache-{{ checksum "uv.lock" }} paths: - ~/.cache/uv - restore_cache: keys: - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - run: name: Install Node dependencies and Playwright # The cimg/python:3.12-browsers image already ships the Chromium system # libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.). # `--with-deps` triggers a redundant apt-get update + install that adds # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors, # so we install just the browser binary. command: | cd ui/litellm-dashboard npm ci npx playwright install chromium - save_cache: key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} paths: - ui/litellm-dashboard/node_modules - ~/.cache/ms-playwright - run: name: Build UI from source # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`. # GNU cp (used on CircleCI's Ubuntu image) interprets that as "copy the # source directory as a child of the destination" when the destination # already exists — silently creating `_experimental/out/out/` instead of # replacing the served bundle. The proxy continued serving whatever was # checked into `_experimental/out/*`, so this job was effectively testing # the pre-build bundle on every run. Replace-and-move guarantees the # freshly built bundle is what the proxy actually serves. command: | cd ui/litellm-dashboard npm run build rm -rf ../../litellm/proxy/_experimental/out mv out ../../litellm/proxy/_experimental/out # Restructure HTML so extensionless routes work (login.html -> login/index.html) find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" done - wait_for_service: url: tcp://localhost:5432 timeout: "30" - run: name: Push Prisma schema command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - run: name: Seed database command: | PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql - run: name: Start mock LLM server command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py background: true - run: name: Start LiteLLM proxy environment: LITELLM_MASTER_KEY: "sk-1234" MOCK_LLM_URL: "http://127.0.0.1:8090/v1" DISABLE_SCHEMA_UPDATE: "true" SERVER_ROOT_PATH: "" # PROXY_LOGOUT_URL is inherited from the job-level environment so the # proxy and proxyLogoutUrl.spec.ts agree on the logout target. # LITELLM_LICENSE is forwarded from the project env so premium-gated # UI flows can be exercised. license.spec.ts asserts the resulting # JWT carries premium_user=true; if it ever stops being passed, that # test fails loudly rather than silently regressing premium coverage. command: | LITELLM_LICENSE="$LITELLM_LICENSE" \ uv run --no-sync python -m litellm.proxy.proxy_cli \ --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ --port 4000 background: true - run: name: Wait for proxy to be ready command: | for i in $(seq 1 60); do HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true) if [ "$HTTP_CODE" = "200" ]; then echo "Proxy is ready" exit 0 fi sleep 2 done echo "Proxy failed to start" exit 1 - run: name: Run Playwright E2E tests # Forward LITELLM_LICENSE so license.spec.ts can detect that the # proxy was launched with a license and assert premium_user=true. command: | cd ui/litellm-dashboard LITELLM_LICENSE="$LITELLM_LICENSE" \ npx playwright test --config e2e_tests/playwright.config.ts no_output_timeout: 10m - store_artifacts: path: ui/litellm-dashboard/test-results destination: e2e-test-results - store_artifacts: path: ui/litellm-dashboard/playwright-report destination: e2e-playwright-report e2e_ui_testing_server_root_path: docker: - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 environment: POSTGRES_USER: e2euser POSTGRES_PASSWORD: e2epassword POSTGRES_DB: litellm_e2e resource_class: large working_directory: ~/project environment: DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" CI: "true" # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH # is read both by the proxy at boot (to rewrite the built UI bundle in place) # and by migration.serverRootPath.config.ts, which refuses to run without it. SERVER_ROOT_PATH: "/litellm" steps: - checkout - setup_google_dns - install_uv - restore_cache: keys: - v1-uv-cache-{{ checksum "uv.lock" }} - run: name: Install Python dependencies command: | uv sync --frozen --all-groups --all-extras --python 3.12 uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - save_cache: key: v1-uv-cache-{{ checksum "uv.lock" }} paths: - ~/.cache/uv - restore_cache: keys: - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - run: name: Install Node dependencies and Playwright command: | cd ui/litellm-dashboard npm ci npx playwright install chromium - save_cache: key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} paths: - ui/litellm-dashboard/node_modules - ~/.cache/ms-playwright - run: name: Build UI from source command: | cd ui/litellm-dashboard npm run build rm -rf ../../litellm/proxy/_experimental/out mv out ../../litellm/proxy/_experimental/out find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" done - wait_for_service: url: tcp://localhost:5432 timeout: "30" - run: name: Push Prisma schema command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - run: name: Seed database command: | PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql - run: name: Start mock LLM server command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py background: true - run: name: Start LiteLLM proxy under a server root path environment: LITELLM_MASTER_KEY: "sk-1234" MOCK_LLM_URL: "http://127.0.0.1:8090/v1" DISABLE_SCHEMA_UPDATE: "true" # Output flows to this step's own log, so a boot crash is visible here # rather than swallowed by a downstream readiness probe. command: | LITELLM_LICENSE="$LITELLM_LICENSE" \ uv run --no-sync python -m litellm.proxy.proxy_cli \ --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ --port 4000 background: true - run: name: Wait for prefixed proxy to be ready command: | for i in $(seq 1 60); do HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer sk-1234" http://127.0.0.1:4000/litellm/health 2>/dev/null || true) if [ "$HTTP_CODE" = "200" ]; then echo "Prefixed proxy is ready" exit 0 fi sleep 2 done echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log" exit 1 - run: name: Run migration smoke under SERVER_ROOT_PATH command: | cd ui/litellm-dashboard LITELLM_LICENSE="$LITELLM_LICENSE" \ npx playwright test --config e2e_tests/migration.serverRootPath.config.ts no_output_timeout: 10m - store_artifacts: path: ui/litellm-dashboard/test-results destination: e2e-server-root-path-test-results - store_artifacts: path: ui/litellm-dashboard/playwright-report destination: e2e-server-root-path-playwright-report build_docker_database_image: machine: image: ubuntu-2204:2024.04.1 resource_class: large working_directory: ~/project steps: - checkout - run: name: Build Docker image command: | docker build \ -t litellm-docker-database:ci \ -f docker/Dockerfile.database . - run: name: Save Docker image to workspace root command: | docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst - persist_to_workspace: root: . paths: - litellm-docker-database.tar.zst test_bad_database_url: machine: image: ubuntu-2204:2024.04.1 resource_class: medium working_directory: ~/project steps: - checkout - attach_workspace: at: ~/project - setup_google_dns - start_postgres - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker tag litellm-docker-database:ci myapp:latest - run: name: Run Docker container with bad DATABASE_URL command: | docker run --name my-app \ -p 4000:4000 \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ myapp:latest \ --port 4000 > docker_output.log 2>&1 || true - run: name: Display Docker logs command: cat docker_output.log - run: name: Check for expected error command: | if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \ (grep -q "Database setup failed after multiple retries" docker_output.log || \ grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then echo "Expected error found. Test passed." else echo "Expected error not found. Test failed." cat docker_output.log exit 1 fi workflows: build_and_test: jobs: - using_litellm_on_windows: filters: &main_branches branches: only: - main - /litellm_.*/ - local_testing_part1: filters: *main_branches - local_testing_part2: filters: *main_branches - langfuse_logging_unit_tests: filters: *main_branches - litellm_assistants_api_testing: filters: *main_branches - litellm_router_testing: filters: *main_branches - litellm_router_unit_testing: filters: *main_branches - ui_build: filters: *main_branches - ui_unit_tests: requires: - ui_build filters: *main_branches - auth_ui_unit_tests: filters: *main_branches - proxy_behavior_tests: filters: *main_branches - proxy_security_tests: filters: *main_branches - schema_migration_check: filters: *main_branches - build_docker_database_image: filters: *main_branches - e2e_ui_testing: filters: *main_branches - e2e_ui_testing_server_root_path: filters: *main_branches - build_and_test: requires: - build_docker_database_image filters: *main_branches - e2e_openai_endpoints: requires: - build_docker_database_image filters: *main_branches - proxy_logging_guardrails_model_info_tests: requires: - build_docker_database_image filters: *main_branches - proxy_spend_accuracy_tests: requires: - build_docker_database_image filters: *main_branches - proxy_multi_instance_tests: requires: - build_docker_database_image filters: *main_branches - proxy_store_model_in_db_tests: requires: - build_docker_database_image filters: *main_branches - proxy_build_from_pip_tests: filters: *main_branches - proxy_pass_through_endpoint_tests: requires: - build_docker_database_image filters: *main_branches - proxy_e2e_anthropic_messages_tests: requires: - build_docker_database_image filters: *main_branches - llm_translation_testing: filters: *main_branches - realtime_translation_testing: filters: *main_branches - agent_testing: filters: *main_branches - guardrails_testing: filters: *main_branches - google_generate_content_endpoint_testing: filters: *main_branches - llm_responses_api_testing: filters: *main_branches - ocr_testing: filters: *main_branches - search_testing: filters: *main_branches - litellm_mapped_enterprise_tests: filters: *main_branches - batches_testing: filters: *main_branches - litellm_utils_testing: filters: *main_branches - pass_through_unit_testing: filters: *main_branches - image_gen_testing: filters: *main_branches - logging_testing: filters: *main_branches - audio_testing: filters: *main_branches - redis_caching_unit_tests: filters: *main_branches - upload-coverage: requires: - realtime_translation_testing - agent_testing - google_generate_content_endpoint_testing - guardrails_testing - ocr_testing - search_testing - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing - pass_through_unit_testing - image_gen_testing - logging_testing - audio_testing - redis_caching_unit_tests - langfuse_logging_unit_tests - local_testing_part1 - local_testing_part2 - litellm_assistants_api_testing - litellm_router_unit_testing - auth_ui_unit_tests - db_migration_disable_update_check: requires: - build_docker_database_image filters: *main_branches - installing_litellm_on_python: filters: *main_branches - installing_litellm_on_python_3_13: filters: *main_branches - installing_litellm_on_python_v2_migration_resolver: filters: *main_branches - helm_chart_testing: requires: - build_docker_database_image filters: *main_branches - test_bad_database_url: requires: - build_docker_database_image filters: *main_branches