From 4b55eacdd3620ac1e4caebc13f708ace72b957ec Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:19:55 +0000 Subject: [PATCH] ci: only run CircleCI and GitHub Actions unit tests on code changes Gate the CircleCI build_and_test workflow behind a path-filtering setup pipeline so it only continues when .py/.ts/.tsx files changed, and add matching pull_request paths filters to the GitHub Actions unit-test workflows. Docs-only and other non-code PRs no longer spin up the full test matrices. --- .circleci/config.yml | 3069 +---------------- .circleci/continue_config.yml | 3069 +++++++++++++++++ .github/workflows/test-unit-core-utils.yml | 9 + .github/workflows/test-unit-documentation.yml | 9 + .../test-unit-enterprise-routing.yml | 9 + .github/workflows/test-unit-integrations.yml | 9 + .github/workflows/test-unit-llm-providers.yml | 9 + .github/workflows/test-unit-misc.yml | 9 + .github/workflows/test-unit-proxy-auth.yml | 9 + .github/workflows/test-unit-proxy-db.yml | 9 + .../workflows/test-unit-proxy-endpoints.yml | 9 + .github/workflows/test-unit-proxy-infra.yml | 9 + .github/workflows/test-unit-proxy-legacy.yml | 9 + .../test-unit-responses-caching-types.yml | 9 + 14 files changed, 3189 insertions(+), 3057 deletions(-) create mode 100644 .circleci/continue_config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index abcdbf45187..9732326bba9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,3062 +1,17 @@ version: 2.1 + +setup: true + orbs: - codecov: codecov/codecov@4.0.1 - node: circleci/node@5.1.0 # Add this line to declare the node orb - win: circleci/windows@5.0 # Add Windows orb - -commands: - setup_google_dns: - steps: - - run: - name: "Configure Google DNS" - command: | - # Backup original resolv.conf - sudo cp /etc/resolv.conf /etc/resolv.conf.backup - # Add both local and Google DNS servers - echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf - echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf - echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf - wait_for_service: - description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)" - parameters: - url: - type: string - timeout: - type: string - default: "60" - steps: - - run: - name: "Wait for << parameters.url >>" - command: | - TIMEOUT=<< parameters.timeout >> - URL="<< parameters.url >>" - ELAPSED=0 - echo "Waiting up to ${TIMEOUT}s for ${URL} ..." - if echo "$URL" | grep -q '^tcp://'; then - HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1) - PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2) - while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do - sleep 2; ELAPSED=$((ELAPSED+2)) - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi - done - else - while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do - sleep 2; ELAPSED=$((ELAPSED+2)) - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi - done - fi - echo "Service ready after ${ELAPSED}s" - install_helm: - steps: - - run: - name: Install Helm v3.17.3 - command: | - curl -sSLf -o /tmp/helm.tar.gz \ - https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz - echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c - - sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm - rm -f /tmp/helm.tar.gz - install_kind: - steps: - - run: - name: Install Kind v0.20.0 - command: | - curl -sSLf -o /tmp/kind \ - https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c - - chmod +x /tmp/kind - sudo mv /tmp/kind /usr/local/bin/kind - install_uv: - description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH." - steps: - - run: - name: Install uv (pinned 0.10.9) - command: | - curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh - echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh - rm -f /tmp/uv-install.sh - echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" - export PATH="$HOME/.local/bin:$PATH" - start_postgres: - description: "Start a postgres-db container on port 5432 and wait until it accepts connections." - parameters: - db_name: - type: string - default: circle_test - steps: - - run: - name: Start PostgreSQL - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=<< parameters.db_name >> \ - -p 5432:5432 \ - postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26 - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - start_redis: - description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys." - steps: - - run: - name: Start Redis - command: | - docker run -d \ - --name redis-cache \ - -p 6379:6379 \ - redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 - - wait_for_service: - url: tcp://localhost:6379 - timeout: "60" - start_openai_record_replay_proxy: - description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." - steps: - - run: - name: Start record/replay proxy - background: true - command: | - CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \ - RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \ - uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090 - - run: - name: Wait for record/replay proxy - command: | - for i in $(seq 1 30); do - if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then - echo "record/replay proxy is up" - exit 0 - fi - sleep 1 - done - echo "record/replay proxy did not become ready" >&2 - exit 1 - start_fake_openai_endpoint: - description: "Start the canned OpenAI mock (tests/_fake_openai_endpoint_server.py) on host port 8190 and wait until healthy. Models whose api_base points here (via FAKE_OPENAI_API_BASE) get well-formed chat/text/embedding responses with realistic usage, so the E2E run neither pays for nor depends on the live provider. A request whose model is '429' returns HTTP 429 for rate-limit/cooldown tests. Run after uv deps are synced." - steps: - - run: - name: Start fake OpenAI endpoint - background: true - command: | - uv run --no-sync python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 - - run: - name: Wait for fake OpenAI endpoint - command: | - for i in $(seq 1 30); do - if curl -sf http://localhost:8190/health >/dev/null 2>&1; then - echo "fake OpenAI endpoint is up" - exit 0 - fi - sleep 1 - done - echo "fake OpenAI endpoint did not become ready" >&2 - exit 1 - setup_litellm_enterprise_pip: - steps: - - run: - name: "Install local version of litellm-enterprise" - command: | - # litellm-enterprise is a uv workspace member and is already installed - # by the main `uv sync --all-groups --all-extras`. Do NOT run - # `uv sync --package litellm-enterprise` here — that overwrites the - # shared .venv and strips out dev/test deps (pytest, prisma, etc.). - uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" - setup_litellm_test_deps: - steps: - - checkout - - setup_google_dns - - install_uv - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - -jobs: - # Add Windows testing job - using_litellm_on_windows: - executor: - name: win/default - shell: powershell.exe - working_directory: ~/project - environment: - UV_PYTHON: "3.11" - steps: - - checkout - - run: - name: Install Python - command: | - choco install python --version=3.11.0 -y --no-progress --force - refreshenv - python --version - environment: - CHOCOLATEY_CONFIRM_ALL: "true" - - run: - name: Install Dependencies - environment: - UV_HTTP_TIMEOUT: "300" - command: | - $installer = Join-Path $env:TEMP "uv-install.ps1" - Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer - $expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d" - $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() - if ($actual -ne $expected) { - throw "uv installer hash mismatch: expected $expected got $actual" - } - & $installer - Remove-Item $installer - $uvBin = Join-Path $HOME ".local\bin" - $env:Path = "$uvBin;$env:Path" - if (!(Test-Path $PROFILE)) { - New-Item -ItemType File -Force -Path $PROFILE | Out-Null - } - if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) { - Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`"" - } - uv sync --frozen --group dev --python 3.11 - - run: - name: Run Windows-specific test - command: | - uv run --no-sync python -m pytest tests/windows_tests/ -v - - run: - name: Guard against MAX_PATH-busting packaged wheel paths - environment: - UV_HTTP_TIMEOUT: "300" - command: | - uv build --wheel --out-dir dist - uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py - - local_testing_part1: - docker: - - &python312_image - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - parallelism: 4 - steps: - - checkout - - setup_google_dns - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - # Run pytest and generate JUnit XML report - - run: - name: Run tests (Part 1 - A-M) - command: | - mkdir test-results - - # Discover test files (A-M) - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py") - - echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv \ - --cov=./litellm \ - --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=20 \ - -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ - -n 4 \ - --timeout=300 \ - --timeout_method=thread" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - # When CI reruns only the failed tests, a parallel node can receive - # zero tests and pytest never writes coverage. Emit empty placeholders - # so persist_to_workspace and the downstream coverage combine stay green. - if [ -f coverage.xml ]; then - mv coverage.xml local_testing_part1_coverage.xml - mv .coverage local_testing_part1_coverage - else - touch local_testing_part1_coverage.xml local_testing_part1_coverage - fi - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - local_testing_part1_coverage.xml - - local_testing_part1_coverage - local_testing_part2: - docker: - - *python312_image - working_directory: ~/project - parallelism: 4 - steps: - - checkout - - setup_google_dns - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - # Run pytest and generate JUnit XML report - - run: - name: Run tests (Part 2 - N-Z) - command: | - mkdir test-results - - # Discover test files (N-Z) - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py") - - echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv \ - --cov=./litellm \ - --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=20 \ - -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ - -n 4 \ - --timeout=300 \ - --timeout_method=thread" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - # When CI reruns only the failed tests, a parallel node can receive - # zero tests and pytest never writes coverage. Emit empty placeholders - # so persist_to_workspace and the downstream coverage combine stay green. - if [ -f coverage.xml ]; then - mv coverage.xml local_testing_part2_coverage.xml - mv .coverage local_testing_part2_coverage - else - touch local_testing_part2_coverage.xml local_testing_part2_coverage - fi - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - local_testing_part2_coverage.xml - - local_testing_part2_coverage - langfuse_logging_unit_tests: - docker: - - *python312_image - working_directory: ~/project - resource_class: medium - - steps: - - checkout - - setup_google_dns - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -k \"langfuse\"" - no_output_timeout: 15m - # Store test results - - store_test_results: - path: test-results - auth_ui_unit_tests: - docker: - - *python312_image - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: litellm_test - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - run: - name: Seed DB schema via prisma db push - command: | - set +e - uv run --no-sync litellm --skip_server_startup --use_prisma_db_push - set -e - - run: - name: Generate Prisma Client - command: uv run --no-sync python -m prisma generate - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -n 2" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml auth_ui_unit_tests_coverage.xml - mv .coverage auth_ui_unit_tests_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - auth_ui_unit_tests_coverage.xml - - auth_ui_unit_tests_coverage - - proxy_behavior_tests: - docker: - - *python312_image - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: litellm_test - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - run: - name: Seed DB schema via prisma db push - command: | - uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Generate Prisma Client - command: uv run --no-sync python -m prisma generate - - run: - name: Run proxy management behavior tests - command: | - mkdir -p test-results - uv run --no-sync python -m pytest tests/proxy_behavior \ - -v --junitxml=test-results/junit.xml --durations=10 - no_output_timeout: 15m - - store_test_results: - path: test-results - - proxy_security_tests: - docker: - - *python312_image - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: litellm_test - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - run: - name: Seed DB schema via prisma db push - command: | - uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Generate Prisma Client - command: uv run --no-sync python -m prisma generate - - run: - name: Run proxy security tests - command: | - mkdir -p test-results - uv run --no-sync python -m pytest tests/proxy_security_tests \ - -v --junitxml=test-results/junit.xml --durations=10 - no_output_timeout: 15m - - store_test_results: - path: test-results - - schema_migration_check: - docker: - - *python312_image - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: litellm_test - working_directory: ~/project - environment: - # An empty database; the test applies every committed migration itself. - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - run: - name: Generate Prisma Client - command: uv run --no-sync python -m prisma generate - - run: - name: Check schema.prisma is in sync with committed migrations - command: | - mkdir -p test-results - uv run --no-sync python -m pytest tests/proxy_migration_tests \ - -v --junitxml=test-results/junit.xml --durations=10 - no_output_timeout: 15m - - store_test_results: - path: test-results - - litellm_router_testing: # Runs all tests with the "router" keyword - docker: - - *python312_image - working_directory: ~/project - resource_class: large - parallelism: 4 - environment: - FAKE_OPENAI_API_BASE: http://127.0.0.1:8190 - steps: - - checkout - - setup_google_dns - - install_uv - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - - start_fake_openai_endpoint - # Run pytest and generate JUnit XML report - - setup_litellm_enterprise_pip - - run: - name: Run tests - command: | - # On a "rerun failed tests" build a parallel node can receive no - # tests, so the test command never creates test-results. Pre-create it - # so store_test_results doesn't fail the node on a missing path. - mkdir -p test-results - - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") - - echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v \ - -k 'router' \ - -n 4 \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - --timeout=300 --timeout_method=thread" - no_output_timeout: 15m - # Store test results - - store_test_results: - path: test-results - - litellm_router_unit_testing: # Runs all tests with the "router" keyword - docker: - - *python312_image - working_directory: ~/project - resource_class: large - - steps: - - checkout - - setup_google_dns - - install_uv - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - # Run pytest and generate JUnit XML report - - setup_litellm_enterprise_pip - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -n 4" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml router_unit_tests_coverage.xml - mv .coverage router_unit_tests_coverage - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - router_unit_tests_coverage.xml - - router_unit_tests_coverage - litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword - docker: - - *python312_image - working_directory: ~/project - resource_class: medium - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - setup_litellm_enterprise_pip - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -k \"assistants\"" - no_output_timeout: 15m - # Store test results - - store_test_results: - path: test-results - llm_translation_testing: - docker: - - *python312_image - working_directory: ~/project - resource_class: xlarge - - steps: - - checkout - - setup_google_dns - - install_uv - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - # Add --timeout to kill hanging tests after 120s (2 min) - # Add --durations=20 to show 20 slowest tests for debugging - # Subdirectories with dedicated jobs (maintain this list as new jobs are added) - mkdir -p test-results - # Glob excludes the realtime/ subdirectory since it has its own job - TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v \ - --junitxml=test-results/junit.xml \ - --durations=20 \ - -n 4 \ - --timeout=120 --timeout_method=thread \ - --retries 2 --retry-delay 5 \ - --max-worker-restart=5" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results - realtime_translation_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run realtime tests - command: | - # Add --timeout to kill hanging tests after 120s (2 min) - # Add --durations=20 to show 20 slowest tests for debugging - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=20 \ - -n 4 \ - --timeout=120 --timeout_method=thread" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml realtime_translation_coverage.xml - mv .coverage realtime_translation_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - realtime_translation_coverage.xml - - realtime_translation_coverage - agent_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml agent_coverage.xml - mv .coverage agent_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - agent_coverage.xml - - agent_coverage - guardrails_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - export LITELLM_LOG=WARNING - TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -n 2 \ - --timeout=120 --timeout_method=thread" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml guardrails_coverage.xml - mv .coverage guardrails_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - guardrails_coverage.xml - - guardrails_coverage - - google_generate_content_endpoint_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - --retries 3 --retry-delay 5" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml google_generate_content_endpoint_coverage.xml - mv .coverage google_generate_content_endpoint_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - google_generate_content_endpoint_coverage.xml - - google_generate_content_endpoint_coverage - - llm_responses_api_testing: - docker: - - *python312_image - working_directory: ~/project - resource_class: large - - steps: - - checkout - - setup_google_dns - - install_uv - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -n 8" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results - ocr_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -n 4" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml ocr_coverage.xml - mv .coverage ocr_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - ocr_coverage.xml - - ocr_coverage - search_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -n 4" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml search_coverage.xml - mv .coverage search_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - search_coverage.xml - - search_coverage - litellm_mapped_enterprise_tests: - docker: - - *python312_image - working_directory: ~/project - resource_class: large - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - run: - name: Run enterprise tests - command: | - uv run --no-sync python -m prisma generate - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ - --junitxml=test-results/junit-enterprise.xml \ - --durations=10 \ - -n 4" - no_output_timeout: 15m - # Store test results - - store_test_results: - path: test-results - batches_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -n 2" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml batches_coverage.xml - mv .coverage batches_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - batches_coverage.xml - - batches_coverage - litellm_utils_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -n 2" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_utils_coverage.xml - mv .coverage litellm_utils_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_utils_coverage.xml - - litellm_utils_coverage - - pass_through_unit_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -n 4" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml pass_through_unit_tests_coverage.xml - mv .coverage pass_through_unit_tests_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - pass_through_unit_tests_coverage.xml - - pass_through_unit_tests_coverage - image_gen_testing: - docker: - - *python312_image - working_directory: ~/project - resource_class: large - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -n 4" - no_output_timeout: 15m - # Store test results - - store_test_results: - path: test-results - logging_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - setup_litellm_enterprise_pip - - run: - name: Run tests - command: | - mkdir -p test-results - export LITELLM_LOG=WARNING - TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv \ - --cov=./litellm --cov-report=xml \ - -n 4 \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - --timeout=120 --timeout_method=thread" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml logging_coverage.xml || true - mv .coverage logging_coverage || true - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - logging_coverage.xml - - logging_coverage - audio_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml audio_coverage.xml - mv .coverage audio_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - audio_coverage.xml - - audio_coverage - redis_caching_unit_tests: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(printf "%s\n" \ - tests/local_testing/test_dual_cache.py \ - tests/local_testing/test_redis_batch_optimizations.py \ - tests/local_testing/test_router_utils.py) - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ - --cov=./litellm --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=5 -n 2 \ - --reruns 2 --reruns-delay 1" - no_output_timeout: 20m - - run: - name: Rename the coverage files - command: | - mv coverage.xml redis_caching_coverage.xml - mv .coverage redis_caching_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - redis_caching_coverage.xml - - redis_caching_coverage - installing_litellm_on_python: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - run: - name: Run tests - command: | - uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver" - - installing_litellm_on_python_3_13: - docker: - - image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - resource_class: medium - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.13 - - run: - name: Run tests - command: | - uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver" - - installing_litellm_on_python_v2_migration_resolver: - docker: - - *python312_image - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: litellm_test - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - run: - name: Run v2 migration resolver proxy smoke test - command: | - uv run --no-sync python -m pytest -vv \ - tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver - - helm_chart_testing: - machine: - image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker - resource_class: medium - working_directory: ~/project - - steps: - - checkout - - attach_workspace: - at: ~/project - - setup_google_dns - - install_helm - - install_kind - - # Install kubectl (pinned version with hardcoded checksum) - - run: - name: Install kubectl v1.31.4 - command: | - curl -sSLf -o /tmp/kubectl \ - https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl - echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c - - chmod +x /tmp/kubectl - sudo mv /tmp/kubectl /usr/local/bin/ - - # Create kind cluster - - run: - name: Create Kind Cluster - command: | - kind create cluster --name litellm-test - - - run: - name: Load Docker Database Image for helm tests - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - IMAGE_TAG=${CIRCLE_SHA1:-ci} - docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG} - - - run: - name: Load Docker image into Kind - command: | - IMAGE_TAG=${CIRCLE_SHA1:-ci} - kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test - - # Run helm lint - - run: - name: Run helm lint - command: | - helm lint ./deploy/charts/litellm-helm - - # Run helm tests - - run: - name: Run helm tests - command: | - IMAGE_TAG=${CIRCLE_SHA1:-ci} - helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \ - --set image.repository=litellm-ci \ - --set image.tag=${IMAGE_TAG} \ - --set image.pullPolicy=Never - # Wait for pod to be ready - echo "Waiting 30 seconds for pod to be ready..." - sleep 30 - - # Print pod logs before running tests - echo "Printing pod logs..." - kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") - - # Run the helm tests - helm test litellm --logs - - # Cleanup - - run: - name: Cleanup - command: | - kind delete cluster --name litellm-test - when: always # This ensures cleanup runs even if previous steps fail - - db_migration_disable_update_check: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: medium - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_fake_openai_endpoint - - start_postgres: - db_name: litellm_test - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Seed database with real schema - command: | - docker run -d \ - -p 4001:4000 \ - -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ - -e LITELLM_MASTER_KEY="sk-1234" \ - --name schema-seed \ - --add-host=host.docker.internal:host-gateway \ - -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --use_prisma_db_push - - wait_for_service: - url: http://localhost:4001 - timeout: "300" - - run: - name: Stop schema seed container - command: docker stop schema-seed && docker rm schema-seed - - run: - name: Run Docker container with bad schema and disabled updates - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ - -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ - -e DISABLE_SCHEMA_UPDATE="True" \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - --name my-app \ - --add-host=host.docker.internal:host-gateway \ - -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ - -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ - -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 - - wait_for_service: - url: http://localhost:4000 - timeout: "60" - - run: - name: Check container logs for expected message - command: | - echo "=== Printing Full Container Startup Logs ===" - LOG_OUTPUT="$(docker logs my-app 2>&1)" - printf '%s\n' "$LOG_OUTPUT" - echo "=== End of Full Container Startup Logs ===" - - if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then - echo "Expected message found in logs. Test passed." - else - echo "Expected message not found in logs. Test failed." - exit 1 - fi - - run: - name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ - --junitxml=test-results/junit-2.xml \ - --durations=5" - no_output_timeout: 15m - - store_test_results: - path: test-results - - build_and_test: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - attach_workspace: - at: ~/project - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker tag litellm-docker-database:ci my-app:latest - - start_openai_record_replay_proxy - - start_fake_openai_endpoint - - run: - name: Run Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e USE_PRISMA_MIGRATE=True \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e AZURE_API_KEY=$AZURE_API_KEY \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ - -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ - -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ - -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ - -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - -e GROQ_API_KEY=$GROQ_API_KEY \ - -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ - -e COHERE_API_KEY=$COHERE_API_KEY \ - -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e AUTO_INFER_REGION=True \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e USE_DDTRACE=True \ - -e DD_API_KEY=$DD_API_KEY \ - -e DD_SITE=$DD_SITE \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ - -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ - -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ - -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ - -e RECORDER_OPENAI_BASE_URL=http://host.docker.internal:8090/v1 \ - --add-host host.docker.internal:host-gateway \ - --name my-app \ - -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ - my-app:latest \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - run: - name: Run tests - command: | - mkdir -p test-results - # Original used `tests/*.py` (top-level only); the `--ignore=...` - # flags were vestigial since shell globbing did not descend into - # subdirectories. Replicate by globbing only top-level test files. - TEST_FILES=$(circleci tests glob "tests/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -s -v -x \ - --junitxml=test-results/junit.xml \ - -n 4 \ - --durations=5" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results - e2e_openai_endpoints: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Run Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e AZURE_API_KEY=$AZURE_API_KEY \ - -e AZURE_API_BASE=$AZURE_API_BASE \ - -e AZURE_API_VERSION="2024-05-01-preview" \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ - -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ - -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ - -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ - -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - -e GROQ_API_KEY=$GROQ_API_KEY \ - -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ - -e COHERE_API_KEY=$COHERE_API_KEY \ - -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e AUTO_INFER_REGION=True \ - -e USE_DDTRACE=True \ - -e DD_API_KEY=$DD_API_KEY \ - -e DD_SITE=$DD_SITE \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ - -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ - -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ - -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ - --add-host host.docker.internal:host-gateway \ - --name my-app \ - -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -s -vv \ - --junitxml=test-results/junit.xml \ - --durations=5" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results - proxy_logging_guardrails_model_info_tests: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - start_openai_record_replay_proxy - - start_fake_openai_endpoint - - run: - name: Run Docker container - # intentionally give bad redis credentials here - # the OTEL test - should get this as a trace - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - -e OTEL_EXPORTER="in_memory" \ - -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ - -e USE_DDTRACE=True \ - -e DD_API_KEY=$DD_API_KEY \ - -e DD_SITE=$DD_SITE \ - -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e COHERE_API_KEY=$COHERE_API_KEY \ - -e RECORDER_COHERE_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.cohere.com \ - -e GCS_FLUSH_INTERVAL="1" \ - --add-host host.docker.internal:host-gateway \ - --name my-app \ - -v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \ - -v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v \ - --junitxml=test-results/junit.xml \ - --durations=5" - no_output_timeout: 15m - # Clean up first container - - run: - name: Stop and remove first container - command: | - docker stop my-app - docker rm my-app - - # Second Docker Container Run with Different Config - # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license - - run: - name: Run Second Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e LITELLM_LICENSE="bad-license" \ - --add-host host.docker.internal:host-gateway \ - --name my-app-3 \ - -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug - - - run: - name: Start outputting logs for second container - command: docker logs -f my-app-3 - background: true - - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - - run: - name: Run second round of tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ - --junitxml=test-results/junit-2.xml \ - --durations=5" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results - proxy_spend_accuracy_tests: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - start_redis - - start_fake_openai_endpoint - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Run Docker container - # Point the proxy at the job-local Redis (start_redis) instead of the - # shared remote Redis. The Redis transaction buffer uses a single - # global pod-lock key (cronjob_lock:db_spend_update_job) and a single - # global buffer list (litellm_spend_update_buffer); sharing those - # across concurrent CI pipelines causes spend flushes to stall or - # land in the wrong DB, which is what makes this test flaky. - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=host.docker.internal \ - -e REDIS_PORT=6379 \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - -e USE_DDTRACE=True \ - -e DD_API_KEY=$DD_API_KEY \ - -e DD_SITE=$DD_SITE \ - -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e PROXY_BATCH_WRITE_AT=2 \ - --add-host host.docker.internal:host-gateway \ - --name my-app \ - -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ - --junitxml=test-results/junit.xml \ - --durations=5" - no_output_timeout: 15m - - store_test_results: - path: test-results - - run: - name: Stop and remove first container - when: always - command: | - docker stop my-app - docker rm my-app - docker stop redis-cache - docker rm redis-cache - - proxy_multi_instance_tests: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - start_fake_openai_endpoint - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Run Docker container 1 - # intentionally give bad redis credentials here - # the OTEL test - should get this as a trace - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - -e USE_DDTRACE=True \ - -e DD_API_KEY=$DD_API_KEY \ - -e DD_SITE=$DD_SITE \ - --add-host host.docker.internal:host-gateway \ - --name my-app \ - -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ - - run: - name: Run Docker container 2 - command: | - docker run -d \ - -p 4001:4001 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - -e USE_DDTRACE=True \ - -e DD_API_KEY=$DD_API_KEY \ - -e DD_SITE=$DD_SITE \ - --add-host host.docker.internal:host-gateway \ - --name my-app-2 \ - -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4001 \ - --detailed_debug - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - wait_for_service: - url: http://localhost:4001 - timeout: "300" - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ - --junitxml=test-results/junit.xml \ - --durations=5" - no_output_timeout: 15m - # Clean up first container - # Store test results - - store_test_results: - path: test-results - - proxy_store_model_in_db_tests: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - start_fake_openai_endpoint - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Run Docker container - # intentionally give bad redis credentials here - # the OTEL test - should get this as a trace - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e STORE_MODEL_IN_DB="True" \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - --add-host host.docker.internal:host-gateway \ - --name my-app \ - -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ - --junitxml=test-results/junit.xml \ - --durations=5" - no_output_timeout: 15m - - run: - name: Stop and remove containers - command: | - docker stop my-app || true - docker rm my-app || true - docker stop postgres-db || true - docker rm postgres-db || true - when: always - - store_test_results: - path: test-results - - proxy_build_from_pip_tests: - # Change from docker to machine executor - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_google_dns - # Remove Docker CLI installation since it's already available in machine executor - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - run: - name: Build Docker image - command: | - docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . - - start_postgres - - start_fake_openai_endpoint - - run: - name: Run Docker container - # intentionally give bad redis credentials here - # the OTEL test - should get this as a trace - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - -e OTEL_EXPORTER="in_memory" \ - -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e COHERE_API_KEY=$COHERE_API_KEY \ - -e USE_DDTRACE=True \ - -e DD_API_KEY=$DD_API_KEY \ - -e DD_SITE=$DD_SITE \ - -e GCS_FLUSH_INTERVAL="1" \ - --add-host host.docker.internal:host-gateway \ - --name my-app \ - -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ - my-app:latest \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x \ - --junitxml=test-results/junit-2.xml \ - --durations=5" - no_output_timeout: 15m - # Clean up first container - - store_test_results: - path: test-results - - run: - name: Stop and remove first container - command: | - docker stop my-app || true - docker rm my-app || true - docker stop postgres-db || true - docker rm postgres-db || true - when: always - proxy_pass_through_endpoint_tests: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Run Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e GEMINI_API_KEY=$GEMINI_API_KEY \ - -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ - -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \ - -e AZURE_API_KEY=$AZURE_API_KEY \ - -e AZURE_API_BASE=$AZURE_API_BASE \ - -e USE_DDTRACE=True \ - -e DD_API_KEY=$DD_API_KEY \ - -e DD_SITE=$DD_SITE \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \ - --add-host host.docker.internal:host-gateway \ - --name my-app \ - -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ - -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - # Add Ruby installation and testing before the existing Node.js and Python tests - - run: - name: Install Ruby and Bundler - command: | - # Clone RVM at pinned tag and verify the commit SHA matches the - # published tag before running its install script. - RVM_VERSION="1.29.12" - RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81" - git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm - RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)" - if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then - echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2 - exit 1 - fi - - # Import RVM signing keys (used by `rvm install` to verify Ruby tarballs) - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB - - # Install RVM from the verified checkout. The install script - # sources `scripts/functions/installer` using paths relative to - # its own working directory, so it must be run from /tmp/rvm. - (cd /tmp/rvm && ./install --path "$HOME/.rvm") - source "$HOME/.rvm/scripts/rvm" - - # Install Ruby 3.2.2 (RVM verifies the tarball PGP signature) - rvm install 3.2.2 - rvm use 3.2.2 --default - - # Install latest Bundler - gem install bundler - - - run: - name: Run Ruby tests - command: | - source $HOME/.rvm/scripts/rvm - cd tests/pass_through_tests/ruby_passthrough_tests - bundle install - bundle exec rspec - no_output_timeout: 30m - # Install Node.js directly from nodejs.org with SHA256 verification, - # instead of piping NodeSource's setup_18.x apt-repo installer into - # sudo bash (which runs a mutable upstream script unattended). - - run: - name: Install Node.js 18.20.8 - command: | - NODE_VERSION="18.20.8" - NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz" - NODE_EXPECTED_SHA="5467ee62d6af1411d46b6a10e3fb5cacc92734dbcef465fea14e7b90993001c9" - curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" - echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c - - sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /usr/local --strip-components=1 - rm -f "/tmp/${NODE_TARBALL}" - node --version - npm --version - - - run: - name: Install Node.js test dependencies - command: | - cd tests/pass_through_tests - npm ci - - - run: - name: Run Vertex AI, Google AI Studio Node.js tests - command: | - cd tests/pass_through_tests - NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose - no_output_timeout: 30m - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -v -x \ - --junitxml=test-results/junit.xml \ - --durations=5" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results - - proxy_e2e_anthropic_messages_tests: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - start_openai_record_replay_proxy - - run: - name: Run Docker container with test config - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ - -e RECORDER_ANTHROPIC_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.anthropic.com \ - -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - -e AWS_REGION_NAME="us-east-1" \ - -e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \ - --add-host host.docker.internal:host-gateway \ - --name my-app \ - -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - run: - name: Run Claude Agent SDK E2E Tests - command: | - mkdir -p test-results - export LITELLM_PROXY_URL="http://localhost:4000" - export LITELLM_API_KEY="sk-1234" - TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ - -vv -x -s \ - --junitxml=test-results/junit.xml \ - --durations=5" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results - - upload-coverage: - docker: - - *python312_image - steps: - - checkout - - attach_workspace: - at: . - # Check file locations - - run: - name: Check coverage file location - command: | - echo "Current directory:" - ls -la - echo "\nContents of tests/llm_translation:" - ls -la tests/llm_translation - - install_uv - - run: - name: Combine Coverage - command: | - uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage - uv tool run --from 'coverage[toml]==7.10.6' coverage xml - - codecov/upload: - file: ./coverage.xml - flags: circleci - - ui_build: - docker: - - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: medium+ - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - restore_cache: - keys: - - ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - - ui-build-deps-v1- - - restore_cache: - keys: - - ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - - ui-nextjs-cache-v1- - - run: - name: Install dependencies - command: | - cd ui/litellm-dashboard - npm ci - - save_cache: - key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - paths: - - ui/litellm-dashboard/node_modules - - run: - name: Build UI - command: | - cd ui/litellm-dashboard - source ./build_ui.sh - - save_cache: - key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - paths: - - ui/litellm-dashboard/.next/cache - - persist_to_workspace: - root: . - paths: - - litellm/proxy/_experimental/out - - ui_unit_tests: - docker: - - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: xlarge - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - restore_cache: - keys: - - ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - - ui-unit-deps-v1- - - run: - name: Install dependencies - command: | - cd ui/litellm-dashboard - npm ci - - save_cache: - key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - paths: - - ui/litellm-dashboard/node_modules - - run: - name: Run UI unit tests (Vitest) - command: | - cd ui/litellm-dashboard - - CI=true npm run test -- --run \ - --pool forks --poolOptions.forks.maxForks=8 - - e2e_ui_testing: - docker: - - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: e2euser - POSTGRES_PASSWORD: e2epassword - POSTGRES_DB: litellm_e2e - resource_class: large - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" - CI: "true" - # Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can - # assert the redirect. Set at job level so both the proxy boot step and the - # Playwright step (whose skip guard reads this) see the same value. Safe for - # the rest of the suite: nothing else performs a logout. - PROXY_LOGOUT_URL: "https://www.example.com" - steps: - - checkout - - setup_google_dns - - install_uv - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Python dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - - save_cache: - key: v1-uv-cache-{{ checksum "uv.lock" }} - paths: - - ~/.cache/uv - - restore_cache: - keys: - - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - - run: - name: Install Node dependencies and Playwright - # The cimg/python:3.12-browsers image already ships the Chromium system - # libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.). - # `--with-deps` triggers a redundant apt-get update + install that adds - # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors, - # so we install just the browser binary. - command: | - cd ui/litellm-dashboard - npm ci - npx playwright install chromium - - save_cache: - key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - paths: - - ui/litellm-dashboard/node_modules - - ~/.cache/ms-playwright - - run: - name: Build UI from source - # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`. - # GNU cp (used on CircleCI's Ubuntu image) interprets that as "copy the - # source directory as a child of the destination" when the destination - # already exists — silently creating `_experimental/out/out/` instead of - # replacing the served bundle. The proxy continued serving whatever was - # checked into `_experimental/out/*`, so this job was effectively testing - # the pre-build bundle on every run. Replace-and-move guarantees the - # freshly built bundle is what the proxy actually serves. - command: | - cd ui/litellm-dashboard - npm run build - rm -rf ../../litellm/proxy/_experimental/out - mv out ../../litellm/proxy/_experimental/out - # Restructure HTML so extensionless routes work (login.html -> login/index.html) - find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do - d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" - done - - wait_for_service: - url: tcp://localhost:5432 - timeout: "30" - - run: - name: Push Prisma schema - command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql - - run: - name: Start mock LLM server - command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py - background: true - - run: - name: Start LiteLLM proxy - environment: - LITELLM_MASTER_KEY: "sk-1234" - MOCK_LLM_URL: "http://127.0.0.1:8090/v1" - DISABLE_SCHEMA_UPDATE: "true" - SERVER_ROOT_PATH: "" - # PROXY_LOGOUT_URL is inherited from the job-level environment so the - # proxy and proxyLogoutUrl.spec.ts agree on the logout target. - # LITELLM_LICENSE is forwarded from the project env so premium-gated - # UI flows can be exercised. license.spec.ts asserts the resulting - # JWT carries premium_user=true; if it ever stops being passed, that - # test fails loudly rather than silently regressing premium coverage. - command: | - LITELLM_LICENSE="$LITELLM_LICENSE" \ - uv run --no-sync python -m litellm.proxy.proxy_cli \ - --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ - --port 4000 - background: true - - run: - name: Wait for proxy to be ready - command: | - for i in $(seq 1 60); do - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true) - if [ "$HTTP_CODE" = "200" ]; then - echo "Proxy is ready" - exit 0 - fi - sleep 2 - done - echo "Proxy failed to start" - exit 1 - - run: - name: Run Playwright E2E tests - # Forward LITELLM_LICENSE so license.spec.ts can detect that the - # proxy was launched with a license and assert premium_user=true. - command: | - cd ui/litellm-dashboard - LITELLM_LICENSE="$LITELLM_LICENSE" \ - npx playwright test --config e2e_tests/playwright.config.ts - no_output_timeout: 10m - - store_artifacts: - path: ui/litellm-dashboard/test-results - destination: e2e-test-results - - store_artifacts: - path: ui/litellm-dashboard/playwright-report - destination: e2e-playwright-report - - e2e_ui_testing_server_root_path: - docker: - - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: e2euser - POSTGRES_PASSWORD: e2epassword - POSTGRES_DB: litellm_e2e - resource_class: large - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" - CI: "true" - # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH - # is read both by the proxy at boot (to rewrite the built UI bundle in place) - # and by migration.serverRootPath.config.ts, which refuses to run without it. - SERVER_ROOT_PATH: "/litellm" - steps: - - checkout - - setup_google_dns - - install_uv - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Python dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - - save_cache: - key: v1-uv-cache-{{ checksum "uv.lock" }} - paths: - - ~/.cache/uv - - restore_cache: - keys: - - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - - run: - name: Install Node dependencies and Playwright - command: | - cd ui/litellm-dashboard - npm ci - npx playwright install chromium - - save_cache: - key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - paths: - - ui/litellm-dashboard/node_modules - - ~/.cache/ms-playwright - - run: - name: Build UI from source - command: | - cd ui/litellm-dashboard - npm run build - rm -rf ../../litellm/proxy/_experimental/out - mv out ../../litellm/proxy/_experimental/out - find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do - d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" - done - - wait_for_service: - url: tcp://localhost:5432 - timeout: "30" - - run: - name: Push Prisma schema - command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql - - run: - name: Start mock LLM server - command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py - background: true - - run: - name: Start LiteLLM proxy under a server root path - environment: - LITELLM_MASTER_KEY: "sk-1234" - MOCK_LLM_URL: "http://127.0.0.1:8090/v1" - DISABLE_SCHEMA_UPDATE: "true" - # Output flows to this step's own log, so a boot crash is visible here - # rather than swallowed by a downstream readiness probe. - command: | - LITELLM_LICENSE="$LITELLM_LICENSE" \ - uv run --no-sync python -m litellm.proxy.proxy_cli \ - --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ - --port 4000 - background: true - - run: - name: Wait for prefixed proxy to be ready - command: | - for i in $(seq 1 60); do - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer sk-1234" http://127.0.0.1:4000/litellm/health 2>/dev/null || true) - if [ "$HTTP_CODE" = "200" ]; then - echo "Prefixed proxy is ready" - exit 0 - fi - sleep 2 - done - echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log" - exit 1 - - run: - name: Run migration smoke under SERVER_ROOT_PATH - command: | - cd ui/litellm-dashboard - LITELLM_LICENSE="$LITELLM_LICENSE" \ - npx playwright test --config e2e_tests/migration.serverRootPath.config.ts - no_output_timeout: 10m - - store_artifacts: - path: ui/litellm-dashboard/test-results - destination: e2e-server-root-path-test-results - - store_artifacts: - path: ui/litellm-dashboard/playwright-report - destination: e2e-server-root-path-playwright-report - - build_docker_database_image: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - - run: - name: Build Docker image - command: | - docker build \ - -t litellm-docker-database:ci \ - -f docker/Dockerfile.database . - - - run: - name: Save Docker image to workspace root - command: | - docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst - - - persist_to_workspace: - root: . - paths: - - litellm-docker-database.tar.zst - - test_bad_database_url: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: medium - working_directory: ~/project - steps: - - checkout - - attach_workspace: - at: ~/project - - setup_google_dns - - start_postgres - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker tag litellm-docker-database:ci myapp:latest - - run: - name: Run Docker container with bad DATABASE_URL - command: | - docker run --name my-app \ - -p 4000:4000 \ - -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ - -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ - myapp:latest \ - --port 4000 > docker_output.log 2>&1 || true - - run: - name: Display Docker logs - command: cat docker_output.log - - run: - name: Check for expected error - command: | - if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \ - (grep -q "Database setup failed after multiple retries" docker_output.log || \ - grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then - echo "Expected error found. Test passed." - else - echo "Expected error not found. Test failed." - cat docker_output.log - exit 1 - fi + path-filtering: circleci/path-filtering@1.3.0 workflows: - build_and_test: + setup: jobs: - - using_litellm_on_windows: - filters: &main_branches - branches: - only: - - main - - /litellm_.*/ - - local_testing_part1: - filters: *main_branches - - local_testing_part2: - filters: *main_branches - - langfuse_logging_unit_tests: - filters: *main_branches - - litellm_assistants_api_testing: - filters: *main_branches - - litellm_router_testing: - filters: *main_branches - - litellm_router_unit_testing: - filters: *main_branches - - ui_build: - filters: *main_branches - - ui_unit_tests: - requires: - - ui_build - filters: *main_branches - - auth_ui_unit_tests: - filters: *main_branches - - proxy_behavior_tests: - filters: *main_branches - - proxy_security_tests: - filters: *main_branches - - schema_migration_check: - filters: *main_branches - - build_docker_database_image: - filters: *main_branches - - e2e_ui_testing: - filters: *main_branches - - e2e_ui_testing_server_root_path: - filters: *main_branches - - build_and_test: - requires: - - build_docker_database_image - filters: *main_branches - - e2e_openai_endpoints: - requires: - - build_docker_database_image - filters: *main_branches - - proxy_logging_guardrails_model_info_tests: - requires: - - build_docker_database_image - filters: *main_branches - - proxy_spend_accuracy_tests: - requires: - - build_docker_database_image - filters: *main_branches - - proxy_multi_instance_tests: - requires: - - build_docker_database_image - filters: *main_branches - - proxy_store_model_in_db_tests: - requires: - - build_docker_database_image - filters: *main_branches - - proxy_build_from_pip_tests: - filters: *main_branches - - proxy_pass_through_endpoint_tests: - requires: - - build_docker_database_image - filters: *main_branches - - proxy_e2e_anthropic_messages_tests: - requires: - - build_docker_database_image - filters: *main_branches - - llm_translation_testing: - filters: *main_branches - - realtime_translation_testing: - filters: *main_branches - - agent_testing: - filters: *main_branches - - guardrails_testing: - filters: *main_branches - - google_generate_content_endpoint_testing: - filters: *main_branches - - llm_responses_api_testing: - filters: *main_branches - - ocr_testing: - filters: *main_branches - - search_testing: - filters: *main_branches - - litellm_mapped_enterprise_tests: - filters: *main_branches - - batches_testing: - filters: *main_branches - - litellm_utils_testing: - filters: *main_branches - - pass_through_unit_testing: - filters: *main_branches - - image_gen_testing: - filters: *main_branches - - logging_testing: - filters: *main_branches - - audio_testing: - filters: *main_branches - - redis_caching_unit_tests: - filters: *main_branches - - upload-coverage: - requires: - - realtime_translation_testing - - agent_testing - - google_generate_content_endpoint_testing - - guardrails_testing - - ocr_testing - - search_testing - - litellm_mapped_enterprise_tests - - batches_testing - - litellm_utils_testing - - pass_through_unit_testing - - image_gen_testing - - logging_testing - - audio_testing - - redis_caching_unit_tests - - langfuse_logging_unit_tests - - local_testing_part1 - - local_testing_part2 - - litellm_assistants_api_testing - - litellm_router_unit_testing - - auth_ui_unit_tests - - db_migration_disable_update_check: - requires: - - build_docker_database_image - filters: *main_branches - - installing_litellm_on_python: - filters: *main_branches - - installing_litellm_on_python_3_13: - filters: *main_branches - - installing_litellm_on_python_v2_migration_resolver: - filters: *main_branches - - helm_chart_testing: - requires: - - build_docker_database_image - filters: *main_branches - - test_bad_database_url: - requires: - - build_docker_database_image - filters: *main_branches + - path-filtering/filter: + base-revision: main + config-path: .circleci/continue_config.yml + mapping: | + .*\.py$ run-code-tests true + .*\.ts$ run-code-tests true + .*\.tsx$ run-code-tests true diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml new file mode 100644 index 00000000000..743d42edbf5 --- /dev/null +++ b/.circleci/continue_config.yml @@ -0,0 +1,3069 @@ +version: 2.1 + +parameters: + run-code-tests: + type: boolean + default: false + +orbs: + codecov: codecov/codecov@4.0.1 + node: circleci/node@5.1.0 # Add this line to declare the node orb + win: circleci/windows@5.0 # Add Windows orb + +commands: + setup_google_dns: + steps: + - run: + name: "Configure Google DNS" + command: | + # Backup original resolv.conf + sudo cp /etc/resolv.conf /etc/resolv.conf.backup + # Add both local and Google DNS servers + echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf + echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf + echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf + wait_for_service: + description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)" + parameters: + url: + type: string + timeout: + type: string + default: "60" + steps: + - run: + name: "Wait for << parameters.url >>" + command: | + TIMEOUT=<< parameters.timeout >> + URL="<< parameters.url >>" + ELAPSED=0 + echo "Waiting up to ${TIMEOUT}s for ${URL} ..." + if echo "$URL" | grep -q '^tcp://'; then + HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1) + PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2) + while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do + sleep 2; ELAPSED=$((ELAPSED+2)) + if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi + done + else + while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do + sleep 2; ELAPSED=$((ELAPSED+2)) + if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi + done + fi + echo "Service ready after ${ELAPSED}s" + install_helm: + steps: + - run: + name: Install Helm v3.17.3 + command: | + curl -sSLf -o /tmp/helm.tar.gz \ + https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz + echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c - + sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm + rm -f /tmp/helm.tar.gz + install_kind: + steps: + - run: + name: Install Kind v0.20.0 + command: | + curl -sSLf -o /tmp/kind \ + https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 + echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c - + chmod +x /tmp/kind + sudo mv /tmp/kind /usr/local/bin/kind + install_uv: + description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH." + steps: + - run: + name: Install uv (pinned 0.10.9) + command: | + curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh + echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - + env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh + rm -f /tmp/uv-install.sh + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + start_postgres: + description: "Start a postgres-db container on port 5432 and wait until it accepts connections." + parameters: + db_name: + type: string + default: circle_test + steps: + - run: + name: Start PostgreSQL + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=<< parameters.db_name >> \ + -p 5432:5432 \ + postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26 + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + start_redis: + description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys." + steps: + - run: + name: Start Redis + command: | + docker run -d \ + --name redis-cache \ + -p 6379:6379 \ + redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 + - wait_for_service: + url: tcp://localhost:6379 + timeout: "60" + start_openai_record_replay_proxy: + description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." + steps: + - run: + name: Start record/replay proxy + background: true + command: | + CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \ + RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \ + uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090 + - run: + name: Wait for record/replay proxy + command: | + for i in $(seq 1 30); do + if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then + echo "record/replay proxy is up" + exit 0 + fi + sleep 1 + done + echo "record/replay proxy did not become ready" >&2 + exit 1 + start_fake_openai_endpoint: + description: "Start the canned OpenAI mock (tests/_fake_openai_endpoint_server.py) on host port 8190 and wait until healthy. Models whose api_base points here (via FAKE_OPENAI_API_BASE) get well-formed chat/text/embedding responses with realistic usage, so the E2E run neither pays for nor depends on the live provider. A request whose model is '429' returns HTTP 429 for rate-limit/cooldown tests. Run after uv deps are synced." + steps: + - run: + name: Start fake OpenAI endpoint + background: true + command: | + uv run --no-sync python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 + - run: + name: Wait for fake OpenAI endpoint + command: | + for i in $(seq 1 30); do + if curl -sf http://localhost:8190/health >/dev/null 2>&1; then + echo "fake OpenAI endpoint is up" + exit 0 + fi + sleep 1 + done + echo "fake OpenAI endpoint did not become ready" >&2 + exit 1 + setup_litellm_enterprise_pip: + steps: + - run: + name: "Install local version of litellm-enterprise" + command: | + # litellm-enterprise is a uv workspace member and is already installed + # by the main `uv sync --all-groups --all-extras`. Do NOT run + # `uv sync --package litellm-enterprise` here — that overwrites the + # shared .venv and strips out dev/test deps (pytest, prisma, etc.). + uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" + setup_litellm_test_deps: + steps: + - checkout + - setup_google_dns + - install_uv + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + +jobs: + # Add Windows testing job + using_litellm_on_windows: + executor: + name: win/default + shell: powershell.exe + working_directory: ~/project + environment: + UV_PYTHON: "3.11" + steps: + - checkout + - run: + name: Install Python + command: | + choco install python --version=3.11.0 -y --no-progress --force + refreshenv + python --version + environment: + CHOCOLATEY_CONFIRM_ALL: "true" + - run: + name: Install Dependencies + environment: + UV_HTTP_TIMEOUT: "300" + command: | + $installer = Join-Path $env:TEMP "uv-install.ps1" + Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer + $expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d" + $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() + if ($actual -ne $expected) { + throw "uv installer hash mismatch: expected $expected got $actual" + } + & $installer + Remove-Item $installer + $uvBin = Join-Path $HOME ".local\bin" + $env:Path = "$uvBin;$env:Path" + if (!(Test-Path $PROFILE)) { + New-Item -ItemType File -Force -Path $PROFILE | Out-Null + } + if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) { + Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`"" + } + uv sync --frozen --group dev --python 3.11 + - run: + name: Run Windows-specific test + command: | + uv run --no-sync python -m pytest tests/windows_tests/ -v + - run: + name: Guard against MAX_PATH-busting packaged wheel paths + environment: + UV_HTTP_TIMEOUT: "300" + command: | + uv build --wheel --out-dir dist + uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py + + local_testing_part1: + docker: + - &python312_image + image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + parallelism: 4 + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + # Run pytest and generate JUnit XML report + - run: + name: Run tests (Part 1 - A-M) + command: | + mkdir test-results + + # Discover test files (A-M) + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=timings \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv \ + --cov=./litellm \ + --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ + -n 4 \ + --timeout=300 \ + --timeout_method=thread" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + # When CI reruns only the failed tests, a parallel node can receive + # zero tests and pytest never writes coverage. Emit empty placeholders + # so persist_to_workspace and the downstream coverage combine stay green. + if [ -f coverage.xml ]; then + mv coverage.xml local_testing_part1_coverage.xml + mv .coverage local_testing_part1_coverage + else + touch local_testing_part1_coverage.xml local_testing_part1_coverage + fi + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - local_testing_part1_coverage.xml + - local_testing_part1_coverage + local_testing_part2: + docker: + - *python312_image + working_directory: ~/project + parallelism: 4 + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + # Run pytest and generate JUnit XML report + - run: + name: Run tests (Part 2 - N-Z) + command: | + mkdir test-results + + # Discover test files (N-Z) + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=timings \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv \ + --cov=./litellm \ + --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ + -n 4 \ + --timeout=300 \ + --timeout_method=thread" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + # When CI reruns only the failed tests, a parallel node can receive + # zero tests and pytest never writes coverage. Emit empty placeholders + # so persist_to_workspace and the downstream coverage combine stay green. + if [ -f coverage.xml ]; then + mv coverage.xml local_testing_part2_coverage.xml + mv .coverage local_testing_part2_coverage + else + touch local_testing_part2_coverage.xml local_testing_part2_coverage + fi + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - local_testing_part2_coverage.xml + - local_testing_part2_coverage + langfuse_logging_unit_tests: + docker: + - *python312_image + working_directory: ~/project + resource_class: medium + + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -k \"langfuse\"" + no_output_timeout: 15m + # Store test results + - store_test_results: + path: test-results + auth_ui_unit_tests: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Seed DB schema via prisma db push + command: | + set +e + uv run --no-sync litellm --skip_server_startup --use_prisma_db_push + set -e + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 2" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml auth_ui_unit_tests_coverage.xml + mv .coverage auth_ui_unit_tests_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - auth_ui_unit_tests_coverage.xml + - auth_ui_unit_tests_coverage + + proxy_behavior_tests: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Seed DB schema via prisma db push + command: | + uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate + - run: + name: Run proxy management behavior tests + command: | + mkdir -p test-results + uv run --no-sync python -m pytest tests/proxy_behavior \ + -v --junitxml=test-results/junit.xml --durations=10 + no_output_timeout: 15m + - store_test_results: + path: test-results + + proxy_security_tests: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Seed DB schema via prisma db push + command: | + uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate + - run: + name: Run proxy security tests + command: | + mkdir -p test-results + uv run --no-sync python -m pytest tests/proxy_security_tests \ + -v --junitxml=test-results/junit.xml --durations=10 + no_output_timeout: 15m + - store_test_results: + path: test-results + + schema_migration_check: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + # An empty database; the test applies every committed migration itself. + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate + - run: + name: Check schema.prisma is in sync with committed migrations + command: | + mkdir -p test-results + uv run --no-sync python -m pytest tests/proxy_migration_tests \ + -v --junitxml=test-results/junit.xml --durations=10 + no_output_timeout: 15m + - store_test_results: + path: test-results + + litellm_router_testing: # Runs all tests with the "router" keyword + docker: + - *python312_image + working_directory: ~/project + resource_class: large + parallelism: 4 + environment: + FAKE_OPENAI_API_BASE: http://127.0.0.1:8190 + steps: + - checkout + - setup_google_dns + - install_uv + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + - start_fake_openai_endpoint + # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip + - run: + name: Run tests + command: | + # On a "rerun failed tests" build a parallel node can receive no + # tests, so the test command never creates test-results. Pre-create it + # so store_test_results doesn't fail the node on a missing path. + mkdir -p test-results + + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=timings \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v \ + -k 'router' \ + -n 4 \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + --timeout=300 --timeout_method=thread" + no_output_timeout: 15m + # Store test results + - store_test_results: + path: test-results + + litellm_router_unit_testing: # Runs all tests with the "router" keyword + docker: + - *python312_image + working_directory: ~/project + resource_class: large + + steps: + - checkout + - setup_google_dns + - install_uv + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 4" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml router_unit_tests_coverage.xml + mv .coverage router_unit_tests_coverage + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - router_unit_tests_coverage.xml + - router_unit_tests_coverage + litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword + docker: + - *python312_image + working_directory: ~/project + resource_class: medium + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -k \"assistants\"" + no_output_timeout: 15m + # Store test results + - store_test_results: + path: test-results + llm_translation_testing: + docker: + - *python312_image + working_directory: ~/project + resource_class: xlarge + + steps: + - checkout + - setup_google_dns + - install_uv + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + # Add --timeout to kill hanging tests after 120s (2 min) + # Add --durations=20 to show 20 slowest tests for debugging + # Subdirectories with dedicated jobs (maintain this list as new jobs are added) + mkdir -p test-results + # Glob excludes the realtime/ subdirectory since it has its own job + TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -n 4 \ + --timeout=120 --timeout_method=thread \ + --retries 2 --retry-delay 5 \ + --max-worker-restart=5" + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + realtime_translation_testing: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - run: + name: Run realtime tests + command: | + # Add --timeout to kill hanging tests after 120s (2 min) + # Add --durations=20 to show 20 slowest tests for debugging + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -n 4 \ + --timeout=120 --timeout_method=thread" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml realtime_translation_coverage.xml + mv .coverage realtime_translation_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - realtime_translation_coverage.xml + - realtime_translation_coverage + agent_testing: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml agent_coverage.xml + mv .coverage agent_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - agent_coverage.xml + - agent_coverage + guardrails_testing: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + export LITELLM_LOG=WARNING + TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 2 \ + --timeout=120 --timeout_method=thread" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml guardrails_coverage.xml + mv .coverage guardrails_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - guardrails_coverage.xml + - guardrails_coverage + + google_generate_content_endpoint_testing: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + --retries 3 --retry-delay 5" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml google_generate_content_endpoint_coverage.xml + mv .coverage google_generate_content_endpoint_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - google_generate_content_endpoint_coverage.xml + - google_generate_content_endpoint_coverage + + llm_responses_api_testing: + docker: + - *python312_image + working_directory: ~/project + resource_class: large + + steps: + - checkout + - setup_google_dns + - install_uv + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 8" + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + ocr_testing: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 4" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml ocr_coverage.xml + mv .coverage ocr_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - ocr_coverage.xml + - ocr_coverage + search_testing: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 4" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml search_coverage.xml + mv .coverage search_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - search_coverage.xml + - search_coverage + litellm_mapped_enterprise_tests: + docker: + - *python312_image + working_directory: ~/project + resource_class: large + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - setup_litellm_enterprise_pip + - run: + name: Run enterprise tests + command: | + uv run --no-sync python -m prisma generate + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit-enterprise.xml \ + --durations=10 \ + -n 4" + no_output_timeout: 15m + # Store test results + - store_test_results: + path: test-results + batches_testing: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 2" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml batches_coverage.xml + mv .coverage batches_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - batches_coverage.xml + - batches_coverage + litellm_utils_testing: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 2" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_utils_coverage.xml + mv .coverage litellm_utils_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_utils_coverage.xml + - litellm_utils_coverage + + pass_through_unit_testing: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 4" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml pass_through_unit_tests_coverage.xml + mv .coverage pass_through_unit_tests_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - pass_through_unit_tests_coverage.xml + - pass_through_unit_tests_coverage + image_gen_testing: + docker: + - *python312_image + working_directory: ~/project + resource_class: large + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 4" + no_output_timeout: 15m + # Store test results + - store_test_results: + path: test-results + logging_testing: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip + - run: + name: Run tests + command: | + mkdir -p test-results + export LITELLM_LOG=WARNING + TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv \ + --cov=./litellm --cov-report=xml \ + -n 4 \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + --timeout=120 --timeout_method=thread" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml logging_coverage.xml || true + mv .coverage logging_coverage || true + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - logging_coverage.xml + - logging_coverage + audio_testing: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml audio_coverage.xml + mv .coverage audio_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - audio_coverage.xml + - audio_coverage + redis_caching_unit_tests: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - save_cache: + paths: + - ~/.cache/uv + key: v1-uv-cache-{{ checksum "uv.lock" }} + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(printf "%s\n" \ + tests/local_testing/test_dual_cache.py \ + tests/local_testing/test_redis_batch_optimizations.py \ + tests/local_testing/test_router_utils.py) + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=./litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 -n 2 \ + --reruns 2 --reruns-delay 1" + no_output_timeout: 20m + - run: + name: Rename the coverage files + command: | + mv coverage.xml redis_caching_coverage.xml + mv .coverage redis_caching_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - redis_caching_coverage.xml + - redis_caching_coverage + installing_litellm_on_python: + docker: + - *python312_image + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - setup_litellm_enterprise_pip + - run: + name: Run tests + command: | + uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver" + + installing_litellm_on_python_3_13: + docker: + - image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: medium + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.13 + - run: + name: Run tests + command: | + uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver" + + installing_litellm_on_python_v2_migration_resolver: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - setup_litellm_enterprise_pip + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Run v2 migration resolver proxy smoke test + command: | + uv run --no-sync python -m pytest -vv \ + tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver + + helm_chart_testing: + machine: + image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker + resource_class: medium + working_directory: ~/project + + steps: + - checkout + - attach_workspace: + at: ~/project + - setup_google_dns + - install_helm + - install_kind + + # Install kubectl (pinned version with hardcoded checksum) + - run: + name: Install kubectl v1.31.4 + command: | + curl -sSLf -o /tmp/kubectl \ + https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl + echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c - + chmod +x /tmp/kubectl + sudo mv /tmp/kubectl /usr/local/bin/ + + # Create kind cluster + - run: + name: Create Kind Cluster + command: | + kind create cluster --name litellm-test + + - run: + name: Load Docker Database Image for helm tests + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + IMAGE_TAG=${CIRCLE_SHA1:-ci} + docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG} + + - run: + name: Load Docker image into Kind + command: | + IMAGE_TAG=${CIRCLE_SHA1:-ci} + kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test + + # Run helm lint + - run: + name: Run helm lint + command: | + helm lint ./deploy/charts/litellm-helm + + # Run helm tests + - run: + name: Run helm tests + command: | + IMAGE_TAG=${CIRCLE_SHA1:-ci} + helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \ + --set image.repository=litellm-ci \ + --set image.tag=${IMAGE_TAG} \ + --set image.pullPolicy=Never + # Wait for pod to be ready + echo "Waiting 30 seconds for pod to be ready..." + sleep 30 + + # Print pod logs before running tests + echo "Printing pod logs..." + kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") + + # Run the helm tests + helm test litellm --logs + + # Cleanup + - run: + name: Cleanup + command: | + kind delete cluster --name litellm-test + when: always # This ensures cleanup runs even if previous steps fail + + db_migration_disable_update_check: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: medium + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_fake_openai_endpoint + - start_postgres: + db_name: litellm_test + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Seed database with real schema + command: | + docker run -d \ + -p 4001:4000 \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + --name schema-seed \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --use_prisma_db_push + - wait_for_service: + url: http://localhost:4001 + timeout: "300" + - run: + name: Stop schema seed container + command: docker stop schema-seed && docker rm schema-seed + - run: + name: Run Docker container with bad schema and disabled updates + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ + -e DISABLE_SCHEMA_UPDATE="True" \ + -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ + --name my-app \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ + -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ + -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 + - wait_for_service: + url: http://localhost:4000 + timeout: "60" + - run: + name: Check container logs for expected message + command: | + echo "=== Printing Full Container Startup Logs ===" + LOG_OUTPUT="$(docker logs my-app 2>&1)" + printf '%s\n' "$LOG_OUTPUT" + echo "=== End of Full Container Startup Logs ===" + + if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then + echo "Expected message found in logs. Test passed." + else + echo "Expected message not found in logs. Test failed." + exit 1 + fi + - run: + name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit-2.xml \ + --durations=5" + no_output_timeout: 15m + - store_test_results: + path: test-results + + build_and_test: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - attach_workspace: + at: ~/project + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker tag litellm-docker-database:ci my-app:latest + - start_openai_record_replay_proxy + - start_fake_openai_endpoint + - run: + name: Run Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e USE_PRISMA_MIGRATE=True \ + -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ + -e AZURE_API_KEY=$AZURE_API_KEY \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ + -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ + -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e GROQ_API_KEY=$GROQ_API_KEY \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e COHERE_API_KEY=$COHERE_API_KEY \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e AUTO_INFER_REGION=True \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ + -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ + -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ + -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + -e RECORDER_OPENAI_BASE_URL=http://host.docker.internal:8090/v1 \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ + my-app:latest \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + - run: + name: Run tests + command: | + mkdir -p test-results + # Original used `tests/*.py` (top-level only); the `--ignore=...` + # flags were vestigial since shell globbing did not descend into + # subdirectories. Replicate by globbing only top-level test files. + TEST_FILES=$(circleci tests glob "tests/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -s -v -x \ + --junitxml=test-results/junit.xml \ + -n 4 \ + --durations=5" + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + e2e_openai_endpoints: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e AZURE_API_KEY=$AZURE_API_KEY \ + -e AZURE_API_BASE=$AZURE_API_BASE \ + -e AZURE_API_VERSION="2024-05-01-preview" \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ + -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ + -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e GROQ_API_KEY=$GROQ_API_KEY \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e COHERE_API_KEY=$COHERE_API_KEY \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e AUTO_INFER_REGION=True \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ + -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ + -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ + -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -s -vv \ + --junitxml=test-results/junit.xml \ + --durations=5" + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + proxy_logging_guardrails_model_info_tests: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - start_openai_record_replay_proxy + - start_fake_openai_endpoint + - run: + name: Run Docker container + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e OTEL_EXPORTER="in_memory" \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e COHERE_API_KEY=$COHERE_API_KEY \ + -e RECORDER_COHERE_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.cohere.com \ + -e GCS_FLUSH_INTERVAL="1" \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \ + -v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v \ + --junitxml=test-results/junit.xml \ + --durations=5" + no_output_timeout: 15m + # Clean up first container + - run: + name: Stop and remove first container + command: | + docker stop my-app + docker rm my-app + + # Second Docker Container Run with Different Config + # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license + - run: + name: Run Second Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ + -e LITELLM_LICENSE="bad-license" \ + --add-host host.docker.internal:host-gateway \ + --name my-app-3 \ + -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug + + - run: + name: Start outputting logs for second container + command: docker logs -f my-app-3 + background: true + + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + + - run: + name: Run second round of tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit-2.xml \ + --durations=5" + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + proxy_spend_accuracy_tests: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres + - start_redis + - start_fake_openai_endpoint + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + # Point the proxy at the job-local Redis (start_redis) instead of the + # shared remote Redis. The Redis transaction buffer uses a single + # global pod-lock key (cronjob_lock:db_spend_update_job) and a single + # global buffer list (litellm_spend_update_buffer); sharing those + # across concurrent CI pipelines causes spend flushes to stall or + # land in the wrong DB, which is what makes this test flaky. + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=host.docker.internal \ + -e REDIS_PORT=6379 \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e PROXY_BATCH_WRITE_AT=2 \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --junitxml=test-results/junit.xml \ + --durations=5" + no_output_timeout: 15m + - store_test_results: + path: test-results + - run: + name: Stop and remove first container + when: always + command: | + docker stop my-app + docker rm my-app + docker stop redis-cache + docker rm redis-cache + + proxy_multi_instance_tests: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres + - start_fake_openai_endpoint + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container 1 + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Run Docker container 2 + command: | + docker run -d \ + -p 4001:4001 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + --add-host host.docker.internal:host-gateway \ + --name my-app-2 \ + -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4001 \ + --detailed_debug + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + - wait_for_service: + url: http://localhost:4001 + timeout: "300" + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --junitxml=test-results/junit.xml \ + --durations=5" + no_output_timeout: 15m + # Clean up first container + # Store test results + - store_test_results: + path: test-results + + proxy_store_model_in_db_tests: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres + - start_fake_openai_endpoint + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e STORE_MODEL_IN_DB="True" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --junitxml=test-results/junit.xml \ + --durations=5" + no_output_timeout: 15m + - run: + name: Stop and remove containers + command: | + docker stop my-app || true + docker rm my-app || true + docker stop postgres-db || true + docker rm postgres-db || true + when: always + - store_test_results: + path: test-results + + proxy_build_from_pip_tests: + # Change from docker to machine executor + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + # Remove Docker CLI installation since it's already available in machine executor + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - run: + name: Build Docker image + command: | + docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . + - start_postgres + - start_fake_openai_endpoint + - run: + name: Run Docker container + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e OTEL_EXPORTER="in_memory" \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e COHERE_API_KEY=$COHERE_API_KEY \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e GCS_FLUSH_INTERVAL="1" \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ + my-app:latest \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --junitxml=test-results/junit-2.xml \ + --durations=5" + no_output_timeout: 15m + # Clean up first container + - store_test_results: + path: test-results + - run: + name: Stop and remove first container + command: | + docker stop my-app || true + docker rm my-app || true + docker stop postgres-db || true + docker rm postgres-db || true + when: always + proxy_pass_through_endpoint_tests: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e GEMINI_API_KEY=$GEMINI_API_KEY \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \ + -e AZURE_API_KEY=$AZURE_API_KEY \ + -e AZURE_API_BASE=$AZURE_API_BASE \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ + -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + # Add Ruby installation and testing before the existing Node.js and Python tests + - run: + name: Install Ruby and Bundler + command: | + # Clone RVM at pinned tag and verify the commit SHA matches the + # published tag before running its install script. + RVM_VERSION="1.29.12" + RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81" + git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm + RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)" + if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then + echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2 + exit 1 + fi + + # Import RVM signing keys (used by `rvm install` to verify Ruby tarballs) + gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB + + # Install RVM from the verified checkout. The install script + # sources `scripts/functions/installer` using paths relative to + # its own working directory, so it must be run from /tmp/rvm. + (cd /tmp/rvm && ./install --path "$HOME/.rvm") + source "$HOME/.rvm/scripts/rvm" + + # Install Ruby 3.2.2 (RVM verifies the tarball PGP signature) + rvm install 3.2.2 + rvm use 3.2.2 --default + + # Install latest Bundler + gem install bundler + + - run: + name: Run Ruby tests + command: | + source $HOME/.rvm/scripts/rvm + cd tests/pass_through_tests/ruby_passthrough_tests + bundle install + bundle exec rspec + no_output_timeout: 30m + # Install Node.js directly from nodejs.org with SHA256 verification, + # instead of piping NodeSource's setup_18.x apt-repo installer into + # sudo bash (which runs a mutable upstream script unattended). + - run: + name: Install Node.js 18.20.8 + command: | + NODE_VERSION="18.20.8" + NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz" + NODE_EXPECTED_SHA="5467ee62d6af1411d46b6a10e3fb5cacc92734dbcef465fea14e7b90993001c9" + curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" + echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c - + sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /usr/local --strip-components=1 + rm -f "/tmp/${NODE_TARBALL}" + node --version + npm --version + + - run: + name: Install Node.js test dependencies + command: | + cd tests/pass_through_tests + npm ci + + - run: + name: Run Vertex AI, Google AI Studio Node.js tests + command: | + cd tests/pass_through_tests + NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose + no_output_timeout: 30m + - run: + name: Run tests + command: | + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5" + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + + proxy_e2e_anthropic_messages_tests: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - start_postgres + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - start_openai_record_replay_proxy + - run: + name: Run Docker container with test config + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e RECORDER_ANTHROPIC_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.anthropic.com \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME="us-east-1" \ + -e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - wait_for_service: + url: http://localhost:4000 + timeout: "300" + - run: + name: Run Claude Agent SDK E2E Tests + command: | + mkdir -p test-results + export LITELLM_PROXY_URL="http://localhost:4000" + export LITELLM_API_KEY="sk-1234" + TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --junitxml=test-results/junit.xml \ + --durations=5" + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + + upload-coverage: + docker: + - *python312_image + steps: + - checkout + - attach_workspace: + at: . + # Check file locations + - run: + name: Check coverage file location + command: | + echo "Current directory:" + ls -la + echo "\nContents of tests/llm_translation:" + ls -la tests/llm_translation + - install_uv + - run: + name: Combine Coverage + command: | + uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage + uv tool run --from 'coverage[toml]==7.10.6' coverage xml + - codecov/upload: + file: ./coverage.xml + flags: circleci + + ui_build: + docker: + - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + resource_class: medium+ + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - ui-build-deps-v1- + - restore_cache: + keys: + - ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - ui-nextjs-cache-v1- + - run: + name: Install dependencies + command: | + cd ui/litellm-dashboard + npm ci + - save_cache: + key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules + - run: + name: Build UI + command: | + cd ui/litellm-dashboard + source ./build_ui.sh + - save_cache: + key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/.next/cache + - persist_to_workspace: + root: . + paths: + - litellm/proxy/_experimental/out + + ui_unit_tests: + docker: + - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - ui-unit-deps-v1- + - run: + name: Install dependencies + command: | + cd ui/litellm-dashboard + npm ci + - save_cache: + key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules + - run: + name: Run UI unit tests (Vitest) + command: | + cd ui/litellm-dashboard + + CI=true npm run test -- --run \ + --pool forks --poolOptions.forks.maxForks=8 + + e2e_ui_testing: + docker: + - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: e2euser + POSTGRES_PASSWORD: e2epassword + POSTGRES_DB: litellm_e2e + resource_class: large + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" + CI: "true" + # Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can + # assert the redirect. Set at job level so both the proxy boot step and the + # Playwright step (whose skip guard reads this) see the same value. Safe for + # the rest of the suite: nothing else performs a logout. + PROXY_LOGOUT_URL: "https://www.example.com" + steps: + - checkout + - setup_google_dns + - install_uv + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Install Python dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma + - save_cache: + key: v1-uv-cache-{{ checksum "uv.lock" }} + paths: + - ~/.cache/uv + - restore_cache: + keys: + - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - run: + name: Install Node dependencies and Playwright + # The cimg/python:3.12-browsers image already ships the Chromium system + # libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.). + # `--with-deps` triggers a redundant apt-get update + install that adds + # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors, + # so we install just the browser binary. + command: | + cd ui/litellm-dashboard + npm ci + npx playwright install chromium + - save_cache: + key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules + - ~/.cache/ms-playwright + - run: + name: Build UI from source + # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`. + # GNU cp (used on CircleCI's Ubuntu image) interprets that as "copy the + # source directory as a child of the destination" when the destination + # already exists — silently creating `_experimental/out/out/` instead of + # replacing the served bundle. The proxy continued serving whatever was + # checked into `_experimental/out/*`, so this job was effectively testing + # the pre-build bundle on every run. Replace-and-move guarantees the + # freshly built bundle is what the proxy actually serves. + command: | + cd ui/litellm-dashboard + npm run build + rm -rf ../../litellm/proxy/_experimental/out + mv out ../../litellm/proxy/_experimental/out + # Restructure HTML so extensionless routes work (login.html -> login/index.html) + find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do + d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" + done + - wait_for_service: + url: tcp://localhost:5432 + timeout: "30" + - run: + name: Push Prisma schema + command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Seed database + command: | + PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ + -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql + - run: + name: Start mock LLM server + command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py + background: true + - run: + name: Start LiteLLM proxy + environment: + LITELLM_MASTER_KEY: "sk-1234" + MOCK_LLM_URL: "http://127.0.0.1:8090/v1" + DISABLE_SCHEMA_UPDATE: "true" + SERVER_ROOT_PATH: "" + # PROXY_LOGOUT_URL is inherited from the job-level environment so the + # proxy and proxyLogoutUrl.spec.ts agree on the logout target. + # LITELLM_LICENSE is forwarded from the project env so premium-gated + # UI flows can be exercised. license.spec.ts asserts the resulting + # JWT carries premium_user=true; if it ever stops being passed, that + # test fails loudly rather than silently regressing premium coverage. + command: | + LITELLM_LICENSE="$LITELLM_LICENSE" \ + uv run --no-sync python -m litellm.proxy.proxy_cli \ + --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ + --port 4000 + background: true + - run: + name: Wait for proxy to be ready + command: | + for i in $(seq 1 60); do + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true) + if [ "$HTTP_CODE" = "200" ]; then + echo "Proxy is ready" + exit 0 + fi + sleep 2 + done + echo "Proxy failed to start" + exit 1 + - run: + name: Run Playwright E2E tests + # Forward LITELLM_LICENSE so license.spec.ts can detect that the + # proxy was launched with a license and assert premium_user=true. + command: | + cd ui/litellm-dashboard + LITELLM_LICENSE="$LITELLM_LICENSE" \ + npx playwright test --config e2e_tests/playwright.config.ts + no_output_timeout: 10m + - store_artifacts: + path: ui/litellm-dashboard/test-results + destination: e2e-test-results + - store_artifacts: + path: ui/litellm-dashboard/playwright-report + destination: e2e-playwright-report + + e2e_ui_testing_server_root_path: + docker: + - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: e2euser + POSTGRES_PASSWORD: e2epassword + POSTGRES_DB: litellm_e2e + resource_class: large + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" + CI: "true" + # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH + # is read both by the proxy at boot (to rewrite the built UI bundle in place) + # and by migration.serverRootPath.config.ts, which refuses to run without it. + SERVER_ROOT_PATH: "/litellm" + steps: + - checkout + - setup_google_dns + - install_uv + - restore_cache: + keys: + - v1-uv-cache-{{ checksum "uv.lock" }} + - run: + name: Install Python dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma + - save_cache: + key: v1-uv-cache-{{ checksum "uv.lock" }} + paths: + - ~/.cache/uv + - restore_cache: + keys: + - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - run: + name: Install Node dependencies and Playwright + command: | + cd ui/litellm-dashboard + npm ci + npx playwright install chromium + - save_cache: + key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules + - ~/.cache/ms-playwright + - run: + name: Build UI from source + command: | + cd ui/litellm-dashboard + npm run build + rm -rf ../../litellm/proxy/_experimental/out + mv out ../../litellm/proxy/_experimental/out + find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do + d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" + done + - wait_for_service: + url: tcp://localhost:5432 + timeout: "30" + - run: + name: Push Prisma schema + command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Seed database + command: | + PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ + -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql + - run: + name: Start mock LLM server + command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py + background: true + - run: + name: Start LiteLLM proxy under a server root path + environment: + LITELLM_MASTER_KEY: "sk-1234" + MOCK_LLM_URL: "http://127.0.0.1:8090/v1" + DISABLE_SCHEMA_UPDATE: "true" + # Output flows to this step's own log, so a boot crash is visible here + # rather than swallowed by a downstream readiness probe. + command: | + LITELLM_LICENSE="$LITELLM_LICENSE" \ + uv run --no-sync python -m litellm.proxy.proxy_cli \ + --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \ + --port 4000 + background: true + - run: + name: Wait for prefixed proxy to be ready + command: | + for i in $(seq 1 60); do + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer sk-1234" http://127.0.0.1:4000/litellm/health 2>/dev/null || true) + if [ "$HTTP_CODE" = "200" ]; then + echo "Prefixed proxy is ready" + exit 0 + fi + sleep 2 + done + echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log" + exit 1 + - run: + name: Run migration smoke under SERVER_ROOT_PATH + command: | + cd ui/litellm-dashboard + LITELLM_LICENSE="$LITELLM_LICENSE" \ + npx playwright test --config e2e_tests/migration.serverRootPath.config.ts + no_output_timeout: 10m + - store_artifacts: + path: ui/litellm-dashboard/test-results + destination: e2e-server-root-path-test-results + - store_artifacts: + path: ui/litellm-dashboard/playwright-report + destination: e2e-server-root-path-playwright-report + + build_docker_database_image: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + + - run: + name: Build Docker image + command: | + docker build \ + -t litellm-docker-database:ci \ + -f docker/Dockerfile.database . + + - run: + name: Save Docker image to workspace root + command: | + docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst + + - persist_to_workspace: + root: . + paths: + - litellm-docker-database.tar.zst + + test_bad_database_url: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: medium + working_directory: ~/project + steps: + - checkout + - attach_workspace: + at: ~/project + - setup_google_dns + - start_postgres + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker tag litellm-docker-database:ci myapp:latest + - run: + name: Run Docker container with bad DATABASE_URL + command: | + docker run --name my-app \ + -p 4000:4000 \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ + -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ + myapp:latest \ + --port 4000 > docker_output.log 2>&1 || true + - run: + name: Display Docker logs + command: cat docker_output.log + - run: + name: Check for expected error + command: | + if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \ + (grep -q "Database setup failed after multiple retries" docker_output.log || \ + grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then + echo "Expected error found. Test passed." + else + echo "Expected error not found. Test failed." + cat docker_output.log + exit 1 + fi + +workflows: + build_and_test: + when: << pipeline.parameters.run-code-tests >> + jobs: + - using_litellm_on_windows: + filters: &main_branches + branches: + only: + - main + - /litellm_.*/ + - local_testing_part1: + filters: *main_branches + - local_testing_part2: + filters: *main_branches + - langfuse_logging_unit_tests: + filters: *main_branches + - litellm_assistants_api_testing: + filters: *main_branches + - litellm_router_testing: + filters: *main_branches + - litellm_router_unit_testing: + filters: *main_branches + - ui_build: + filters: *main_branches + - ui_unit_tests: + requires: + - ui_build + filters: *main_branches + - auth_ui_unit_tests: + filters: *main_branches + - proxy_behavior_tests: + filters: *main_branches + - proxy_security_tests: + filters: *main_branches + - schema_migration_check: + filters: *main_branches + - build_docker_database_image: + filters: *main_branches + - e2e_ui_testing: + filters: *main_branches + - e2e_ui_testing_server_root_path: + filters: *main_branches + - build_and_test: + requires: + - build_docker_database_image + filters: *main_branches + - e2e_openai_endpoints: + requires: + - build_docker_database_image + filters: *main_branches + - proxy_logging_guardrails_model_info_tests: + requires: + - build_docker_database_image + filters: *main_branches + - proxy_spend_accuracy_tests: + requires: + - build_docker_database_image + filters: *main_branches + - proxy_multi_instance_tests: + requires: + - build_docker_database_image + filters: *main_branches + - proxy_store_model_in_db_tests: + requires: + - build_docker_database_image + filters: *main_branches + - proxy_build_from_pip_tests: + filters: *main_branches + - proxy_pass_through_endpoint_tests: + requires: + - build_docker_database_image + filters: *main_branches + - proxy_e2e_anthropic_messages_tests: + requires: + - build_docker_database_image + filters: *main_branches + - llm_translation_testing: + filters: *main_branches + - realtime_translation_testing: + filters: *main_branches + - agent_testing: + filters: *main_branches + - guardrails_testing: + filters: *main_branches + - google_generate_content_endpoint_testing: + filters: *main_branches + - llm_responses_api_testing: + filters: *main_branches + - ocr_testing: + filters: *main_branches + - search_testing: + filters: *main_branches + - litellm_mapped_enterprise_tests: + filters: *main_branches + - batches_testing: + filters: *main_branches + - litellm_utils_testing: + filters: *main_branches + - pass_through_unit_testing: + filters: *main_branches + - image_gen_testing: + filters: *main_branches + - logging_testing: + filters: *main_branches + - audio_testing: + filters: *main_branches + - redis_caching_unit_tests: + filters: *main_branches + - upload-coverage: + requires: + - realtime_translation_testing + - agent_testing + - google_generate_content_endpoint_testing + - guardrails_testing + - ocr_testing + - search_testing + - litellm_mapped_enterprise_tests + - batches_testing + - litellm_utils_testing + - pass_through_unit_testing + - image_gen_testing + - logging_testing + - audio_testing + - redis_caching_unit_tests + - langfuse_logging_unit_tests + - local_testing_part1 + - local_testing_part2 + - litellm_assistants_api_testing + - litellm_router_unit_testing + - auth_ui_unit_tests + - db_migration_disable_update_check: + requires: + - build_docker_database_image + filters: *main_branches + - installing_litellm_on_python: + filters: *main_branches + - installing_litellm_on_python_3_13: + filters: *main_branches + - installing_litellm_on_python_v2_migration_resolver: + filters: *main_branches + - helm_chart_testing: + requires: + - build_docker_database_image + filters: *main_branches + - test_bad_database_url: + requires: + - build_docker_database_image + filters: *main_branches diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml index da1267756cd..5e0a560e63d 100644 --- a/.github/workflows/test-unit-core-utils.yml +++ b/.github/workflows/test-unit-core-utils.yml @@ -7,6 +7,15 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" permissions: contents: read diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml index b2a8640223a..8dc2ee37d74 100644 --- a/.github/workflows/test-unit-documentation.yml +++ b/.github/workflows/test-unit-documentation.yml @@ -7,6 +7,15 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" permissions: contents: read diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml index ffc09dd8f94..b02882b4400 100644 --- a/.github/workflows/test-unit-enterprise-routing.yml +++ b/.github/workflows/test-unit-enterprise-routing.yml @@ -7,6 +7,15 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" permissions: contents: read diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml index b316ad5dfdf..d06d8e6f30e 100644 --- a/.github/workflows/test-unit-integrations.yml +++ b/.github/workflows/test-unit-integrations.yml @@ -7,6 +7,15 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" permissions: contents: read diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml index 2a1912ce92d..1d14f38196c 100644 --- a/.github/workflows/test-unit-llm-providers.yml +++ b/.github/workflows/test-unit-llm-providers.yml @@ -7,6 +7,15 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" permissions: contents: read diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml index 2226d519331..3f2b56a0901 100644 --- a/.github/workflows/test-unit-misc.yml +++ b/.github/workflows/test-unit-misc.yml @@ -7,6 +7,15 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" permissions: contents: read diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml index 99882066a8e..2994ba1abe7 100644 --- a/.github/workflows/test-unit-proxy-auth.yml +++ b/.github/workflows/test-unit-proxy-auth.yml @@ -7,6 +7,15 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" permissions: contents: read diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index 2ac9a3b7c1c..99525ef8d48 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -5,6 +5,15 @@ on: branches: - main - litellm_internal_staging + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" permissions: contents: read diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml index d9b6a348b60..f313e278a2a 100644 --- a/.github/workflows/test-unit-proxy-endpoints.yml +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -7,6 +7,15 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" workflow_dispatch: permissions: diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml index 336e53ee3d7..680c35ff4a5 100644 --- a/.github/workflows/test-unit-proxy-infra.yml +++ b/.github/workflows/test-unit-proxy-infra.yml @@ -7,6 +7,15 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" permissions: contents: read diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml index 5768551f9b0..43852ff843b 100644 --- a/.github/workflows/test-unit-proxy-legacy.yml +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -7,6 +7,15 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" permissions: contents: read diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml index 13069be9e3a..b0f41b1da46 100644 --- a/.github/workflows/test-unit-responses-caching-types.yml +++ b/.github/workflows/test-unit-responses-caching-types.yml @@ -7,6 +7,15 @@ on: - litellm_internal_staging - litellm_oss_branch - "litellm_**" + paths: + - "**/*.py" + - "**/*.ts" + - "**/*.tsx" + - "pyproject.toml" + - "uv.lock" + - "litellm/proxy/schema.prisma" + - ".github/workflows/test-unit-*.yml" + - ".github/workflows/_test-unit-base.yml" permissions: contents: read