name: LiteLLM Unit Tests (Matrix) on: pull_request: branches: [main] permissions: contents: read # Cancel in-progress runs for the same PR concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: test: runs-on: ubuntu-latest timeout-minutes: 20 # Increased from 15 to 20 strategy: fail-fast: false matrix: test-group: # tests/test_litellm split by subdirectory (~560 files total) # Vertex AI tests separated for better isolation (prevent auth/env pollution) - name: "llms-vertex" path: "tests/test_litellm/llms/vertex_ai" workers: 1 reruns: 2 - name: "llms-other" path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" workers: 2 reruns: 2 # tests/test_litellm/proxy split by subdirectory (~180 files total) - name: "proxy-guardrails" path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers" workers: 2 reruns: 2 - name: "proxy-core" path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine" workers: 2 reruns: 2 - name: "proxy-misc" path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py" workers: 2 reruns: 2 - name: "integrations" path: "tests/test_litellm/integrations" workers: 2 reruns: 3 # Integration tests tend to be flakier - name: "core-utils" path: "tests/test_litellm/litellm_core_utils" workers: 2 reruns: 1 - name: "other-1" # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" workers: 2 reruns: 2 - name: "other-2" # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils" workers: 2 reruns: 2 - name: "other-3" # remaining dirs ≈ 8.0k lines path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores" workers: 2 reruns: 2 - name: "root" path: "tests/test_litellm/test_*.py" workers: 2 reruns: 2 # tests/proxy_unit_tests split alphabetically (~48 files total) - name: "proxy-unit-a1" # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest path: "tests/proxy_unit_tests/test_[a-j]*.py" workers: 2 reruns: 1 - name: "proxy-unit-a2" # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback path: "tests/proxy_unit_tests/test_[k-o]*.py" workers: 2 reruns: 1 - name: "proxy-unit-b1" # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*) path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" workers: 2 reruns: 1 - name: "proxy-unit-b2" # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests path: "tests/proxy_unit_tests/test_proxy_server.py" workers: 2 reruns: 1 - name: "proxy-unit-b3" # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py" workers: 2 reruns: 1 - name: "proxy-unit-b4" # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter path: "tests/proxy_unit_tests/test_proxy_utils.py" workers: 2 reruns: 1 - name: "proxy-unit-b5" # proxy_token_counter (1279) - runs independently from utils path: "tests/proxy_unit_tests/test_proxy_token_counter.py" workers: 2 reruns: 1 - name: "proxy-unit-b6" # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62) path: "tests/proxy_unit_tests/test_[r-t]*.py" workers: 2 reruns: 1 - name: "proxy-unit-b7" # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157) path: "tests/proxy_unit_tests/test_[u-z]*.py" workers: 2 reruns: 1 name: test (${{ matrix.test-group.name }}) steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install Poetry run: pip install 'poetry==2.3.2' - name: Cache Poetry dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 with: path: | ~/.cache/pypoetry ~/.cache/pip .venv key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} restore-keys: | ${{ runner.os }}-poetry- - name: Install dependencies run: | poetry config virtualenvs.in-project true poetry install --with dev,proxy-dev --extras "proxy semantic-router" # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies poetry run pip install google-genai==1.22.0 \ google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 - name: Setup litellm-enterprise run: | poetry run pip install --force-reinstall --no-deps -e enterprise/ - name: Generate Prisma client env: PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache run: | poetry run pip install nodejs-wheel-binaries==24.13.1 poetry run prisma generate --schema litellm/proxy/schema.prisma - name: Run tests - ${{ matrix.test-group.name }} run: | poetry run pytest ${{ matrix.test-group.path }} \ --tb=short -vv \ --maxfail=10 \ -n ${{ matrix.test-group.workers }} \ --reruns ${{ matrix.test-group.reruns }} \ --reruns-delay 1 \ --dist=loadscope \ --durations=20 \ --cov=litellm \ --cov-report=xml:coverage-${{ matrix.test-group.name }}.xml \ --cov-config=pyproject.toml - name: Save coverage report if: always() uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: coverage-${{ matrix.test-group.name }} path: coverage-${{ matrix.test-group.name }}.xml retention-days: 1 upload-coverage: name: Upload coverage to Codecov needs: test if: always() runs-on: ubuntu-latest permissions: contents: read id-token: write # Required for OIDC tokenless upload pull-requests: write # Required for Codecov PR comments steps: - name: Checkout code uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - name: Download all coverage reports uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 with: pattern: coverage-* path: coverage-reports merge-multiple: true - name: Upload to Codecov uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4 with: use_oidc: true directory: coverage-reports root_dir: ${{ github.workspace }} fail_ci_if_error: false