Merge pull request #24740 from BerriAI/litellm_unit_test_workflow_isolation

[Infra] Isolate unit test workflows with hardened security posture

.github/workflows/_test-unit-base.yml

@@ -0,0 +1,95 @@
+name: _Unit Test Base (Reusable)
+
+on:
+  workflow_call:
+    inputs:
+      test-path:
+        description: "Pytest path(s) to run"
+        required: true
+        type: string
+      workers:
+        description: "Number of pytest-xdist workers"
+        required: false
+        type: number
+        default: 2
+      reruns:
+        description: "Number of reruns for flaky tests"
+        required: false
+        type: number
+        default: 2
+      timeout-minutes:
+        description: "Job timeout in minutes"
+        required: false
+        type: number
+        default: 20
+      max-failures:
+        description: "Stop after this many failures"
+        required: false
+        type: number
+        default: 10
+
+permissions:
+  contents: read
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    timeout-minutes: ${{ inputs.timeout-minutes }}
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+          poetry run pip install google-genai==1.22.0 \
+            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          poetry run pip install nodejs-wheel-binaries==24.13.1
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run tests
+        env:
+          TEST_PATH: ${{ inputs.test-path }}
+          MAX_FAILURES: ${{ inputs.max-failures }}
+          WORKERS: ${{ inputs.workers }}
+          RERUNS: ${{ inputs.reruns }}
+        run: |
+          poetry run pytest ${TEST_PATH} \
+            --tb=short -vv \
+            --maxfail="${MAX_FAILURES}" \
+            -n "${WORKERS}" \
+            --reruns "${RERUNS}" \
+            --reruns-delay 1 \
+            --dist=loadscope \
+            --durations=20

.github/workflows/test-unit-core-utils.yml

@@ -0,0 +1,20 @@
+name: "Unit Tests: Core Utilities"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  core-utils:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/litellm_core_utils"
+      workers: 2
+      reruns: 1

.github/workflows/test-unit-enterprise-routing.yml

@@ -0,0 +1,24 @@
+name: "Unit Tests: Enterprise, Google GenAI & Routing"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  enterprise-routing:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: >-
+        tests/test_litellm/enterprise
+        tests/test_litellm/google_genai
+        tests/test_litellm/router_utils
+        tests/test_litellm/router_strategy
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-integrations.yml

@@ -0,0 +1,20 @@
+name: "Unit Tests: Integrations (Callbacks & Logging)"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  integrations:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/integrations"
+      workers: 2
+      reruns: 3

.github/workflows/test-unit-llm-providers.yml

@@ -0,0 +1,29 @@
+name: "Unit Tests: LLM Provider Transformations"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  vertex-ai:
+    name: Vertex AI
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/llms/vertex_ai"
+      workers: 1
+      reruns: 2
+
+  other-providers:
+    name: All Other Providers
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-misc.yml

@@ -0,0 +1,31 @@
+name: "Unit Tests: MCP, Secrets, Containers & Misc"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  misc:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: >-
+        tests/test_litellm/secret_managers
+        tests/test_litellm/a2a_protocol
+        tests/test_litellm/anthropic_interface
+        tests/test_litellm/completion_extras
+        tests/test_litellm/containers
+        tests/test_litellm/experimental_mcp_client
+        tests/test_litellm/images
+        tests/test_litellm/interactions
+        tests/test_litellm/passthrough
+        tests/test_litellm/vector_stores
+        tests/test_litellm/test_*.py
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-proxy-auth.yml

@@ -0,0 +1,20 @@
+name: "Unit Tests: Proxy Auth & Key Management"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy-auth:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client"
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-proxy-endpoints.yml

@@ -0,0 +1,35 @@
+name: "Unit Tests: Proxy API Endpoints"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy-endpoints:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: >-
+        tests/test_litellm/proxy/management_endpoints
+        tests/test_litellm/proxy/guardrails
+        tests/test_litellm/proxy/management_helpers
+        tests/test_litellm/proxy/anthropic_endpoints
+        tests/test_litellm/proxy/google_endpoints
+        tests/test_litellm/proxy/openai_files_endpoint
+        tests/test_litellm/proxy/response_api_endpoints
+        tests/test_litellm/proxy/image_endpoints
+        tests/test_litellm/proxy/vector_store_endpoints
+        tests/test_litellm/proxy/agent_endpoints
+        tests/test_litellm/proxy/discovery_endpoints
+        tests/test_litellm/proxy/health_endpoints
+        tests/test_litellm/proxy/public_endpoints
+        tests/test_litellm/proxy/prompts
+        tests/test_litellm/proxy/ui_crud_endpoints
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-proxy-infra.yml

@@ -0,0 +1,28 @@
+name: "Unit Tests: Proxy Infrastructure"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy-infra:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: >-
+        tests/test_litellm/proxy/db
+        tests/test_litellm/proxy/middleware
+        tests/test_litellm/proxy/spend_tracking
+        tests/test_litellm/proxy/pass_through_endpoints
+        tests/test_litellm/proxy/_experimental
+        tests/test_litellm/proxy/experimental
+        tests/test_litellm/proxy/common_utils
+        tests/test_litellm/proxy/test_*.py
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-proxy-legacy.yml

@@ -0,0 +1,96 @@
+name: "Unit Tests: Proxy Legacy Tests"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group:
+          - name: "auth-and-jwt"
+            path: "tests/proxy_unit_tests/test_[a-j]*.py"
+          - name: "key-generation"
+            path: "tests/proxy_unit_tests/test_[k-o]*.py"
+          - name: "proxy-config"
+            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+          - name: "proxy-server"
+            path: "tests/proxy_unit_tests/test_proxy_server.py"
+          - name: "proxy-server-extras"
+            path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
+          - name: "proxy-utils"
+            path: "tests/proxy_unit_tests/test_proxy_utils.py"
+          - name: "proxy-token-counter"
+            path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
+          - name: "proxy-response-and-misc"
+            path: "tests/proxy_unit_tests/test_[r-t]*.py"
+          - name: "proxy-user-auth-and-spend"
+            path: "tests/proxy_unit_tests/test_[u-z]*.py"
+
+    name: ${{ matrix.test-group.name }}
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+          poetry run pip install google-genai==1.22.0 \
+            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          poetry run pip install nodejs-wheel-binaries==24.13.1
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run tests - ${{ matrix.test-group.name }}
+        env:
+          TEST_PATH: ${{ matrix.test-group.path }}
+        run: |
+          poetry run pytest ${TEST_PATH} \
+            --tb=short -vv \
+            --maxfail=10 \
+            -n 2 \
+            --reruns 1 \
+            --reruns-delay 1 \
+            --dist=loadscope \
+            --durations=20

.github/workflows/test-unit-responses-caching-types.yml

@@ -0,0 +1,20 @@
+name: "Unit Tests: Responses, Caching & Types"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  responses-caching-types:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
+      workers: 2
+      reruns: 2