litellm/.github/workflows/test-integration.yml

name: integration

on:
  pull_request:
    types: [labeled]
  schedule:
    - cron: "5 8-20/3 * * 1-6"
      timezone: America/Los_Angeles
  workflow_dispatch:
    inputs:
      suite:
        description: Integration group to run
        type: choice
        options:
          - all
          - providers
          - extensions
          - accounting
          - management
          - management-replica
          - database
          - database-replica
          - mcp
          - security
          - cost
          - sdk
          - browser
        default: all
      workers:
        description: Workers per group, blank keeps each group's default
        type: string
        default: ""

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  integration:
    name: integration ${{ matrix.suite }}
    if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
    environment: integration-license
    runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
    env:
      UV_PYTHON: "3.12"
      INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
    services:
      postgres:
        image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
        command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: circle_test
        ports:
          - 5432:5432
        options: >-
          --shm-size 1g
          --health-cmd "pg_isready -U postgres"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 20
      redis:
        image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 20
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false

      - name: Set up Python
        timeout-minutes: 3
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: ${{ env.UV_PYTHON }}

      - name: Set up uv
        timeout-minutes: 3
        uses: ./.github/actions/setup-uv-with-retries
        with:
          version: "0.10.9"

      - name: Cache the Rust build
        timeout-minutes: 5
        uses: ./.github/actions/cache-cargo-build

      - name: Cache Prisma binaries
        timeout-minutes: 5
        uses: ./.github/actions/cache-prisma-binaries

      - name: Install dependencies
        timeout-minutes: 20
        run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras

      - name: Install redis-server 7.2.16
        timeout-minutes: 3
        run: |
          mkdir -p "$RUNNER_TEMP/redis-bin"
          cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
          docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
          docker rm "$cid"
          "$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
          echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"

      - name: Install MCP SDK1 peer
        if: matrix.suite == 'mcp'
        timeout-minutes: 5
        run: |
          uv venv --python 3.12 .venv-mcp-peer
          uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
          echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"

      - name: Set up Node.js
        if: matrix.suite == 'browser'
        timeout-minutes: 5
        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
        with:
          node-version-file: ui/litellm-dashboard/.nvmrc
          cache: "npm"
          cache-dependency-path: |
            ui/litellm-dashboard/package-lock.json
            tests/e2e/ui/package-lock.json

      - name: Install browser dependencies and build the dashboard
        if: matrix.suite == 'browser'
        timeout-minutes: 20
        run: |
          (cd ui/litellm-dashboard && npm ci)
          (cd tests/e2e/ui && npm ci)
          sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
            node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
          node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
          (cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)

      - name: Run integration contracts
        timeout-minutes: 45
        env:
          SUITE: ${{ matrix.suite }}
          INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
          INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
          LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
        run: |
          case "$SUITE" in
            *-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
            *) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
          esac

      - name: Verify every case ran exactly once
        if: ${{ !cancelled() && matrix.suite != 'browser' }}
        timeout-minutes: 2
        run: |
          record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
          test -n "$record"
          mkdir -p "$RUNNER_TEMP/shards/0"
          cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
          .venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1

      - name: Report MCP coverage
        if: ${{ !cancelled() && matrix.suite == 'mcp' }}
        timeout-minutes: 5
        run: |
          coverage_dir=test-results/integration-mcp/coverage
          test -f "$coverage_dir/coverage.txt" && exit 0
          export COVERAGE_FILE="$PWD/$coverage_dir/data"
          .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
          .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
          .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
          tail -n 1 "$coverage_dir/coverage.txt"

      - name: Check results for the license
        id: secret-scan
        if: ${{ !cancelled() }}
        timeout-minutes: 5
        env:
          SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
        run: python .github/scripts/assert_secret_absent.py test-results

      - name: Upload results
        if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
        timeout-minutes: 10
        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
        with:
          name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
          path: test-results/
          retention-days: 14