From 6549f3eb1a94b1672f7d66e9fe825834ccf3b1c9 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 28 Mar 2026 12:06:45 -0700 Subject: [PATCH 1/4] [Infra] Add unit test workflows for Postgres, Redis, and security test suites Add three new GHA workflows for tests requiring service containers, plus a reusable base workflow that provides Postgres and cloud Redis support. New workflows: - test-unit-proxy-db.yml: proxy DB tests (key generation, auth checks, remaining) using a local Postgres container with a 3-way descriptive matrix - test-unit-caching-redis.yml: caching tests that need Redis but no provider API keys, using cloud Redis via the integration-redis environment - test-unit-security.yml: proxy security tests using a local Postgres container Reusable base (_test-unit-services-base.yml): - Local Postgres pinned by digest (postgres@sha256:705a5d5b...) - Cloud Redis credentials scoped to the integration-redis GHA environment - Environment binding is derived from enable-redis flag inside the base (not caller-controllable) to prevent secret scope bypass - Supports workers=0 for tests that cannot run in parallel Security hardening: - All actions pinned to commit SHAs - persist-credentials: false on all checkouts - permissions: contents: read only - Postgres-only workflows (proxy-db, security) use zero secrets and trigger on both pull_request and push to main/litellm_* - Redis workflow triggers on push only (not pull_request) to prevent external PRs from accessing Redis Cloud credentials - Added ${TEST_PATH:?} guard to both _test-unit-base.yml and _test-unit-services-base.yml to fail fast on empty test paths - All files pass zizmor --pedantic with zero findings Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/_test-unit-base.yml | 3 +- .../workflows/_test-unit-services-base.yml | 151 ++++++++++++++++++ .github/workflows/test-unit-caching-redis.yml | 35 ++++ .github/workflows/test-unit-proxy-db.yml | 42 +++++ .github/workflows/test-unit-security.yml | 25 +++ 5 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/_test-unit-services-base.yml create mode 100644 .github/workflows/test-unit-caching-redis.yml create mode 100644 .github/workflows/test-unit-proxy-db.yml create mode 100644 .github/workflows/test-unit-security.yml diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 7a62fd37c4c..f1ae30e67d7 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -33,6 +33,7 @@ permissions: jobs: run: + name: Run tests runs-on: ubuntu-latest timeout-minutes: ${{ inputs.timeout-minutes }} @@ -85,7 +86,7 @@ jobs: WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} run: | - poetry run pytest ${TEST_PATH} \ + poetry run pytest ${TEST_PATH:?} \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ -n "${WORKERS}" \ diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml new file mode 100644 index 00000000000..86f51173be3 --- /dev/null +++ b/.github/workflows/_test-unit-services-base.yml @@ -0,0 +1,151 @@ +name: _Unit Test Services Base (Reusable) + +on: + workflow_call: + inputs: + test-path: + description: "Pytest path(s) to run" + required: true + type: string + workers: + description: "Number of pytest-xdist workers (0 = no parallelism)" + required: false + type: number + default: 2 + reruns: + description: "Number of reruns for flaky tests" + required: false + type: number + default: 2 + timeout-minutes: + description: "Job timeout in minutes" + required: false + type: number + default: 20 + max-failures: + description: "Stop after this many failures" + required: false + type: number + default: 10 + enable-redis: + description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars" + required: false + type: boolean + default: false + enable-postgres: + description: "Start a local Postgres service container and run Prisma migrations" + required: false + type: boolean + default: false + secrets: + REDIS_HOST: + required: false + REDIS_PORT: + required: false + REDIS_PASSWORD: + required: false + +permissions: + contents: read + +jobs: + run: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + # Environment is derived from enable-redis, not caller-controllable. + # This prevents callers from passing arbitrary environment names to bypass secret scoping. + environment: ${{ inputs.enable-redis && 'integration-redis' || '' }} + + services: + postgres: + image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-services-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry-services- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run Prisma migrations + if: ${{ inputs.enable-postgres }} + env: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + run: | + poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + + - name: Run tests + env: + TEST_PATH: ${{ inputs.test-path }} + MAX_FAILURES: ${{ inputs.max-failures }} + WORKERS: ${{ inputs.workers }} + RERUNS: ${{ inputs.reruns }} + DATABASE_URL: ${{ inputs.enable-postgres && 'postgresql://postgres:postgres@localhost:5432/litellm_test' || '' }} + REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }} + REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }} + REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }} + run: | + if [ "${WORKERS}" = "0" ]; then + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --durations=20 + else + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 + fi diff --git a/.github/workflows/test-unit-caching-redis.yml b/.github/workflows/test-unit-caching-redis.yml new file mode 100644 index 00000000000..4475dbf1b7d --- /dev/null +++ b/.github/workflows/test-unit-caching-redis.yml @@ -0,0 +1,35 @@ +name: "Unit Tests: Caching (Redis)" + +# Uses cloud Redis credentials — only runs on trusted branches, not PRs. +# This prevents external PRs from accessing Redis credentials. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + caching-redis: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + # Redis-only tests that do NOT require provider API keys. + # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py, + # test_router_caching.py) are in Phase 3 integration workflows. + test-path: >- + tests/local_testing/test_dual_cache.py + tests/local_testing/test_redis_batch_optimizations.py + tests/local_testing/test_router_utils.py + workers: 2 + reruns: 2 + timeout-minutes: 20 + enable-redis: true + enable-postgres: false + secrets: + REDIS_HOST: ${{ secrets.REDIS_HOST }} + REDIS_PORT: ${{ secrets.REDIS_PORT }} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml new file mode 100644 index 00000000000..0231da3f6e4 --- /dev/null +++ b/.github/workflows/test-unit-proxy-db.yml @@ -0,0 +1,42 @@ +name: "Unit Tests: Proxy DB Operations" + +on: + pull_request: + branches: [main] + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-db: + strategy: + fail-fast: false + matrix: + include: + # Key generation tests must NOT run in parallel (event loop conflicts with logging worker) + - test-group: key-generation + test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py" + workers: 0 + timeout: 30 + - test-group: auth-checks + test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + - test-group: remaining + test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: ${{ matrix.test-path }} + workers: ${{ matrix.workers }} + reruns: 2 + timeout-minutes: ${{ matrix.timeout }} + enable-redis: false + enable-postgres: true diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml new file mode 100644 index 00000000000..63eed494a73 --- /dev/null +++ b/.github/workflows/test-unit-security.yml @@ -0,0 +1,25 @@ +name: "Unit Tests: Security" + +on: + pull_request: + branches: [main] + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + security: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: "tests/proxy_security_tests/" + workers: 1 + reruns: 2 + timeout-minutes: 20 + enable-redis: false + enable-postgres: true From d42e2f64291136b4853908a534db8579da9edd2a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 28 Mar 2026 13:28:41 -0700 Subject: [PATCH 2/4] [Fix] Move Postgres DATABASE_URL to environment secret to avoid credential leak warnings The hardcoded postgresql://postgres:postgres@localhost connection string was being flagged by secret scanners. Move DATABASE_URL to a GHA environment secret (integration-postgres) so the password is never in the workflow file. Changes: - _test-unit-services-base.yml: DATABASE_URL now comes from secrets, environment is derived from enable-* flags (integration-postgres, integration-redis, or integration-redis-postgres) - test-unit-proxy-db.yml: switched to push-only trigger (uses secrets now) - test-unit-security.yml: switched to push-only trigger (uses secrets now) Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/_test-unit-services-base.yml | 16 ++++++++++++---- .github/workflows/test-unit-proxy-db.yml | 7 ++++--- .github/workflows/test-unit-security.yml | 7 ++++--- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml index 86f51173be3..042008f24a8 100644 --- a/.github/workflows/_test-unit-services-base.yml +++ b/.github/workflows/_test-unit-services-base.yml @@ -44,6 +44,8 @@ on: required: false REDIS_PASSWORD: required: false + DATABASE_URL: + required: false permissions: contents: read @@ -53,9 +55,15 @@ jobs: name: Run tests runs-on: ubuntu-latest timeout-minutes: ${{ inputs.timeout-minutes }} - # Environment is derived from enable-redis, not caller-controllable. + # Environment is derived from the enable-* flags, not caller-controllable. # This prevents callers from passing arbitrary environment names to bypass secret scoping. - environment: ${{ inputs.enable-redis && 'integration-redis' || '' }} + environment: >- + ${{ + (inputs.enable-redis && inputs.enable-postgres) && 'integration-redis-postgres' || + inputs.enable-redis && 'integration-redis' || + inputs.enable-postgres && 'integration-postgres' || + '' + }} services: postgres: @@ -117,7 +125,7 @@ jobs: - name: Run Prisma migrations if: ${{ inputs.enable-postgres }} env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + DATABASE_URL: ${{ secrets.DATABASE_URL }} run: | poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss @@ -127,7 +135,7 @@ jobs: MAX_FAILURES: ${{ inputs.max-failures }} WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} - DATABASE_URL: ${{ inputs.enable-postgres && 'postgresql://postgres:postgres@localhost:5432/litellm_test' || '' }} + DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }} REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }} REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }} REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }} diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index 0231da3f6e4..db0eef1cf66 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -1,8 +1,7 @@ name: "Unit Tests: Proxy DB Operations" +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. on: - pull_request: - branches: [main] push: branches: [main, "litellm_*"] @@ -10,7 +9,7 @@ permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -40,3 +39,5 @@ jobs: timeout-minutes: ${{ matrix.timeout }} enable-redis: false enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml index 63eed494a73..1a08608740a 100644 --- a/.github/workflows/test-unit-security.yml +++ b/.github/workflows/test-unit-security.yml @@ -1,8 +1,7 @@ name: "Unit Tests: Security" +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. on: - pull_request: - branches: [main] push: branches: [main, "litellm_*"] @@ -10,7 +9,7 @@ permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -23,3 +22,5 @@ jobs: timeout-minutes: 20 enable-redis: false enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} From 3ae80407dd1e9c1ea3c55fcb3cbd485a253c94c8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 28 Mar 2026 13:31:58 -0700 Subject: [PATCH 3/4] [Fix] Move Postgres username and password to environment secrets Move POSTGRES_USER and POSTGRES_PASSWORD from hardcoded values to environment secrets so no credentials appear in workflow files at all. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/_test-unit-services-base.yml | 10 +++++++--- .github/workflows/test-unit-proxy-db.yml | 2 ++ .github/workflows/test-unit-security.yml | 2 ++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml index 042008f24a8..f5bd0e2ea66 100644 --- a/.github/workflows/_test-unit-services-base.yml +++ b/.github/workflows/_test-unit-services-base.yml @@ -46,6 +46,10 @@ on: required: false DATABASE_URL: required: false + POSTGRES_USER: + required: false + POSTGRES_PASSWORD: + required: false permissions: contents: read @@ -69,13 +73,13 @@ jobs: postgres: image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14 env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} POSTGRES_DB: litellm_test ports: - 5432:5432 options: >- - --health-cmd "pg_isready -U postgres" + --health-cmd "pg_isready" --health-interval 10s --health-timeout 5s --health-retries 5 diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index db0eef1cf66..bdfb6efeef1 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -41,3 +41,5 @@ jobs: enable-postgres: true secrets: DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml index 1a08608740a..b38c82b1c24 100644 --- a/.github/workflows/test-unit-security.yml +++ b/.github/workflows/test-unit-security.yml @@ -24,3 +24,5 @@ jobs: enable-postgres: true secrets: DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} From 3b5b98327e466b0b1e7350d5056a0c0fb3b327d2 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 28 Mar 2026 14:25:29 -0700 Subject: [PATCH 4/4] [Fix] Use integration-redis-postgres env for Redis workflows since Postgres always starts GHA doesn't support conditional service containers, so the Postgres container always starts even for Redis-only jobs. Use integration-redis-postgres environment for any workflow with enable-redis so the Postgres container gets valid credentials. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/_test-unit-services-base.yml | 5 +++-- .github/workflows/test-unit-caching-redis.yml | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml index f5bd0e2ea66..d53a9e8822a 100644 --- a/.github/workflows/_test-unit-services-base.yml +++ b/.github/workflows/_test-unit-services-base.yml @@ -61,10 +61,11 @@ jobs: timeout-minutes: ${{ inputs.timeout-minutes }} # Environment is derived from the enable-* flags, not caller-controllable. # This prevents callers from passing arbitrary environment names to bypass secret scoping. + # Note: Postgres service container always starts (GHA limitation), so any Redis job + # also needs Postgres secrets → uses integration-redis-postgres, not integration-redis. environment: >- ${{ - (inputs.enable-redis && inputs.enable-postgres) && 'integration-redis-postgres' || - inputs.enable-redis && 'integration-redis' || + inputs.enable-redis && 'integration-redis-postgres' || inputs.enable-postgres && 'integration-postgres' || '' }} diff --git a/.github/workflows/test-unit-caching-redis.yml b/.github/workflows/test-unit-caching-redis.yml index 4475dbf1b7d..ca274324f2f 100644 --- a/.github/workflows/test-unit-caching-redis.yml +++ b/.github/workflows/test-unit-caching-redis.yml @@ -33,3 +33,6 @@ jobs: REDIS_HOST: ${{ secrets.REDIS_HOST }} REDIS_PORT: ${{ secrets.REDIS_PORT }} REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}