From c83d2c407c9e6afd9f39f36b9b5b9d28d5bca61a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 4 Apr 2026 23:58:08 -0700 Subject: [PATCH] chore: fixes --- .circleci/config.yml | 24 +- .github/workflows/_test-unit-base.yml | 25 +- .../workflows/_test-unit-services-base.yml | 28 +- .../workflows/run_llm_translation_tests.py | 0 .github/workflows/scorecard.yml | 2 +- .github/workflows/test-litellm-matrix.yml | 214 ++++++++++++++ .github/workflows/test-litellm.yml | 4 +- .github/workflows/test-unit-core-utils.yml | 3 - .../test-unit-enterprise-routing.yml | 3 - .github/workflows/test-unit-integrations.yml | 3 - .github/workflows/test-unit-llm-providers.yml | 4 - .github/workflows/test-unit-misc.yml | 3 - .github/workflows/test-unit-proxy-auth.yml | 3 - .../workflows/test-unit-proxy-endpoints.yml | 3 - .github/workflows/test-unit-proxy-infra.yml | 3 - .github/workflows/test-unit-proxy-legacy.yml | 17 +- .../test-unit-responses-caching-types.yml | 3 - .pre-commit-config.yaml | 40 --- .trivyignore | 12 - ci_cd/.grype.yaml | 36 --- ci_cd/publish-proxy-extras.sh | 19 -- ci_cd/security_scans.sh | 262 ------------------ docs/my-website/.trivyignore | 7 - scripts/install.sh | 4 +- ui/litellm-dashboard/.trivyignore | 7 - 25 files changed, 224 insertions(+), 505 deletions(-) mode change 100755 => 100644 .github/workflows/run_llm_translation_tests.py create mode 100644 .github/workflows/test-litellm-matrix.yml delete mode 100644 .pre-commit-config.yaml delete mode 100644 .trivyignore delete mode 100644 ci_cd/.grype.yaml delete mode 100644 ci_cd/publish-proxy-extras.sh delete mode 100755 ci_cd/security_scans.sh delete mode 100644 docs/my-website/.trivyignore delete mode 100644 ui/litellm-dashboard/.trivyignore diff --git a/.circleci/config.yml b/.circleci/config.yml index 5bbabad9107..307247651e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3233,31 +3233,9 @@ jobs: name: Install Playwright Browsers command: | npx playwright install - - run: - name: Install Neon CLI - command: | - npm i -g neonctl - - run: - name: Create Neon branch - command: | - export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ") - echo "Expires at: $EXPIRES_AT" - neon branches create \ - --project-id $NEON_PROJECT_ID \ - --name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \ - --expires-at $EXPIRES_AT \ - --parent br-fancy-paper-ad1olsb3 \ - --api-key $NEON_API_KEY || true - run: name: Run Docker container command: | - E2E_UI_TEST_DATABASE_URL=$(neon connection-string \ - --project-id $NEON_PROJECT_ID \ - --api-key $NEON_API_KEY \ - --branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \ - --database-name yuneng-trial-db \ - --role neondb_owner) - echo $E2E_UI_TEST_DATABASE_URL docker run -d \ -p 4000:4000 \ -e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \ @@ -3531,7 +3509,7 @@ workflows: only: - main - /litellm_.*/ - # - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli + # - e2e_ui_testing: # name: e2e_ui_testing_chromium # browser: chromium # context: e2e_ui_tests diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 966e5eef574..f1ae30e67d7 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -27,16 +27,9 @@ on: required: false type: number default: 10 - coverage-name: - description: "Coverage report name for Codecov (empty = skip coverage)" - required: false - type: string - default: "" permissions: contents: read - id-token: write - pull-requests: write jobs: run: @@ -92,12 +85,7 @@ jobs: MAX_FAILURES: ${{ inputs.max-failures }} WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} - COVERAGE_NAME: ${{ inputs.coverage-name }} run: | - COV_FLAGS="" - if [ -n "${COVERAGE_NAME}" ]; then - COV_FLAGS="--cov=litellm --cov-report=xml:coverage-${COVERAGE_NAME}.xml --cov-config=pyproject.toml" - fi poetry run pytest ${TEST_PATH:?} \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ @@ -105,15 +93,4 @@ jobs: --reruns "${RERUNS}" \ --reruns-delay 1 \ --dist=loadscope \ - --durations=20 \ - ${COV_FLAGS} - - - name: Upload coverage to Codecov - if: ${{ always() && inputs.coverage-name != '' }} - uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4 - with: - use_oidc: true - files: coverage-${{ inputs.coverage-name }}.xml - flags: ${{ inputs.coverage-name }} - root_dir: ${{ github.workspace }} - fail_ci_if_error: false + --durations=20 diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml index a1d2f7c49b9..d53a9e8822a 100644 --- a/.github/workflows/_test-unit-services-base.yml +++ b/.github/workflows/_test-unit-services-base.yml @@ -37,11 +37,6 @@ on: required: false type: boolean default: false - coverage-name: - description: "Coverage report name for Codecov (empty = skip coverage)" - required: false - type: string - default: "" secrets: REDIS_HOST: required: false @@ -58,8 +53,6 @@ on: permissions: contents: read - id-token: write - pull-requests: write jobs: run: @@ -147,24 +140,18 @@ jobs: MAX_FAILURES: ${{ inputs.max-failures }} WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} - COVERAGE_NAME: ${{ inputs.coverage-name }} DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }} REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }} REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }} REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }} run: | - COV_FLAGS="" - if [ -n "${COVERAGE_NAME}" ]; then - COV_FLAGS="--cov=litellm --cov-report=xml:coverage-${COVERAGE_NAME}.xml --cov-config=pyproject.toml" - fi if [ "${WORKERS}" = "0" ]; then poetry run pytest ${TEST_PATH:?} \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ --reruns "${RERUNS}" \ --reruns-delay 1 \ - --durations=20 \ - ${COV_FLAGS} + --durations=20 else poetry run pytest ${TEST_PATH:?} \ --tb=short -vv \ @@ -173,16 +160,5 @@ jobs: --reruns "${RERUNS}" \ --reruns-delay 1 \ --dist=loadscope \ - --durations=20 \ - ${COV_FLAGS} + --durations=20 fi - - - name: Upload coverage to Codecov - if: ${{ always() && inputs.coverage-name != '' }} - uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4 - with: - use_oidc: true - files: coverage-${{ inputs.coverage-name }}.xml - flags: ${{ inputs.coverage-name }} - root_dir: ${{ github.workspace }} - fail_ci_if_error: false diff --git a/.github/workflows/run_llm_translation_tests.py b/.github/workflows/run_llm_translation_tests.py old mode 100755 new mode 100644 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 7cd12bb219c..3a00064c3bd 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -42,6 +42,6 @@ jobs: retention-days: 5 - name: Upload to code scanning - uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1 + uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1 with: sarif_file: results.sarif diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml new file mode 100644 index 00000000000..dafabe4d83e --- /dev/null +++ b/.github/workflows/test-litellm-matrix.yml @@ -0,0 +1,214 @@ +name: LiteLLM Unit Tests (Matrix) + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +# Cancel in-progress runs for the same PR +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 20 # Increased from 15 to 20 + strategy: + fail-fast: false + matrix: + test-group: + # tests/test_litellm split by subdirectory (~560 files total) + # Vertex AI tests separated for better isolation (prevent auth/env pollution) + - name: "llms-vertex" + path: "tests/test_litellm/llms/vertex_ai" + workers: 1 + reruns: 2 + - name: "llms-other" + path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" + workers: 2 + reruns: 2 + # tests/test_litellm/proxy split by subdirectory (~180 files total) + - name: "proxy-guardrails" + path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers" + workers: 2 + reruns: 2 + - name: "proxy-core" + path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine" + workers: 2 + reruns: 2 + - name: "proxy-misc" + path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py" + workers: 2 + reruns: 2 + - name: "integrations" + path: "tests/test_litellm/integrations" + workers: 2 + reruns: 3 # Integration tests tend to be flakier + - name: "core-utils" + path: "tests/test_litellm/litellm_core_utils" + workers: 2 + reruns: 1 + - name: "other-1" + # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines + path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" + workers: 2 + reruns: 2 + - name: "other-2" + # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines + path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils" + workers: 2 + reruns: 2 + - name: "other-3" + # remaining dirs ≈ 8.0k lines + path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores" + workers: 2 + reruns: 2 + - name: "root" + path: "tests/test_litellm/test_*.py" + workers: 2 + reruns: 2 + # tests/proxy_unit_tests split alphabetically (~48 files total) + - name: "proxy-unit-a1" + # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest + path: "tests/proxy_unit_tests/test_[a-j]*.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-a2" + # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback + path: "tests/proxy_unit_tests/test_[k-o]*.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b1" + # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*) + path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b2" + # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests + path: "tests/proxy_unit_tests/test_proxy_server.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b3" + # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests + path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b4" + # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter + path: "tests/proxy_unit_tests/test_proxy_utils.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b5" + # proxy_token_counter (1279) - runs independently from utils + path: "tests/proxy_unit_tests/test_proxy_token_counter.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b6" + # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62) + path: "tests/proxy_unit_tests/test_[r-t]*.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b7" + # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157) + path: "tests/proxy_unit_tests/test_[u-z]*.py" + workers: 2 + reruns: 1 + + name: test (${{ matrix.test-group.name }}) + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests - ${{ matrix.test-group.name }} + run: | + poetry run pytest ${{ matrix.test-group.path }} \ + --tb=short -vv \ + --maxfail=10 \ + -n ${{ matrix.test-group.workers }} \ + --reruns ${{ matrix.test-group.reruns }} \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 \ + --cov=litellm \ + --cov-report=xml:coverage-${{ matrix.test-group.name }}.xml \ + --cov-config=pyproject.toml + + - name: Save coverage report + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: coverage-${{ matrix.test-group.name }} + path: coverage-${{ matrix.test-group.name }}.xml + retention-days: 1 + + upload-coverage: + name: Upload coverage to Codecov + needs: test + if: always() + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # Required for OIDC tokenless upload + pull-requests: write # Required for Codecov PR comments + + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Download all coverage reports + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + with: + pattern: coverage-* + path: coverage-reports + merge-multiple: true + + - name: Upload to Codecov + uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4 + with: + use_oidc: true + directory: coverage-reports + root_dir: ${{ github.workspace }} + fail_ci_if_error: false diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index a7072b63fbb..0c040b3ebe7 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -1,7 +1,7 @@ name: LiteLLM Mock Tests (folder - tests/test_litellm) -# DEPRECATED: This workflow is replaced by the individual test-unit-*.yml -# workflows which run the same tests with better readability. +# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs +# the same tests in parallel across 10 jobs for faster CI times. # Kept for manual debugging only. on: workflow_dispatch: # Manual trigger only diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml index a021236b174..2f3698fdf60 100644 --- a/.github/workflows/test-unit-core-utils.yml +++ b/.github/workflows/test-unit-core-utils.yml @@ -6,8 +6,6 @@ on: permissions: contents: read - id-token: write - pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -20,4 +18,3 @@ jobs: test-path: "tests/test_litellm/litellm_core_utils" workers: 2 reruns: 1 - coverage-name: "core-utils" diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml index d6dc433e9f0..13ae3efedba 100644 --- a/.github/workflows/test-unit-enterprise-routing.yml +++ b/.github/workflows/test-unit-enterprise-routing.yml @@ -6,8 +6,6 @@ on: permissions: contents: read - id-token: write - pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -24,4 +22,3 @@ jobs: tests/test_litellm/router_strategy workers: 2 reruns: 2 - coverage-name: "enterprise-routing" diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml index aa00b0a1eaa..2789f99d81c 100644 --- a/.github/workflows/test-unit-integrations.yml +++ b/.github/workflows/test-unit-integrations.yml @@ -6,8 +6,6 @@ on: permissions: contents: read - id-token: write - pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -20,4 +18,3 @@ jobs: test-path: "tests/test_litellm/integrations" workers: 2 reruns: 3 - coverage-name: "integrations" diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml index db183092af6..6c00272b0c8 100644 --- a/.github/workflows/test-unit-llm-providers.yml +++ b/.github/workflows/test-unit-llm-providers.yml @@ -6,8 +6,6 @@ on: permissions: contents: read - id-token: write - pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -21,7 +19,6 @@ jobs: test-path: "tests/test_litellm/llms/vertex_ai" workers: 1 reruns: 2 - coverage-name: "llms-vertex" other-providers: name: All Other Providers @@ -30,4 +27,3 @@ jobs: test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" workers: 2 reruns: 2 - coverage-name: "llms-other" diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml index b4ed1552f19..9228decd7cc 100644 --- a/.github/workflows/test-unit-misc.yml +++ b/.github/workflows/test-unit-misc.yml @@ -6,8 +6,6 @@ on: permissions: contents: read - id-token: write - pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -31,4 +29,3 @@ jobs: tests/test_litellm/test_*.py workers: 2 reruns: 2 - coverage-name: "misc" diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml index df1f7015dd6..e71821db701 100644 --- a/.github/workflows/test-unit-proxy-auth.yml +++ b/.github/workflows/test-unit-proxy-auth.yml @@ -6,8 +6,6 @@ on: permissions: contents: read - id-token: write - pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -20,4 +18,3 @@ jobs: test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client" workers: 2 reruns: 2 - coverage-name: "proxy-auth" diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml index dda1b099623..caff3b3ae06 100644 --- a/.github/workflows/test-unit-proxy-endpoints.yml +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -6,8 +6,6 @@ on: permissions: contents: read - id-token: write - pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -35,4 +33,3 @@ jobs: tests/test_litellm/proxy/ui_crud_endpoints workers: 2 reruns: 2 - coverage-name: "proxy-endpoints" diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml index aa91b179be3..4dfbbe317ed 100644 --- a/.github/workflows/test-unit-proxy-infra.yml +++ b/.github/workflows/test-unit-proxy-infra.yml @@ -6,8 +6,6 @@ on: permissions: contents: read - id-token: write - pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -28,4 +26,3 @@ jobs: tests/test_litellm/proxy/test_*.py workers: 2 reruns: 2 - coverage-name: "proxy-infra" diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml index 30ec8b11d60..a9391137263 100644 --- a/.github/workflows/test-unit-proxy-legacy.yml +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -6,8 +6,6 @@ on: permissions: contents: read - id-token: write - pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -95,17 +93,4 @@ jobs: --reruns 1 \ --reruns-delay 1 \ --dist=loadscope \ - --durations=20 \ - --cov=litellm \ - --cov-report=xml:coverage-proxy-legacy-${{ matrix.test-group.name }}.xml \ - --cov-config=pyproject.toml - - - name: Upload coverage to Codecov - if: always() - uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4 - with: - use_oidc: true - files: coverage-proxy-legacy-${{ matrix.test-group.name }}.xml - flags: proxy-legacy-${{ matrix.test-group.name }} - root_dir: ${{ github.workspace }} - fail_ci_if_error: false + --durations=20 diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml index b2c21fd0547..7f3acac2803 100644 --- a/.github/workflows/test-unit-responses-caching-types.yml +++ b/.github/workflows/test-unit-responses-caching-types.yml @@ -6,8 +6,6 @@ on: permissions: contents: read - id-token: write - pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -20,4 +18,3 @@ jobs: test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" workers: 2 reruns: 2 - coverage-name: "responses-caching-types" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 2bc361bc48f..00000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,40 +0,0 @@ -repos: -- repo: local - hooks: - - id: pyright - name: pyright - entry: pyright - language: system - types: [python] - files: ^(litellm/|litellm_proxy_extras/|enterprise/) - - id: isort - name: isort - entry: isort - language: system - types: [python] - files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py - exclude: ^litellm/__init__.py$ - - id: black - name: black - entry: poetry run black - language: system - types: [python] - files: (litellm/|litellm_proxy_extras/).*\.py -- repo: https://github.com/pycqa/flake8 - rev: 7.0.0 # The version of flake8 to use - hooks: - - id: flake8 - exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/ - additional_dependencies: [flake8-print] - files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py -- repo: https://github.com/python-poetry/poetry - rev: 1.8.0 - hooks: - - id: poetry-check - files: ^(pyproject.toml|litellm-proxy-extras/pyproject.toml)$ -- repo: local - hooks: - - id: check-files-match - name: Check if files match - entry: python3 ci_cd/check_files_match.py - language: system \ No newline at end of file diff --git a/.trivyignore b/.trivyignore deleted file mode 100644 index 0d04ecacdb5..00000000000 --- a/.trivyignore +++ /dev/null @@ -1,12 +0,0 @@ -# LiteLLM Trivy Ignore File -# CVEs listed here are temporarily allowlisted pending fixes - -# Next.js vulnerabilities in UI dashboard (next@14.2.35) -# Allowlisted: 2026-01-31, 7-day fix timeline -# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+ - -# HIGH: DoS via request deserialization -GHSA-h25m-26qc-wcjf - -# MEDIUM: Image Optimizer DoS -CVE-2025-59471 diff --git a/ci_cd/.grype.yaml b/ci_cd/.grype.yaml deleted file mode 100644 index b9bc9db58f5..00000000000 --- a/ci_cd/.grype.yaml +++ /dev/null @@ -1,36 +0,0 @@ -ignore: - - vulnerability: CVE-2026-22184 - reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists - # Wolfi base image: Python 3.13 and Node from apk have no fixed builds in Wolfi yet / not applicable - - vulnerability: CVE-2025-55130 - reason: Node in Wolfi apk; only used for Admin UI build/prisma - - vulnerability: CVE-2025-59465 - reason: Node in Wolfi apk; only used for Admin UI build/prisma - - vulnerability: CVE-2025-55131 - reason: Node in Wolfi apk; only used for Admin UI build/prisma - - vulnerability: CVE-2025-59466 - reason: Node in Wolfi apk; only used for Admin UI build/prisma - - vulnerability: CVE-2026-21637 - reason: Node in Wolfi apk; only used for Admin UI build/prisma - - vulnerability: CVE-2025-55132 - reason: Node in Wolfi apk; only used for Admin UI build/prisma - - vulnerability: GHSA-hx9q-6w63-j58v - reason: orjson dumps recursion; allowlisted - - vulnerability: GHSA-73rr-hh4g-fpgx - reason: diff npm transitive dep; override in package.json, allowlisted - - vulnerability: CVE-2026-0865 - reason: Python 3.13 in Wolfi base; no fixed apk build yet - - vulnerability: CVE-2025-15282 - reason: Python 3.13 in Wolfi base; no fixed apk build yet - - vulnerability: CVE-2026-0672 - reason: Python 3.13 in Wolfi base; no fixed apk build yet - - vulnerability: CVE-2025-15366 - reason: Python 3.13 in Wolfi base; no fixed apk build yet - - vulnerability: CVE-2025-15367 - reason: Python 3.13 in Wolfi base; no fixed apk build yet - - vulnerability: CVE-2025-11468 - reason: Python 3.13 in Wolfi base; no fixed apk build yet - - vulnerability: CVE-2025-12781 - reason: Python 3.13 in Wolfi base; no fixed apk build yet - - vulnerability: CVE-2026-1299 - reason: Python 3.13 in Wolfi base; no fixed apk build yet diff --git a/ci_cd/publish-proxy-extras.sh b/ci_cd/publish-proxy-extras.sh deleted file mode 100644 index 6c83d1f9212..00000000000 --- a/ci_cd/publish-proxy-extras.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Exit on error -set -e - -echo "🚀 Building and publishing litellm-proxy-extras" - -# Navigate to litellm-proxy-extras directory -cd "$(dirname "$0")/../litellm-proxy-extras" - -# Build the package -echo "📦 Building package..." -poetry build - -# Publish to PyPI -echo "🌎 Publishing to PyPI..." -poetry publish - -echo "✅ Done! Package published successfully" \ No newline at end of file diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh deleted file mode 100755 index ee33393c8b1..00000000000 --- a/ci_cd/security_scans.sh +++ /dev/null @@ -1,262 +0,0 @@ -#!/bin/bash - -# Security Scans Script for LiteLLM -# This script runs comprehensive security scans including Trivy and Grype - -set -e - -echo "Starting security scans for LiteLLM..." - -# Function to install Trivy and required tools -install_trivy() { - echo "Installing Trivy and required tools..." - TRIVY_VERSION="0.35.0" - sudo apt-get update - sudo apt-get install -y wget jq curl bsdmainutils - wget -qO trivy.deb "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb" - sudo dpkg -i trivy.deb - rm trivy.deb - echo "Trivy ${TRIVY_VERSION} installed successfully" -} - -# Function to install Grype -install_grype() { - echo "Installing Grype..." - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin - echo "Grype installed successfully" -} - -# Function to install ggshield -install_ggshield() { - echo "Installing ggshield..." - pip3 install --upgrade pip - pip3 install ggshield - echo "ggshield installed successfully" -} - -# # Function to run secret detection scans -# run_secret_detection() { -# echo "Running secret detection scans..." - -# if ! command -v ggshield &> /dev/null; then -# install_ggshield -# fi - -# # Check if GITGUARDIAN_API_KEY is set (required for CI/CD) -# if [ -z "$GITGUARDIAN_API_KEY" ]; then -# echo "Warning: GITGUARDIAN_API_KEY environment variable is not set." -# echo "ggshield requires a GitGuardian API key to scan for secrets." -# echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables." -# exit 1 -# fi - -# echo "Scanning codebase for secrets..." -# echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)" -# echo "ggshield will automatically handle rate limits and retry as needed." -# echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml" - -# # Use --recursive for directory scanning and auto-confirm if prompted -# # .gitguardian.yaml will automatically exclude binary files, wheel files, etc. -# # GITGUARDIAN_API_KEY environment variable will be used for authentication -# echo y | ggshield secret scan path . --recursive || { -# echo "" -# echo "==========================================" -# echo "ERROR: Secret Detection Failed" -# echo "==========================================" -# echo "ggshield has detected secrets in the codebase." -# echo "Please review discovered secrets above, revoke any actively used secrets" -# echo "from underlying systems and make changes to inject secrets dynamically at runtime." -# echo "" -# echo "For more information, see: https://docs.gitguardian.com/secrets-detection/" -# echo "==========================================" -# echo "" -# exit 1 -# } - -# echo "Secret detection scans completed successfully" -# } - -# Function to run Trivy scans -run_trivy_scans() { - echo "Running Trivy scans..." - - echo "Scanning LiteLLM Docs..." - trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/ - - echo "Scanning LiteLLM UI..." - trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/ - - echo "Trivy scans completed successfully" -} - -# Function to build and scan Docker images with Grype -run_grype_scans() { - echo "Running Grype scans..." - - # Temporarily add wheel files to .dockerignore for security scans - echo "Temporarily modifying .dockerignore to exclude problematic wheel files..." - cp .dockerignore .dockerignore.backup 2>/dev/null || touch .dockerignore.backup - echo "/*.whl" >> .dockerignore - - # Build and scan Dockerfile.database - echo "Building and scanning Dockerfile.database..." - docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database . - grype litellm-database:latest --config ci_cd/.grype.yaml --fail-on critical - - # Build and scan main Dockerfile - echo "Building and scanning main Dockerfile..." - docker build --no-cache -t litellm:latest . - grype litellm:latest --config ci_cd/.grype.yaml --fail-on critical - - # Restore original .dockerignore - echo "Restoring original .dockerignore..." - mv .dockerignore.backup .dockerignore - - # Scan the locally built LiteLLM image for vulnerabilities with CVSS >= 4.0 - echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..." - echo "Using locally built image: litellm:latest" - - # Allowlist of CVEs to be ignored in failure threshold/reporting - # - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix - # - GHSA-4xh5-x5gv-qwph: GitHub Security Advisory alias for CVE-2025-8869 - # - GHSA-5j98-mcp5-4vw2: glob CLI command injection via -c/--cmd; glob CLI is not used in the litellm runtime image, - # and the vulnerable versions are pulled in only via OS-level/node tooling outside of our application code - ALLOWED_CVES=( - "CVE-2025-8869" - "GHSA-4xh5-x5gv-qwph" - "CVE-2025-8291" # no fix available as of Oct 11, 2025 - "GHSA-5j98-mcp5-4vw2" - "CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image - "CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image - "CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image - "CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet - "CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build - "CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build - "CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build - "CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build - "CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build - "CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet - "GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+) - "GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code - "GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit - "GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel - "CVE-2025-59465" # Node only used for Admin UI build/prisma - "CVE-2025-55131" # Node only used for Admin UI build/prisma - "CVE-2025-59466" # Node only used for Admin UI build/prisma - "CVE-2025-55130" # Node only used for Admin UI build/prisma - "CVE-2025-59467" # Node only used for Admin UI build/prisma - "CVE-2026-21637" # Node only used for Admin UI build/prisma - "CVE-2025-55132" # Node only used for Admin UI build/prisma - "GHSA-hx9q-6w63-j58v" # orjson dumps recursion; allowlisted - "CVE-2025-15281" # No fix available yet - "CVE-2026-0865" # No fix available yet - "CVE-2025-15282" # No fix available yet - "CVE-2026-0672" # No fix available yet - "CVE-2025-15366" # No fix available yet - "CVE-2025-15367" # No fix available yet - "CVE-2025-12781" # No fix available yet - "CVE-2025-11468" # No fix available yet - "CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization - "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time - "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code - "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code - "CVE-2026-25639" # axios DoS via __proto__ in mergeConfig - transitive dev dep via @neondatabase/api-client, not imported in application code - "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image - "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code - "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet - "CVE-2026-3644" # Python 3.13 vulnerability - no fix available in base image - "CVE-2026-4224" # Python 3.13 Expat parser stack overflow in ElementDeclHandler - no fix available in base image - ) - - # Build JSON array of allowlisted CVE IDs for jq - ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .) - - echo "Checking for vulnerabilities with CVSS score >= 4.0..." - echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}" - echo "" - - # Show all high-severity vulnerabilities for transparency - TOTAL_HIGH_SEVERITY=$(grype litellm:latest -o json | jq -r ' - .matches[] - | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) - | .vulnerability.id' | wc -l) - - if [ "$TOTAL_HIGH_SEVERITY" -gt 0 ]; then - echo "Total vulnerabilities found with CVSS >= 4.0: $TOTAL_HIGH_SEVERITY" - echo "" - echo "All high-severity vulnerabilities (including allowlisted):" - grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r ' - ["Package", "Version", "Vulnerability ID", "CVSS Score", "Allowlisted"], - (.matches[] - | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) - | [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, (if (.vulnerability.id as $id | $allow | index($id)) then "YES" else "NO" end)]) - | @tsv' | column -t -s $'\t' - echo "" - fi - - HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r ' - .matches[] - | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) - | select((.vulnerability.id as $id | $allow | index($id) | not)) - | .vulnerability.id' | wc -l) - - if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then - echo "" - echo "==========================================" - echo "ERROR: Security Scan Failed" - echo "==========================================" - echo "Found $HIGH_SEVERITY_COUNT non-allowlisted vulnerabilities with CVSS score >= 4.0 in litellm:latest" - echo "" - echo "These vulnerabilities are NOT in the allowlist and must be addressed." - echo "Current allowlisted CVEs: ${ALLOWED_CVES[*]}" - echo "" - echo "Detailed vulnerability report:" - echo "" - grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r ' - ["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"], - (.matches[] - | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) - | select((.vulnerability.id as $id | $allow | index($id) | not)) - | [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description]) - | @tsv' | column -t -s $'\t' - echo "" - echo "==========================================" - echo "Action Required:" - echo "==========================================" - echo "1. If a fix is available, update the package to the fixed version" - echo "2. If the vulnerability is not applicable or has no fix:" - echo " - Add the CVE/GHSA ID to ALLOWED_CVES array in ci_cd/security_scans.sh" - echo " - Add a comment explaining why it's safe to ignore" - echo "" - echo "Note: Some vulnerabilities may have multiple IDs (CVE-XXXX and GHSA-XXXX)." - echo "Add all relevant IDs to the allowlist if they refer to the same issue." - echo "==========================================" - echo "" - exit 1 - else - echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest" - fi - - echo "Grype scans completed successfully" -} - -# Main execution -main() { - echo "Installing security scanning tools..." - install_trivy - install_grype - - # echo "Running secret detection scans..." - # run_secret_detection - - echo "Running filesystem vulnerability scans..." - run_trivy_scans - - echo "Running Docker image vulnerability scans..." - run_grype_scans - - echo "All security scans completed successfully!" -} - -# Execute main function -main "$@" diff --git a/docs/my-website/.trivyignore b/docs/my-website/.trivyignore deleted file mode 100644 index 977504f2670..00000000000 --- a/docs/my-website/.trivyignore +++ /dev/null @@ -1,7 +0,0 @@ -# js-yaml CVE-2025-64718 -# This vulnerability is not applicable because we've forced js-yaml to version 4.1.1 -# via npm overrides in package.json. Trivy incorrectly reports this based on -# dependency requirements in the lockfile, but the actual installed version is 4.1.1. -# Verified with: npm list js-yaml -CVE-2025-64718 - diff --git a/scripts/install.sh b/scripts/install.sh index b9912287b70..03ae31cd18b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -83,8 +83,8 @@ echo "" header "Installing litellm[proxy]…" echo "" -"$PYTHON_BIN" -m pip install --upgrade "${LITELLM_PACKAGE}" \ - || die "pip install failed. Try manually: $PYTHON_BIN -m pip install '${LITELLM_PACKAGE}'" +"$PYTHON_BIN" -m pip install --only-binary :all: --upgrade "${LITELLM_PACKAGE}" \ + || die "pip install failed. Try manually: $PYTHON_BIN -m pip install --only-binary :all: '${LITELLM_PACKAGE}'" # ── find the litellm binary installed by pip for this Python ─────────────── # sysconfig.get_path('scripts') is where pip puts console scripts — reliable diff --git a/ui/litellm-dashboard/.trivyignore b/ui/litellm-dashboard/.trivyignore deleted file mode 100644 index 977504f2670..00000000000 --- a/ui/litellm-dashboard/.trivyignore +++ /dev/null @@ -1,7 +0,0 @@ -# js-yaml CVE-2025-64718 -# This vulnerability is not applicable because we've forced js-yaml to version 4.1.1 -# via npm overrides in package.json. Trivy incorrectly reports this based on -# dependency requirements in the lockfile, but the actual installed version is 4.1.1. -# Verified with: npm list js-yaml -CVE-2025-64718 -