Merge remote-tracking branch 'origin/main' into litellm_rag_team_provider_credentials

.circleci/config.yml

@@ -243,7 +243,7 @@ commands:
             mkdir -p "$HOME/.local/bin"
             mv /tmp/codecov "$HOME/.local/bin/codecov"
   start_openai_record_replay_proxy:
-    description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream/<host>/ so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced."
+    description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay Redis cache hits. Cache misses call the live provider and store its response, so these legacy E2E jobs still require provider availability and may incur provider charges. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream/<host>/ so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced."
     steps:
       - run:
           name: Start record/replay proxy
@@ -379,50 +379,6 @@ commands:
       - save_cargo_target
 
 jobs:
-  # Add Windows testing job
-  using_litellm_on_windows:
-    executor:
-      name: win/default
-      shell: powershell.exe
-    working_directory: ~/project
-    environment:
-      UV_PYTHON: "3.11"
-      CARGO_HTTP_MULTIPLEXING: "false"
-      CARGO_NET_RETRY: "5"
-    steps:
-      - checkout
-      - run:
-          name: Install Python
-          command: |
-            choco install python --version=3.11.0 -y --no-progress --force
-            refreshenv
-            python --version
-          environment:
-            CHOCOLATEY_CONFIRM_ALL: "true"
-      - install_windows_toolchain
-      - run:
-          name: Install Dependencies
-          no_output_timeout: 30m
-          environment:
-            UV_HTTP_TIMEOUT: "300"
-          command: |
-            $env:Path = "$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
-            for ($attempt = 1; $attempt -le 5; $attempt++) {
-              Write-Host "uv sync attempt $attempt/5"
-              uv sync --frozen --group dev --python 3.11
-              if ($LASTEXITCODE -eq 0) {
-                break
-              }
-              if ($attempt -eq 5) {
-                exit $LASTEXITCODE
-              }
-              Start-Sleep -Seconds 15
-            }
-      - run:
-          name: Run Windows-specific test
-          command: |
-            uv run --no-sync python -m pytest --tb=short tests/windows_tests/ -v
-
   windows_release_wheel:
     executor:
       name: win/default
@@ -808,7 +764,7 @@ jobs:
       - *python312_image
     working_directory: ~/project
     resource_class: large
-    parallelism: 4
+    parallelism: 1
     environment:
       FAKE_OPENAI_API_BASE: http://127.0.0.1:8190
     steps:
@@ -1384,89 +1340,6 @@ jobs:
           paths:
             - audio_coverage.xml
             - audio_coverage
-  installing_litellm_on_python:
-    docker:
-      - *python312_image
-      - *postgres16_sidecar
-    working_directory: ~/project
-    environment: *sidecar_database_url
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - setup_litellm_enterprise_pip
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Run tests
-          command: |
-            uv run --no-sync python -m pytest --tb=short -vv tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
-
-  installing_litellm_on_python_3_13:
-    docker:
-      - image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13
-        auth:
-          username: ${DOCKERHUB_USERNAME}
-          password: ${DOCKERHUB_PASSWORD}
-      - *postgres16_sidecar
-    working_directory: ~/project
-    environment: *sidecar_database_url
-    resource_class: medium
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.13
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Run tests
-          command: |
-            uv run --no-sync python -m pytest --tb=short -v tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
-
-  installing_litellm_on_python_v2_migration_resolver:
-    docker:
-      - *python312_image
-      - *postgres16_sidecar
-    working_directory: ~/project
-    environment: *sidecar_database_url
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - setup_litellm_enterprise_pip
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Run both migration resolvers against Postgres
-          command: |
-            uv run --no-sync python -m pytest --tb=short -vv \
-              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings \
-              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_legacy_resolver
-
   helm_chart_testing:
     machine:
       image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker
@@ -2159,95 +2032,6 @@ jobs:
       - store_test_results:
           path: test-results
 
-  proxy_build_from_pip_tests:
-    # Change from docker to machine executor
-    machine:
-      image: ubuntu-2204:2024.04.1
-    resource_class: large
-    working_directory: ~/project
-    steps:
-      - checkout
-      - run:
-          name: Generate LiteLLM master key
-          command: |
-            key="$(openssl rand -hex 16)"
-            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      # Remove Docker CLI installation since it's already available in machine executor
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - run:
-          name: Build Docker image
-          command: |
-            docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip .
-      - start_postgres
-      - start_fake_openai_endpoint
-      - run:
-          name: Run Docker container
-          # intentionally give bad redis credentials here
-          # the OTEL test - should get this as a trace
-          command: |
-            docker run -d \
-              -p 4000:4000 \
-              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-              -e REDIS_HOST=$REDIS_HOST \
-              -e REDIS_PASSWORD=$REDIS_PASSWORD \
-              -e REDIS_PORT=$REDIS_PORT \
-              -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-              -e OPENAI_API_KEY=$OPENAI_API_KEY \
-              -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-              -e LITELLM_LICENSE=$LITELLM_LICENSE \
-              -e OTEL_EXPORTER="in_memory" \
-              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-              -e AWS_REGION_NAME=$AWS_REGION_NAME \
-              -e COHERE_API_KEY=$COHERE_API_KEY \
-              -e USE_DDTRACE=True \
-              -e DD_API_KEY=$DD_API_KEY \
-              -e DD_SITE=$DD_SITE \
-              -e GCS_FLUSH_INTERVAL="1" \
-              -e LITELLM_LOG=ERROR \
-              --add-host host.docker.internal:host-gateway \
-              --name my-app \
-              -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
-              my-app:latest \
-              --config /app/config.yaml \
-              --port 4000
-      - run:
-          name: Start outputting logs
-          command: docker logs -f my-app
-          background: true
-      - wait_for_service:
-          url: http://localhost:4000
-          timeout: "300"
-      - run:
-          name: Run tests
-          command: |
-            mkdir -p test-results
-            TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
-            echo "$TEST_FILES" | circleci tests run \
-              --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-                -vv \
-                --junitxml=test-results/junit-2.xml \
-                --durations=5"
-          no_output_timeout: 15m
-            # Clean up first container
-      - store_test_results:
-          path: test-results
-      - run:
-          name: Stop and remove first container
-          command: |
-            docker stop my-app || true
-            docker rm my-app || true
-            docker stop postgres-db || true
-            docker rm postgres-db || true
-          when: always
   proxy_pass_through_endpoint_tests:
     machine:
       image: ubuntu-2204:2024.04.1
@@ -3028,11 +2812,72 @@ jobs:
             docker logs redis-cache > test-results/services-<< parameters.suite >>-<< parameters.mode >>/redis.log 2>&1 || true
             docker rm -f postgres-db redis-cache
             test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
+      - when:
+          condition:
+            not:
+              equal: [1, << parameters.parallelism >>]
+          steps:
+            - run:
+                name: Prepare this shard's execution record
+                command: |
+                  results="test-results/integration-<< parameters.suite >>"
+                  if [ "<< parameters.mode >>" = replica ]; then
+                    results="$results-replica"
+                  fi
+                  mkdir -p "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX"
+                  cp "$results/execution.json" "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX/execution.json"
+                  if [ "<< parameters.suite >>" = mcp ]; then
+                    mkdir -p "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage"
+                    cp test-results/integration-mcp/coverage/data "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage/data.$CIRCLE_NODE_INDEX"
+                  fi
+            - persist_to_workspace:
+                root: /tmp/integration-shards
+                paths:
+                  - << parameters.suite >>-<< parameters.mode >>
       - store_test_results:
           path: test-results
       - store_artifacts:
           path: test-results
 
+  integration_shard_coverage:
+    parameters:
+      suite:
+        type: enum
+        enum: [management, accounting, database, extensions, providers, mcp]
+      mode:
+        type: enum
+        enum: [standard, replica]
+        default: standard
+      shards:
+        type: integer
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: medium
+    working_directory: ~/project
+    steps:
+      - setup_litellm_test_deps
+      - attach_workspace:
+          at: /tmp/integration-shards
+      - run:
+          name: Verify every << parameters.suite >> case ran exactly once
+          command: |
+            .venv/bin/python -m tests.integration._support.sharding /tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >> \
+              --shards << parameters.shards >>
+      - when:
+          condition:
+            equal: [mcp, << parameters.suite >>]
+          steps:
+            - run:
+                name: Combine coverage from every MCP shard
+                command: |
+                  mkdir -p test-results/integration-mcp/coverage
+                  export COVERAGE_FILE="$PWD/test-results/integration-mcp/coverage/data"
+                  .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml /tmp/integration-shards/mcp-standard/*/coverage
+                  .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > test-results/integration-mcp/coverage/coverage.txt
+                  .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d test-results/integration-mcp/coverage/html
+            - store_artifacts:
+                path: test-results/integration-mcp/coverage
+
   routing_parity:
     parameters:
       suite:
@@ -3114,6 +2959,9 @@ jobs:
         default: ""
       timeout_minutes:
         type: integer
+      workers:
+        type: integer
+        default: 1
     machine:
       image: ubuntu-2204:2024.04.1
     resource_class: large
@@ -3141,12 +2989,16 @@ jobs:
           command: |
             mkdir -p test-results
             coverage_args=()
+            worker_args=()
+            if [ "<< parameters.workers >>" -gt 1 ]; then
+              worker_args=(-n << parameters.workers >> --dist=worksteal)
+            fi
             if [ -n "<< parameters.coverage_flag >>" ]; then
               coverage_args=(--cov=./litellm --cov-report=xml:coverage.xml)
             fi
             timeout --signal=TERM << parameters.timeout_minutes >>m uv run --no-sync pytest \
               << parameters.test_path >> -vv --tb=short --durations=10 -o junit_family=xunit1 \
-              --junitxml=test-results/junit.xml "${coverage_args[@]}"
+              --junitxml=test-results/junit.xml "${coverage_args[@]}" "${worker_args[@]}"
       - when:
           condition: << parameters.coverage_flag >>
           steps:
@@ -3244,8 +3096,8 @@ jobs:
               timeout --signal=TERM 15m uv run --no-sync pytest \
               tests/unit/test_redis.py tests/unit/caching/test_redis_connection_pool.py \
               tests/unit/caching/test_redis_cluster_cache.py tests/unit/caching/test_evicted_client_closer.py \
-              tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \
-              tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \
+              tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_azure_credentials \
+              tests/integration/sdk/test_redis_cluster_iam_auth.py::test_sync_cluster_authenticates_with_gcp_credentials \
               --tb=short -vv --reruns 2 --reruns-delay 1 --durations=20 --cov=./litellm \
               --cov-report=xml:coverage.xml -o junit_family=xunit1 --junitxml=test-results/junit.xml
       - when:
@@ -3323,21 +3175,93 @@ workflows:
           name: integration-<< matrix.suite >>
           matrix:
             parameters:
-              suite: [management, accounting, database, providers, sdk, cost, security, browser]
+              suite: [sdk, cost, browser]
+          parallelism: 1
+      - integration_contracts:
+          name: integration-security
+          suite: security
+          parallelism: 1
+          resource_class: xlarge
+      - integration_contracts:
+          name: integration-management
+          suite: management
+          parallelism: 4
+      - integration_shard_coverage:
+          name: integration-management-coverage
+          suite: management
+          shards: 4
+          requires: [integration-management]
+      - integration_contracts:
+          name: integration-accounting
+          suite: accounting
+          parallelism: 4
+      - integration_shard_coverage:
+          name: integration-accounting-coverage
+          suite: accounting
+          shards: 4
+          requires: [integration-accounting]
+      - integration_contracts:
+          name: integration-providers
+          suite: providers
+          parallelism: 8
+      - integration_shard_coverage:
+          name: integration-providers-coverage
+          suite: providers
+          shards: 8
+          requires:
+            - integration-providers
+      - integration_contracts:
+          name: integration-database
+          suite: database
+          parallelism: 2
+      - integration_shard_coverage:
+          name: integration-database-coverage
+          suite: database
+          shards: 2
+          requires: [integration-database]
       - integration_contracts:
           name: integration-mcp
           suite: mcp
           resource_class: xlarge
+          parallelism: 2
+      - integration_shard_coverage:
+          name: integration-mcp-coverage
+          suite: mcp
+          shards: 2
+          requires:
+            - integration-mcp
       - integration_contracts:
           name: integration-extensions
           suite: extensions
-          parallelism: 4
+          parallelism: 8
+      - integration_shard_coverage:
+          name: integration-extensions-coverage
+          suite: extensions
+          shards: 8
+          requires:
+            - integration-extensions
       - integration_contracts:
-          name: integration-<< matrix.suite >>-replica
-          matrix:
-            parameters:
-              suite: [management, database]
-              mode: [replica]
+          name: integration-management-replica
+          suite: management
+          mode: replica
+          parallelism: 4
+      - integration_shard_coverage:
+          name: integration-management-replica-coverage
+          suite: management
+          mode: replica
+          shards: 4
+          requires: [integration-management-replica]
+      - integration_contracts:
+          name: integration-database-replica
+          suite: database
+          mode: replica
+          parallelism: 2
+      - integration_shard_coverage:
+          name: integration-database-replica-coverage
+          suite: database
+          mode: replica
+          shards: 2
+          requires: [integration-database-replica]
       - postgres_suite:
           name: proxy-behavior
           test_path: tests/proxy_behavior
@@ -3354,6 +3278,7 @@ workflows:
           test_path: tests/proxy_migration_tests
           seed: false
           timeout_minutes: 20
+          workers: 2
       - postgres_suite:
           name: roi-database
           test_path: tests/integration/database/test_roi_observed.py
@@ -3371,6 +3296,8 @@ workflows:
                 - "6.4.0"
                 - "7.4.1"
                 - "8.0.1"
+      - windows_release_wheel
+      - base_sdk_install
   build_and_test:
     unless:
       or:
@@ -3378,10 +3305,7 @@ workflows:
         - not:
             equal: ["", << pipeline.parameters.routing_parity_base >>]
     jobs:
-      - using_litellm_on_windows
-      - windows_release_wheel
       - provider_replay_harness
-      - base_sdk_install
       - local_testing_part1
       - local_testing_part2
       - langfuse_logging_unit_tests
@@ -3405,7 +3329,6 @@ workflows:
       - proxy_store_model_in_db_tests:
           requires:
             - build_docker_database_image
-      - proxy_build_from_pip_tests
       - proxy_pass_through_endpoint_tests:
           requires:
             - build_docker_database_image
@@ -3442,9 +3365,6 @@ workflows:
       - db_migration_disable_update_check:
           requires:
             - build_docker_database_image
-      - installing_litellm_on_python
-      - installing_litellm_on_python_3_13
-      - installing_litellm_on_python_v2_migration_resolver
       - helm_chart_testing:
           requires:
             - build_docker_database_image

.circleci/scripts/classify_changes.sh

@@ -15,7 +15,7 @@ outside_cost_map_set=false
 while IFS= read -r file || [ -n "$file" ]; do
   [ -n "$file" ] || continue
   case "$file" in
-    litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/local_testing/test_caching.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock)
+    litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock)
       has_redis_compat=true ;;
   esac
   case "$file" in

.circleci/scripts/run_integration.sh

@@ -134,6 +134,9 @@ upstream_pid=$!
 if [ "$suite" = cost ]; then
   export INTEGRATION_WORKERS=8
 fi
+if [ "$suite" = security ]; then
+  export INTEGRATION_WORKERS=4
+fi
 if [ "$suite" = mcp ]; then
   export INTEGRATION_WORKERS=4 INTEGRATION_COVERAGE=1
 fi
@@ -213,13 +216,11 @@ if [ "$suite" = browser ]; then
   exit 0
 fi
 
-node_files=()
+shard_arguments=()
 if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then
-  split="$(.venv/bin/python tests/integration/run.py "$suite" --list \
-    | circleci tests split --split-by=timings --timings-type=filename)"
-  read -r -a node_files <<< "$(printf '%s' "$split" | tr '\n' ' ')"
-  test "${#node_files[@]}" -gt 0
-  printf '%s\n' "${node_files[@]}" > "$results/node-files.txt"
+  .venv/bin/python tests/integration/run.py "$suite" --list \
+    | circleci tests split --split-by=timings --timings-type=filename > "$results/node-files.txt"
+  shard_arguments=(--shard-files "$results/node-files.txt")
 fi
 
 env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
@@ -232,11 +233,13 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
   INTEGRATION_SEED="$INTEGRATION_SEED" \
   INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
+  INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \
   LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \
   INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \
   INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \
   INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
-  .venv/bin/python tests/integration/run.py "$suite" --results "$results" "${node_files[@]}"
+  .venv/bin/python tests/integration/run.py "$suite" --results "$results" \
+  "${shard_arguments[@]}"
 
 if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
   for covered_pid in "$proxy_pid" "$peer_pid"; do

.github/ci-coverage-allowlist.yml

@@ -21,17 +21,13 @@ test_paths:
       Live-provider caching cases in tests/local_testing that remain outside CI. Jobs that
       glob that directory either deselect them (local_testing_part1 and part2 carry `-k "... and
       not caching and not cache"`) or keep only another keyword (langfuse, router, assistants).
-      Separately, the CircleCI redis-compat jobs select two IAM cluster authentication tests in
-      test_caching.py by node ID. It does not run that file's other tests.
-      The gap was eight files and 118 tests when measured 2026-08-20; the five keyless files now
-      run in the caching-local shard, leaving live cases in these three. Measured 2026-08-21 with no provider
-      credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
-      test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
-      test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
-      split that porting tests/local_testing off CircleCI will force, not a job that is red by
-      construction
+      The two Redis Cluster IAM authentication tests moved to tests/integration/sdk and remain
+      selected by node ID in redis_compat, where redis-server is installed.
+      The five keyless files now run in the caching-local shard, leaving live cases in these two.
+      Measured 2026-08-21 with no provider credentials: test_disk_cache_unit_tests.py needs
+      OPENAI_API_KEY for 2 of its 4 tests, and test_gcs_cache_unit_tests.py needs GCS credentials
+      for all 4. Keep these live cases outside PR CI rather than making a job red by construction
     paths:
-      - tests/local_testing/test_caching.py
       - tests/local_testing/test_disk_cache_unit_tests.py
       - tests/local_testing/test_gcs_cache_unit_tests.py
   - reason: >-

.github/scripts/assert_ci_coverage.py

@@ -9,6 +9,8 @@ import sys
 import warnings
 from collections.abc import Callable, Iterable, Mapping, Sequence
 from dataclasses import dataclass
+from functools import lru_cache
+from types import MappingProxyType
 from typing import Final
 
 import yaml
@@ -316,6 +318,7 @@ def _keyword_terms(
 def _slices() -> tuple[Slice, ...]:
     if not CIRCLECI_CONFIG.exists():
         return ()
+    test_files: Final = frozenset(_test_files())
     jobs: Final = yaml.safe_load(CIRCLECI_CONFIG.read_text()).get("jobs", {})
     return tuple(
         Slice(job=job, globs=globs, named=named, required=required, excluded=excluded, understood=understood)
@@ -323,7 +326,7 @@ def _slices() -> tuple[Slice, ...]:
         for text in ("\n".join(_strings(body)),)
         if "pytest" in text
         for globs in (tuple(GLOB_CALL_RE.findall(text)),)
-        for named in (frozenset(TEST_TOKEN_RE.findall(text)) & frozenset(_test_files()),)
+        for named in (frozenset(TEST_TOKEN_RE.findall(text)) & test_files,)
         for required, excluded, understood in (
             _keyword_terms(tuple(KEYWORD_RE.findall(text)), attributable=len(globs) < 2),
         )
@@ -363,6 +366,7 @@ def _workflow_named_tokens() -> frozenset[str]:
 
 
 def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
+    matchable_names: Final = lru_cache(maxsize=None)(_matchable_names)
     slices: Final = _slices()
     named_by_workflow: Final = _workflow_named_tokens()
     globbed: Final = tuple(
@@ -376,7 +380,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
         for path in globbed
         if not allowlist.covers_test(path)
         and not any(_token_covers(token, path) for token in named_by_workflow)
-        and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices)
+        and not any(slice_.claims(path, matchable_names(path)) for slice_ in slices)
     )
 
 

.github/workflows/helm_unit_test.yml

@@ -1,54 +0,0 @@
-name: Helm unit test
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  unit-test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Set up Helm 3.11.1
-        uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
-        with:
-          version: "3.11.1"
-
-      - name: Download and verify Helm Unit Test Plugin
-        run: |
-          curl -fsSLo "$RUNNER_TEMP/helm-unittest.tgz" https://github.com/helm-unittest/helm-unittest/releases/download/v0.8.2/helm-unittest-linux-amd64-0.8.2.tgz
-          echo "56ab3091e6fa52a7c92ee951def9bed957f295d9ce98483aed404e748d7b3a94  $RUNNER_TEMP/helm-unittest.tgz" | sha256sum -c -
-
-      - name: Install Helm Unit Test Plugin
-        run: |
-          PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest"
-          mkdir -p "$PLUGIN_DIR"
-          tar -xzf "$RUNNER_TEMP/helm-unittest.tgz" -C "$PLUGIN_DIR"
-          helm plugin list
-
-      - name: Run unit tests
-        run: |
-          for chart in helm/litellm-helm helm/litellm; do
-            declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')"
-            output="$(mktemp)"
-            helm unittest -f 'tests/*.yaml' "$chart" | tee "$output"
-            executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")"
-            if [ "$declared" != "$executed" ]; then
-              echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute."
-              exit 1
-            fi
-            echo "$chart: all $declared declared test suites ran"
-          done

.github/workflows/required-checks-legacy.yml

@@ -1,397 +0,0 @@
-name: Required checks (legacy)
-
-on:
-  pull_request:
-    branches:
-      - main
-      - "litellm_**"
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: true
-
-jobs:
-  lint:
-    name: lint
-    permissions:
-      contents: read
-      pull-requests: read
-      actions: read
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 1
-          clean: true
-          persist-credentials: false
-
-      - name: Detect relevant changes
-        id: changes
-        uses: ./.github/actions/detect-changes
-
-      - name: Fetch gate base (merge-base with target branch)
-        if: steps.changes.outputs.decision != 'skip'
-        env:
-          GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
-          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-        run: |
-          retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
-          MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
-          test -n "$MERGE_BASE"
-          retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
-          echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
-
-      - name: Set up Python
-        if: steps.changes.outputs.decision != 'skip'
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        if: steps.changes.outputs.decision != 'skip'
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Cache uv dependencies
-        if: steps.changes.outputs.decision != 'skip'
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: |
-            ~/.cache/uv
-            .venv
-          key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-uv-lint-
-
-      - name: Clean Python cache
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          find . -type d -name "__pycache__" -exec rm -rf {} + || true
-          find . -name "*.pyc" -delete || true
-
-      - name: Check uv.lock is up to date
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1)
-
-      - name: Cache the Rust build
-        if: steps.changes.outputs.decision != 'skip'
-        uses: ./.github/actions/cache-cargo-build
-
-      - name: Install dependencies
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          uv sync --frozen --group proxy-dev --group e2e-dev
-
-      - name: Cache Prisma binaries
-        if: steps.changes.outputs.decision != 'skip'
-        uses: ./.github/actions/cache-prisma-binaries
-
-      - name: Generate Prisma client
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
-
-      - name: Check ruff format
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- ':(glob)litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
-          if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
-            echo "No changed litellm Python files to check with ruff format."
-            exit 0
-          fi
-          xargs uv run --no-sync ruff format --check --exclude '/enterprise/' < "$RUNNER_TEMP/ruff_format_files.txt"
-
-      - name: Debug - Check file state
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          echo "Current branch:"
-          git branch --show-current
-          echo "Last 3 commits:"
-          git log --oneline -3
-          echo "File content around line 43:"
-          head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
-
-      - name: Check MCP operation boundary
-        if: steps.changes.outputs.decision != 'skip'
-        run: uv run --no-sync python scripts/check_mcp_operation_boundary.py
-
-      - name: Run Ruff linting
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          cd litellm
-          uv run --no-sync ruff check .
-          cd ..
-
-      - name: Run Ruff linting (test tree)
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          uv run --no-sync ruff check --config ruff-tests.toml tests
-
-      - name: Check strict ruff rules (delta vs merge-base counts)
-        if: steps.changes.outputs.decision != 'skip'
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
-
-      - name: Check type discipline (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs merge-base counts)
-        if: steps.changes.outputs.decision != 'skip'
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
-
-      - name: Check test quality (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs merge-base counts)
-        if: steps.changes.outputs.decision != 'skip'
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA"
-
-      - name: Print OpenAI version
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
-
-      - name: Check basedpyright (delta vs merge-base counts)
-        if: steps.changes.outputs.decision != 'skip'
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA"
-
-      - name: Check tests/e2e basedpyright (zero errors)
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/**/*.py' ':(glob)tests/e2e_harness/**/*.py' pyrightconfig.json | grep -q .; then
-            uv run --no-sync basedpyright tests/e2e tests/e2e_harness
-          else
-            echo "No changed tests/e2e Python files; skipping."
-          fi
-
-      - name: Run the e2e harness tests
-        if: steps.changes.outputs.decision != 'skip'
-        env:
-          LITELLM_MASTER_KEY: sk-e2e-harness-tests-reach-no-proxy
-        run: |
-          if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- tests/e2e tests/e2e_harness ':(exclude)tests/e2e/ui' pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
-            echo "No changed e2e harness files; skipping."
-            exit 0
-          fi
-          retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
-          CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
-          tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
-          PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
-
-      - name: Check for circular imports
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          cd litellm
-          uv run --no-sync python ../tests/documentation_tests/test_circular_imports.py
-          cd ..
-
-      - name: Check import safety
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
-
-  frontend-lint:
-    name: frontend-lint
-    permissions:
-      contents: read
-    runs-on: ubuntu-latest
-    timeout-minutes: 8
-    defaults:
-      run:
-        working-directory: ui/litellm-dashboard
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 1
-          persist-credentials: false
-
-      - name: Collect changed files
-        id: changed
-        env:
-          GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
-          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-        run: |
-          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
-          test -n "$merge_base"
-          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
-          : > "$RUNNER_TEMP/prettier_files.txt"
-          : > "$RUNNER_TEMP/eslint_files.txt"
-          while IFS= read -r f; do
-            [ -f "$f" ] || continue
-            case "$f" in
-              *.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs)
-                printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt"
-                printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;;
-              *.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
-                printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
-            esac
-          done < <(git diff --name-only --diff-filter=ACMR --relative "$merge_base" "$HEAD_SHA" -- .)
-          if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
-            echo "has_files=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "has_files=false" >> "$GITHUB_OUTPUT"
-            echo "No lintable UI files changed in this PR; nothing to check."
-          fi
-
-      - name: Setup Node.js
-        if: steps.changed.outputs.has_files == 'true'
-        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
-        with:
-          node-version-file: ui/litellm-dashboard/.nvmrc
-          cache: "npm"
-          cache-dependency-path: ui/litellm-dashboard/package-lock.json
-
-      - name: Install dependencies
-        if: steps.changed.outputs.has_files == 'true'
-        run: npm ci
-
-      - name: Lint changed files (prettier + eslint)
-        if: steps.changed.outputs.has_files == 'true'
-        run: |
-          prettier_files=()
-          eslint_files=()
-          while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt"
-          while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt"
-          status=0
-          if [ ${#prettier_files[@]} -gt 0 ]; then
-            echo "::group::Prettier (${#prettier_files[@]} files)"
-            npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; }
-            echo "::endgroup::"
-          fi
-          if [ ${#eslint_files[@]} -gt 0 ]; then
-            echo "::group::ESLint (${#eslint_files[@]} files)"
-            npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1
-            echo "::endgroup::"
-          fi
-          exit $status
-
-      - name: Check lint budgets
-        if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
-        run: |
-          npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true
-          node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json
-
-      - name: Check for dead code (knip)
-        if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
-        run: npm run knip:ci
-
-  assert-ci-coverage:
-    name: assert-ci-coverage
-    permissions:
-      contents: read
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Assert every test file and Dockerfile is invoked by a job
-        run: |
-          python -m pip install "pyyaml==6.0.3"
-          python .github/scripts/assert_ci_coverage.py
-
-      - name: Assert no -k expression deselects a file from every job that globs it
-        run: python .github/scripts/assert_ci_coverage.py --slices
-
-      - name: Assert .github/workflows/ holds only workflows, correctly named
-        run: python .github/scripts/assert_workflow_dir_hygiene.py
-
-  dashboard-build:
-    name: Dashboard build
-    runs-on: ubuntu-24.04
-    timeout-minutes: 30
-    steps:
-      - name: Checkout
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Build the dashboard stage
-        run: docker build --target ui-builder -f Dockerfile .
-
-  core-checks:
-    name: Core checks (Python ${{ matrix.python-version }})
-    runs-on: ubuntu-24.04
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
-    env:
-      LITELLM_LOCAL_MODEL_COST_MAP: "True"
-    steps:
-      - name: Checkout
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Install dependencies
-        run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }}
-
-      - name: Create the loopback-only network namespace
-        run: |
-          sudo ip netns add smoke
-          sudo ip netns exec smoke ip link set lo up
-          cat > "${RUNNER_TEMP}/in-netns" <<'WRAP'
-          #!/usr/bin/env bash
-          set -euo pipefail
-          exec sudo --preserve-env=LITELLM_LOCAL_MODEL_COST_MAP ip netns exec smoke setpriv --reuid "$(id -u)" --regid "$(id -g)" --init-groups -- env HOME="${HOME}" PATH="${PATH}" "$@"
-          WRAP
-          chmod +x "${RUNNER_TEMP}/in-netns"
-          echo "IN_NETNS=${RUNNER_TEMP}/in-netns" >> "${GITHUB_ENV}"
-
-      - name: Verify namespace isolation
-        run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py verify-isolation
-
-      - name: Verify interpreter version
-        run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py interpreter --expect ${{ matrix.python-version }}
-
-      - name: Import and CLI checks
-        run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli
-
-      - name: Proxy startup check
-        run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics"
-
-      - name: Upload smoke diagnostics
-        if: always()
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
-        with:
-          name: merge-smoke-diagnostics-py${{ matrix.python-version }}
-          path: ${{ runner.temp }}/smoke-diagnostics
-          if-no-files-found: ignore
-
-      - name: Remove the network namespace
-        if: always()
-        run: sudo ip netns delete smoke

.github/workflows/test-linting.yml

@@ -645,9 +645,32 @@ jobs:
 
       - name: Assert .github/workflows/ holds only workflows, correctly named
         run: python .github/scripts/assert_workflow_dir_hygiene.py
+  semgrep:
+    name: semgrep
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Run Semgrep (custom rules)
+        run: uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error
   lint-passed:
     name: lint passed
-    needs: [python, secret-scan, ui, ui-api-types, code-quality, ci-coverage]
+    needs: [python, secret-scan, ui, ui-api-types, code-quality, ci-coverage, semgrep]
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 2

.github/workflows/test-litellm-ui-build.yml

@@ -1,40 +0,0 @@
-name: UI Build Check
-permissions:
-  contents: read
-  pull-requests: read
-
-on:
-  pull_request:
-    branches:
-      - main
-      - "litellm_**"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  build-ui:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Detect relevant changes
-        id: changes
-        uses: ./.github/actions/detect-changes
-        with:
-          category: ui
-
-      # Built through the image stage rather than the checkout, because the
-      # stage copies ui/litellm-dashboard/ alone: an import reaching above the
-      # dashboard root resolves in a checkout and fails in every image we ship.
-      # Dockerfile, docker/Dockerfile.non_root and ui/Dockerfile share this
-      # stage verbatim, so building one covers all three.
-      - name: Build the dashboard as the shipped images build it
-        if: steps.changes.outputs.decision != 'skip'
-        run: docker build --target ui-builder -f Dockerfile .

.github/workflows/test-merge-smoke.yml

@@ -91,9 +91,158 @@ jobs:
         if: always()
         run: sudo ip netns delete smoke
 
+  all-extras:
+    name: all-extras-py${{ matrix.python-version }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.13"]
+    env:
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install every extra and dependency group
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python ${{ matrix.python-version }}
+
+      - name: Import the SDK, the proxy server and the enterprise package
+        run: .venv/bin/python -c "import litellm, litellm.proxy.proxy_server, litellm_enterprise"
+
+  windows-sdk:
+    name: windows-sdk
+    runs-on: windows-2025
+    timeout-minutes: 10
+    env:
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.11"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --group dev --python 3.11
+
+      - name: Run the Windows tests
+        run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests
+
+  proxy-build-from-pip:
+    name: proxy-build-from-pip
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    services:
+      postgres:
+        image: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: circle_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 2s
+          --health-timeout 5s
+          --health-retries 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install the test client dependencies
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --extra proxy --group dev --python 3.12
+
+      - name: Build the image from the published package
+        run: docker build -t litellm-build-from-pip:smoke -f docker/build_from_pip/Dockerfile.build_from_pip .
+
+      - name: Start the fake OpenAI endpoint
+        run: |
+          nohup .venv/bin/python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 > "${RUNNER_TEMP}/fake-openai.log" 2>&1 &
+          for _ in $(seq 1 30); do
+            curl -sf http://localhost:8190/health > /dev/null && exit 0
+            sleep 1
+          done
+          cat "${RUNNER_TEMP}/fake-openai.log"
+          exit 1
+
+      - name: Generate a master key
+        run: |
+          key="sk-$(openssl rand -hex 16)"
+          echo "::add-mask::${key}"
+          echo "LITELLM_MASTER_KEY=${key}" >> "${GITHUB_ENV}"
+
+      - name: Start the proxy container
+        run: |
+          docker run -d \
+            -p 4000:4000 \
+            -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
+            -e LITELLM_MASTER_KEY \
+            -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
+            -e OTEL_EXPORTER=in_memory \
+            -e USE_DDTRACE=True \
+            -e GCS_FLUSH_INTERVAL=1 \
+            -e LITELLM_LOG=ERROR \
+            --add-host host.docker.internal:host-gateway \
+            --name litellm-build-from-pip \
+            -v "${GITHUB_WORKSPACE}/docker/build_from_pip/litellm_config.yaml:/app/config.yaml" \
+            litellm-build-from-pip:smoke \
+            --config /app/config.yaml \
+            --port 4000
+          for _ in $(seq 1 150); do
+            curl -sf --max-time 5 -o /dev/null http://localhost:4000 && exit 0
+            sleep 2
+          done
+          exit 1
+
+      - name: Run the startup tests
+        run: .venv/bin/python -m pytest --tb=short -vv tests/basic_proxy_startup_tests
+
+      - name: Show the proxy logs
+        if: always()
+        run: docker logs litellm-build-from-pip || true
+
   smoke-passed:
     name: smoke passed
-    needs: [dashboard-build, core-checks]
+    needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip]
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 2

.github/workflows/test-semgrep.yml

@@ -1,37 +0,0 @@
-name: Semgrep
-
-on:
-  pull_request:
-    branches:
-      - main
-      - "litellm_**"
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  semgrep:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Run Semgrep (custom rules)
-        run: uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error

.github/workflows/test-unit.yml

@@ -214,7 +214,7 @@ jobs:
             job-timeout-minutes: 60
             dist: loadscope
 
-          - shard: llms-a-to-g
+          - shard: llms-providers
             test-path: |-
               tests/unit/llms/test_cache_control_and_reasoning.py
               tests/unit/llms/test_custom_llm.py
@@ -272,16 +272,6 @@ jobs:
               tests/unit/llms/github_copilot
               tests/unit/llms/gradient_ai
               tests/unit/llms/groq
-              --ignore=tests/unit/llms/base_llm/batches/base_batches_config_test.py
-            python-version: "3.12"
-            workers: 4
-            reruns: 0
-            timeout-minutes: 20
-            job-timeout-minutes: 60
-            dist: loadscope
-
-          - shard: llms-h-to-z
-            test-path: |-
               tests/unit/llms/heroku
               tests/unit/llms/hosted_vllm
               tests/unit/llms/huggingface
@@ -337,6 +327,7 @@ jobs:
               tests/unit/llms/sagemaker
               tests/unit/llms/sail
               tests/unit/llms/sambanova
+              tests/unit/llms/scaledown
               tests/unit/llms/sap
               tests/unit/llms/scaleway
               tests/unit/llms/searchapi
@@ -345,6 +336,7 @@ jobs:
               tests/unit/llms/snowflake
               tests/unit/llms/soniox
               tests/unit/llms/stability
+              tests/unit/llms/strands_decider
               tests/unit/llms/tavily
               tests/unit/llms/tencent
               tests/unit/llms/tinyfish
@@ -362,6 +354,7 @@ jobs:
               tests/unit/llms/xinference
               tests/unit/llms/you_com
               tests/unit/llms/zai
+              --ignore=tests/unit/llms/base_llm/batches/base_batches_config_test.py
             python-version: "3.12"
             workers: 4
             reruns: 0
@@ -935,6 +928,7 @@ jobs:
         timeout-minutes: 3
         with:
           persist-credentials: false
+          fetch-depth: ${{ matrix.shard == 'db-and-spend' && '0' || '1' }}
 
       - name: Detect relevant changes
         id: changes
@@ -967,11 +961,13 @@ jobs:
         with:
           artifact: ${{ needs.rust-bridge.outputs.artifact }}
 
-      - name: Install dependencies
+      - name: Install dependencies and check schema
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 8
         env:
           RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
+          UNIT_SHARD: ${{ matrix.shard }}
+          SCHEMA_BASE_REF: origin/${{ github.event.repository.default_branch }}
         run: |
           diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
           if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
@@ -983,6 +979,9 @@ jobs:
             uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
           fi
           uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
+          if [ "$UNIT_SHARD" = "db-and-spend" ]; then
+            uv run --no-sync python scripts/check_schema_compatibility.py --base-ref "$SCHEMA_BASE_REF"
+          fi
 
       - name: Cache Prisma binaries
         if: steps.changes.outputs.decision != 'skip'
@@ -1235,6 +1234,117 @@ jobs:
           uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
           uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
           uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py
+  helm:
+    name: helm
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Helm 3.11.1
+        uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
+        with:
+          version: "3.11.1"
+
+      - name: Download and verify Helm Unit Test Plugin
+        run: |
+          curl -fsSLo "$RUNNER_TEMP/helm-unittest.tgz" https://github.com/helm-unittest/helm-unittest/releases/download/v0.8.2/helm-unittest-linux-amd64-0.8.2.tgz
+          echo "56ab3091e6fa52a7c92ee951def9bed957f295d9ce98483aed404e748d7b3a94  $RUNNER_TEMP/helm-unittest.tgz" | sha256sum -c -
+
+      - name: Install Helm Unit Test Plugin
+        run: |
+          PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest"
+          mkdir -p "$PLUGIN_DIR"
+          tar -xzf "$RUNNER_TEMP/helm-unittest.tgz" -C "$PLUGIN_DIR"
+          helm plugin list
+
+      - name: Run unit tests
+        run: |
+          for chart in helm/litellm-helm helm/litellm; do
+            declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')"
+            output="$(mktemp)"
+            helm unittest -f 'tests/*.yaml' "$chart" | tee "$output"
+            executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")"
+            if [ "$declared" != "$executed" ]; then
+              echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute."
+              exit 1
+            fi
+            echo "$chart: all $declared declared test suites ran"
+          done
+  openai-sdk-compat:
+    name: OpenAI ${{ matrix.openai-version }} compatibility
+    permissions:
+      contents: read
+      pull-requests: read
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        openai-version: ["2.20.0", "3.0.0", "3.25.0"]
+    env:
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      PYTHONPATH: ${{ github.workspace }}
+      UV_PYTHON: "3.10"
+      OPENAI_VERSION: ${{ matrix.openai-version }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Detect relevant changes
+        id: changes
+        timeout-minutes: 2
+        uses: ./.github/actions/detect-changes
+
+      - name: Set up Python
+        if: steps.changes.outputs.decision != 'skip'
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: ${{ env.UV_PYTHON }}
+
+      - name: Set up uv
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group dev --group proxy-dev --extra proxy --no-install-workspace
+          uv pip install "openai==${OPENAI_VERSION:?}" --exclude-newer-package openai=2026-10-07T00:00:00Z
+          uv run --no-sync python -c 'import os, openai; assert openai.__version__ == os.environ["OPENAI_VERSION"], openai.__version__'
+
+      - name: Run OpenAI SDK compatibility tests
+        if: steps.changes.outputs.decision != 'skip'
+        run: |
+          uv run --no-sync pytest \
+            tests/unit/llms/openai/test_openai.py \
+            tests/unit/llms/openai/test_openai_common_utils.py \
+            tests/unit/llms/openai/test_openai_file_content_streaming.py \
+            tests/unit/test_openai_embedding_encoding_format_default.py \
+            tests/unit/llms/inception/test_inception_completion_transformation.py::test_inception_fim_targets_fim_endpoint \
+            tests/unit/llms/inception/test_inception_completion_transformation.py::test_inception_fim_async \
+            tests/unit/llms/azure/test_azure_exception_mapping.py \
+            tests/unit/test_completion_timeout_resolution.py \
+            tests/unit/types/test_router.py \
+            tests/unit/test_exception_header_preservation.py \
+            tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py::test_speech_binary_response_still_streaming_reports_the_bytes_downloaded_so_far \
+            --tb=short -q
+
+      - name: Run workload identity tests with SDK 3
+        if: steps.changes.outputs.decision != 'skip' && startsWith(matrix.openai-version, '3.')
+        run: uv run --no-sync pytest tests/unit/llms/openai/test_openai_workload_identity.py --tb=short -q
+
+      - name: Check consumer types with the installed SDK
+        if: steps.changes.outputs.decision != 'skip'
+        run: uv run --no-sync basedpyright --pythonpath .venv/bin/python --project tests/typing/pyrightconfig.json
   coverage:
     name: coverage
     needs: unit
@@ -1283,7 +1393,7 @@ jobs:
   unit-passed:
     name: unit passed
     permissions: {}
-    needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs]
+    needs: [rust-bridge, assert-shard-coverage, unit, ui-unit, docs, helm, openai-sdk-compat]
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 2

AGENTS.md

@@ -25,7 +25,7 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
 
 Never test structure of code only function of it
 
-A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
+A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent". Assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
 
 `tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
 
@@ -33,20 +33,20 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions
 
 When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
 
-When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule. A section you have nothing to put in (Relevant issues, Affected release, Linear ticket, Caveats, QA runbook, and so on) is removed entirely, heading included, never left as an empty title
+When writing a PR body, treat the comments and imperative instructions inside .github/PULL_REQUEST_TEMPLATE/general.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/PULL_REQUEST_TEMPLATE/general.md from disk before writing a PR body to make sure you see every comment rule. A section you have nothing to put in (Relevant issues, Affected release, Linear ticket, Caveats, QA runbook, and so on) is removed entirely, heading included, never left as an empty title
 
 Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
 
-Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
+Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
 
 If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
 - don't use emojis
-- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
+- don't use "—". Don't use ";" or ":" unless it would be nonsensical not to. Instead, reach for ",", ".", conjunction words, etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them also feels AI-y. A word cap does not penalize you for adding more sentences
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
 - unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
-- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
+- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a ".". Exception is that docs (i.e., litellm-docs) should have trailing "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
 - use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
 
@@ -58,17 +58,17 @@ The four lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gat
 
 `make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
 
-`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
+`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy" followed by "queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
 
 If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their cap in `ANY_CAPS`, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
 
-If you get an LIT001 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
+If you get an LIT001 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used. Reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
 
 Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType]  # stubs lack async overload` or `# noqa: TID251  # <reason>`. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing
 
 Commit and push your work when you're done without asking
 
-When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
+When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified. Treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
 
 Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
 
@@ -84,7 +84,7 @@ Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer de
 
 Do not put names of customers or customer company names in code, PR descriptions, issue bodies, etc. This means never mention literally any company name. Especially if you're about to say a sentence mentioning that the reason the PR exists was a feature/model/bug fix/etc. requested by a company. That's the indication that you should replace that company name with "the customer". e.g. not "Model request from Acme (Pylon #1234)" but "Model request from a customer (Pylon #1234)". This is because the codebase is public. The only exception is for publicly known providers or vendors such as OpenAI, Anthropic, AWS Bedrock, etc. only IF we're adding support for that provider/vendor in general and NOT if that PR or whatnot was a request by one of them, and they're actually one of our customers.
 
-CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
+CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`). Download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
 
 Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
 
@@ -92,19 +92,19 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 
 - Composition over inheritance
 - Never-nester: early returns over deep nesting
-- Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
-- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc.
-  - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
+- Don't throw. Model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
+- No mutation. Don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc.
+  - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011). `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
   - Qualify every TypedDict field with `ReadOnly[...]` (LIT012), which nests freely with `Required` / `NotRequired` / `Annotated` in any order. If making the key writable is truly unavoidable, suppress with `# writable-ok: <reason>`
-  - Comprehensions take at most one `for` clause and one `if` clause (LIT014); split stacked clauses into a helper generator, a named intermediate, or a plain loop. Suppress with `# comprehension-ok: <reason>` only when unavoidable
+  - Comprehensions take at most one `for` clause and one `if` clause (LIT014). Split stacked clauses into a helper generator, a named intermediate, or a plain loop. Suppress with `# comprehension-ok: <reason>` only when unavoidable
   - Every pydantic model must be frozen (LIT015), set via `model_config = ConfigDict(frozen=True)`, a dict-literal `model_config`, an inner `class Config`, or the class keywords. Subclasses inherit it unless they override it. Replace in-place field writes with `model_copy(update=...)`. If making the model mutable is truly unavoidable, suppress with `# frozen-ok: <reason>` on the `class` line
 - Use dependency injection
-- Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
+- Fully typed. No `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
 - Use tagged unions + match
 - No monster files or god objects
 - No file sprawl: deliberate file and folder structure
-- Standard over hand-rolled: use the official SDK or a library where one exists; where none does, follow industry standards instead of inventing local conventions
-- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module; if none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
+- Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
+- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
 
 Follow conventional commits for commit names and PR titles
 
@@ -130,4 +130,4 @@ Before implementing:
 
 Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify
 
-Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or raise `ANY_CAPS` to satisfy a check
+Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review. Never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or raise `ANY_CAPS` to satisfy a check

README.md

@@ -407,6 +407,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [ScaleDown (`scaledown`)](https://docs.litellm.ai/docs/providers/scaledown) | ✅ |  |  |  |  |  |  |  |  |  |
 | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) |  |  |  |  |  |  |  |  |  |  |
 | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

ci_cd/cost_map_guard.py

@@ -14,8 +14,9 @@ import argparse
 import json
 import subprocess
 import sys
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
+from types import MappingProxyType
 from typing import Final
 
 from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
@@ -25,6 +26,7 @@ BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
 SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
 GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
 BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
 
 CostMap = dict[str, object]
 
@@ -51,7 +53,31 @@ def _rendered_schema(cost_map: CostMap) -> str:
         return str(error)
 
 
-def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]:
+def _evaluation_output_price_failure(
+    key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
+) -> str | None:
+    if not isinstance(entry, dict) or entry.get("mode") != "evaluation":
+        return None
+    output_cost: Final = entry.get("output_cost_per_token")
+    if (
+        not isinstance(output_cost, (int, float))
+        or isinstance(output_cost, bool)
+        or output_cost <= 0
+        or key in evaluation_output_price_sources
+    ):
+        return None
+    return (
+        f"{key}: mode evaluation rows bill input only; output_cost_per_token must be 0 unless the vendor "
+        "prices output tokens for this model (add the source to EVALUATION_OUTPUT_PRICE_SOURCES)"
+    )
+
+
+def _file_failures(
+    head: Snapshot,
+    head_map: CostMap,
+    *,
+    evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
+) -> tuple[str, ...]:
     schema_text: Final = _rendered_schema(head_map)
     if not schema_text.startswith("{"):
         return (schema_text,)
@@ -75,6 +101,16 @@ def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]:
             f"{COST_MAP_PATH} does not validate against its schema: {error}"
             for error in validation_errors(head_map, json.loads(schema_text))[:20]
         ),
+        *(
+            failure
+            for key, entry in head_map.items()
+            if (
+                failure := _evaluation_output_price_failure(
+                    key, entry, evaluation_output_price_sources
+                )
+            )
+            is not None
+        ),
     )
 
 
@@ -121,13 +157,25 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
     return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
 
 
-def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
+def guard_failures(
+    base: Snapshot,
+    head: Snapshot,
+    changed_files: Sequence[str],
+    bot: bool,
+    *,
+    evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
+) -> tuple[str, ...]:
     if not bot and not touches_cost_map(changed_files):
         return ()
     head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
     if isinstance(head_map, str):
         return (head_map,)
-    return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
+    return (
+        *_file_failures(
+            head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
+        ),
+        *(_bot_failures(base, head_map, changed_files) if bot else ()),
+    )
 
 
 def _git(*args: str) -> str | None:

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -12,7 +12,7 @@ Endpoints for /project operations
 
 import json
 from collections.abc import Mapping, Sequence
-from typing import TYPE_CHECKING, Final
+from typing import TYPE_CHECKING, Final, cast
 
 from fastapi import APIRouter, Depends, HTTPException, Request
 from pydantic import TypeAdapter
@@ -31,6 +31,7 @@ from litellm.proxy.management_helpers.utils import (
     management_endpoint_wrapper,
 )
 from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
+from litellm.repositories.base_repository import record_to_dict
 from litellm.repositories.budget_repository import BudgetRepository
 from litellm.repositories.object_permission_repository import ObjectPermissionRepository
 from litellm.repositories.prisma_protocols import TableActions
@@ -57,6 +58,15 @@ def _project_table(prisma_client: PrismaClient) -> TableActions["prisma_models.L
     return ProjectRepository(prisma_client).table
 
 
+def _writer_project_table(
+    prisma_client: PrismaClient,
+) -> TableActions["prisma_models.LiteLLM_ProjectTable"]:
+    return cast(  # cast-ok: writer_db exposes generated Prisma tables through a dynamic wrapper
+        "TableActions[prisma_models.LiteLLM_ProjectTable]",
+        prisma_client.writer_db.litellm_projecttable,
+    )
+
+
 def _verification_token_table(
     prisma_client: PrismaClient,
 ) -> TableActions["prisma_models.LiteLLM_VerificationToken"]:
@@ -824,6 +834,38 @@ async def update_project(
             **({"max_budget": None} if "max_budget" in data.model_fields_set and data.max_budget is None else {}),
         }
 
+        object_permission_data: Final = update_data.pop("object_permission", None)
+        object_permission_payload: Final = (
+            _OBJECT_PERMISSION_PAYLOAD.validate_python(object_permission_data) if object_permission_data else None
+        )
+
+        if data.team_id is not None:
+            current_project_record: Final = await _writer_project_table(prisma_client).find_unique(
+                where={"project_id": data.project_id}
+            )
+            current_project: Final = (
+                LiteLLM_ProjectTable.model_validate(record_to_dict(current_project_record))
+                if current_project_record is not None
+                else None
+            )
+            if current_project is not None and data.team_id != current_project.team_id:
+                mismatched_key_count: Final = await prisma_client.writer_db.litellm_verificationtoken.count(
+                    where={
+                        "project_id": data.project_id,
+                        "OR": [{"team_id": {"not": data.team_id}}, {"team_id": None}],
+                    }
+                )
+                if mismatched_key_count > 0:
+                    raise ProxyException(
+                        message=(
+                            f"Project {data.project_id} has {mismatched_key_count} key(s) that do not belong to "
+                            f"team {data.team_id}. Detach or delete them before moving the project."
+                        ),
+                        type="bad_request",
+                        code=400,
+                        param="team_id",
+                    )
+
         if budget_updates and existing_project.budget_id:
             # Update existing budget
             await _budget_table(prisma_client).update(
@@ -837,23 +879,17 @@ async def update_project(
             for field in budget_updates.keys():
                 update_data.pop(field, None)
 
-        # Handle object permissions
-        if "object_permission" in update_data:
-            object_permission_data = update_data.pop("object_permission")
-            if object_permission_data:
-                object_permission_payload: Final = _OBJECT_PERMISSION_PAYLOAD.validate_python(object_permission_data)
-                if existing_project.object_permission_id:
-                    # Update existing permission
-                    await _object_permission_table(prisma_client).update(
-                        where={"object_permission_id": existing_project.object_permission_id},
-                        data=object_permission_payload,
-                    )
-                else:
-                    # Create new permission
-                    created_permission: Final = await _object_permission_table(prisma_client).create(
-                        data=object_permission_payload,
-                    )
-                    update_data["object_permission_id"] = created_permission.object_permission_id
+        if object_permission_payload is not None:
+            if existing_project.object_permission_id:
+                await _object_permission_table(prisma_client).update(
+                    where={"object_permission_id": existing_project.object_permission_id},
+                    data=object_permission_payload,
+                )
+            else:
+                created_permission: Final = await _object_permission_table(prisma_client).create(
+                    data=object_permission_payload,
+                )
+                update_data["object_permission_id"] = created_permission.object_permission_id
 
         # Handle metadata fields
         for field in LiteLLM_ManagementEndpoint_MetadataFields:

litellm-proxy-extras/litellm_proxy_extras/migration_lock.py

@@ -1,4 +1,5 @@
 import random
+import re
 import time
 from collections.abc import Generator, Mapping
 from contextlib import contextmanager
@@ -7,25 +8,50 @@ from typing import TYPE_CHECKING, Final
 from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 
 from litellm_proxy_extras._logging import logger
-from litellm_proxy_extras.prisma_toolchain import MIGRATION_LOCK_TIMEOUT_ENV_VAR, migration_lock_timeout
+from litellm_proxy_extras.prisma_toolchain import (
+    MIGRATION_LOCK_TIMEOUT_ENV_VAR,
+    migration_ddl_lock_timeout,
+    migration_lock_timeout,
+)
 
 MIGRATION_LOCK_KEY: Final = int.from_bytes(b"llm_mig2", "big")
+_LOCK_TIMEOUT_PINNED_RE: Final = re.compile(r"(?:-c\s*|--)lock_timeout=")
 
 if TYPE_CHECKING:
     import psycopg
 
 
-def migration_environment(environment: Mapping[str, str]) -> Mapping[str, str]:
-    database_url: Final = environment.get("DATABASE_URL")
-    direct_url: Final = environment.get("DIRECT_URL")
-    if not database_url or not direct_url:
-        return environment
+def _migration_url(database_url: str, direct_url: "str | None") -> str:
+    if not direct_url:
+        return database_url
     schema: Final = next((value for key, value in parse_qsl(urlsplit(database_url).query) if key == "schema"), "public")
     direct: Final = urlsplit(direct_url)
     parameters: Final = tuple((key, value) for key, value in parse_qsl(direct.query) if key != "schema")
+    return urlunsplit(direct._replace(query=urlencode((*parameters, ("schema", schema)))))
+
+
+def _with_ddl_lock_timeout(url: str) -> str:
+    split: Final = urlsplit(url)
+    pairs: Final = tuple(parse_qsl(split.query))
+    if any(key == "pgbouncer" and value == "true" for key, value in pairs):
+        return url
+    existing: Final = next((value for key, value in pairs if key == "options"), "")
+    if _LOCK_TIMEOUT_PINNED_RE.search(existing):
+        return url
+    timeout_ms: Final = max(1, int(migration_ddl_lock_timeout() * 1000))
+    options: Final = f"{existing} -c lock_timeout={timeout_ms}".strip()
+    rewritten: Final = tuple((key, value) for key, value in pairs if key != "options") + (("options", options),)
+    return urlunsplit(split._replace(query=urlencode(rewritten)))
+
+
+def migration_environment(environment: Mapping[str, str]) -> Mapping[str, str]:
+    database_url: Final = environment.get("DATABASE_URL")
+    if not database_url:
+        return environment
+    migration_url: Final = _migration_url(database_url, environment.get("DIRECT_URL"))
     return {
         **environment,
-        "DATABASE_URL": urlunsplit(direct._replace(query=urlencode((*parameters, ("schema", schema))))),
+        "DATABASE_URL": _with_ddl_lock_timeout(migration_url),
     }
 
 

litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py

@@ -60,11 +60,13 @@ PRISMA_COMMAND_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_COMMAND_TIMEOUT"
 PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_BOOTSTRAP_TIMEOUT"
 PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT"
 MIGRATION_LOCK_TIMEOUT_ENV_VAR = "LITELLM_MIGRATION_LOCK_TIMEOUT"
+MIGRATION_DDL_LOCK_TIMEOUT_ENV_VAR = "LITELLM_MIGRATION_DDL_LOCK_TIMEOUT"
 NODEENV_CACHE_DIR_ENV_VAR = "PRISMA_NODEENV_CACHE_DIR"
 
 DEFAULT_PRISMA_COMMAND_TIMEOUT = 60.0
 DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT = 600.0
 DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT = 600.0
+DEFAULT_MIGRATION_DDL_LOCK_TIMEOUT = 10.0
 
 BOOTSTRAP_ARG = "--version"
 PRISMA_CONSOLE_SCRIPT = "prisma"
@@ -111,6 +113,11 @@ def migration_lock_timeout() -> float:
     return _timeout_from_env(MIGRATION_LOCK_TIMEOUT_ENV_VAR, 600.0)
 
 
+def migration_ddl_lock_timeout() -> float:
+    """Seconds one migration statement may wait on a table lock before Postgres cancels it."""
+    return _timeout_from_env(MIGRATION_DDL_LOCK_TIMEOUT_ENV_VAR, DEFAULT_MIGRATION_DDL_LOCK_TIMEOUT)
+
+
 def prisma_bootstrap_timeout() -> float:
     """Seconds the one-time Node toolchain install may run for."""
     return _timeout_from_env(

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -18,6 +18,7 @@ from litellm_proxy_extras import prisma_toolchain
 from litellm_proxy_extras._logging import logger
 from litellm_proxy_extras.migration_lock import held_migration_lock
 from litellm_proxy_extras.prisma_toolchain import (
+    MIGRATION_DDL_LOCK_TIMEOUT_ENV_VAR,
     PRISMA_COMMAND_TIMEOUT_ENV_VAR,
     PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR,
     ensure_prisma_toolchain,
@@ -62,6 +63,7 @@ def _get_prisma_env() -> dict:
 _MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
 
 _MIGRATION_DEADLOCK_MARKER = "deadlock detected"
+_MIGRATION_LOCK_TIMEOUT_MARKER = "canceling statement due to lock timeout"
 INDEX_REPAIR_ADVISORY_LOCK_KEY: Final = int.from_bytes(b"litellm", "big")
 _TRANSIENT_INDEX_SUFFIX_RE: Final = re.compile(r"_cc(?:new|old)\d*$")
 _INVALID_LITELLM_INDEXES_SQL: Final = (
@@ -1162,9 +1164,11 @@ class ProxyExtrasDBManager:
 
             raise RuntimeError(
                 f"Database migration failed after {MAX_MIGRATE_DEPLOY_ATTEMPTS} "
-                "attempts that made no progress (timeouts or deadlock retries). Check database connectivity, "
-                "load, and _prisma_migrations ledger state, and raise "
-                f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out."
+                "attempts that made no progress (timeouts, lock timeouts or deadlock retries). "
+                "Check database connectivity, load, long-running transactions on the migrated tables, "
+                "and _prisma_migrations ledger state. Raise "
+                f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out, or "
+                f"{MIGRATION_DDL_LOCK_TIMEOUT_ENV_VAR} if they timed out waiting for a table lock."
             )
         finally:
             os.chdir(original_dir)
@@ -1225,6 +1229,13 @@ class ProxyExtrasDBManager:
                     )
                     ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name)
                     return budget.spend()
+                if ledger_logs and _MIGRATION_LOCK_TIMEOUT_MARKER in ledger_logs:
+                    logger.info(
+                        "Migration %s timed out waiting for a table lock, rolling its ledger row back and retrying",
+                        migration_name,
+                    )
+                    ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name)
+                    return budget.spend()
                 if ProxyExtrasDBManager._failed_migration_recovered(migration_name, started_at):
                     logger.info(
                         "Migration %s started at %s was already rolled back or completed by a concurrent "
@@ -1270,6 +1281,16 @@ class ProxyExtrasDBManager:
                 ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name)
                 return budget.spend()
 
+            if migration_name and _MIGRATION_LOCK_TIMEOUT_MARKER in stderr:
+                ProxyExtrasDBManager._log_migration_lock_holders(migration_name)
+                logger.warning(
+                    "Migration %s timed out waiting for a table lock, rolling its ledger row back and retrying. "
+                    f"Raise {MIGRATION_DDL_LOCK_TIMEOUT_ENV_VAR} if the database needs longer.",
+                    migration_name,
+                )
+                ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name)
+                return budget.spend()
+
             raise RuntimeError(
                 "Database migration failed and cannot be auto-recovered. "
                 f"Manual intervention required.\n\nPrisma error:\n{stderr}"
@@ -1282,6 +1303,13 @@ class ProxyExtrasDBManager:
             )
             return budget.spend()
 
+        if _MIGRATION_LOCK_TIMEOUT_MARKER in stderr:
+            logger.info(
+                "Waiting for the advisory lock held by another Prisma migration; "
+                "contention does not spend a migration failure attempt"
+            )
+            return budget.after_contention(attempt_seconds)
+
         if "P1002" in stderr and "advisory lock" in stderr:
             logger.info(
                 "Waiting for the advisory lock held by another Prisma migration; "
@@ -1294,6 +1322,68 @@ class ProxyExtrasDBManager:
             f"Manual intervention required.\n\nPrisma error:\n{stderr}"
         ) from error
 
+    @staticmethod
+    def _log_migration_lock_holders(migration_name: str) -> None:
+        """Best-effort log of the sessions holding locks on the tables a timed-out migration touches."""
+        try:
+            migration_sql: Final = (
+                Path(ProxyExtrasDBManager._get_prisma_dir()) / "migrations" / migration_name / "migration.sql"
+            ).read_text()
+        except OSError:
+            return
+        relations: Final = sorted(set(re.findall(r'"(LiteLLM_\w+)"', migration_sql)))
+        database_url: Final = os.getenv("DATABASE_URL")
+        if not relations or not database_url:
+            return
+        try:
+            import psycopg
+        except ImportError:
+            return
+        try:
+            with psycopg.connect(
+                ProxyExtrasDBManager._strip_prisma_query_params(database_url),
+                connect_timeout=10,
+                autocommit=True,
+                options="-c statement_timeout=5000",
+            ) as conn:
+                rows: Final = conn.execute(
+                    "SELECT l.pid, c.relname, l.mode, a.state, a.application_name, "
+                    "date_trunc('second', now() - a.xact_start) "
+                    "FROM pg_locks l "
+                    "JOIN pg_class c ON c.oid = l.relation "
+                    "JOIN pg_namespace n ON n.oid = c.relnamespace "
+                    "JOIN pg_stat_activity a ON a.pid = l.pid "
+                    "WHERE l.granted AND n.nspname = %s AND c.relname = ANY(%s) AND l.pid <> pg_backend_pid() "
+                    "ORDER BY a.xact_start NULLS LAST "
+                    "LIMIT 10",
+                    (
+                        ProxyExtrasDBManager._prisma_schema_param(database_url) or "public",
+                        relations,
+                    ),
+                ).fetchall()
+        except psycopg.Error:
+            return
+        if not rows:
+            logger.warning(
+                "Migration %s timed out waiting for a lock, but no current lock holder "
+                "was found on %s; the holder has likely since committed or rolled back",
+                migration_name,
+                ", ".join(relations),
+            )
+            return
+        for pid, relname, mode, state, application_name, age in rows:
+            logger.warning(
+                "Migration %s timed out waiting for a lock on %s held by pid %s "
+                "(mode %s, state %s, application %s, transaction open for %s)",
+                migration_name,
+                relname,
+                pid,
+                mode,
+                state,
+                application_name,
+                age,
+            )
+
     @staticmethod
     def _mark_migration_applied(name: str) -> None:
         """Roll a failed ledger row back if it is still there, then mark it applied."""

litellm-rust/AGENTS.md

@@ -4,7 +4,7 @@ For diagnostic tracing changes, follow [.agents/skills/rust-tracing/SKILL.md](.a
 
 For string-valued enums and their Serde conversions, follow [.agents/skills/rust-string-enums/SKILL.md](.agents/skills/rust-string-enums/SKILL.md)
 
-For fieldless enums, derive `strum::VariantArray` and use `VARIANTS` instead of a hand-listed `ALL` array; derive Strum string conversions instead of hand-written variant-to-string matches
+For fieldless enums, derive `strum::VariantArray` and use `VARIANTS` instead of a hand-listed `ALL` array. Derive Strum string conversions instead of hand-written variant-to-string matches
 
 Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `str::parse`) with no `as_str`/`parse` wrapper that only delegates, and spell each variant with explicit `#[strum(serialize = "...")]` instead of `serialize_all`
 
@@ -15,20 +15,20 @@ Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `st
 - A test that only uses the crate's public API lives in `crates/<crate>/tests/<subject>.rs`, next to `src/`
 - Split a mixed test file along that line instead of widening visibility to move it
 - A test for another crate's item belongs in that crate, not in a downstream one
-- Never set `autotests = false` or hand-list `[[test]]` targets; every file directly under `tests/` is discovered by cargo, and a shared helper goes in `tests/<name>/mod.rs` or `tests/<subject>/support.rs` so it is not picked up as a test crate of its own
+- Never set `autotests = false` or hand-list `[[test]]` targets. Every file directly under `tests/` is discovered by cargo, and a shared helper goes in `tests/<name>/mod.rs` or `tests/<subject>/support.rs` so it is not picked up as a test crate of its own
 
 ## Test fixtures and cases
 
 Use [`#[rstest]`](https://docs.rs/rstest/latest/rstest/attr.rstest.html) for new and updated tests and [`#[fixture]`](https://docs.rs/rstest/latest/rstest/attr.fixture.html) for reusable setup, injected through typed test arguments. Express input variations as named `#[case::name(...)]` cases instead of loops or duplicated tests so each failure identifies its case. Keep behavior assertions in the test body and fixtures focused on setup. Use the workspace `rstest` dependency
 
-- Never loop over inputs (`for`, `.iter().for_each`, `.all`) inside a test body; give each input its own `#[case::name(...)]`, or use `#[values(...)]` for a cross product
+- Never loop over inputs (`for`, `.iter().for_each`, `.all`) inside a test body. Give each input its own `#[case::name(...)]`, or use `#[values(...)]` for a cross product
 - Exception: a test pinning a Rust table against a repo-owned data file (for example `include_str!` of a JSON config) may iterate that file's entries
 
 ## Error definitions
 
 - A crate's errors live in `src/error.rs`, defined with `thiserror`, and re-exported from `lib.rs`
-- Put message templates in the variant's `#[error(...)]` declaration. Callers pass only the small typed arguments needed to fill them, never `Error::Variant(format!(...))` or a preformatted message. Keep the smallest set of neutral variants that callers need to distinguish; different wording or providers do not justify new variants
-- Default to one top-level `Error` enum per crate, with one variant per failure mode and a `#[error(...)]` message on each. A failure mode is something a caller handles differently (phase, status code, retry, a message Python parity pins exactly); failures no caller tells apart share one variant and differ only in its message
+- Put message templates in the variant's `#[error(...)]` declaration. Callers pass only the small typed arguments needed to fill them, never `Error::Variant(format!(...))` or a preformatted message. Keep the smallest set of neutral variants that callers need to distinguish. Different wording or providers do not justify new variants
+- Default to one top-level `Error` enum per crate, with one variant per failure mode and a `#[error(...)]` message on each. A failure mode is something a caller handles differently (phase, status code, retry, a message Python parity pins exactly). Failures no caller tells apart share one variant and differ only in its message
 - Keep shared error enums minimal and provider-neutral. Provider names, credential types, configuration fields, and setup guidance belong in caller-supplied data, not dedicated variants or hardcoded shared messages. Reuse a variant for the same failure mode across providers, such as `MissingApiBase { provider: "Azure", guidance: "..." }`. An exact parity message does not justify a provider-specific variant when caller-supplied context can preserve it
 - Wrap a lower-level error as a variant with `#[from]` or `#[source]` instead of flattening it to a string
 - Exception: split into separate types when different functions fail in disjoint ways, especially when different callers see them. A shared enum would force every caller to match variants its function can never return

litellm-rust/Cargo.lock

@@ -3378,10 +3378,12 @@ dependencies = [
  "moka",
  "reqwest 0.12.28",
  "rstest",
+ "serde",
  "serde_json",
  "sha2 0.10.9",
  "thiserror 2.0.19",
  "tokio",
+ "veil",
 ]
 
 [[package]]
@@ -3410,9 +3412,12 @@ dependencies = [
  "litellm-auth-types",
  "moka",
  "rstest",
+ "serde",
  "serde_json",
  "sha2 0.10.9",
+ "tempfile",
  "tokio",
+ "veil",
 ]
 
 [[package]]
@@ -3421,6 +3426,7 @@ version = "0.1.0"
 dependencies = [
  "rstest",
  "serde",
+ "serde_json",
  "subtle",
  "thiserror 2.0.19",
  "tokio",
@@ -3640,6 +3646,7 @@ name = "litellm-config"
 version = "0.1.0"
 dependencies = [
  "litellm-auth-types",
+ "litellm-router-types",
  "rstest",
  "serde",
  "serde_json",
@@ -4024,6 +4031,7 @@ dependencies = [
  "litellm-inference-testing",
  "litellm-llms",
  "litellm-llms-types",
+ "litellm-router-types",
  "litellm-secrets",
  "litellm-tracing",
  "reqwest 0.12.28",
@@ -4144,6 +4152,7 @@ dependencies = [
  "litellm-http",
  "litellm-llms-types",
  "litellm-python-compat",
+ "litellm-router-types",
  "litellm-secrets",
  "reqwest 0.12.28",
  "rstest",
@@ -4215,6 +4224,7 @@ dependencies = [
  "litellm-inference-transcription",
  "litellm-llms",
  "litellm-llms-types",
+ "litellm-router-types",
  "litellm-secrets",
  "litellm-secrets-aws",
  "litellm-secrets-types",
@@ -4263,9 +4273,21 @@ dependencies = [
  "litellm-config",
  "litellm-inference",
  "litellm-inference-messages",
+ "litellm-router-types",
  "rstest",
 ]
 
+[[package]]
+name = "litellm-router-types"
+version = "0.1.0"
+dependencies = [
+ "litellm-auth-types",
+ "rstest",
+ "serde",
+ "serde_json",
+ "serde_with",
+]
+
 [[package]]
 name = "litellm-secrets"
 version = "0.1.0"

litellm-rust/Cargo.toml

@@ -11,6 +11,7 @@ repository = "https://github.com/BerriAI/litellm"
 [workspace.dependencies]
 litellm-config = { path = "crates/config" }
 litellm-router = { path = "crates/router" }
+litellm-router-types = { path = "crates/router-types" }
 litellm-tracing = { path = "crates/tracing" }
 litellm-spend-clickhouse = { path = "crates/spend-clickhouse" }
 litellm-storage-clickhouse = { path = "crates/storage-clickhouse" }

litellm-rust/crates/auth-aws/Cargo.toml

@@ -10,9 +10,11 @@ litellm-auth-types.workspace = true
 litellm-http.workspace = true
 
 moka = { workspace = true, features = ["sync"] }
-serde_json.workspace = true
+serde.workspace = true
+serde_json = { workspace = true, features = ["preserve_order"] }
 sha2.workspace = true
 thiserror.workspace = true
+veil.workspace = true
 
 aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"] }
 aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] }

litellm-rust/crates/auth-aws/src/aws.rs

@@ -3,7 +3,6 @@ use std::time::Duration;
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use moka::sync::Cache;
-use serde_json::{Map, Value};
 use sha2::{Digest, Sha256};
 
 use aws_credential_types::Credentials;
@@ -14,10 +13,12 @@ use aws_sigv4::http_request::{
 use aws_sigv4::sign::v4;
 use aws_smithy_runtime_api::client::identity::Identity;
 
+use litellm_auth_types::AwsParams;
+
 use super::Error;
 use super::constants::{
-    AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION, AWS_REGION_NAME,
-    AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN,
+    AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_ROLE_ARN,
+    AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN,
     AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, AWS_WEB_IDENTITY_TOKEN_FILE,
     DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES,
 };
@@ -543,50 +544,39 @@ fn is_bedrock_region(value: &str) -> bool {
 /// region, then the environment. Each service decides what a missing one means.
 pub fn resolve_aws_region(
     model_region: Option<&str>,
-    optional_params: &Map<String, Value>,
+    params: &AwsParams,
     env_lookup: &dyn Fn(&str) -> Option<String>,
 ) -> Option<String> {
-    optional_params
-        .get("aws_region_name")
-        .and_then(Value::as_str)
-        .or(model_region)
-        .map(str::to_string)
-        .or_else(|| env_lookup(AWS_REGION_NAME))
-        .or_else(|| env_lookup(AWS_REGION))
+    params
+        .resolve(&AwsParams::REGION, &|_| None)
+        .or_else(|| model_region.map(str::to_string))
+        .or_else(|| AwsParams::REGION.resolve(&|_| None, env_lookup))
 }
 
 pub fn resolve_bedrock_region(
     model_region: Option<&str>,
-    optional_params: &Map<String, Value>,
+    params: &AwsParams,
     env_lookup: &dyn Fn(&str) -> Option<String>,
 ) -> String {
-    resolve_aws_region(model_region, optional_params, env_lookup)
+    resolve_aws_region(model_region, params, env_lookup)
         .unwrap_or_else(|| DEFAULT_BEDROCK_REGION.to_string())
 }
 
 pub fn aws_auth_config(
-    optional_params: &Map<String, Value>,
+    params: &AwsParams,
     env_lookup: &dyn Fn(&str) -> Option<String>,
 ) -> AwsAuthConfig {
-    let value = |key: &str| {
-        optional_params
-            .get(key)
-            .and_then(Value::as_str)
-            .map(str::to_string)
-    };
-    let env = |key: &str| env_lookup(key);
     AwsAuthConfig {
-        access_key_id: value("aws_access_key_id").or_else(|| env("AWS_ACCESS_KEY_ID")),
-        secret_access_key: value("aws_secret_access_key").or_else(|| env("AWS_SECRET_ACCESS_KEY")),
-        session_token: value("aws_session_token").or_else(|| env("AWS_SESSION_TOKEN")),
-        region_name: value("aws_region_name").or_else(|| env(AWS_REGION_NAME)),
-        session_name: value("aws_session_name").or_else(|| env("AWS_SESSION_NAME")),
-        profile_name: value("aws_profile_name").or_else(|| env("AWS_PROFILE_NAME")),
-        role_name: value("aws_role_name").or_else(|| env("AWS_ROLE_NAME")),
-        web_identity_token: value("aws_web_identity_token")
-            .or_else(|| env("AWS_WEB_IDENTITY_TOKEN")),
-        sts_endpoint: value("aws_sts_endpoint").or_else(|| env("AWS_STS_ENDPOINT")),
-        external_id: value("aws_external_id").or_else(|| env("AWS_EXTERNAL_ID")),
+        access_key_id: params.resolve(&AwsParams::ACCESS_KEY_ID, env_lookup),
+        secret_access_key: params.resolve(&AwsParams::SECRET_ACCESS_KEY, env_lookup),
+        session_token: params.resolve(&AwsParams::SESSION_TOKEN, env_lookup),
+        region_name: params.resolve(&AwsParams::REGION, env_lookup),
+        session_name: params.resolve(&AwsParams::SESSION_NAME, env_lookup),
+        profile_name: params.resolve(&AwsParams::PROFILE_NAME, env_lookup),
+        role_name: params.resolve(&AwsParams::ROLE_NAME, env_lookup),
+        web_identity_token: params.resolve(&AwsParams::WEB_IDENTITY_TOKEN, env_lookup),
+        sts_endpoint: params.resolve(&AwsParams::STS_ENDPOINT, env_lookup),
+        external_id: params.resolve(&AwsParams::EXTERNAL_ID, env_lookup),
     }
 }
 
@@ -599,13 +589,10 @@ pub enum AwsCredentialSource {
 }
 
 impl AwsCredentialSource {
-    pub fn from_params(
-        optional_params: &Map<String, Value>,
-        env_lookup: &dyn Fn(&str) -> Option<String>,
-    ) -> Self {
-        match host_supplied_credentials(optional_params) {
+    pub fn from_params(params: &AwsParams, env_lookup: &dyn Fn(&str) -> Option<String>) -> Self {
+        match host_supplied_credentials(params) {
             Some(credentials) => Self::HostSupplied(credentials),
-            None => Self::Chain(aws_auth_config(optional_params, env_lookup)),
+            None => Self::Chain(aws_auth_config(params, env_lookup)),
         }
     }
 
@@ -629,20 +616,20 @@ impl AwsCredentialSource {
 /// state, where an unrelated `AWS_ROLE_NAME` or `AWS_PROFILE_NAME` in the
 /// environment outranks explicit keys in [`classify_auth`] and the two sides
 /// would sign as different principals.
-pub fn host_supplied_credentials(optional_params: &Map<String, Value>) -> Option<Credentials> {
-    let value = |key: &str| {
-        optional_params
-            .get(key)
-            .and_then(Value::as_str)
+pub fn host_supplied_credentials(params: &AwsParams) -> Option<Credentials> {
+    let value = |value: &Option<String>| {
+        value
+            .as_deref()
             .map(str::trim)
             .filter(|value| !value.is_empty())
+            .map(str::to_string)
     };
-    let access_key_id = value("aws_access_key_id")?;
-    let secret_access_key = value("aws_secret_access_key")?;
+    let access_key_id = value(&params.aws_access_key_id)?;
+    let secret_access_key = value(&params.aws_secret_access_key)?;
     Some(Credentials::new(
         access_key_id,
         secret_access_key,
-        value("aws_session_token").map(str::to_string),
+        value(&params.aws_session_token),
         None,
         "litellm-host-supplied",
     ))
@@ -651,7 +638,7 @@ pub fn host_supplied_credentials(optional_params: &Map<String, Value>) -> Option
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::constants::BEDROCK_SERVICE;
+    use crate::constants::{AWS_REGION, BEDROCK_SERVICE};
 
     fn no_env(_: &str) -> Option<String> {
         None
@@ -667,37 +654,80 @@ mod tests {
             recorded.lock().unwrap().insert(name.to_string());
             None
         };
-        resolve_aws_region(None, &Map::new(), &env);
-        aws_auth_config(&Map::new(), &env);
+        resolve_aws_region(None, &AwsParams::default(), &env);
+        aws_auth_config(&AwsParams::default(), &env);
         assert!(
             seen.lock()
                 .unwrap()
                 .iter()
-                .all(|name| crate::constants::SECRET_NAMES.contains(&name.as_str()))
+                .all(|name| AwsParams::secret_names().contains(&name.as_str()))
         );
     }
 
-    #[test]
-    fn a_region_comes_from_the_call_then_the_model_then_the_environment() {
-        let params = Map::from_iter([("aws_region_name".to_string(), Value::from("eu-west-1"))]);
-        let region_name = |key: &str| (key == AWS_REGION_NAME).then(|| "ap-south-1".to_string());
-        let region = |key: &str| (key == AWS_REGION).then(|| "sa-east-1".to_string());
-
-        let resolved = [
-            resolve_aws_region(Some("us-east-2"), &params, &region_name),
-            resolve_aws_region(Some("us-east-2"), &Map::new(), &region_name),
-            resolve_aws_region(None, &Map::new(), &region_name),
-            resolve_aws_region(None, &Map::new(), &region),
-            resolve_aws_region(None, &Map::new(), &no_env),
-        ];
+    #[rstest::rstest]
+    #[case::param_wins(Some("from-params"), &[(AWS_SECRET_ACCESS_KEY, "from-env")], Some("from-params"))]
+    #[case::blank_param_falls_to_environment(Some("  "), &[(AWS_SECRET_ACCESS_KEY, "from-env")], Some("from-env"))]
+    #[case::absent_param_falls_to_environment(None, &[(AWS_SECRET_ACCESS_KEY, "from-env")], Some("from-env"))]
+    #[case::nothing(None, &[], None)]
+    fn auth_config_reads_each_spec_from_params_then_the_environment(
+        #[case] aws_secret_access_key: Option<&str>,
+        #[case] environment: &[(&str, &str)],
+        #[case] expected: Option<&str>,
+    ) {
+        let params = AwsParams {
+            aws_secret_access_key: aws_secret_access_key.map(str::to_string),
+            ..AwsParams::default()
+        };
+        let env = |key: &str| {
+            environment
+                .iter()
+                .find(|(name, _)| *name == key)
+                .map(|(_, value)| value.to_string())
+        };
 
         assert_eq!(
-            resolved.map(|region| region.unwrap_or_else(|| "none".into())),
-            ["eu-west-1", "us-east-2", "ap-south-1", "sa-east-1", "none"]
+            aws_auth_config(&params, &env).secret_access_key.as_deref(),
+            expected
         );
         assert_eq!(
-            resolve_bedrock_region(None, &Map::new(), &no_env),
-            DEFAULT_BEDROCK_REGION
+            aws_auth_config(&AwsParams::default(), &|key: &str| (key == AWS_REGION)
+                .then(|| "sa-east-1".to_string()))
+            .region_name
+            .as_deref(),
+            Some("sa-east-1")
+        );
+    }
+
+    #[rstest::rstest]
+    #[case::call_params(Some("us-east-2"), Some("eu-west-1"), &[(AWS_REGION_NAME, "ap-south-1")], Some("eu-west-1"))]
+    #[case::model_region(Some("us-east-2"), None, &[(AWS_REGION_NAME, "ap-south-1")], Some("us-east-2"))]
+    #[case::aws_region_name(None, None, &[(AWS_REGION_NAME, "ap-south-1")], Some("ap-south-1"))]
+    #[case::aws_region(None, None, &[(AWS_REGION, "sa-east-1")], Some("sa-east-1"))]
+    #[case::nothing(None, None, &[], None)]
+    fn a_region_comes_from_the_call_then_the_model_then_the_environment(
+        #[case] model_region: Option<&str>,
+        #[case] aws_region_name: Option<&str>,
+        #[case] environment: &[(&str, &str)],
+        #[case] expected: Option<&str>,
+    ) {
+        let params = AwsParams {
+            aws_region_name: aws_region_name.map(str::to_string),
+            ..AwsParams::default()
+        };
+        let env = |key: &str| {
+            environment
+                .iter()
+                .find(|(name, _)| *name == key)
+                .map(|(_, value)| value.to_string())
+        };
+
+        assert_eq!(
+            resolve_aws_region(model_region, &params, &env).as_deref(),
+            expected
+        );
+        assert_eq!(
+            resolve_bedrock_region(model_region, &params, &env),
+            expected.unwrap_or(DEFAULT_BEDROCK_REGION)
         );
     }
 

litellm-rust/crates/auth-aws/src/constants.rs

@@ -1,33 +1,21 @@
-pub const AWS_ACCESS_KEY_ID: &str = "AWS_ACCESS_KEY_ID";
-pub const AWS_SECRET_ACCESS_KEY: &str = "AWS_SECRET_ACCESS_KEY";
-pub const AWS_SESSION_TOKEN: &str = "AWS_SESSION_TOKEN";
-pub const AWS_REGION_NAME: &str = "AWS_REGION_NAME";
-pub const AWS_REGION: &str = "AWS_REGION";
+use litellm_auth_types::AwsParams;
+
+pub const AWS_ACCESS_KEY_ID: &str = AwsParams::ACCESS_KEY_ID.env[0];
+pub const AWS_SECRET_ACCESS_KEY: &str = AwsParams::SECRET_ACCESS_KEY.env[0];
+pub const AWS_SESSION_TOKEN: &str = AwsParams::SESSION_TOKEN.env[0];
+pub const AWS_REGION_NAME: &str = AwsParams::REGION.env[0];
+pub const AWS_REGION: &str = AwsParams::REGION.env[1];
 pub const AWS_DEFAULT_REGION: &str = "AWS_DEFAULT_REGION";
-pub const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = "AWS_BEDROCK_RUNTIME_ENDPOINT";
-pub const AWS_SESSION_NAME: &str = "AWS_SESSION_NAME";
-pub const AWS_PROFILE_NAME: &str = "AWS_PROFILE_NAME";
-pub const AWS_ROLE_NAME: &str = "AWS_ROLE_NAME";
-pub const AWS_WEB_IDENTITY_TOKEN: &str = "AWS_WEB_IDENTITY_TOKEN";
+pub const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = AwsParams::BEDROCK_RUNTIME_ENDPOINT.env[0];
+pub const AWS_SESSION_NAME: &str = AwsParams::SESSION_NAME.env[0];
+pub const AWS_PROFILE_NAME: &str = AwsParams::PROFILE_NAME.env[0];
+pub const AWS_ROLE_NAME: &str = AwsParams::ROLE_NAME.env[0];
+pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsParams::WEB_IDENTITY_TOKEN.env[0];
 pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN";
 pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
-pub const AWS_STS_ENDPOINT: &str = "AWS_STS_ENDPOINT";
-pub const AWS_EXTERNAL_ID: &str = "AWS_EXTERNAL_ID";
+pub const AWS_STS_ENDPOINT: &str = AwsParams::STS_ENDPOINT.env[0];
+pub const AWS_EXTERNAL_ID: &str = AwsParams::EXTERNAL_ID.env[0];
 pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK";
-pub const SECRET_NAMES: &[&str] = &[
-    AWS_ACCESS_KEY_ID,
-    AWS_SECRET_ACCESS_KEY,
-    AWS_SESSION_TOKEN,
-    AWS_REGION_NAME,
-    AWS_REGION,
-    AWS_SESSION_NAME,
-    AWS_PROFILE_NAME,
-    AWS_ROLE_NAME,
-    AWS_WEB_IDENTITY_TOKEN,
-    AWS_STS_ENDPOINT,
-    AWS_EXTERNAL_ID,
-];
-
 /// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors
 /// Python's `_filter_headers_for_aws_signature` allowlist.
 pub const AWS_SIGNED_HEADER_NAMES: &[&str] = &[

litellm-rust/crates/auth-gcp/Cargo.toml

@@ -12,9 +12,11 @@ google-sdk = ["dep:google-cloud-auth", "dep:http"]
 litellm-auth-types.workspace = true
 
 moka.workspace = true
+serde.workspace = true
 serde_json.workspace = true
 sha2.workspace = true
 tokio.workspace = true
+veil.workspace = true
 
 gcp_auth = "0.12.7"
 google-cloud-auth = { workspace = true, optional = true }
@@ -22,3 +24,4 @@ http = { workspace = true, optional = true }
 
 [dev-dependencies]
 rstest.workspace = true
+tempfile.workspace = true

litellm-rust/crates/auth-gcp/src/auth.rs

@@ -0,0 +1,274 @@
+use std::{future::Future, path::Path, pin::Pin, sync::Arc};
+
+use gcp_auth::{CustomServiceAccount, TokenProvider};
+use litellm_auth_types::{CredentialPlacement, Error, ErrorDetail, http::apply_credential};
+use moka::future::Cache;
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+
+use crate::{
+    config::{
+        CredentialSource, VertexConfig, credential_source, get_vertex_ai_project, non_empty_env,
+    },
+    constants::{
+        CLOUD_PLATFORM_SCOPE, GOOGLE_OAUTH_TOKEN_ENDPOINT, VERTEX_AI_API_KEY_ENV,
+        VERTEXAI_API_KEY_ENV,
+    },
+};
+
+pub struct VertexEnvironment {
+    pub headers: Vec<(String, String)>,
+    pub project_id: String,
+}
+
+struct VertexAccessToken {
+    token: String,
+    project_id: String,
+}
+
+#[derive(Clone)]
+pub struct VertexAuth {
+    providers: Cache<CredentialCacheKey, Arc<dyn VertexTokenSource>>,
+    loader: Arc<dyn VertexProviderLoader>,
+}
+
+impl Default for VertexAuth {
+    fn default() -> Self {
+        Self::new(Arc::new(GcpProviderLoader))
+    }
+}
+
+impl VertexAuth {
+    pub fn new(loader: Arc<dyn VertexProviderLoader>) -> Self {
+        Self {
+            providers: Cache::builder().max_capacity(64).build(),
+            loader,
+        }
+    }
+
+    pub async fn access_token(
+        &self,
+        config: &VertexConfig,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<String, Error> {
+        self.load_provider(config, env_lookup).await?.token().await
+    }
+
+    pub async fn validate_environment(
+        &self,
+        headers: Vec<(String, String)>,
+        api_key: Option<&str>,
+        config: &VertexConfig,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<VertexEnvironment, Error> {
+        let has_authorization = headers
+            .iter()
+            .any(|(name, _)| name.eq_ignore_ascii_case("Authorization"));
+        let static_token = api_key
+            .map(str::trim)
+            .filter(|value| !value.is_empty())
+            .map(str::to_string)
+            .or_else(|| non_empty_env(env_lookup, VERTEX_AI_API_KEY_ENV))
+            .or_else(|| non_empty_env(env_lookup, VERTEXAI_API_KEY_ENV));
+        let project_id = get_vertex_ai_project(config, env_lookup);
+
+        if !has_authorization && static_token.is_none() {
+            let access = self.get_access_token(config, env_lookup).await?;
+            return Ok(VertexEnvironment {
+                headers: apply_credential(headers, &access.token, CredentialPlacement::Bearer)?,
+                project_id: project_id.unwrap_or(access.project_id),
+            });
+        }
+
+        let project_id = match project_id {
+            Some(project_id) => project_id,
+            None => {
+                self.load_provider(config, env_lookup)
+                    .await?
+                    .project_id()
+                    .await?
+            }
+        };
+        let headers = if has_authorization {
+            headers
+        } else {
+            apply_credential(
+                headers,
+                static_token.as_deref().expect("static token was checked"),
+                CredentialPlacement::Bearer,
+            )?
+        };
+        Ok(VertexEnvironment {
+            headers,
+            project_id,
+        })
+    }
+
+    async fn get_access_token(
+        &self,
+        config: &VertexConfig,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<VertexAccessToken, Error> {
+        let provider = self.load_provider(config, env_lookup).await?;
+        let (token, project_id) = tokio::try_join!(provider.token(), provider.project_id())?;
+        Ok(VertexAccessToken { token, project_id })
+    }
+
+    async fn load_provider(
+        &self,
+        config: &VertexConfig,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<Arc<dyn VertexTokenSource>, Error> {
+        let source = credential_source(config, env_lookup);
+        let key = source.cache_key();
+        self.providers
+            .try_get_with(key, self.loader.load(source))
+            .await
+            .map_err(|error| (*error).clone())
+    }
+}
+
+pub trait VertexTokenSource: Send + Sync {
+    fn project_id(&self) -> VertexAuthFuture<'_, String>;
+    fn token(&self) -> VertexAuthFuture<'_, String>;
+}
+
+pub trait VertexProviderLoader: Send + Sync {
+    fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>>;
+}
+
+pub type VertexAuthFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
+
+struct GcpTokenSource(Arc<dyn TokenProvider>);
+
+impl VertexTokenSource for GcpTokenSource {
+    fn project_id(&self) -> VertexAuthFuture<'_, String> {
+        Box::pin(async move {
+            self.0
+                .project_id()
+                .await
+                .map(|project| project.to_string())
+                .map_err(auth_acquisition_error)
+        })
+    }
+
+    fn token(&self) -> VertexAuthFuture<'_, String> {
+        Box::pin(async move {
+            self.0
+                .token(&[CLOUD_PLATFORM_SCOPE])
+                .await
+                .map(|token| token.as_str().to_string())
+                .map_err(auth_acquisition_error)
+        })
+    }
+}
+
+struct GcpProviderLoader;
+
+impl VertexProviderLoader for GcpProviderLoader {
+    fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>> {
+        Box::pin(async move {
+            let provider: Arc<dyn TokenProvider> = match source {
+                CredentialSource::Inline(configured) => Arc::new(
+                    CustomServiceAccount::from_json(validate_request_credentials(
+                        configured.expose(),
+                    )?)
+                    .map_err(auth_acquisition_error)?,
+                ),
+                CredentialSource::Trusted(configured) => {
+                    let configured = configured.expose();
+                    let service_account = if Path::new(configured).is_file() {
+                        CustomServiceAccount::from_file(configured)
+                    } else {
+                        CustomServiceAccount::from_json(configured)
+                    }
+                    .map_err(auth_acquisition_error)?;
+                    Arc::new(service_account)
+                }
+                CredentialSource::ApplicationCredentials(path) => {
+                    Arc::new(CustomServiceAccount::from_file(path).map_err(auth_acquisition_error)?)
+                }
+                CredentialSource::Adc => {
+                    gcp_auth::provider().await.map_err(auth_acquisition_error)?
+                }
+            };
+            Ok(Arc::new(GcpTokenSource(provider)) as Arc<dyn VertexTokenSource>)
+        })
+    }
+}
+
+fn validate_request_credentials(configured: &str) -> Result<&str, Error> {
+    let token_uri = serde_json::from_str::<Value>(configured)
+        .ok()
+        .and_then(|credentials| {
+            credentials
+                .get("token_uri")
+                .and_then(Value::as_str)
+                .map(str::to_string)
+        });
+    if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) {
+        return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType {
+            field: "request-controlled vertex_credentials token_uri".into(),
+            expected: "the canonical Google OAuth token endpoint",
+        }));
+    }
+    Ok(configured)
+}
+
+impl CredentialSource {
+    fn cache_key(&self) -> CredentialCacheKey {
+        match self {
+            Self::Inline(configured) => {
+                CredentialCacheKey::Inline(Sha256::digest(configured.expose()).into())
+            }
+            Self::Trusted(configured) => {
+                CredentialCacheKey::Trusted(Sha256::digest(configured.expose()).into())
+            }
+            Self::ApplicationCredentials(path) => {
+                CredentialCacheKey::ApplicationCredentials(path.clone())
+            }
+            Self::Adc => CredentialCacheKey::Adc,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+enum CredentialCacheKey {
+    Inline([u8; 32]),
+    Trusted([u8; 32]),
+    ApplicationCredentials(String),
+    Adc,
+}
+
+fn auth_acquisition_error(error: gcp_auth::Error) -> Error {
+    Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::failed(
+        "Vertex AI credentials",
+        error,
+    ))
+}
+
+#[cfg(test)]
+mod tests {
+    use litellm_auth_types::SecretValue;
+
+    use super::*;
+
+    #[rstest::rstest]
+    #[case::canonical_endpoint(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#, true)]
+    #[case::noncanonical_endpoint(r#"{"token_uri":"http://127.0.0.1/token"}"#, false)]
+    #[case::missing_endpoint("{}", false)]
+    fn request_credentials_require_canonical_token_endpoint(
+        #[case] credentials: &str,
+        #[case] accepted: bool,
+    ) {
+        assert_eq!(validate_request_credentials(credentials).is_ok(), accepted);
+    }
+
+    #[rstest::rstest]
+    fn inline_and_trusted_credentials_never_share_a_cache_entry() {
+        assert_ne!(
+            CredentialSource::Inline(SecretValue::new("same-value")).cache_key(),
+            CredentialSource::Trusted(SecretValue::new("same-value")).cache_key()
+        );
+    }
+}

litellm-rust/crates/auth-gcp/src/config.rs

@@ -0,0 +1,311 @@
+use std::{collections::BTreeMap, path::Path};
+
+use litellm_auth_types::{Error, ErrorDetail, InputSource, SecretValue, Sourced};
+use serde_json::{Map, Value};
+
+use litellm_auth_types::VertexParams;
+
+use crate::constants::GOOGLE_APPLICATION_CREDENTIALS_ENV;
+
+#[derive(Clone, Debug, Default)]
+pub struct VertexConfig {
+    credentials: Option<Sourced<SecretValue>>,
+    project_id: Option<String>,
+    location: Option<String>,
+}
+
+impl VertexConfig {
+    pub fn new(
+        credentials: Option<Sourced<SecretValue>>,
+        project_id: Option<String>,
+        location: Option<String>,
+    ) -> Self {
+        Self {
+            credentials: credentials.filter(|value| !value.value().expose().trim().is_empty()),
+            project_id: project_id.filter(|value| !value.trim().is_empty()),
+            location: location.filter(|value| !value.trim().is_empty()),
+        }
+    }
+
+    pub fn from_sourced_optional_params(
+        params: &Map<String, Value>,
+        sources: &BTreeMap<String, InputSource>,
+    ) -> Result<Self, Error> {
+        Ok(Self::new(
+            optional_credentials(params, sources, VertexParams::CREDENTIALS.wire)?,
+            optional_string(params, VertexParams::PROJECT.wire)?,
+            optional_string(params, VertexParams::LOCATION.wire)?,
+        ))
+    }
+
+    /// The deployment's `litellm_params`, which a host already trusts the way it trusts its
+    /// own configuration.
+    pub fn from_params(params: &VertexParams) -> Self {
+        Self::new(
+            params
+                .credentials()
+                .map(|value| Sourced::new(SecretValue::new(value), InputSource::Deployment)),
+            params.project(),
+            params.location(),
+        )
+    }
+
+    pub fn project_id(&self) -> Option<&str> {
+        self.project_id.as_deref()
+    }
+
+    pub fn location(&self) -> Option<&str> {
+        self.location.as_deref()
+    }
+}
+
+pub fn get_vertex_ai_project(
+    config: &VertexConfig,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> Option<String> {
+    config
+        .project_id()
+        .map(str::to_string)
+        .or_else(|| VertexParams::PROJECT.resolve(&|_| None, env_lookup))
+}
+
+/// The project a request is billed to when nothing names it: the `project_id` inside the
+/// service account the call would authenticate with. Application default credentials carry
+/// no project that can be read without a token exchange, so they resolve to `None`.
+pub fn get_vertex_ai_project_from_credentials(
+    config: &VertexConfig,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> Option<String> {
+    let text = match credential_source(config, env_lookup) {
+        CredentialSource::Inline(configured) => configured.expose().to_string(),
+        CredentialSource::Trusted(configured) => {
+            let configured = configured.expose();
+            match std::fs::read_to_string(configured) {
+                Ok(contents) if Path::new(configured).is_file() => contents,
+                _ => configured.to_string(),
+            }
+        }
+        CredentialSource::ApplicationCredentials(path) => std::fs::read_to_string(path).ok()?,
+        CredentialSource::Adc => return None,
+    };
+    serde_json::from_str::<Value>(&text)
+        .ok()?
+        .get("project_id")
+        .and_then(Value::as_str)
+        .map(str::to_string)
+}
+
+pub fn get_vertex_ai_location(
+    config: &VertexConfig,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> Option<String> {
+    config
+        .location()
+        .map(str::to_string)
+        .or_else(|| VertexParams::LOCATION.resolve(&|_| None, env_lookup))
+}
+
+#[derive(Clone, Debug)]
+pub enum CredentialSource {
+    Inline(SecretValue),
+    Trusted(SecretValue),
+    ApplicationCredentials(String),
+    Adc,
+}
+
+pub(crate) fn credential_source(
+    config: &VertexConfig,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> CredentialSource {
+    if let Some(configured) = config.credentials.clone() {
+        return match configured.source() {
+            InputSource::Request => CredentialSource::Inline(configured.into_value()),
+            InputSource::Deployment | InputSource::Environment => {
+                CredentialSource::Trusted(configured.into_value())
+            }
+        };
+    }
+    if let Some(configured) = VertexParams::CREDENTIALS.resolve(&|_| None, env_lookup) {
+        return CredentialSource::Trusted(SecretValue::new(configured));
+    }
+    non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV)
+        .map(CredentialSource::ApplicationCredentials)
+        .unwrap_or(CredentialSource::Adc)
+}
+
+fn optional_credentials(
+    params: &Map<String, Value>,
+    sources: &BTreeMap<String, InputSource>,
+    names: &[&str],
+) -> Result<Option<Sourced<SecretValue>>, Error> {
+    for name in names {
+        let source = source_for(sources, name);
+        match params.get(*name) {
+            None | Some(Value::Null) => continue,
+            Some(Value::String(value)) if value.trim().is_empty() => continue,
+            Some(Value::String(value)) => {
+                return Ok(Some(Sourced::new(SecretValue::new(value), source)));
+            }
+            Some(Value::Object(value)) if value.is_empty() => continue,
+            Some(Value::Object(value)) => {
+                return serde_json::to_string(value)
+                    .map(SecretValue::new)
+                    .map(|value| Sourced::new(value, source))
+                    .map(Some)
+                    .map_err(|error| {
+                        Error::InvalidConfiguration(ErrorDetail::failed(
+                            "credential serialization",
+                            error,
+                        ))
+                    });
+            }
+            Some(_) => {
+                return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType {
+                    field: names[0].into(),
+                    expected: "a string or null",
+                }));
+            }
+        }
+    }
+    Ok(None)
+}
+
+fn source_for(sources: &BTreeMap<String, InputSource>, name: &str) -> InputSource {
+    sources.get(name).copied().unwrap_or_default()
+}
+
+fn optional_string(params: &Map<String, Value>, names: &[&str]) -> Result<Option<String>, Error> {
+    for name in names {
+        match params.get(*name) {
+            None | Some(Value::Null) => continue,
+            Some(Value::String(value)) if value.trim().is_empty() => continue,
+            Some(Value::String(value)) => return Ok(Some(value.clone())),
+            Some(_) => {
+                return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType {
+                    field: names[0].into(),
+                    expected: "a string or null",
+                }));
+            }
+        }
+    }
+    Ok(None)
+}
+
+pub(crate) fn non_empty_env(
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+    name: &str,
+) -> Option<String> {
+    env_lookup(name)
+        .map(|value| value.trim().to_string())
+        .filter(|value| !value.is_empty())
+}
+
+#[cfg(test)]
+mod tests {
+    use serde_json::json;
+
+    use super::*;
+    use crate::constants::VERTEXAI_CREDENTIALS_ENV;
+
+    fn config(value: Value) -> VertexConfig {
+        VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new())
+            .unwrap()
+    }
+
+    #[rstest::rstest]
+    fn empty_primary_values_fall_back_to_python_aliases() {
+        let config = config(json!({
+            "vertex_credentials": null,
+            "vertex_ai_credentials": "alias-credentials",
+            "vertex_project": " ",
+            "vertex_ai_project": "alias-project",
+            "vertex_location": null,
+            "vertex_ai_location": "alias-location"
+        }));
+        assert_eq!(
+            config.credentials.as_ref().unwrap().value().expose(),
+            "alias-credentials"
+        );
+        assert_eq!(config.project_id(), Some("alias-project"));
+        assert_eq!(config.location(), Some("alias-location"));
+    }
+
+    #[rstest::rstest]
+    fn typed_config_preserves_source_and_empty_value_fallback() {
+        let configured = VertexConfig::new(
+            Some(Sourced::new(
+                SecretValue::new("inline-json"),
+                InputSource::Request,
+            )),
+            Some("project".into()),
+            Some("location".into()),
+        );
+        assert!(matches!(
+            credential_source(&configured, &|_| Some("environment-json".into())),
+            CredentialSource::Inline(value) if value.expose() == "inline-json"
+        ));
+        let empty = VertexConfig::new(
+            Some(Sourced::new(SecretValue::new(" "), InputSource::Request)),
+            Some(" ".into()),
+            Some(" ".into()),
+        );
+        assert!(matches!(
+            credential_source(&empty, &|_| None),
+            CredentialSource::Adc
+        ));
+        assert_eq!(
+            get_vertex_ai_project(&empty, &|_| Some("env-project".into())).as_deref(),
+            Some("env-project")
+        );
+        assert_eq!(
+            get_vertex_ai_location(&empty, &|_| Some("env-location".into())).as_deref(),
+            Some("env-location")
+        );
+    }
+
+    #[rstest::rstest]
+    fn credential_discovery_prefers_input_then_environment_then_adc() {
+        let params = json!({"vertex_credentials":"input-json"});
+        let sources = BTreeMap::from([("vertex_credentials".to_string(), InputSource::Request)]);
+        let configured =
+            VertexConfig::from_sourced_optional_params(params.as_object().unwrap(), &sources)
+                .unwrap();
+        assert!(
+            matches!(credential_source(&configured, &|_| Some("environment-value".into())), CredentialSource::Inline(value) if value.expose() == "input-json")
+        );
+        let empty = VertexConfig::default();
+        assert!(
+            matches!(credential_source(&empty, &|name| (name == VERTEXAI_CREDENTIALS_ENV).then(|| "environment-json".into())), CredentialSource::Trusted(value) if value.expose() == "environment-json")
+        );
+        assert!(
+            matches!(credential_source(&empty, &|name| (name == GOOGLE_APPLICATION_CREDENTIALS_ENV).then(|| "adc.json".into())), CredentialSource::ApplicationCredentials(path) if path == "adc.json")
+        );
+        assert!(matches!(
+            credential_source(&empty, &|_| None),
+            CredentialSource::Adc
+        ));
+    }
+
+    #[rstest::rstest]
+    fn params_become_a_deployment_sourced_config() {
+        let params = VertexParams {
+            vertex_credentials: Some("deployment-json".into()),
+            vertex_project: Some("project-1".into()),
+            vertex_ai_location: Some("europe-west4".into()),
+            ..VertexParams::default()
+        };
+        let config = VertexConfig::from_params(&params);
+        assert_eq!(config.project_id(), Some("project-1"));
+        assert_eq!(config.location(), Some("europe-west4"));
+        assert!(
+            matches!(credential_source(&config, &|_| Some("environment-json".into())), CredentialSource::Trusted(value) if value.expose() == "deployment-json")
+        );
+        assert!(matches!(
+            credential_source(
+                &VertexConfig::from_params(&VertexParams::default()),
+                &|_| None
+            ),
+            CredentialSource::Adc
+        ));
+    }
+}

litellm-rust/crates/auth-gcp/src/constants.rs

@@ -0,0 +1,36 @@
+use std::sync::LazyLock;
+
+use litellm_auth_types::VertexParams;
+
+pub const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform";
+pub const GOOGLE_OAUTH_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token";
+pub const GOOGLE_APPLICATION_CREDENTIALS_ENV: &str = "GOOGLE_APPLICATION_CREDENTIALS";
+pub const VERTEX_AI_API_KEY_ENV: &str = "VERTEX_AI_API_KEY";
+pub const VERTEXAI_API_KEY_ENV: &str = "VERTEXAI_API_KEY";
+pub const VERTEXAI_CREDENTIALS_ENV: &str = VertexParams::CREDENTIALS.env[0];
+pub const VERTEX_LOCATION_ENV: &str = VertexParams::LOCATION.env[1];
+
+/// Every environment name the Vertex params and the token acquisition read, for hosts
+/// that resolve secrets up front.
+pub fn secret_names() -> &'static [&'static str] {
+    static NAMES: LazyLock<Vec<&'static str>> = LazyLock::new(|| {
+        [
+            VERTEX_AI_API_KEY_ENV,
+            VERTEXAI_API_KEY_ENV,
+            GOOGLE_APPLICATION_CREDENTIALS_ENV,
+        ]
+        .into_iter()
+        .chain(
+            VertexParams::SPECS
+                .iter()
+                .flat_map(|spec| spec.env.iter().copied()),
+        )
+        .fold(Vec::new(), |mut names, name| {
+            if !names.contains(&name) {
+                names.push(name);
+            }
+            names
+        })
+    });
+    &NAMES
+}

litellm-rust/crates/auth-gcp/src/lib.rs

@@ -1,708 +1,16 @@
-use std::{collections::BTreeMap, future::Future, path::Path, pin::Pin, sync::Arc};
-
-use gcp_auth::{CustomServiceAccount, TokenProvider};
-use litellm_auth_types::{
-    CredentialPlacement, Error, InputSource, SecretValue, Sourced, http::apply_credential,
-};
-use moka::future::Cache;
-use serde_json::{Map, Value};
-use sha2::{Digest, Sha256};
-
+mod auth;
+mod config;
+pub mod constants;
 #[cfg(feature = "google-sdk")]
 mod sdk;
+
+pub use auth::{
+    VertexAuth, VertexAuthFuture, VertexEnvironment, VertexProviderLoader, VertexTokenSource,
+};
+pub use config::{
+    CredentialSource, VertexConfig, get_vertex_ai_location, get_vertex_ai_project,
+    get_vertex_ai_project_from_credentials,
+};
+pub use constants::secret_names;
 #[cfg(feature = "google-sdk")]
 pub use sdk::GoogleCredentials;
-
-const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform";
-const GOOGLE_OAUTH_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token";
-const GOOGLE_APPLICATION_CREDENTIALS_ENV: &str = "GOOGLE_APPLICATION_CREDENTIALS";
-const VERTEX_AI_API_KEY_ENV: &str = "VERTEX_AI_API_KEY";
-const VERTEXAI_API_KEY_ENV: &str = "VERTEXAI_API_KEY";
-const VERTEXAI_CREDENTIALS_ENV: &str = "VERTEXAI_CREDENTIALS";
-const VERTEXAI_PROJECT_ENV: &str = "VERTEXAI_PROJECT";
-const VERTEXAI_LOCATION_ENV: &str = "VERTEXAI_LOCATION";
-const VERTEX_LOCATION_ENV: &str = "VERTEX_LOCATION";
-
-pub const SECRET_NAMES: &[&str] = &[
-    VERTEX_AI_API_KEY_ENV,
-    VERTEXAI_API_KEY_ENV,
-    VERTEXAI_CREDENTIALS_ENV,
-    GOOGLE_APPLICATION_CREDENTIALS_ENV,
-    VERTEXAI_PROJECT_ENV,
-    VERTEXAI_LOCATION_ENV,
-    VERTEX_LOCATION_ENV,
-];
-
-#[derive(Clone, Debug, Default)]
-pub struct VertexConfig {
-    credentials: Option<Sourced<SecretValue>>,
-    project_id: Option<String>,
-    location: Option<String>,
-}
-
-impl VertexConfig {
-    pub fn new(
-        credentials: Option<Sourced<SecretValue>>,
-        project_id: Option<String>,
-        location: Option<String>,
-    ) -> Self {
-        Self {
-            credentials: credentials.filter(|value| !value.value().expose().trim().is_empty()),
-            project_id: project_id.filter(|value| !value.trim().is_empty()),
-            location: location.filter(|value| !value.trim().is_empty()),
-        }
-    }
-
-    pub fn from_sourced_optional_params(
-        params: &Map<String, Value>,
-        sources: &BTreeMap<String, InputSource>,
-    ) -> Result<Self, Error> {
-        Ok(Self::new(
-            optional_credentials(
-                params,
-                sources,
-                &["vertex_credentials", "vertex_ai_credentials"],
-            )?,
-            optional_string(params, &["vertex_project", "vertex_ai_project"])?,
-            optional_string(params, &["vertex_location", "vertex_ai_location"])?,
-        ))
-    }
-
-    pub fn or_configured(self, project_id: Option<&str>, location: Option<&str>) -> Self {
-        let configured =
-            |value: Option<&str>| value.filter(|value| !value.is_empty()).map(str::to_string);
-        Self {
-            project_id: self.project_id.or_else(|| configured(project_id)),
-            location: self.location.or_else(|| configured(location)),
-            ..self
-        }
-    }
-
-    pub fn project_id(&self) -> Option<&str> {
-        self.project_id.as_deref()
-    }
-
-    pub fn location(&self) -> Option<&str> {
-        self.location.as_deref()
-    }
-}
-
-pub struct VertexEnvironment {
-    pub headers: Vec<(String, String)>,
-    pub project_id: String,
-}
-
-struct VertexAccessToken {
-    token: String,
-    project_id: String,
-}
-
-pub fn get_vertex_ai_project(
-    config: &VertexConfig,
-    env_lookup: &dyn Fn(&str) -> Option<String>,
-) -> Option<String> {
-    config
-        .project_id()
-        .map(str::to_string)
-        .or_else(|| non_empty_env(env_lookup, VERTEXAI_PROJECT_ENV))
-}
-
-pub fn get_vertex_ai_location(
-    config: &VertexConfig,
-    env_lookup: &dyn Fn(&str) -> Option<String>,
-) -> Option<String> {
-    config
-        .location()
-        .map(str::to_string)
-        .or_else(|| non_empty_env(env_lookup, VERTEXAI_LOCATION_ENV))
-        .or_else(|| non_empty_env(env_lookup, VERTEX_LOCATION_ENV))
-}
-
-#[derive(Clone)]
-pub struct VertexAuth {
-    providers: Cache<CredentialCacheKey, Arc<dyn VertexTokenSource>>,
-    loader: Arc<dyn VertexProviderLoader>,
-}
-
-impl Default for VertexAuth {
-    fn default() -> Self {
-        Self::new(Arc::new(GcpProviderLoader))
-    }
-}
-
-impl VertexAuth {
-    pub fn new(loader: Arc<dyn VertexProviderLoader>) -> Self {
-        Self {
-            providers: Cache::builder().max_capacity(64).build(),
-            loader,
-        }
-    }
-
-    pub async fn access_token(
-        &self,
-        config: &VertexConfig,
-        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
-    ) -> Result<String, Error> {
-        self.load_provider(config, env_lookup).await?.token().await
-    }
-
-    pub async fn validate_environment(
-        &self,
-        headers: Vec<(String, String)>,
-        api_key: Option<&str>,
-        config: &VertexConfig,
-        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
-    ) -> Result<VertexEnvironment, Error> {
-        let has_authorization = headers
-            .iter()
-            .any(|(name, _)| name.eq_ignore_ascii_case("Authorization"));
-        let static_token = api_key
-            .map(str::trim)
-            .filter(|value| !value.is_empty())
-            .map(str::to_string)
-            .or_else(|| non_empty_env(env_lookup, VERTEX_AI_API_KEY_ENV))
-            .or_else(|| non_empty_env(env_lookup, VERTEXAI_API_KEY_ENV));
-        let project_id = get_vertex_ai_project(config, env_lookup);
-
-        if !has_authorization && static_token.is_none() {
-            let access = self.get_access_token(config, env_lookup).await?;
-            return Ok(VertexEnvironment {
-                headers: apply_credential(headers, &access.token, CredentialPlacement::Bearer)?,
-                project_id: project_id.unwrap_or(access.project_id),
-            });
-        }
-
-        let project_id = match project_id {
-            Some(project_id) => project_id,
-            None => {
-                self.load_provider(config, env_lookup)
-                    .await?
-                    .project_id()
-                    .await?
-            }
-        };
-        let headers = if has_authorization {
-            headers
-        } else {
-            apply_credential(
-                headers,
-                static_token.as_deref().expect("static token was checked"),
-                CredentialPlacement::Bearer,
-            )?
-        };
-        Ok(VertexEnvironment {
-            headers,
-            project_id,
-        })
-    }
-
-    async fn get_access_token(
-        &self,
-        config: &VertexConfig,
-        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
-    ) -> Result<VertexAccessToken, Error> {
-        let provider = self.load_provider(config, env_lookup).await?;
-        let (token, project_id) = tokio::try_join!(provider.token(), provider.project_id())?;
-        Ok(VertexAccessToken { token, project_id })
-    }
-
-    async fn load_provider(
-        &self,
-        config: &VertexConfig,
-        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
-    ) -> Result<Arc<dyn VertexTokenSource>, Error> {
-        let source = credential_source(config, env_lookup);
-        let key = source.cache_key();
-        self.providers
-            .try_get_with(key, self.loader.load(source))
-            .await
-            .map_err(|error| (*error).clone())
-    }
-}
-
-pub trait VertexTokenSource: Send + Sync {
-    fn project_id(&self) -> VertexAuthFuture<'_, String>;
-    fn token(&self) -> VertexAuthFuture<'_, String>;
-}
-
-pub trait VertexProviderLoader: Send + Sync {
-    fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>>;
-}
-
-pub type VertexAuthFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
-
-struct GcpTokenSource(Arc<dyn TokenProvider>);
-
-impl VertexTokenSource for GcpTokenSource {
-    fn project_id(&self) -> VertexAuthFuture<'_, String> {
-        Box::pin(async move {
-            self.0
-                .project_id()
-                .await
-                .map(|project| project.to_string())
-                .map_err(auth_acquisition_error)
-        })
-    }
-
-    fn token(&self) -> VertexAuthFuture<'_, String> {
-        Box::pin(async move {
-            self.0
-                .token(&[CLOUD_PLATFORM_SCOPE])
-                .await
-                .map(|token| token.as_str().to_string())
-                .map_err(auth_acquisition_error)
-        })
-    }
-}
-
-struct GcpProviderLoader;
-
-impl VertexProviderLoader for GcpProviderLoader {
-    fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>> {
-        Box::pin(async move {
-            let provider: Arc<dyn TokenProvider> = match source {
-                CredentialSource::Inline(configured) => Arc::new(
-                    CustomServiceAccount::from_json(validate_request_credentials(
-                        configured.expose(),
-                    )?)
-                    .map_err(auth_acquisition_error)?,
-                ),
-                CredentialSource::Trusted(configured) => {
-                    let configured = configured.expose();
-                    let service_account = if Path::new(configured).is_file() {
-                        CustomServiceAccount::from_file(configured)
-                    } else {
-                        CustomServiceAccount::from_json(configured)
-                    }
-                    .map_err(auth_acquisition_error)?;
-                    Arc::new(service_account)
-                }
-                CredentialSource::ApplicationCredentials(path) => {
-                    Arc::new(CustomServiceAccount::from_file(path).map_err(auth_acquisition_error)?)
-                }
-                CredentialSource::Adc => {
-                    gcp_auth::provider().await.map_err(auth_acquisition_error)?
-                }
-            };
-            Ok(Arc::new(GcpTokenSource(provider)) as Arc<dyn VertexTokenSource>)
-        })
-    }
-}
-
-fn validate_request_credentials(configured: &str) -> Result<&str, Error> {
-    let token_uri = serde_json::from_str::<Value>(configured)
-        .ok()
-        .and_then(|credentials| {
-            credentials
-                .get("token_uri")
-                .and_then(Value::as_str)
-                .map(str::to_string)
-        });
-    if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) {
-        return Err(Error::InvalidConfiguration("request-controlled Vertex credentials must use the canonical Google OAuth token endpoint".into()));
-    }
-    Ok(configured)
-}
-
-#[derive(Clone, Debug)]
-pub enum CredentialSource {
-    Inline(SecretValue),
-    Trusted(SecretValue),
-    ApplicationCredentials(String),
-    Adc,
-}
-
-impl CredentialSource {
-    fn cache_key(&self) -> CredentialCacheKey {
-        match self {
-            Self::Inline(configured) => {
-                CredentialCacheKey::Inline(Sha256::digest(configured.expose()).into())
-            }
-            Self::Trusted(configured) => {
-                CredentialCacheKey::Trusted(Sha256::digest(configured.expose()).into())
-            }
-            Self::ApplicationCredentials(path) => {
-                CredentialCacheKey::ApplicationCredentials(path.clone())
-            }
-            Self::Adc => CredentialCacheKey::Adc,
-        }
-    }
-}
-
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-enum CredentialCacheKey {
-    Inline([u8; 32]),
-    Trusted([u8; 32]),
-    ApplicationCredentials(String),
-    Adc,
-}
-
-fn credential_source(
-    config: &VertexConfig,
-    env_lookup: &dyn Fn(&str) -> Option<String>,
-) -> CredentialSource {
-    if let Some(configured) = config.credentials.clone() {
-        return match configured.source() {
-            InputSource::Request => CredentialSource::Inline(configured.into_value()),
-            InputSource::Deployment | InputSource::Environment => {
-                CredentialSource::Trusted(configured.into_value())
-            }
-        };
-    }
-    if let Some(configured) = non_empty_env(env_lookup, VERTEXAI_CREDENTIALS_ENV) {
-        return CredentialSource::Trusted(SecretValue::new(configured));
-    }
-    non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV)
-        .map(CredentialSource::ApplicationCredentials)
-        .unwrap_or(CredentialSource::Adc)
-}
-
-fn optional_credentials(
-    params: &Map<String, Value>,
-    sources: &BTreeMap<String, InputSource>,
-    names: &[&str],
-) -> Result<Option<Sourced<SecretValue>>, Error> {
-    for name in names {
-        let source = source_for(sources, name);
-        match params.get(*name) {
-            None | Some(Value::Null) => continue,
-            Some(Value::String(value)) if value.trim().is_empty() => continue,
-            Some(Value::String(value)) => {
-                return Ok(Some(Sourced::new(SecretValue::new(value), source)));
-            }
-            Some(Value::Object(value)) if value.is_empty() => continue,
-            Some(Value::Object(value)) => {
-                return serde_json::to_string(value)
-                    .map(SecretValue::new)
-                    .map(|value| Sourced::new(value, source))
-                    .map(Some)
-                    .map_err(|error| {
-                        Error::InvalidConfiguration(litellm_auth_types::ErrorDetail::failed(
-                            "credential serialization",
-                            error,
-                        ))
-                    });
-            }
-            Some(_) => {
-                return Err(Error::InvalidConfiguration(
-                    litellm_auth_types::ErrorDetail::InvalidType {
-                        field: names[0].into(),
-                        expected: "a string or null",
-                    },
-                ));
-            }
-        }
-    }
-    Ok(None)
-}
-
-fn source_for(sources: &BTreeMap<String, InputSource>, name: &str) -> InputSource {
-    sources.get(name).copied().unwrap_or_default()
-}
-
-fn optional_string(params: &Map<String, Value>, names: &[&str]) -> Result<Option<String>, Error> {
-    for name in names {
-        match params.get(*name) {
-            None | Some(Value::Null) => continue,
-            Some(Value::String(value)) if value.trim().is_empty() => continue,
-            Some(Value::String(value)) => return Ok(Some(value.clone())),
-            Some(_) => {
-                return Err(Error::InvalidConfiguration(
-                    litellm_auth_types::ErrorDetail::InvalidType {
-                        field: names[0].into(),
-                        expected: "a string or null",
-                    },
-                ));
-            }
-        }
-    }
-    Ok(None)
-}
-
-fn non_empty_env(env_lookup: &dyn Fn(&str) -> Option<String>, name: &str) -> Option<String> {
-    env_lookup(name)
-        .map(|value| value.trim().to_string())
-        .filter(|value| !value.is_empty())
-}
-
-fn auth_acquisition_error(error: gcp_auth::Error) -> Error {
-    Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::failed(
-        "Vertex AI credentials",
-        error,
-    ))
-}
-
-#[cfg(test)]
-mod tests {
-    use std::collections::BTreeSet;
-    use std::sync::atomic::{AtomicUsize, Ordering};
-
-    use serde_json::json;
-
-    use super::*;
-
-    struct FakeProvider {
-        calls: Arc<AtomicUsize>,
-    }
-
-    impl VertexTokenSource for FakeProvider {
-        fn project_id(&self) -> VertexAuthFuture<'_, String> {
-            self.calls.fetch_add(1, Ordering::SeqCst);
-            Box::pin(async { Ok("adc-project".into()) })
-        }
-
-        fn token(&self) -> VertexAuthFuture<'_, String> {
-            self.calls.fetch_add(1, Ordering::SeqCst);
-            Box::pin(async { Ok("adc-token".into()) })
-        }
-    }
-
-    struct FakeLoader {
-        loads: Arc<AtomicUsize>,
-        provider: Arc<dyn VertexTokenSource>,
-    }
-
-    impl VertexProviderLoader for FakeLoader {
-        fn load(
-            &self,
-            _source: CredentialSource,
-        ) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>> {
-            let loads = self.loads.clone();
-            let provider = self.provider.clone();
-            Box::pin(async move {
-                loads.fetch_add(1, Ordering::SeqCst);
-                Ok(provider)
-            })
-        }
-    }
-
-    fn config(value: Value) -> VertexConfig {
-        VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new())
-            .unwrap()
-    }
-
-    fn auth(calls: Arc<AtomicUsize>, loads: Arc<AtomicUsize>) -> VertexAuth {
-        let provider: Arc<dyn VertexTokenSource> = Arc::new(FakeProvider { calls });
-        VertexAuth::new(Arc::new(FakeLoader { loads, provider }))
-    }
-
-    #[test]
-    fn config_is_typed_and_secrets_are_redacted() {
-        let config = config(json!({
-            "vertex_credentials":{"private_key":"secret-key"},
-            "vertex_project":"project-1",
-            "vertex_location":"europe-west4"
-        }));
-        assert_eq!(config.project_id(), Some("project-1"));
-        assert_eq!(config.location(), Some("europe-west4"));
-        assert!(!format!("{config:?}").contains("secret-key"));
-        assert!(
-            VertexConfig::from_sourced_optional_params(
-                json!({"vertex_credentials":true}).as_object().unwrap(),
-                &BTreeMap::new()
-            )
-            .is_err()
-        );
-    }
-
-    #[tokio::test]
-    async fn secret_names_cover_environment_reads() {
-        let seen = Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
-        let recorded = seen.clone();
-        let env = |name: &str| {
-            recorded.lock().unwrap().insert(name.to_string());
-            None
-        };
-        let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
-        auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env)
-            .await
-            .unwrap();
-        get_vertex_ai_location(&VertexConfig::default(), &env);
-        assert!(
-            seen.lock()
-                .unwrap()
-                .iter()
-                .all(|name| SECRET_NAMES.contains(&name.as_str()))
-        );
-    }
-
-    #[test]
-    fn empty_primary_values_fall_back_to_python_aliases() {
-        let config = config(json!({
-            "vertex_credentials": null,
-            "vertex_ai_credentials": "alias-credentials",
-            "vertex_project": " ",
-            "vertex_ai_project": "alias-project",
-            "vertex_location": null,
-            "vertex_ai_location": "alias-location"
-        }));
-        assert_eq!(
-            config.credentials.as_ref().unwrap().value().expose(),
-            "alias-credentials"
-        );
-        assert_eq!(config.project_id(), Some("alias-project"));
-        assert_eq!(config.location(), Some("alias-location"));
-    }
-
-    #[test]
-    fn typed_config_preserves_source_and_empty_value_fallback() {
-        let configured = VertexConfig::new(
-            Some(Sourced::new(
-                SecretValue::new("inline-json"),
-                InputSource::Request,
-            )),
-            Some("project".into()),
-            Some("location".into()),
-        );
-        assert!(matches!(
-            credential_source(&configured, &|_| Some("environment-json".into())),
-            CredentialSource::Inline(value) if value.expose() == "inline-json"
-        ));
-        let empty = VertexConfig::new(
-            Some(Sourced::new(SecretValue::new(" "), InputSource::Request)),
-            Some(" ".into()),
-            Some(" ".into()),
-        );
-        assert!(matches!(
-            credential_source(&empty, &|_| None),
-            CredentialSource::Adc
-        ));
-        assert_eq!(
-            get_vertex_ai_project(&empty, &|_| Some("env-project".into())).as_deref(),
-            Some("env-project")
-        );
-        assert_eq!(
-            get_vertex_ai_location(&empty, &|_| Some("env-location".into())).as_deref(),
-            Some("env-location")
-        );
-    }
-
-    #[test]
-    fn project_and_location_prefer_input_then_environment() {
-        let configured =
-            config(json!({"vertex_project":"input-project","vertex_location":"input-location"}));
-        let env = |name: &str| Some(format!("env-{name}"));
-        assert_eq!(
-            get_vertex_ai_project(&configured, &env).as_deref(),
-            Some("input-project")
-        );
-        assert_eq!(
-            get_vertex_ai_location(&configured, &env).as_deref(),
-            Some("input-location")
-        );
-        let empty = VertexConfig::default();
-        assert_eq!(
-            get_vertex_ai_project(&empty, &|_| Some("env-project".into())).as_deref(),
-            Some("env-project")
-        );
-        assert_eq!(
-            get_vertex_ai_location(&empty, &|name| (name == VERTEX_LOCATION_ENV)
-                .then(|| "fallback-location".into()))
-            .as_deref(),
-            Some("fallback-location")
-        );
-    }
-
-    #[test]
-    fn credential_discovery_prefers_input_then_environment_then_adc() {
-        let params = json!({"vertex_credentials":"input-json"});
-        let sources = BTreeMap::from([("vertex_credentials".to_string(), InputSource::Request)]);
-        let configured =
-            VertexConfig::from_sourced_optional_params(params.as_object().unwrap(), &sources)
-                .unwrap();
-        assert!(
-            matches!(credential_source(&configured, &|_| Some("environment-value".into())), CredentialSource::Inline(value) if value.expose() == "input-json")
-        );
-        let empty = VertexConfig::default();
-        assert!(
-            matches!(credential_source(&empty, &|name| (name == VERTEXAI_CREDENTIALS_ENV).then(|| "environment-json".into())), CredentialSource::Trusted(value) if value.expose() == "environment-json")
-        );
-        assert!(
-            matches!(credential_source(&empty, &|name| (name == GOOGLE_APPLICATION_CREDENTIALS_ENV).then(|| "adc.json".into())), CredentialSource::ApplicationCredentials(path) if path == "adc.json")
-        );
-        assert!(matches!(
-            credential_source(&empty, &|_| None),
-            CredentialSource::Adc
-        ));
-        assert_ne!(
-            CredentialSource::Inline(SecretValue::new("same-value")).cache_key(),
-            CredentialSource::Trusted(SecretValue::new("same-value")).cache_key()
-        );
-    }
-
-    #[rstest::rstest]
-    #[case::canonical_endpoint(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#, true)]
-    #[case::noncanonical_endpoint(r#"{"token_uri":"http://127.0.0.1/token"}"#, false)]
-    #[case::missing_endpoint("{}", false)]
-    fn request_credentials_require_canonical_token_endpoint(
-        #[case] credentials: &str,
-        #[case] accepted: bool,
-    ) {
-        assert_eq!(validate_request_credentials(credentials).is_ok(), accepted);
-    }
-
-    #[tokio::test]
-    async fn explicit_token_and_header_do_not_acquire_adc() {
-        let loads = Arc::new(AtomicUsize::new(0));
-        let auth = auth(Arc::new(AtomicUsize::new(0)), loads.clone());
-        let configured = config(json!({"vertex_project":"project-1"}));
-        let explicit = auth
-            .validate_environment(Vec::new(), Some("access-token"), &configured, &|_| None)
-            .await
-            .unwrap();
-        assert_eq!(explicit.headers[0].1, "Bearer access-token");
-        let existing = auth
-            .validate_environment(
-                vec![("authorization".into(), "Bearer existing".into())],
-                None,
-                &configured,
-                &|_| None,
-            )
-            .await
-            .unwrap();
-        assert_eq!(existing.headers[0].1, "Bearer existing");
-        assert_eq!(loads.load(Ordering::SeqCst), 0);
-    }
-
-    #[tokio::test]
-    async fn provider_is_reused_across_authentication_calls() {
-        let calls = Arc::new(AtomicUsize::new(0));
-        let loads = Arc::new(AtomicUsize::new(0));
-        let auth = auth(calls.clone(), loads.clone());
-        for _ in 0..2 {
-            let environment = auth
-                .validate_environment(Vec::new(), None, &VertexConfig::default(), &|_| None)
-                .await
-                .unwrap();
-            assert_eq!(environment.project_id, "adc-project");
-            assert_eq!(environment.headers[0].1, "Bearer adc-token");
-        }
-        assert_eq!(loads.load(Ordering::SeqCst), 1);
-        assert_eq!(calls.load(Ordering::SeqCst), 4);
-    }
-
-    #[test]
-    fn configured_defaults_sit_between_call_params_and_the_environment() {
-        let env = |name: &str| Some(format!("env-{name}"));
-        let from_config =
-            VertexConfig::default().or_configured(Some("global-project"), Some("global-location"));
-        assert_eq!(
-            get_vertex_ai_project(&from_config, &env).as_deref(),
-            Some("global-project")
-        );
-        assert_eq!(
-            get_vertex_ai_location(&from_config, &env).as_deref(),
-            Some("global-location")
-        );
-        let from_call =
-            config(json!({"vertex_project":"call-project","vertex_location":"call-location"}))
-                .or_configured(Some("global-project"), Some("global-location"));
-        assert_eq!(from_call.project_id(), Some("call-project"));
-        assert_eq!(from_call.location(), Some("call-location"));
-        let empty_global = VertexConfig::default().or_configured(Some(""), None);
-        assert_eq!(
-            get_vertex_ai_project(&empty_global, &env).as_deref(),
-            Some("env-VERTEXAI_PROJECT")
-        );
-    }
-}

litellm-rust/crates/auth-gcp/tests/vertex_auth.rs

@@ -0,0 +1,120 @@
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    },
+};
+
+use litellm_auth_gcp::{
+    CredentialSource, VertexAuth, VertexAuthFuture, VertexConfig, VertexProviderLoader,
+    VertexTokenSource, get_vertex_ai_location, secret_names,
+};
+use rstest::rstest;
+use serde_json::{Value, json};
+
+struct FakeProvider {
+    calls: Arc<AtomicUsize>,
+}
+
+impl VertexTokenSource for FakeProvider {
+    fn project_id(&self) -> VertexAuthFuture<'_, String> {
+        self.calls.fetch_add(1, Ordering::SeqCst);
+        Box::pin(async { Ok("adc-project".into()) })
+    }
+
+    fn token(&self) -> VertexAuthFuture<'_, String> {
+        self.calls.fetch_add(1, Ordering::SeqCst);
+        Box::pin(async { Ok("adc-token".into()) })
+    }
+}
+
+struct FakeLoader {
+    loads: Arc<AtomicUsize>,
+    provider: Arc<dyn VertexTokenSource>,
+}
+
+impl VertexProviderLoader for FakeLoader {
+    fn load(&self, _source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>> {
+        let loads = self.loads.clone();
+        let provider = self.provider.clone();
+        Box::pin(async move {
+            loads.fetch_add(1, Ordering::SeqCst);
+            Ok(provider)
+        })
+    }
+}
+
+fn config(value: Value) -> VertexConfig {
+    VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new())
+        .unwrap()
+}
+
+fn auth(calls: Arc<AtomicUsize>, loads: Arc<AtomicUsize>) -> VertexAuth {
+    let provider: Arc<dyn VertexTokenSource> = Arc::new(FakeProvider { calls });
+    VertexAuth::new(Arc::new(FakeLoader { loads, provider }))
+}
+
+#[rstest]
+#[tokio::test]
+async fn secret_names_cover_environment_reads() {
+    let seen = Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
+    let recorded = seen.clone();
+    let env = |name: &str| {
+        recorded.lock().unwrap().insert(name.to_string());
+        None
+    };
+    let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
+    auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env)
+        .await
+        .unwrap();
+    get_vertex_ai_location(&VertexConfig::default(), &env);
+    assert!(
+        seen.lock()
+            .unwrap()
+            .iter()
+            .all(|name| secret_names().contains(&name.as_str()))
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn explicit_token_and_header_do_not_acquire_adc() {
+    let loads = Arc::new(AtomicUsize::new(0));
+    let auth = auth(Arc::new(AtomicUsize::new(0)), loads.clone());
+    let configured = config(json!({"vertex_project":"project-1"}));
+    let explicit = auth
+        .validate_environment(Vec::new(), Some("access-token"), &configured, &|_| None)
+        .await
+        .unwrap();
+    assert_eq!(explicit.headers[0].1, "Bearer access-token");
+    let existing = auth
+        .validate_environment(
+            vec![("authorization".into(), "Bearer existing".into())],
+            None,
+            &configured,
+            &|_| None,
+        )
+        .await
+        .unwrap();
+    assert_eq!(existing.headers[0].1, "Bearer existing");
+    assert_eq!(loads.load(Ordering::SeqCst), 0);
+}
+
+#[rstest]
+#[tokio::test]
+async fn provider_is_reused_across_authentication_calls() {
+    let calls = Arc::new(AtomicUsize::new(0));
+    let loads = Arc::new(AtomicUsize::new(0));
+    let auth = auth(calls.clone(), loads.clone());
+    for _ in 0..2 {
+        let environment = auth
+            .validate_environment(Vec::new(), None, &VertexConfig::default(), &|_| None)
+            .await
+            .unwrap();
+        assert_eq!(environment.project_id, "adc-project");
+        assert_eq!(environment.headers[0].1, "Bearer adc-token");
+    }
+    assert_eq!(loads.load(Ordering::SeqCst), 1);
+    assert_eq!(calls.load(Ordering::SeqCst), 4);
+}

litellm-rust/crates/auth-gcp/tests/vertex_config.rs

@@ -0,0 +1,95 @@
+use std::collections::BTreeMap;
+
+use litellm_auth_gcp::{
+    VertexConfig,
+    constants::{GOOGLE_APPLICATION_CREDENTIALS_ENV, VERTEX_LOCATION_ENV},
+    get_vertex_ai_location, get_vertex_ai_project, get_vertex_ai_project_from_credentials,
+};
+use litellm_auth_types::VertexParams;
+use rstest::rstest;
+use serde_json::{Value, json};
+
+fn config(value: Value) -> VertexConfig {
+    VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new())
+        .unwrap()
+}
+
+#[rstest]
+fn config_is_typed_and_secrets_are_redacted() {
+    let config = config(json!({
+        "vertex_credentials":{"private_key":"secret-key"},
+        "vertex_project":"project-1",
+        "vertex_location":"europe-west4"
+    }));
+    assert_eq!(config.project_id(), Some("project-1"));
+    assert_eq!(config.location(), Some("europe-west4"));
+    assert!(!format!("{config:?}").contains("secret-key"));
+    assert!(
+        VertexConfig::from_sourced_optional_params(
+            json!({"vertex_credentials":true}).as_object().unwrap(),
+            &BTreeMap::new()
+        )
+        .is_err()
+    );
+}
+
+#[rstest]
+fn project_and_location_prefer_input_then_environment() {
+    let configured =
+        config(json!({"vertex_project":"input-project","vertex_location":"input-location"}));
+    let env = |name: &str| Some(format!("env-{name}"));
+    assert_eq!(
+        get_vertex_ai_project(&configured, &env).as_deref(),
+        Some("input-project")
+    );
+    assert_eq!(
+        get_vertex_ai_location(&configured, &env).as_deref(),
+        Some("input-location")
+    );
+    let empty = VertexConfig::default();
+    assert_eq!(
+        get_vertex_ai_project(&empty, &|_| Some("env-project".into())).as_deref(),
+        Some("env-project")
+    );
+    assert_eq!(
+        get_vertex_ai_location(&empty, &|name| (name == VERTEX_LOCATION_ENV)
+            .then(|| "fallback-location".into()))
+        .as_deref(),
+        Some("fallback-location")
+    );
+}
+
+#[rstest]
+fn project_is_read_from_the_credentials_that_would_authenticate() {
+    let key = tempfile::NamedTempFile::new().unwrap();
+    std::fs::write(key.path(), r#"{"project_id":"file-project"}"#).unwrap();
+    let path = key.path().to_str().unwrap().to_string();
+    let inline = VertexConfig::from_params(&VertexParams {
+        vertex_credentials: Some(r#"{"project_id":"inline-project"}"#.into()),
+        ..VertexParams::default()
+    });
+    assert_eq!(
+        get_vertex_ai_project_from_credentials(&inline, &|_| None).as_deref(),
+        Some("inline-project")
+    );
+    let from_file = VertexConfig::from_params(&VertexParams {
+        vertex_credentials: Some(path.clone()),
+        ..VertexParams::default()
+    });
+    assert_eq!(
+        get_vertex_ai_project_from_credentials(&from_file, &|_| None).as_deref(),
+        Some("file-project")
+    );
+    let empty = VertexConfig::default();
+    assert_eq!(
+        get_vertex_ai_project_from_credentials(&empty, &|name| (name
+            == GOOGLE_APPLICATION_CREDENTIALS_ENV)
+            .then(|| path.clone()))
+        .as_deref(),
+        Some("file-project")
+    );
+    assert_eq!(
+        get_vertex_ai_project_from_credentials(&empty, &|_| None),
+        None
+    );
+}

litellm-rust/crates/auth-types/Cargo.toml

@@ -7,6 +7,7 @@ repository.workspace = true
 
 [dependencies]
 serde.workspace = true
+serde_json.workspace = true
 subtle.workspace = true
 thiserror.workspace = true
 veil.workspace = true

litellm-rust/crates/auth-types/src/error.rs

@@ -19,6 +19,11 @@ pub enum Error {
         provider: &'static str,
         environment_variable: &'static str,
     },
+    #[error("Missing {provider} {} - {}", .spec.setting, .spec.guidance())]
+    MissingParam {
+        provider: &'static str,
+        spec: &'static crate::ParamSpec,
+    },
     #[error("Missing {provider} API Base - {guidance}")]
     MissingApiBase {
         provider: &'static str,

litellm-rust/crates/auth-types/src/lib.rs

@@ -3,6 +3,7 @@
 mod credential;
 mod error;
 pub mod http;
+mod params;
 mod policy;
 mod secret;
 mod token;
@@ -52,6 +53,7 @@ pub use credential::{
 };
 pub use error::{Error, ErrorDetail, ErrorSource};
 pub use http::CredentialPlacement;
+pub use params::{AwsParams, ParamSpec, VertexParams};
 pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
 pub use secret::SecretValue;
 pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};

litellm-rust/crates/auth-types/src/params/aws.rs

@@ -0,0 +1,174 @@
+use std::sync::LazyLock;
+
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value};
+use veil::Redact;
+
+use super::ParamSpec;
+
+const fn spec(
+    setting: &'static str,
+    wire: &'static [&'static str],
+    env: &'static [&'static str],
+) -> ParamSpec {
+    ParamSpec {
+        setting,
+        wire,
+        module_global: None,
+        env,
+    }
+}
+
+/// The `aws_*` fields of Python's `GenericLiteLLMParams`.
+#[derive(Redact, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct AwsParams {
+    #[serde(default)]
+    pub aws_access_key_id: Option<String>,
+    #[serde(default)]
+    #[redact(with = "[REDACTED]")]
+    pub aws_secret_access_key: Option<String>,
+    #[serde(default)]
+    #[redact(with = "[REDACTED]")]
+    pub aws_session_token: Option<String>,
+    #[serde(default)]
+    pub aws_region_name: Option<String>,
+    #[serde(default)]
+    pub aws_session_name: Option<String>,
+    #[serde(default)]
+    pub aws_profile_name: Option<String>,
+    #[serde(default)]
+    pub aws_role_name: Option<String>,
+    #[serde(default)]
+    #[redact(with = "[REDACTED]")]
+    pub aws_web_identity_token: Option<String>,
+    #[serde(default)]
+    pub aws_sts_endpoint: Option<String>,
+    #[serde(default)]
+    #[redact(with = "[REDACTED]")]
+    pub aws_external_id: Option<String>,
+    #[serde(default)]
+    pub aws_bedrock_runtime_endpoint: Option<String>,
+}
+
+impl AwsParams {
+    pub const ACCESS_KEY_ID: ParamSpec = spec(
+        "access key id",
+        &["aws_access_key_id"],
+        &["AWS_ACCESS_KEY_ID"],
+    );
+    pub const SECRET_ACCESS_KEY: ParamSpec = spec(
+        "secret access key",
+        &["aws_secret_access_key"],
+        &["AWS_SECRET_ACCESS_KEY"],
+    );
+    pub const SESSION_TOKEN: ParamSpec = spec(
+        "session token",
+        &["aws_session_token"],
+        &["AWS_SESSION_TOKEN"],
+    );
+    pub const REGION: ParamSpec = spec(
+        "region",
+        &["aws_region_name"],
+        &["AWS_REGION_NAME", "AWS_REGION"],
+    );
+    pub const SESSION_NAME: ParamSpec =
+        spec("session name", &["aws_session_name"], &["AWS_SESSION_NAME"]);
+    pub const PROFILE_NAME: ParamSpec =
+        spec("profile name", &["aws_profile_name"], &["AWS_PROFILE_NAME"]);
+    pub const ROLE_NAME: ParamSpec = spec("role name", &["aws_role_name"], &["AWS_ROLE_NAME"]);
+    pub const WEB_IDENTITY_TOKEN: ParamSpec = spec(
+        "web identity token",
+        &["aws_web_identity_token"],
+        &["AWS_WEB_IDENTITY_TOKEN"],
+    );
+    pub const STS_ENDPOINT: ParamSpec =
+        spec("STS endpoint", &["aws_sts_endpoint"], &["AWS_STS_ENDPOINT"]);
+    pub const EXTERNAL_ID: ParamSpec =
+        spec("external id", &["aws_external_id"], &["AWS_EXTERNAL_ID"]);
+    pub const BEDROCK_RUNTIME_ENDPOINT: ParamSpec = spec(
+        "Bedrock runtime endpoint",
+        &["aws_bedrock_runtime_endpoint"],
+        &["AWS_BEDROCK_RUNTIME_ENDPOINT"],
+    );
+
+    /// Every `aws_*` param Python reads, in declaration order, with the environment names
+    /// each falls back to.
+    pub const SPECS: [ParamSpec; 11] = [
+        Self::ACCESS_KEY_ID,
+        Self::SECRET_ACCESS_KEY,
+        Self::SESSION_TOKEN,
+        Self::REGION,
+        Self::SESSION_NAME,
+        Self::PROFILE_NAME,
+        Self::ROLE_NAME,
+        Self::WEB_IDENTITY_TOKEN,
+        Self::STS_ENDPOINT,
+        Self::EXTERNAL_ID,
+        Self::BEDROCK_RUNTIME_ENDPOINT,
+    ];
+
+    /// The wire names a host projects out of a caller's kwargs, derived from [`Self::SPECS`].
+    pub fn fields() -> impl Iterator<Item = &'static str> {
+        Self::SPECS
+            .iter()
+            .flat_map(|spec| spec.wire.iter().copied())
+    }
+
+    /// The environment names the specs read, for hosts that resolve secrets up front.
+    pub fn secret_names() -> &'static [&'static str] {
+        static NAMES: LazyLock<Vec<&'static str>> = LazyLock::new(|| {
+            AwsParams::SPECS
+                .iter()
+                .flat_map(|spec| spec.env.iter().copied())
+                .fold(Vec::new(), |mut names, name| {
+                    if !names.contains(&name) {
+                        names.push(name);
+                    }
+                    names
+                })
+        });
+        &NAMES
+    }
+
+    /// The value under a wire name, read through serde so the names can never drift from
+    /// the struct.
+    pub fn get(&self, wire: &str) -> Option<String> {
+        serde_json::to_value(self)
+            .ok()?
+            .get(wire)?
+            .as_str()
+            .map(str::to_string)
+    }
+
+    /// The spec's value from these params, then the environment.
+    pub fn resolve(
+        &self,
+        spec: &ParamSpec,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Option<String> {
+        spec.resolve(&|name| self.get(name), env_lookup)
+    }
+
+    /// Reads the string-valued `aws_*` keys of an untyped params map, ignoring anything else.
+    pub fn from_optional_params(optional_params: &Map<String, Value>) -> Self {
+        let value = |key: &str| {
+            optional_params
+                .get(key)
+                .and_then(Value::as_str)
+                .map(str::to_string)
+        };
+        Self {
+            aws_access_key_id: value("aws_access_key_id"),
+            aws_secret_access_key: value("aws_secret_access_key"),
+            aws_session_token: value("aws_session_token"),
+            aws_region_name: value("aws_region_name"),
+            aws_session_name: value("aws_session_name"),
+            aws_profile_name: value("aws_profile_name"),
+            aws_role_name: value("aws_role_name"),
+            aws_web_identity_token: value("aws_web_identity_token"),
+            aws_sts_endpoint: value("aws_sts_endpoint"),
+            aws_external_id: value("aws_external_id"),
+            aws_bedrock_runtime_endpoint: value("aws_bedrock_runtime_endpoint"),
+        }
+    }
+}

litellm-rust/crates/auth-types/src/params/mod.rs

@@ -0,0 +1,48 @@
+mod aws;
+mod vertex;
+
+pub use aws::AwsParams;
+pub use vertex::VertexParams;
+
+/// One connection param of Python's `CredentialLiteLLMParams` and every place Python reads
+/// it from, in order: the wire names on a call or a deployment, the `litellm.<name>` module
+/// global a host folds in, then the environment names.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct ParamSpec {
+    pub setting: &'static str,
+    pub wire: &'static [&'static str],
+    pub module_global: Option<&'static str>,
+    pub env: &'static [&'static str],
+}
+
+impl ParamSpec {
+    /// The first non-blank value: each wire name through `params`, then each environment
+    /// name through `env`.
+    pub fn resolve(
+        &self,
+        params: &dyn Fn(&str) -> Option<String>,
+        env: &dyn Fn(&str) -> Option<String>,
+    ) -> Option<String> {
+        self.wire
+            .iter()
+            .map(|name| params(name))
+            .chain(self.env.iter().map(|name| env(name)))
+            .find_map(|value| value.filter(|value| !value.trim().is_empty()))
+    }
+
+    pub fn missing(&'static self, provider: &'static str) -> crate::Error {
+        crate::Error::MissingParam {
+            provider,
+            spec: self,
+        }
+    }
+
+    pub(crate) fn guidance(&self) -> String {
+        let wire = self.wire.join(" or ");
+        let env = self.env.join(" or ");
+        match self.module_global {
+            Some(global) => format!("pass {wire}, set litellm.{global}, or set {env}"),
+            None => format!("pass {wire} or set {env}"),
+        }
+    }
+}

litellm-rust/crates/auth-types/src/params/vertex.rs

@@ -0,0 +1,118 @@
+use serde::{Deserialize, Deserializer, Serialize, de::Unexpected};
+use serde_json::Value;
+use veil::Redact;
+
+use super::ParamSpec;
+
+/// The Vertex AI fields of Python's `GenericLiteLLMParams`, both the current names and the
+/// `vertex_ai_*` spellings `VertexBase.safe_get_vertex_ai_*` still read.
+#[derive(Redact, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct VertexParams {
+    #[serde(default, deserialize_with = "optional_credential_text")]
+    #[redact(with = "[REDACTED]")]
+    pub vertex_credentials: Option<String>,
+    #[serde(default)]
+    pub vertex_project: Option<String>,
+    #[serde(default)]
+    pub vertex_location: Option<String>,
+    #[serde(default, deserialize_with = "optional_credential_text")]
+    #[redact(with = "[REDACTED]")]
+    pub vertex_ai_credentials: Option<String>,
+    #[serde(default)]
+    pub vertex_ai_project: Option<String>,
+    #[serde(default)]
+    pub vertex_ai_location: Option<String>,
+}
+
+impl VertexParams {
+    pub const CREDENTIALS: ParamSpec = ParamSpec {
+        setting: "credentials",
+        wire: &["vertex_credentials", "vertex_ai_credentials"],
+        module_global: None,
+        env: &["VERTEXAI_CREDENTIALS"],
+    };
+    pub const PROJECT: ParamSpec = ParamSpec {
+        setting: "project",
+        wire: &["vertex_project", "vertex_ai_project"],
+        module_global: Some("vertex_project"),
+        env: &["VERTEXAI_PROJECT"],
+    };
+    pub const LOCATION: ParamSpec = ParamSpec {
+        setting: "location",
+        wire: &["vertex_location", "vertex_ai_location"],
+        module_global: Some("vertex_location"),
+        env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"],
+    };
+
+    /// Every Vertex AI param Python reads, with both spellings, the module global
+    /// `VertexBase.get_vertex_ai_*` consults, and the environment names it falls back to.
+    pub const SPECS: [ParamSpec; 3] = [Self::CREDENTIALS, Self::PROJECT, Self::LOCATION];
+
+    /// The wire names a host projects out of a caller's kwargs, derived from [`Self::SPECS`].
+    pub fn fields() -> impl Iterator<Item = &'static str> {
+        Self::SPECS
+            .iter()
+            .flat_map(|spec| spec.wire.iter().copied())
+    }
+
+    /// The value under a wire name, read through serde so the names can never drift from
+    /// the struct.
+    pub fn get(&self, wire: &str) -> Option<String> {
+        serde_json::to_value(self)
+            .ok()?
+            .get(wire)?
+            .as_str()
+            .map(str::to_string)
+    }
+
+    /// The spec's value from these params, then the environment.
+    pub fn resolve(
+        &self,
+        spec: &ParamSpec,
+        env_lookup: &dyn Fn(&str) -> Option<String>,
+    ) -> Option<String> {
+        spec.resolve(&|name| self.get(name), env_lookup)
+    }
+
+    pub fn credentials(&self) -> Option<String> {
+        self.resolve(&Self::CREDENTIALS, &|_| None)
+    }
+
+    pub fn project(&self) -> Option<String> {
+        self.resolve(&Self::PROJECT, &|_| None)
+    }
+
+    pub fn location(&self) -> Option<String> {
+        self.resolve(&Self::LOCATION, &|_| None)
+    }
+}
+
+/// A credential is a service account JSON text or a path to one; a YAML or kwargs mapping is
+/// accepted as the JSON text it spells, the way Python passes a dict through.
+fn optional_credential_text<'de, D: Deserializer<'de>>(
+    deserializer: D,
+) -> Result<Option<String>, D::Error> {
+    match Option::<Value>::deserialize(deserializer)? {
+        None | Some(Value::Null) => Ok(None),
+        Some(Value::String(text)) => Ok(Some(text)),
+        Some(Value::Object(object)) if object.is_empty() => Ok(None),
+        Some(Value::Object(object)) => serde_json::to_string(&object)
+            .map(Some)
+            .map_err(serde::de::Error::custom),
+        Some(other) => Err(serde::de::Error::invalid_type(
+            Unexpected::Other(value_kind(&other)),
+            &"a string or an object",
+        )),
+    }
+}
+
+fn value_kind(value: &Value) -> &'static str {
+    match value {
+        Value::Null => "null",
+        Value::Bool(_) => "a boolean",
+        Value::Number(_) => "a number",
+        Value::String(_) => "a string",
+        Value::Array(_) => "a list",
+        Value::Object(_) => "an object",
+    }
+}

litellm-rust/crates/auth-types/tests/aws_params.rs

@@ -0,0 +1,26 @@
+use litellm_auth_types::AwsParams;
+use rstest::rstest;
+use serde_json::{Map, Value};
+
+#[rstest]
+fn fields_name_every_param_once_and_in_declaration_order() {
+    let filled: Map<String, Value> = AwsParams::fields()
+        .map(|name| (name.to_string(), Value::from(format!("value-of-{name}"))))
+        .collect();
+    let typed = AwsParams::from_optional_params(&filled);
+    let serialized = serde_json::to_value(&typed).unwrap();
+    assert_eq!(
+        serialized
+            .as_object()
+            .unwrap()
+            .keys()
+            .map(String::as_str)
+            .collect::<Vec<_>>(),
+        AwsParams::fields().collect::<Vec<_>>()
+    );
+    assert_eq!(serialized, Value::Object(filled.clone()));
+    assert_eq!(
+        serde_json::from_value::<AwsParams>(Value::Object(filled)).unwrap(),
+        typed
+    );
+}

litellm-rust/crates/auth-types/tests/params.rs

@@ -0,0 +1,72 @@
+use litellm_auth_types::{AwsParams, ParamSpec};
+use rstest::rstest;
+
+const LOCATION: ParamSpec = ParamSpec {
+    setting: "location",
+    wire: &["vertex_location", "vertex_ai_location"],
+    module_global: Some("vertex_location"),
+    env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"],
+};
+
+fn lookup(entries: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
+    move |name| {
+        entries
+            .iter()
+            .find(|(key, _)| *key == name)
+            .map(|(_, value)| value.to_string())
+    }
+}
+
+#[rstest]
+#[case::current_spelling_first(
+    &[("vertex_location", "eu"), ("vertex_ai_location", "us")],
+    &[("VERTEXAI_LOCATION", "env")],
+    Some("eu")
+)]
+#[case::legacy_spelling_before_the_environment(
+    &[("vertex_ai_location", "us")],
+    &[("VERTEXAI_LOCATION", "env")],
+    Some("us")
+)]
+#[case::blank_values_are_skipped(
+    &[("vertex_location", "  "), ("vertex_ai_location", "")],
+    &[("VERTEXAI_LOCATION", " "), ("VERTEX_LOCATION", "fallback")],
+    Some("fallback")
+)]
+#[case::environment_names_in_order(
+    &[],
+    &[("VERTEX_LOCATION", "second"), ("VERTEXAI_LOCATION", "first")],
+    Some("first")
+)]
+#[case::nothing_set(&[], &[], None)]
+fn resolution_walks_wire_names_then_environment_names(
+    #[case] params: &'static [(&'static str, &'static str)],
+    #[case] env: &'static [(&'static str, &'static str)],
+    #[case] expected: Option<&str>,
+) {
+    assert_eq!(
+        LOCATION.resolve(&lookup(params), &lookup(env)).as_deref(),
+        expected
+    );
+}
+
+#[rstest]
+#[case::without_a_module_global(
+    &AwsParams::REGION,
+    "Missing AWS region - pass aws_region_name or set AWS_REGION_NAME or AWS_REGION"
+)]
+#[case::with_a_module_global(
+    &LOCATION,
+    "Missing AWS location - pass vertex_location or vertex_ai_location, set litellm.vertex_location, or set VERTEXAI_LOCATION or VERTEX_LOCATION"
+)]
+fn a_missing_param_names_every_place_the_spec_reads(
+    #[case] spec: &'static ParamSpec,
+    #[case] expected: &str,
+) {
+    let message = spec.missing("AWS").to_string();
+
+    assert_eq!(message, expected);
+    assert!(spec.wire.iter().all(|name| message.contains(name)));
+    assert!(spec.env.iter().all(|name| message.contains(name)));
+    assert_eq!(message.contains("litellm."), spec.module_global.is_some());
+}

litellm-rust/crates/auth-types/tests/vertex_params.rs

@@ -0,0 +1,126 @@
+use std::collections::BTreeSet;
+
+use litellm_auth_types::VertexParams;
+use rstest::rstest;
+use serde_json::{Map, Value, json};
+
+#[rstest]
+#[case::current_spelling(
+    VertexParams { vertex_project: Some("p".into()), vertex_ai_project: Some("old".into()), ..VertexParams::default() },
+    Some("p")
+)]
+#[case::legacy_spelling(
+    VertexParams { vertex_ai_project: Some("old".into()), ..VertexParams::default() },
+    Some("old")
+)]
+#[case::blank_falls_through(
+    VertexParams { vertex_project: Some("  ".into()), vertex_ai_project: Some("old".into()), ..VertexParams::default() },
+    Some("old")
+)]
+#[case::unset(VertexParams::default(), None)]
+fn params_prefer_the_current_spelling_over_the_legacy_one(
+    #[case] params: VertexParams,
+    #[case] project: Option<&str>,
+) {
+    assert_eq!(params.project().as_deref(), project);
+    let mirrored = VertexParams {
+        vertex_credentials: params.vertex_project.clone(),
+        vertex_ai_credentials: params.vertex_ai_project.clone(),
+        vertex_location: params.vertex_project.clone(),
+        vertex_ai_location: params.vertex_ai_project.clone(),
+        ..VertexParams::default()
+    };
+    assert_eq!(mirrored.credentials().as_deref(), project);
+    assert_eq!(mirrored.location().as_deref(), project);
+}
+
+#[rstest]
+#[case::text(json!({"vertex_credentials": "/path/to/key.json"}), Some("/path/to/key.json"))]
+#[case::object(json!({"vertex_credentials": {"type": "service_account"}}), Some(r#"{"type":"service_account"}"#))]
+#[case::null(json!({"vertex_credentials": null}), None)]
+#[case::empty_object_falls_through_to_the_legacy_spelling(
+    json!({"vertex_credentials": {}, "vertex_ai_credentials": "/legacy/key.json"}),
+    Some("/legacy/key.json")
+)]
+#[case::absent(json!({}), None)]
+fn credentials_deserialize_from_text_or_an_object(
+    #[case] params: Value,
+    #[case] credentials: Option<&str>,
+) {
+    let params: VertexParams = serde_json::from_value(params).unwrap();
+    assert_eq!(params.credentials().as_deref(), credentials);
+}
+
+#[rstest]
+#[case::number(json!({"vertex_credentials": 7}))]
+#[case::list(json!({"vertex_ai_credentials": ["a"]}))]
+#[case::project(json!({"vertex_project": ["p"]}))]
+fn a_param_of_the_wrong_type_is_rejected(#[case] params: Value) {
+    assert!(serde_json::from_value::<VertexParams>(params).is_err());
+}
+
+#[rstest]
+fn fields_name_every_param_once() {
+    let filled: Value = VertexParams::fields()
+        .map(|name| (name.to_string(), Value::from(name)))
+        .collect::<Map<_, _>>()
+        .into();
+    let params: VertexParams = serde_json::from_value(filled).unwrap();
+    let expected = VertexParams {
+        vertex_credentials: Some("vertex_credentials".into()),
+        vertex_project: Some("vertex_project".into()),
+        vertex_location: Some("vertex_location".into()),
+        vertex_ai_credentials: Some("vertex_ai_credentials".into()),
+        vertex_ai_project: Some("vertex_ai_project".into()),
+        vertex_ai_location: Some("vertex_ai_location".into()),
+    };
+    assert_eq!(params, expected);
+    assert_eq!(
+        VertexParams::fields().collect::<BTreeSet<_>>().len(),
+        VertexParams::fields().count()
+    );
+}
+
+#[rstest]
+fn debug_output_does_not_expose_credentials() {
+    let params = VertexParams {
+        vertex_credentials: Some("current-secret".into()),
+        vertex_ai_credentials: Some("legacy-secret".into()),
+        vertex_project: Some("project-1".into()),
+        ..VertexParams::default()
+    };
+    let debug = format!("{params:?}");
+
+    assert!(!debug.contains("current-secret"));
+    assert!(!debug.contains("legacy-secret"));
+    assert!(debug.contains("project-1"));
+}
+
+#[rstest]
+#[case::current_spelling(Some("p"), Some("legacy"), &[("VERTEXAI_PROJECT", "env")], Some("p"))]
+#[case::legacy_spelling(None, Some("legacy"), &[("VERTEXAI_PROJECT", "env")], Some("legacy"))]
+#[case::blank_params_fall_to_the_environment(Some(" "), None, &[("VERTEXAI_PROJECT", "env")], Some("env"))]
+#[case::nothing(None, None, &[], None)]
+fn a_spec_resolves_from_the_params_then_the_environment(
+    #[case] vertex_project: Option<&str>,
+    #[case] vertex_ai_project: Option<&str>,
+    #[case] environment: &[(&str, &str)],
+    #[case] expected: Option<&str>,
+) {
+    let params = VertexParams {
+        vertex_project: vertex_project.map(str::to_string),
+        vertex_ai_project: vertex_ai_project.map(str::to_string),
+        ..VertexParams::default()
+    };
+    let env = |name: &str| {
+        environment
+            .iter()
+            .find(|(key, _)| *key == name)
+            .map(|(_, value)| value.to_string())
+    };
+
+    assert_eq!(
+        params.resolve(&VertexParams::PROJECT, &env).as_deref(),
+        expected
+    );
+}

litellm-rust/crates/cache-response/AGENTS.md

@@ -24,6 +24,6 @@ Keep unary caching independent of stream-only methods. Store streams only after
 
 Test each contract in its owner: storage capabilities in backend tests, envelopes and freshness here, reuse and replay in core, Python callback and fallback behavior at the bridge, and HTTP behavior at the gateway. Run backend contract checks and Python response-codec fixtures before exposing a new backend
 
-`ScopedCache` requires an explicit shared or isolated scope at construction. Per-call `CachePolicy` controls reads, writes, expiry, and freshness without replacing the attached scope or service. `CacheOptions` binds that policy to an explicit scope for storage requests and has no default sharing policy. Versioned native envelopes reject incompatible API surfaces and versions as misses; this envelope is distinct from the legacy Python response codec
+`ScopedCache` requires an explicit shared or isolated scope at construction. Per-call `CachePolicy` controls reads, writes, expiry, and freshness without replacing the attached scope or service. `CacheOptions` binds that policy to an explicit scope for storage requests and has no default sharing policy. Versioned native envelopes reject incompatible API surfaces and versions as misses. This envelope is distinct from the legacy Python response codec
 
 Response storage is not the source of budget or rate-limit coordination dependencies. Keep counters, reservations, and atomic admission operations out of `ResponseCacheService`, including when both services happen to use Redis

litellm-rust/crates/callbacks-legacy-python/AGENTS.md

@@ -1,20 +1,20 @@
 - Target invariants, not completion claims
 - This crate owns compatibility for all existing Python callbacks and loggers, including `CustomLogger`. `mapping.rs` owns the executable call bindings and the inventory of Python-owned hooks. A Python-owned entry records an existing path, never permission to invoke it a second time. The native call adapter preserves the `Logging` contract (`function_setup`, the deployment hooks, `pre_call`/`post_call`, the sync and async success and failure fan-out, the deferred proxy release, the argument sharing those callbacks rely on)
   - Smell test: if a future callback host (`callbacks-v1-python`, WASM, in-process Rust) could share a piece of this crate, it does not belong here
-  - SDK request policy (credential inheritance, the budget and retry-count limits) is a separate hook supplied by `python-bridge`; compose it after this adapter so logging adopts the final keyword view before policy mutates or rejects it
-  - The driver in `litellm-host-python`, the routes and core see one `PythonCallHooks` using the shared `CallEvent`; they never learn which Python objects consume a call
+  - SDK request policy (credential inheritance, the budget and retry-count limits) is a separate hook supplied by `python-bridge`. Compose it after this adapter so logging adopts the final keyword view before policy mutates or rejects it
+  - The driver in `litellm-host-python`, the routes and core see one `PythonCallHooks` using the shared `CallEvent`. They never learn which Python objects consume a call
 - Every litellm Python internal Rust still borrows is a variant of `LegacyPython`, grouped by subsystem, with its signature pinned in `python_contract.json`
   - The enum only shrinks: when Rust owns a subsystem, delete its group rather than adding a Rust path beside it
   - Calling a user's own callback directly is permanent Python surface and gets its own type outside `LegacyPython`
-- `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the call rewrites it (setup, deployment hook, preflight) and the bound request object backing omitted keywords; shared bridge composition hands it to `LegacyLogging`; routes use the neutral call boundary
+- `PublicCall` is the caller's call as `Logging` sees it: the positional arguments, the keyword view as the call rewrites it (setup, deployment hook, preflight) and the bound request object backing omitted keywords. Shared bridge composition hands it to `LegacyLogging`. Routes use the neutral call boundary
 - `LoggingOperation` selects legacy logging entrypoints and response handling. It belongs here rather than in shared inference data contracts
-- `setup` reuses a `Logging` passed as `litellm_logging_obj` (the proxy and Router) and otherwise builds one through `function_setup`; which callbacks run is `Logging`'s decision, never this crate's
-- Callbacks receive the caller's own objects and may mutate them; this crate alone carries that obligation
-  - Retain complete boundary arguments, opaque values, aliases, omitted/default distinctions and deliberate copies; preserve the deployment-hook kwargs view
+- `setup` reuses a `Logging` passed as `litellm_logging_obj` (the proxy and Router) and otherwise builds one through `function_setup`. Which callbacks run is `Logging`'s decision, never this crate's
+- Callbacks receive the caller's own objects and may mutate them. This crate alone carries that obligation
+  - Retain complete boundary arguments, opaque values, aliases, omitted/default distinctions and deliberate copies. Preserve the deployment-hook kwargs view
   - Before `pre_call`, re-alias every body key whose value equals the caller's argument to the caller's own object, resolved through `litellm_host_python::lookup`
-  - Retain body/header roots from `pre_call` to `post_call`; in-place mutation reaches the wire, envelope field replacement is visible to later callbacks only
+  - Retain body/header roots from `pre_call` to `post_call`. In-place mutation reaches the wire, envelope field replacement is visible to later callbacks only
 - Success and failure handlers receive the exact selected public response or exception
-  - A failure-handler error cannot suppress the other eligible family or replace the mapped provider error; a cancellation ends the call with no further dispatch
-  - Dispatch errors never replay provider work or trigger the opposite outcome; the proxy releases deferred success at most once
+  - A failure-handler error cannot suppress the other eligible family or replace the mapped provider error. A cancellation ends the call with no further dispatch
+  - Dispatch errors never replay provider work or trigger the opposite outcome. The proxy releases deferred success at most once
   - Delivery follows the registry, not the callable's type: direct, awaited, executor-submitted, logging-worker and deferred paths stay distinct
-- Traverse every retained Python edge; `close` is idempotent and restores the correlation context once
+- Traverse every retained Python edge. `close` is idempotent and restores the correlation context once

litellm-rust/crates/config/Cargo.toml

@@ -7,6 +7,7 @@ repository.workspace = true
 
 [dependencies]
 litellm-auth-types.workspace = true
+litellm-router-types.workspace = true
 serde.workspace = true
 serde_yaml_ng = "0.10.0"
 strum.workspace = true

litellm-rust/crates/config/src/lib.rs

@@ -10,13 +10,14 @@ use std::{fmt, path::Path};
 use serde::Deserialize;
 
 pub use error::Error;
+pub use litellm_router_types::{LitellmParams, Spelled};
 pub use mcp::{McpAuth, McpServer, McpTransport};
-pub use model::{LiteLlmParams, Model};
+pub use model::Model;
 pub use settings::{
     ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings,
     TracingStoreSettings,
 };
-pub use value::{AdditionalFields, Flag, NumberOrString, Object, OneOrMany, Value};
+pub use value::{AdditionalFields, Object, Value};
 
 #[derive(Clone, Default, Deserialize)]
 #[serde(default)]

litellm-rust/crates/config/src/model.rs

@@ -1,14 +1,14 @@
 use std::fmt;
 
-use litellm_auth_types::SecretValue;
+use litellm_router_types::LitellmParams;
 use serde::Deserialize;
 
-use crate::{AdditionalFields, Flag, NumberOrString, Object};
+use crate::{AdditionalFields, Object};
 
 #[derive(Clone, Deserialize)]
 pub struct Model {
     pub model_name: String,
-    pub litellm_params: LiteLlmParams,
+    pub litellm_params: LitellmParams,
     #[serde(default)]
     pub model_info: Object,
     pub blocked: Option<bool>,
@@ -28,68 +28,3 @@ impl fmt::Debug for Model {
             .finish()
     }
 }
-
-#[derive(Clone, Deserialize)]
-pub struct LiteLlmParams {
-    pub model: String,
-    pub api_key: Option<SecretValue>,
-    pub api_base: Option<String>,
-    pub api_version: Option<String>,
-    pub custom_llm_provider: Option<String>,
-    pub timeout: Option<NumberOrString>,
-    pub stream_timeout: Option<NumberOrString>,
-    pub max_retries: Option<NumberOrString>,
-    pub tpm: Option<NumberOrString>,
-    pub rpm: Option<NumberOrString>,
-    pub itpm: Option<NumberOrString>,
-    pub otpm: Option<NumberOrString>,
-    pub max_parallel_requests: Option<u64>,
-    pub organization: Option<serde_yaml_ng::Value>,
-    pub drop_params: Option<Flag>,
-    pub tags: Option<Box<[String]>>,
-    pub tag_regex: Option<Box<[String]>>,
-    pub max_budget: Option<f64>,
-    pub budget_duration: Option<String>,
-    pub default_api_key_tpm_limit: Option<u64>,
-    pub default_api_key_rpm_limit: Option<u64>,
-    pub use_in_pass_through: Option<bool>,
-    pub use_chat_completions_api: Option<bool>,
-    pub litellm_credential_name: Option<String>,
-    pub provider_affinity_header: Option<String>,
-    #[serde(flatten)]
-    pub additional_fields: AdditionalFields,
-}
-
-impl fmt::Debug for LiteLlmParams {
-    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
-        formatter
-            .debug_struct("LiteLlmParams")
-            .field("model", &self.model)
-            .field("api_key", &self.api_key)
-            .field("api_base", &self.api_base)
-            .field("api_version", &self.api_version)
-            .field("custom_llm_provider", &self.custom_llm_provider)
-            .field("timeout", &self.timeout)
-            .field("stream_timeout", &self.stream_timeout)
-            .field("max_retries", &self.max_retries)
-            .field("tpm", &self.tpm)
-            .field("rpm", &self.rpm)
-            .field("itpm", &self.itpm)
-            .field("otpm", &self.otpm)
-            .field("max_parallel_requests", &self.max_parallel_requests)
-            .field("organization", &self.organization)
-            .field("drop_params", &self.drop_params)
-            .field("tags", &self.tags)
-            .field("tag_regex", &self.tag_regex)
-            .field("max_budget", &self.max_budget)
-            .field("budget_duration", &self.budget_duration)
-            .field("default_api_key_tpm_limit", &self.default_api_key_tpm_limit)
-            .field("default_api_key_rpm_limit", &self.default_api_key_rpm_limit)
-            .field("use_in_pass_through", &self.use_in_pass_through)
-            .field("use_chat_completions_api", &self.use_chat_completions_api)
-            .field("litellm_credential_name", &self.litellm_credential_name)
-            .field("provider_affinity_header", &self.provider_affinity_header)
-            .field("additional_fields", &self.additional_fields.keys())
-            .finish()
-    }
-}

litellm-rust/crates/config/src/settings.rs

@@ -3,7 +3,7 @@ use std::fmt;
 use litellm_auth_types::SecretValue;
 use serde::Deserialize;
 
-use crate::{AdditionalFields, Flag, NumberOrString, Object, OneOrMany, Value};
+use crate::{AdditionalFields, Object, Spelled, Value, value::one_or_many};
 
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "lowercase")]
@@ -18,7 +18,7 @@ pub struct ClickHouseStoreSettings {
     pub kind: TracingStoreKind,
     pub url: Option<SecretValue>,
     pub database: Option<String>,
-    pub retention_days: Option<NumberOrString>,
+    pub retention_days: Option<Spelled<f64>>,
 }
 
 impl fmt::Debug for ClickHouseStoreSettings {
@@ -207,7 +207,7 @@ impl fmt::Debug for RouterSettings {
 #[derive(Clone, Default, Deserialize)]
 #[serde(default)]
 pub struct LiteLlmSettings {
-    pub ssl_verify: Option<Flag>,
+    pub ssl_verify: Option<Spelled<bool>>,
     pub ssl_certificate: Option<String>,
     pub ssl_security_level: Option<String>,
     pub ssl_ecdh_curve: Option<String>,
@@ -216,14 +216,17 @@ pub struct LiteLlmSettings {
     pub aiohttp_trust_env: Option<bool>,
     pub disable_aiohttp_trust_env: Option<bool>,
     pub disable_aiohttp_transport: Option<bool>,
-    pub drop_params: Option<Flag>,
-    pub request_timeout: Option<NumberOrString>,
+    pub drop_params: Option<Spelled<bool>>,
+    pub request_timeout: Option<Spelled<f64>>,
     pub num_retries: Option<u64>,
     pub cache: Option<bool>,
     pub cache_params: Option<Object>,
-    pub callbacks: Option<OneOrMany<Value>>,
-    pub success_callback: Option<OneOrMany<Value>>,
-    pub failure_callback: Option<OneOrMany<Value>>,
+    #[serde(deserialize_with = "one_or_many")]
+    pub callbacks: Option<Vec<Value>>,
+    #[serde(deserialize_with = "one_or_many")]
+    pub success_callback: Option<Vec<Value>>,
+    #[serde(deserialize_with = "one_or_many")]
+    pub failure_callback: Option<Vec<Value>>,
     pub json_logs: Option<bool>,
     pub set_verbose: Option<bool>,
     #[serde(flatten)]

litellm-rust/crates/config/src/value.rs

@@ -1,6 +1,6 @@
 use std::{collections::BTreeMap, fmt, ops::Deref};
 
-use serde::Deserialize;
+use serde::{Deserialize, Deserializer};
 
 pub type Value = serde_yaml_ng::Value;
 pub type AdditionalFields = BTreeMap<String, Value>;
@@ -40,39 +40,14 @@ impl fmt::Debug for Object {
     }
 }
 
-#[derive(Clone, Debug, Deserialize, PartialEq)]
-#[serde(untagged)]
-pub enum NumberOrString {
-    Number(f64),
-    String(String),
-}
-
-#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
-#[serde(untagged)]
-pub enum Flag {
-    Boolean(bool),
-    String(String),
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(untagged)]
-pub enum OneOrMany<T> {
-    Many(Box<[T]>),
-    One(T),
-}
-
-impl<T> OneOrMany<T> {
-    pub fn len(&self) -> usize {
-        match self {
-            Self::Many(values) => values.len(),
-            Self::One(_) => 1,
-        }
-    }
-
-    pub fn is_empty(&self) -> bool {
-        match self {
-            Self::Many(values) => values.is_empty(),
-            Self::One(_) => false,
-        }
-    }
+/// Python's callback shorthand: one entry or a list of them, kept as the values written.
+pub(crate) fn one_or_many<'de, D: Deserializer<'de>>(
+    deserializer: D,
+) -> Result<Option<Vec<Value>>, D::Error> {
+    Ok(
+        Option::<Value>::deserialize(deserializer)?.map(|value| match value {
+            Value::Sequence(values) => values,
+            one => vec![one],
+        }),
+    )
 }

litellm-rust/crates/config/tests/config.rs

@@ -1,4 +1,4 @@
-use litellm_config::{Config, Error, Flag, NumberOrString, TracingStoreSettings};
+use litellm_config::{Config, Error, Spelled, TracingStoreSettings};
 use rstest::{fixture, rstest};
 use tempfile::TempDir;
 
@@ -75,6 +75,9 @@ fn config_debug_redacts_api_keys() {
 #[case::malformed_yaml("model_list: [")]
 #[case::missing_params("model_list: [{model_name: assistant}]")]
 #[case::missing_model("model_list: [{model_name: assistant, litellm_params: {api_key: key}}]")]
+#[case::provider_param_of_the_wrong_type(
+    "model_list: [{model_name: assistant, litellm_params: {model: test, aws_region_name: [eu-central-1]}}]"
+)]
 fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) {
     assert!(matches!(Config::from_yaml(yaml), Err(Error::Parse(_))));
 }
@@ -128,7 +131,7 @@ fn tracing_settings_are_typed_and_redact_the_url() {
         "https://writer:password@example.com"
     );
     assert_eq!(store.database.as_deref(), Some("analytics"));
-    assert_eq!(store.retention_days, Some(NumberOrString::Number(7.0)));
+    assert_eq!(store.retention_days, Some(Spelled::Value(7.0)));
     assert!(!format!("{config:?}").contains("password"));
 }
 
@@ -145,9 +148,7 @@ fn tracing_settings_accept_environment_references() {
     };
     assert_eq!(
         store.retention_days,
-        Some(NumberOrString::String(
-            "os.environ/RETENTION_DAYS".to_owned()
-        ))
+        Some(Spelled::Text("os.environ/RETENTION_DAYS".to_owned()))
     );
 }
 
@@ -190,6 +191,9 @@ model_list:
       rpm: 5
       drop_params: "true"
       vertex_project: test-project
+      vertex_credentials:
+        type: service_account
+      future_param: 1
     model_info:
       mode: chat
     access_groups: [internal]
@@ -214,25 +218,26 @@ future_section:
     let model = &config.model_list[0];
     assert_eq!(
         model.litellm_params.timeout,
-        Some(NumberOrString::String(
-            "os.environ/REQUEST_TIMEOUT".to_string()
-        ))
+        Some(Spelled::Text("os.environ/REQUEST_TIMEOUT".to_string()))
     );
     assert_eq!(
         model.litellm_params.tpm,
-        Some(NumberOrString::String("os.environ/TPM_LIMIT".to_string()))
+        Some(Spelled::Text("os.environ/TPM_LIMIT".to_string()))
     );
-    assert_eq!(model.litellm_params.rpm, Some(NumberOrString::Number(5.0)));
+    assert_eq!(model.litellm_params.rpm, Some(Spelled::Value(5.0)));
     assert_eq!(
         model.litellm_params.drop_params,
-        Some(Flag::String("true".to_string()))
+        Some(Spelled::Text("true".to_string()))
     );
-    assert!(
-        model
-            .litellm_params
-            .additional_fields
-            .contains_key("vertex_project")
+    assert_eq!(
+        model.litellm_params.vertex.vertex_project.as_deref(),
+        Some("test-project")
     );
+    assert_eq!(
+        model.litellm_params.vertex.vertex_credentials.as_deref(),
+        Some(r#"{"type":"service_account"}"#)
+    );
+    assert!(model.litellm_params.extra.contains_key("future_param"));
     assert!(model.additional_fields.contains_key("access_groups"));
     assert_eq!(config.router_settings.allowed_fails, Some(2));
     assert!(
@@ -345,7 +350,9 @@ fn accepts_python_callback_shorthand(#[case] setting: &str, #[case] expected_len
 
 #[rstest]
 #[case::api_key("api_key", "provider-secret")]
-#[case::provider_extension("aws_secret_access_key", "aws-secret")]
+#[case::aws_secret("aws_secret_access_key", "aws-secret")]
+#[case::vertex_credentials("vertex_credentials", "vertex-secret")]
+#[case::provider_extension("azure_ad_token", "azure-secret")]
 #[case::general_extension("custom_auth_secret", "auth-secret")]
 #[case::router_extension("redis_password", "redis-secret")]
 #[case::litellm_extension("callback_token", "callback-secret")]
@@ -358,6 +365,12 @@ fn debug_output_does_not_expose_config_values(#[case] field: &str, #[case] secre
         "aws_secret_access_key" => format!(
             "model_list: [{{model_name: assistant, litellm_params: {{model: test, aws_secret_access_key: {secret}}}}}]"
         ),
+        "vertex_credentials" => format!(
+            "model_list: [{{model_name: assistant, litellm_params: {{model: test, vertex_credentials: {secret}}}}}]"
+        ),
+        "azure_ad_token" => format!(
+            "model_list: [{{model_name: assistant, litellm_params: {{model: test, azure_ad_token: {secret}}}}}]"
+        ),
         "custom_auth_secret" => format!("general_settings: {{{field}: {secret}}}"),
         "redis_password" => format!("router_settings: {{{field}: {secret}}}"),
         "callback_token" => format!("litellm_settings: {{{field}: {secret}}}"),

litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs

@@ -18,6 +18,8 @@ pub enum LlmProviders {
     Bedrock,
     #[strum(serialize = "cohere")]
     Cohere,
+    #[strum(serialize = "deepseek")]
+    Deepseek,
     #[strum(serialize = "mistral")]
     Mistral,
     #[strum(serialize = "openai")]

litellm-rust/crates/coroutine/AGENTS.md

@@ -1,9 +1,9 @@
 # Requirements
 
-Core must pause mid-call to ask the host for things it cannot do itself (Python callbacks, secret and token reads, `before_send` rewrites, stream demand), then continue where it stopped. Any change to this crate must keep every requirement below; the alternatives section says which one each rejected design breaks
+Core must pause mid-call to ask the host for things it cannot do itself (Python callbacks, secret and token reads, `before_send` rewrites, stream demand), then continue where it stopped. Any change to this crate must keep every requirement below. The alternatives section says which one each rejected design breaks
 
 - R1 Core never calls the host: it names an op and waits for the answer, so it stays free of PyO3 and of any other host runtime
-- R2 Async host work is awaited by the host's own driver in the caller's asyncio task (`litellm/rust_bridge/lifecycle.py`), so `contextvars` writes reach the caller; a Rust-side `into_future` would run it in a copied context
+- R2 Async host work is awaited by the host's own driver in the caller's asyncio task (`litellm/rust_bridge/lifecycle.py`), so `contextvars` writes reach the caller. A Rust-side `into_future` would run it in a copied context
 - R3 The body awaits real I/O (HTTP, `spawn_blocking`, timers) between yields, so `resume` is itself a future driven by the caller's runtime
 - R4 Route code stays straight-line async (`host.route(OcrOp::ReadDocument).await?`) instead of hand-written states
 - R5 Each op fixes its answer type at compile time: a host cannot answer `ReadDocument` with a token, and core never matches a result variant it did not ask for
@@ -20,7 +20,7 @@ Core must pause mid-call to ask the host for things it cannot do itself (Python
 - `corosensei` and other stackful coroutines: sync bodies on their own stack, no async I/O inside (R3)
 - A hand-written phase enum with an `advance` match (the old `HostPhase`): every await point becomes a state (R4)
 - An injected host trait with `async fn`s: core would call the host itself (R1, R2)
-- Sans-IO, where core does no I/O and HTTP becomes one more host op: keeps every requirement and makes `resume` a pure step function, but HTTP, streaming, retries and timeouts would move out of core into every bridge; the one real alternative, not taken
+- Sans-IO, where core does no I/O and HTTP becomes one more host op: keeps every requirement and makes `resume` a pure step function, but HTTP, streaming, retries and timeouts would move out of core into every bridge. The one real alternative, not taken
 - Temporal's Rust workflow SDK (`WorkflowFuture`, `WfContext`) is the closest precedent: an `async fn` polled in place, commands sent over a channel with a oneshot to unblock them. Roles are inverted there (the language SDK owns the program, core answers), and its workflow body may not do real I/O
 
 # Tradeoffs accepted

litellm-rust/crates/gateway-auth/AGENTS.md

@@ -2,9 +2,9 @@
 
 Inbound authentication uses a verifier, identity resolver, and authorizer injected into `Auth`. `Auth::from_config` composes the current master-key implementation. `Auth::new` accepts alternate implementations and an injected clock
 
-`CredentialExtractor` selects a token or transport credential from HTTP request parts. The default `Bearer` extractor is strict; a host can supply another extractor with `Auth::with_extractor` for a separate route profile. A transport credential is only a selection marker, never proof of identity
+`CredentialExtractor` selects a token or transport credential from HTTP request parts. The default `Bearer` extractor is strict. A host can supply another extractor with `Auth::with_extractor` for a separate route profile. A transport credential is only a selection marker, never proof of identity
 
-`Authenticator` receives that selected credential and HTTP request parts. Transport verifiers must obtain proof from trusted server extensions, not client-supplied identity headers. It returns `VerifiedIdentity` only after verification. Request parts provide headers and trusted transport extensions for integrations; the verifier must validate any client-controlled claims before treating them as identity. There is no fallback chain after verification failure
+`Authenticator` receives that selected credential and HTTP request parts. Transport verifiers must obtain proof from trusted server extensions, not client-supplied identity headers. It returns `VerifiedIdentity` only after verification. Request parts provide headers and trusted transport extensions for integrations. The verifier must validate any client-controlled claims before treating them as identity. There is no fallback chain after verification failure
 
 `IdentityResolver` maps verified identity into an internal principal and permissions. The caller retains the original authentication evidence and credential restrictions independently of that mapping. Principals include an authority, subject, and kind. External subjects must remain authority-scoped unless the resolver explicitly maps them to a shared internal identity
 
@@ -14,13 +14,13 @@ Inbound authentication uses a verifier, identity resolver, and authorizer inject
 
 The Axum `authenticate` middleware currently accepts one Authorization header using the existing Bearer format. Missing, duplicate, empty, or malformed credentials fail. Authentication replaces any preexisting caller extension and checks the method and matched route before dispatch. Handlers extract `AuthenticatedRequest` and authorize their parsed operation before calling a provider. Missing authenticated context fails closed
 
-The gateway also inserts an MCP `Authorization` extension and a `SessionOwner` derived from the verified principal and credential reference. The MCP host checks the injected policy at initialization and before sending resolved upstream operations. Standalone MCP library hosts retain their existing explicitly trusted registry behavior when no policy is installed; production mounts must install authentication and the policy extension together
+The gateway also inserts an MCP `Authorization` extension and a `SessionOwner` derived from the verified principal and credential reference. The MCP host checks the injected policy at initialization and before sending resolved upstream operations. Standalone MCP library hosts retain their existing explicitly trusted registry behavior when no policy is installed. Production mounts must install authentication and the policy extension together
 
 Local UI login continues using axum-login and tower-sessions. Only after session and CSRF validation does `UiSession` expose an authenticated caller. Its credentials are restricted to session-info and logout operations, so the UI CSRF bearer cannot authorize inference. Future UI management routes must extend that explicit scope
 
 Authentication evidence and principals contain no raw token, password, request body, or mutable accounting state. Session ownership is scoped by principal authority, subject, verifier, and credential ID, so separate credentials do not silently share an MCP session. Credential rotation may retain ownership when the verifier preserves a stable credential ID. Scope and expiry checks still run for each operation
 
-Failures distinguish invalid or expired credentials, forbidden operations, unavailable authentication services, and missing server configuration/context. HTTP adapters map those outcomes to status codes; inference keeps its API-specific error envelopes and MCP translates policy failures to its own protocol
+Failures distinguish invalid or expired credentials, forbidden operations, unavailable authentication services, and missing server configuration/context. HTTP adapters map those outcomes to status codes. Inference keeps its API-specific error envelopes and MCP translates policy failures to its own protocol
 
 Virtual-key storage, JWT verification, OAuth2 introspection, trusted-proxy validation, SSO, and custom Python hook adapters are not implemented here yet. They should supply these contracts rather than bypassing the shared authorization boundary. Request-body-dependent custom hooks will need a bounded endpoint adapter after parsing
 

litellm-rust/crates/gateway-inference/AGENTS.md

@@ -1,6 +1,6 @@
-- Expose a mountable Axum router; listener binding, server lifecycle, and shared inbound middleware belong to `gateway`
-- Own endpoint paths, request parsing, model alias resolution, and API-specific response and SSE error formats; delegate hosted call execution and HTTP body delivery to host-http
-- Delegate inference execution to the `inference-<fmt>` crates and provider transformations and authentication to `llms` and the auth crates; do not duplicate them in handlers
-- Let the `inference-<fmt>` crates validate inference fields and supported features, then map their errors to HTTP responses; do not add gateway checks for temporary inference limitations
-- Use injected deployments, HTTP pools, settings, and secret sources; do not load process configuration or construct independent clients in handlers
-- Test HTTP contracts here, including status codes, forwarded headers, error envelopes, and streaming behavior; keep inference and provider tests in their owning crates
+- Expose a mountable Axum router. Listener binding, server lifecycle, and shared inbound middleware belong to `gateway`
+- Own endpoint paths, request parsing, model alias resolution, and API-specific response and SSE error formats. Delegate hosted call execution and HTTP body delivery to host-http
+- Delegate inference execution to the `inference-<fmt>` crates and provider transformations and authentication to `llms` and the auth crates. Do not duplicate them in handlers
+- Let the `inference-<fmt>` crates validate inference fields and supported features, then map their errors to HTTP responses. Do not add gateway checks for temporary inference limitations
+- Use injected deployments, HTTP pools, settings, and secret sources. Do not load process configuration or construct independent clients in handlers
+- Test HTTP contracts here, including status codes, forwarded headers, error envelopes, and streaming behavior. Keep inference and provider tests in their owning crates

litellm-rust/crates/gateway-inference/src/messages.rs

@@ -79,6 +79,7 @@ fn project(
         api_key: deployment.api_key.clone(),
         api_base: deployment.api_base.clone(),
         custom_llm_provider: deployment.custom_llm_provider.clone(),
+        litellm_params: deployment.litellm_params.clone(),
         extra_headers: None,
         provider_specific_header: anthropic_api_headers(headers),
         timeout: deployment.timeout,