chore: merge staging into Anthropic usage fix

.circleci/config.yml

@@ -88,6 +88,59 @@ commands:
             rm -f /tmp/uv-install.sh
             echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
             export PATH="$HOME/.local/bin:$PATH"
+  install_node:
+    description: "Install the Node.js version pinned in ui/litellm-dashboard/.nvmrc (24.19.0, which bundles npm 11.17.0) with checksum verification, and prepend it to PATH. Run this on any executor whose image does not already ship that version, or `npm ci` in ui/litellm-dashboard fails EBADENGINE against the engines floor. Installs into /opt/node rather than over /usr/local on purpose: cimg/python:*-browsers ships its own node there, and unpacking the tarball on top of it leaves npm 11.17 files merged with the image's npm 11.9 tree, which reports the new version and then exits 1 on `npm ci` with no error text at all. Requires checkout, which the .nvmrc drift check reads."
+    steps:
+      - run:
+          name: Install Node.js 24.19.0
+          command: |
+            NODE_VERSION="24.19.0"
+            NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz"
+            NODE_EXPECTED_SHA="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647"
+            NVMRC_VERSION="$(tr -d '[:space:]' < ui/litellm-dashboard/.nvmrc)"
+            if [ "$NVMRC_VERSION" != "$NODE_VERSION" ]; then
+              echo "install_node: ui/litellm-dashboard/.nvmrc pins ${NVMRC_VERSION} but this command pins ${NODE_VERSION}; update NODE_VERSION and NODE_EXPECTED_SHA together" >&2
+              exit 1
+            fi
+            curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}"
+            echo "${NODE_EXPECTED_SHA}  /tmp/${NODE_TARBALL}" | sha256sum -c -
+            sudo mkdir -p /opt/node
+            sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /opt/node --strip-components=1
+            rm -f "/tmp/${NODE_TARBALL}"
+            echo 'export PATH="/opt/node/bin:$PATH"' >> "$BASH_ENV"
+            export PATH="/opt/node/bin:$PATH"
+            node --version
+            npm --version
+  install_rust:
+    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.97.1) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
+    steps:
+      - run:
+          name: Install Rust (rustup 1.28.2, toolchain 1.97.1)
+          command: |
+            case "$(uname -m)" in
+              x86_64)
+                RUSTUP_TRIPLE=x86_64-unknown-linux-gnu
+                RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c
+                ;;
+              aarch64)
+                RUSTUP_TRIPLE=aarch64-unknown-linux-gnu
+                RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c
+                ;;
+              *)
+                echo "install_rust: unsupported architecture $(uname -m)" >&2
+                exit 1
+                ;;
+            esac
+            curl -sSLf -o /tmp/rustup-init \
+              "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init"
+            echo "${RUSTUP_SHA256}  /tmp/rustup-init" | sha256sum -c -
+            chmod +x /tmp/rustup-init
+            /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.97.1
+            rm -f /tmp/rustup-init
+            echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
+            export PATH="$HOME/.cargo/bin:$PATH"
+            rustc --version
+            cargo --version
   start_postgres:
     description: "Start a postgres-db container on port 5432 and wait until it accepts connections."
     parameters:
@@ -163,6 +216,26 @@ commands:
             done
             echo "fake OpenAI endpoint did not become ready" >&2
             exit 1
+  start_cost_center_service:
+    description: "Start the stand-in cost center validation service (tests/store_model_in_db_tests/cost_center_service.py) on host port 9414 and wait until healthy. The proxy's team-metadata validator (team_metadata_validator_e2e.py, impl 'http') reaches it via TEAM_METADATA_VALIDATION_SERVICE_URL=http://host.docker.internal:9414/validate. Run after uv deps are synced."
+    steps:
+      - run:
+          name: Start cost center validation service
+          background: true
+          command: |
+            uv run --no-sync python tests/store_model_in_db_tests/cost_center_service.py --host 0.0.0.0 --port 9414
+      - run:
+          name: Wait for cost center validation service
+          command: |
+            for i in $(seq 1 30); do
+              if curl -sf http://localhost:9414/health >/dev/null 2>&1; then
+                echo "cost center validation service is up"
+                exit 0
+              fi
+              sleep 1
+            done
+            echo "cost center validation service did not become ready" >&2
+            exit 1
   setup_litellm_enterprise_pip:
     steps:
       - run:
@@ -178,6 +251,7 @@ commands:
       - checkout
       - setup_google_dns
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -281,6 +355,33 @@ jobs:
             uv build --wheel --out-dir dist
             uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py
 
+  base_sdk_install:
+    docker:
+      - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+    steps:
+      - checkout
+      - setup_google_dns
+      - install_uv
+      - install_rust
+      - run:
+          name: Build the wheel
+          environment:
+            UV_HTTP_TIMEOUT: "300"
+          command: |
+            uv build --wheel --out-dir dist
+      - run:
+          name: Install the wheel with no extras and smoke-check it
+          environment:
+            UV_HTTP_TIMEOUT: "300"
+          command: |
+            uv venv /tmp/base-sdk --python 3.12
+            VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl
+            /tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py
+
   local_testing_part1:
     docker:
       - &python312_image
@@ -298,6 +399,7 @@ jobs:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -371,6 +473,7 @@ jobs:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -445,6 +548,7 @@ jobs:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -496,6 +600,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -562,6 +667,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -602,6 +708,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -643,6 +750,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -676,6 +784,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -726,6 +835,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -777,6 +887,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -810,6 +921,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -856,6 +968,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -902,6 +1015,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -944,6 +1058,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -990,6 +1105,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1037,6 +1153,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -1077,6 +1194,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1122,6 +1240,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1166,6 +1285,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1198,6 +1318,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1241,6 +1362,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1285,6 +1407,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1329,6 +1452,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1360,6 +1484,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1406,6 +1531,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1451,6 +1577,7 @@ jobs:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1501,6 +1628,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1525,6 +1653,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1551,6 +1680,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1652,6 +1782,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1747,6 +1878,7 @@ jobs:
           at: ~/project
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1835,6 +1967,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1918,6 +2051,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2050,6 +2184,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2136,6 +2271,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2232,12 +2368,14 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
             uv sync --frozen --all-groups --all-extras --python 3.12
       - start_postgres
       - start_fake_openai_endpoint
+      - start_cost_center_service
       - attach_workspace:
           at: ~/project
       - run:
@@ -2257,11 +2395,13 @@ jobs:
               -e STORE_MODEL_IN_DB="True" \
               -e LITELLM_MASTER_KEY="sk-1234" \
               -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
+              -e TEAM_METADATA_VALIDATION_SERVICE_URL=http://host.docker.internal:9414/validate \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
               -e LITELLM_LOG=ERROR \
               --add-host host.docker.internal:host-gateway \
               --name my-app \
               -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
+              -v $(pwd)/litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py:/app/team_metadata_validator_e2e.py \
               litellm-docker-database:ci \
               --config /app/config.yaml \
               --port 4000
@@ -2307,6 +2447,7 @@ jobs:
       - setup_google_dns
       # Remove Docker CLI installation since it's already available in machine executor
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2388,6 +2529,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2473,20 +2615,9 @@ jobs:
             bundle exec rspec
           no_output_timeout: 30m
       # Install Node.js directly from nodejs.org with SHA256 verification,
-      # instead of piping NodeSource's setup_18.x apt-repo installer into
+      # instead of piping NodeSource's setup_24.x apt-repo installer into
       # sudo bash (which runs a mutable upstream script unattended).
-      - run:
-          name: Install Node.js 18.20.8
-          command: |
-            NODE_VERSION="18.20.8"
-            NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz"
-            NODE_EXPECTED_SHA="5467ee62d6af1411d46b6a10e3fb5cacc92734dbcef465fea14e7b90993001c9"
-            curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}"
-            echo "${NODE_EXPECTED_SHA}  /tmp/${NODE_TARBALL}" | sha256sum -c -
-            sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /usr/local --strip-components=1
-            rm -f "/tmp/${NODE_TARBALL}"
-            node --version
-            npm --version
+      - install_node
 
       - run:
           name: Install Node.js test dependencies
@@ -2527,6 +2658,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2614,7 +2746,7 @@ jobs:
 
   ui_build:
     docker:
-      - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81
+      - image: cimg/node:24.19@sha256:8966565f07189a67d64d6808a2b127f31dafae566508e3547f55640e1070bfad
         auth:
           username: ${DOCKERHUB_USERNAME}
           password: ${DOCKERHUB_PASSWORD}
@@ -2658,7 +2790,7 @@ jobs:
 
   ui_unit_tests:
     docker:
-      - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81
+      - image: cimg/node:24.19@sha256:8966565f07189a67d64d6808a2b127f31dafae566508e3547f55640e1070bfad
         auth:
           username: ${DOCKERHUB_USERNAME}
           password: ${DOCKERHUB_PASSWORD}
@@ -2716,7 +2848,9 @@ jobs:
       - skip_if_unrelated_changes:
           category: client
       - setup_google_dns
+      - install_node
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -2731,7 +2865,7 @@ jobs:
             - ~/.cache/uv
       - restore_cache:
           keys:
-            - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+            - ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
       - run:
           name: Install Node dependencies and Playwright
           # The cimg/python:3.12-browsers image already ships the Chromium system
@@ -2742,11 +2876,14 @@ jobs:
           command: |
             cd ui/litellm-dashboard
             npm ci
+            cd ../../tests/e2e/ui
+            npm ci
             npx playwright install chromium
       - save_cache:
-          key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+          key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
           paths:
             - ui/litellm-dashboard/node_modules
+            - tests/e2e/ui/node_modules
             - ~/.cache/ms-playwright
       - run:
           name: Build UI from source
@@ -2777,10 +2914,10 @@ jobs:
           name: Seed database
           command: |
             PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
-              -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
+              -f tests/e2e/ui/fixtures/seed.sql
       - run:
           name: Start mock LLM server
-          command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
+          command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
           background: true
       - run:
           name: Start LiteLLM proxy
@@ -2798,7 +2935,7 @@ jobs:
           command: |
             LITELLM_LICENSE="$LITELLM_LICENSE" \
               uv run --no-sync python -m litellm.proxy.proxy_cli \
-                --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
+                --config tests/e2e/ui/fixtures/config.yml \
                 --port 4000
           background: true
       - run:
@@ -2819,15 +2956,15 @@ jobs:
           # Forward LITELLM_LICENSE so license.spec.ts can detect that the
           # proxy was launched with a license and assert premium_user=true.
           command: |
-            cd ui/litellm-dashboard
+            cd tests/e2e/ui
             LITELLM_LICENSE="$LITELLM_LICENSE" \
-              npx playwright test --config e2e_tests/playwright.config.ts
+              npx playwright test --config playwright.config.ts
           no_output_timeout: 10m
       - store_artifacts:
-          path: ui/litellm-dashboard/test-results
+          path: tests/e2e/ui/test-results
           destination: e2e-test-results
       - store_artifacts:
-          path: ui/litellm-dashboard/playwright-report
+          path: tests/e2e/ui/playwright-report
           destination: e2e-playwright-report
 
   e2e_ui_testing_server_root_path:
@@ -2855,7 +2992,9 @@ jobs:
       - skip_if_unrelated_changes:
           category: client
       - setup_google_dns
+      - install_node
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -2870,17 +3009,20 @@ jobs:
             - ~/.cache/uv
       - restore_cache:
           keys:
-            - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+            - ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
       - run:
           name: Install Node dependencies and Playwright
           command: |
             cd ui/litellm-dashboard
             npm ci
+            cd ../../tests/e2e/ui
+            npm ci
             npx playwright install chromium
       - save_cache:
-          key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+          key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
           paths:
             - ui/litellm-dashboard/node_modules
+            - tests/e2e/ui/node_modules
             - ~/.cache/ms-playwright
       - run:
           name: Build UI from source
@@ -2902,10 +3044,10 @@ jobs:
           name: Seed database
           command: |
             PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
-              -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
+              -f tests/e2e/ui/fixtures/seed.sql
       - run:
           name: Start mock LLM server
-          command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
+          command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
           background: true
       - run:
           name: Start LiteLLM proxy under a server root path
@@ -2918,7 +3060,7 @@ jobs:
           command: |
             LITELLM_LICENSE="$LITELLM_LICENSE" \
               uv run --no-sync python -m litellm.proxy.proxy_cli \
-                --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
+                --config tests/e2e/ui/fixtures/config.yml \
                 --port 4000
           background: true
       - run:
@@ -2937,15 +3079,15 @@ jobs:
       - run:
           name: Run migration smoke under SERVER_ROOT_PATH
           command: |
-            cd ui/litellm-dashboard
+            cd tests/e2e/ui
             LITELLM_LICENSE="$LITELLM_LICENSE" \
-              npx playwright test --config e2e_tests/migration.serverRootPath.config.ts
+              npx playwright test --config migration.serverRootPath.config.ts
           no_output_timeout: 10m
       - store_artifacts:
-          path: ui/litellm-dashboard/test-results
+          path: tests/e2e/ui/test-results
           destination: e2e-server-root-path-test-results
       - store_artifacts:
-          path: ui/litellm-dashboard/playwright-report
+          path: tests/e2e/ui/playwright-report
           destination: e2e-server-root-path-playwright-report
 
   build_docker_database_image:
@@ -3025,6 +3167,8 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - base_sdk_install:
+          filters: *main_branches
       - local_testing_part1:
           filters: *main_branches
       - local_testing_part2:

.circleci/scripts/classify_changes.sh

@@ -8,7 +8,7 @@ has_backend=false
 while IFS= read -r file || [ -n "$file" ]; do
   [ -n "$file" ] || continue
   case "$file" in
-    ui/*) has_client=true ;;
+    ui/* | tests/e2e/ui/*) has_client=true ;;
     docs/* | *.md | *.mdx) : ;;
     *) has_backend=true ;;
   esac

.flake8

@@ -1,46 +0,0 @@
-[flake8]
-ignore =
-    # The following ignores can be removed when formatting using black
-    W191,W291,W292,W293,W391,W504
-    E101,E111,E114,E116,E117,E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,
-    E201,E202,E221,E222,E225,E226,E231,E241,E251,E252,E261,E265,E271,E272,E275,
-    E301,E302,E303,E305,E306,
-    # line break before binary operator
-    W503,
-    # inline comment should start with '# '
-    E262,
-    # too many leading '#' for block comment
-    E266,
-    # multiple imports on one line
-    E401,
-    # module level import not at top of file
-    E402,
-    # Line too long (82 > 79 characters)
-    E501,
-    # comparison to None should be 'if cond is None:'
-    E711,
-    # comparison to True should be 'if cond is True:' or 'if cond:'
-    E712,
-    # do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()`
-    E721,
-    # do not use bare 'except'
-    E722,
-    # x is imported but unused
-    F401,
-    # 'from . import *' used; unable to detect undefined names
-    F403,
-    # x may be undefined, or defined from star imports:
-    F405,
-    # f-string is missing placeholders
-    F541,
-    # dictionary key '' repeated with different values
-    F601,
-    # redefinition of unused x from line 123
-    F811,
-    # undefined name x
-    F821,
-    # local variable x is assigned to but never used
-    F841,
-
-# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8
-extend-ignore = E203

.git-blame-ignore-revs

@@ -17,3 +17,24 @@
 
 # style: unify ruff format width on 120 (#31518)
 48b5a5a0cc5a694a11219416ee0b6eb6e620e74e
+
+# refactor(imports): move collections.abc names out of typing (#35495)
+397e8e4918777e4e60a7f5e88699e0a9a7dabb3d
+
+# refactor(lint): apply every safe ruff autofix and zero 28 strict-rule budgets (#35495)
+b604e2b20c6db2099085a2f0e59b7e99e87eed6f
+
+# refactor(logging): drop redundant !s conversion flags from f-strings (#35546)
+7b2d3440cba3160277470f7a0180098ae9b87864
+
+# perf: build log messages lazily so filtered-out log records cost nothing (#35703)
+c9887a1f94bc1e7e4bdfe64d640f0509a0bc19dd
+
+# feat(lint): enforce Final on locals and freeze function parameters (#35807)
+2708620d6a599cc73c1950a942d26ac26a7ed3d4
+
+# chore(lint): remove litellm/types from the ruff lint exclusion (#35926)
+4e32a8bf6a1e1af1e04b67c759841ccef44b2235
+
+# chore(lint): strip inert type: ignore comments and zero LIT009/LIT010/LIT011 headroom (#35928)
+338e411103ad5d7003e97f34f04fa36bca542dbe

.github/CODEOWNERS

@@ -1,2 +1,3 @@
 /ui/ @yuneng-jiang @ryan-crabbe-berri
 /litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri
+/ui/litellm-dashboard/src/lib/http/schema.d.ts

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -30,7 +30,7 @@ body:
     id: steps-to-reproduce
     attributes:
       label: Steps to Reproduce
-      description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
+      description: Please provide a numbered list of the exact steps to reproduce this bug (include a curl/python snippet to reproduce it). Number each step (1., 2., 3., ...) in the order you performed them.
       placeholder: |
         1. config.yaml file/ .env file/ etc.
         2. Run the following code...

.github/ci-coverage-allowlist.yml

@@ -0,0 +1,152 @@
+description: >-
+  Paths deliberately outside CI coverage, each with the reason it is exempt.
+  assert_ci_coverage.py fails when a test file or Dockerfile is neither invoked
+  by a job nor listed here, so every entry below is a decision on the record.
+
+test_paths:
+  - reason: >-
+      The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
+      from a pull request; it needs a live gateway and provider credentials no PR job holds
+    paths:
+      - tests/e2e
+  - reason: >-
+      The documentation and code-quality workflows execute four files in this directory by name as
+      scripts and pytest never collects the directory, so these six run nowhere; listed individually
+      so a seventh cannot inherit the exemption
+    paths:
+      - tests/documentation_tests/test_exception_types.py
+      - tests/documentation_tests/test_general_setting_keys.py
+      - tests/documentation_tests/test_optional_params.py
+      - tests/documentation_tests/test_readme_providers.py
+      - tests/documentation_tests/test_requests_lib_usage.py
+      - tests/documentation_tests/test_standard_logging_payload.py
+  - reason: >-
+      Sibling files here are executed by name from the code-quality workflow; this one is referenced
+      by no job
+    paths:
+      - tests/code_coverage_tests/test_aio_http_image_conversion.py
+  - reason: >-
+      A second mirror of the package tree living beside tests/test_litellm, which is the mirror the
+      repo convention names; only test_no_hardcoded_secrets.py is invoked, from the linting
+      workflow, and whether this directory should exist at all is unresolved
+    paths:
+      - tests/litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_headers.py
+      - tests/litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_transformation.py
+      - tests/litellm/integrations/helicone/test_helicone_gemini.py
+      - tests/litellm/litellm_core_utils/test_json_schema_validation.py
+      - tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py
+      - tests/litellm/llms/anthropic/test_anthropic_schema_filter.py
+      - tests/litellm/llms/azure/test_azure_embedding.py
+      - tests/litellm/llms/bedrock/embed/test_embedding.py
+      - tests/litellm/llms/bedrock/test_nova_imported_models.py
+      - tests/litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py
+      - tests/litellm/llms/gradient_ai/chat/test_gradient_ai_chat_transformation.py
+      - tests/litellm/llms/oci/chat/test_oci_chat_transformation.py
+      - tests/litellm/llms/openai_like/test_abliteration_provider.py
+      - tests/litellm/llms/openai_like/test_assemblyai_provider.py
+      - tests/litellm/llms/openai_like/test_empiriolabs_provider.py
+      - tests/litellm/llms/vertex_ai/agent_engine/test_transformation.py
+      - tests/litellm/llms/vertex_ai/gemini/test_transformation.py
+      - tests/litellm/llms/vertex_ai/text_to_speech/test_transformation.py
+      - tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
+      - tests/litellm/proxy/agent_endpoints/test_agent_rbac.py
+      - tests/litellm/proxy/common_utils/test_rbac_utils.py
+      - tests/litellm/proxy/management_endpoints/test_common_utils.py
+      - tests/litellm/proxy/management_endpoints/test_cost_estimate_endpoint.py
+      - tests/litellm/proxy/test_claude_code_marketplace.py
+      - tests/litellm/proxy/test_init_litellm_callbacks.py
+      - tests/litellm/proxy/test_prisma_engine_watchdog.py
+      - tests/litellm/proxy/vector_store_endpoints/test_vector_store_rbac.py
+      - tests/litellm/test_bedrock_extended_beta_models.py
+      - tests/litellm/test_bedrock_nemotron_super.py
+      - tests/litellm/test_proxy_auth.py
+      - tests/litellm/test_router_retry_backoff_headers.py
+      - tests/litellm/test_sambanova_model_metadata.py
+      - tests/litellm/test_stream_chunk_builder_images.py
+  - reason: >-
+      Legacy proxy suite superseded by the proxy shards; no job invokes it and whether it still
+      describes supported behaviour is unresolved
+    paths:
+      - tests/old_proxy_tests/tests/test_anthropic_context_caching.py
+      - tests/old_proxy_tests/tests/test_anthropic_sdk.py
+      - tests/old_proxy_tests/tests/test_async.py
+      - tests/old_proxy_tests/tests/test_gemini_context_caching.py
+      - tests/old_proxy_tests/tests/test_langchain_embedding.py
+      - tests/old_proxy_tests/tests/test_langchain_request.py
+      - tests/old_proxy_tests/tests/test_llamaindex.py
+      - tests/old_proxy_tests/tests/test_mistral_sdk.py
+      - tests/old_proxy_tests/tests/test_openai_embedding.py
+      - tests/old_proxy_tests/tests/test_openai_exception_request.py
+      - tests/old_proxy_tests/tests/test_openai_request.py
+      - tests/old_proxy_tests/tests/test_openai_request_with_traceparent.py
+      - tests/old_proxy_tests/tests/test_openai_simple_embedding.py
+      - tests/old_proxy_tests/tests/test_openai_tts_request.py
+      - tests/old_proxy_tests/tests/test_pass_through_langfuse.py
+      - tests/old_proxy_tests/tests/test_q.py
+      - tests/old_proxy_tests/tests/test_simple_traceparent_openai.py
+      - tests/old_proxy_tests/tests/test_vertex_sdk_forward_headers.py
+      - tests/old_proxy_tests/tests/test_vtx_embedding.py
+      - tests/old_proxy_tests/tests/test_vtx_sdk_embedding.py
+  - reason: >-
+      No job invokes this suite and its files mix pure transformation tests with ones driving live
+      vendor vector stores, so assigning them needs a per-file decision
+    paths:
+      - tests/vector_store_tests/rag/test_rag_bedrock.py
+      - tests/vector_store_tests/rag/test_rag_openai.py
+      - tests/vector_store_tests/rag/test_rag_s3_vectors.py
+      - tests/vector_store_tests/rag/test_rag_vertex_ai.py
+      - tests/vector_store_tests/test_azure_ai_vector_store.py
+      - tests/vector_store_tests/test_azure_vector_store.py
+      - tests/vector_store_tests/test_bedrock_vector_store.py
+      - tests/vector_store_tests/test_gemini_vector_store.py
+      - tests/vector_store_tests/test_milvus_vector_store.py
+      - tests/vector_store_tests/test_openai_vector_store.py
+      - tests/vector_store_tests/test_ragflow_vector_store.py
+      - tests/vector_store_tests/test_s3_vectors_vector_store.py
+      - tests/vector_store_tests/test_vertex_ai_search_api_vector_store.py
+      - tests/vector_store_tests/test_vertex_ai_vector_store.py
+  - reason: >-
+      Throughput and memory-growth measurements whose runtime and variance make them unsuitable for
+      a per-pull-request job
+    paths:
+      - tests/load_tests/test_datadog_load_test.py
+      - tests/load_tests/test_langsmith_load_test.py
+      - tests/load_tests/test_linear_memory_growth.py
+      - tests/load_tests/test_memory_usage.py
+      - tests/load_tests/test_otel_load_test.py
+      - tests/load_tests/test_vertex_embeddings_load_test.py
+      - tests/load_tests/test_vertex_load_tests.py
+  - reason: >-
+      Third-party integration tests that skip themselves without OCI configuration or sandbox
+      credentials, neither of which a pull request job holds
+    paths:
+      - tests/integration/sandbox/test_e2b_sandbox.py
+      - tests/integration/test_oci_integration.py
+      - tests/integration/test_oci_proxy_integration.py
+  - reason: >-
+      Two prompt-factory tests sitting at the top level of tests/ instead of under the
+      tests/test_litellm mirror the shards enumerate; they need moving rather than a shard entry
+    paths:
+      - tests/litellm_core_utils/test_anthropic_dedup_factory.py
+      - tests/litellm_core_utils/test_bedrock_converse_dedup_factory.py
+  - reason: >-
+      A unit test for the proxy-extras package that no job invokes, while the package's other tests
+      live under tests/proxy_migration_tests
+    paths:
+      - tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
+
+dockerfiles:
+  - reason: >-
+      The dashboard container is a static Next.js export served by nginx, and the dashboard build
+      and lint workflows already exercise that output, so building the image adds no signal about it
+    paths:
+      - ui/Dockerfile
+  - reason: >-
+      The Rust gateway ships as its own chart and package with a separate release pipeline, so its
+      image is not part of this repo's Python image set
+    paths:
+      - litellm-rust/crates/ai-gateway/Dockerfile
+  - reason: >-
+      An example image under cookbook/ that is documentation rather than a shipped artifact
+    paths:
+      - cookbook/litellm-ollama-docker-image/Dockerfile

.github/pull_request_template.md

@@ -1,3 +1,45 @@
+## TLDR
+
+<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
+     This section must be extremely human parsable, comprehensible, and readable: its target audience is humans, not AI agents -->
+
+Problem this solves:
+
+- <blah>
+- ...
+
+How it solves it:
+
+- <blah>
+- ...
+
+## User Flow
+
+<!-- Two ordered lists, Before and After, walking the same end user through the same task, written strictly from that user's seat
+     Read the linked issue, ticket, or customer thread first so the flow reflects the real application and the routes its users actually hit; don't invent a generic scenario
+     Lead each list with one plain sentence saying where the flow fails (Before) or succeeds (After), then number the steps
+     Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen
+     No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. "The upload hands back an ID that looks like OpenAI's own `file-abc123` instead of the scrambled one the gateway returned" is right, "no managed-file row was registered" is wrong
+     Keep the two lists step-for-step identical until they diverge, so the changed step is obvious
+     If the bug had a security or authorization consequence, end each list with what another user could or could no longer do
+     Regenerate this section whenever new commits change the PR's behavior, so it never describes an older revision
+
+Example:
+
+Before: a developer whose app streams chat completions gets no token counts back, so their cost dashboard reads zero
+
+1. They send POST https://litellm-domain/v1/chat/completions with `"stream": true` and no `stream_options`
+2. The last SSE chunk arrives with `"usage": null`, so their app records 0 prompt and 0 completion tokens
+3. They open https://litellm-domain/ui/?page=logs and see the request logged at $0 spend
+
+After: the same request comes back with real token counts, so the dashboard shows real spend
+
+1. The proxy admin sets `always_include_stream_usage: true` and restarts the proxy
+2. The developer sends the same POST https://litellm-domain/v1/chat/completions with `"stream": true` and no `stream_options`
+3. The last SSE chunk now carries a `usage` object with real prompt and completion token counts
+4. https://litellm-domain/ui/?page=logs shows that request at non-zero spend
+-->
+
 ## Relevant issues
 
 <!-- e.g., "Fixes #000" -->
@@ -25,6 +67,7 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
      The proof must be completely e2e with no mocks, using, for example, actual LLM calls costing real $. `pytest` commands are not enough
      For bug fixes: show reproduction before the fix and passing behavior after
      Include the commit hash each proof was captured at, for both the before and the after runs
+     If the change applies to all three LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every single one of them, not just one
      For new features: show the feature working end-to-end
      For UI changes: include before/after screenshots -->
 

.github/scripts/assert_ci_coverage.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+import pathlib
+import re
+import sys
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+
+import yaml
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[2]
+WORKFLOW_DIR = REPO_ROOT / ".github" / "workflows"
+CIRCLECI_CONFIG = REPO_ROOT / ".circleci" / "config.yml"
+ALLOWLIST_FILE = REPO_ROOT / ".github" / "ci-coverage-allowlist.yml"
+TESTS_ROOT = REPO_ROOT / "tests"
+
+ALLOWLIST_KEYS = frozenset({"description", "test_paths", "dockerfiles"})
+PATH_FILTER_KEYS = frozenset({"paths", "paths-ignore"})
+TEST_PATH_KEYS = frozenset({"test-path", "test-paths"})
+DOCKERFILE_INPUT_KEYS = frozenset({"file", "dockerfile"})
+TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bcircleci tests\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s")
+IMAGE_BUILD_RE = re.compile(r"\bdocker\s+(?:buildx\s+)?build\b")
+TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?-]+")
+DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
+COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
+GLOB_CHARS = frozenset("*?")
+
+
+@dataclass(frozen=True, slots=True)
+class AllowEntry:
+    paths: tuple[str, ...]
+    reason: str
+
+
+@dataclass(frozen=True, slots=True)
+class Allowlist:
+    test_paths: tuple[AllowEntry, ...]
+    dockerfiles: tuple[AllowEntry, ...]
+
+    def covers_test(self, relative_path: str) -> bool:
+        return any(_token_covers(path, relative_path) for entry in self.test_paths for path in entry.paths)
+
+    def covers_dockerfile(self, relative_path: str) -> bool:
+        return any(relative_path == path for entry in self.dockerfiles for path in entry.paths)
+
+
+@dataclass(frozen=True, slots=True)
+class Scalar:
+    key: str
+    value: str
+
+
+@dataclass(frozen=True, slots=True)
+class Finding:
+    subject: str
+    detail: str
+
+
+def _scalars(node: object, key: str) -> tuple[Scalar, ...]:
+    if isinstance(node, str):
+        return (Scalar(key=key, value=node),)
+    if isinstance(node, Mapping):
+        return tuple(
+            scalar
+            for child_key, value in node.items()
+            if child_key not in PATH_FILTER_KEYS
+            for scalar in _scalars(value, str(child_key))
+        )
+    if isinstance(node, Sequence):
+        return tuple(scalar for item in node for scalar in _scalars(item, key))
+    return ()
+
+
+def _config_files() -> tuple[pathlib.Path, ...]:
+    workflows = tuple(sorted(path for path in WORKFLOW_DIR.iterdir() if path.suffix in (".yml", ".yaml")))
+    circleci = (CIRCLECI_CONFIG,) if CIRCLECI_CONFIG.is_file() else ()
+    return workflows + circleci
+
+
+def _all_scalars() -> tuple[Scalar, ...]:
+    return tuple(
+        scalar
+        for path in _config_files()
+        for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
+    )
+
+
+def _uncommented(value: str) -> str:
+    return COMMENT_RE.sub("", value)
+
+
+def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
+    return frozenset(
+        match.group(0).rstrip("/")
+        for scalar in scalars
+        if scalar.key in TEST_PATH_KEYS or TEST_RUNNER_RE.search(scalar.value)
+        for match in TEST_TOKEN_RE.finditer(_uncommented(scalar.value))
+    )
+
+
+def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
+    return frozenset(
+        match.group(0)
+        for scalar in scalars
+        if scalar.key in DOCKERFILE_INPUT_KEYS or IMAGE_BUILD_RE.search(scalar.value)
+        for match in DOCKERFILE_TOKEN_RE.finditer(_uncommented(scalar.value))
+    )
+
+
+def _glob_to_regex(token: str) -> re.Pattern[str]:
+    parts = re.split(r"(\*\*/|\*\*|\*|\?)", token)
+    translated = "".join(
+        {"**/": r"(?:.*/)?", "**": r".*", "*": r"[^/]*", "?": r"[^/]"}.get(part, re.escape(part)) for part in parts
+    )
+    return re.compile(rf"{translated}(?:/.*)?$")
+
+
+def _token_covers(token: str, relative_path: str) -> bool:
+    if GLOB_CHARS & set(token):
+        return _glob_to_regex(token).match(relative_path) is not None
+    return relative_path == token or relative_path.startswith(f"{token}/")
+
+
+def _test_files() -> tuple[str, ...]:
+    return tuple(
+        sorted(
+            path.relative_to(REPO_ROOT).as_posix()
+            for path in TESTS_ROOT.rglob("test_*.py")
+            if path.is_file() and "node_modules" not in path.parts
+        )
+    )
+
+
+def _dockerfiles() -> tuple[str, ...]:
+    return tuple(
+        sorted(
+            path.relative_to(REPO_ROOT).as_posix()
+            for path in REPO_ROOT.rglob("Dockerfile*")
+            if path.is_file()
+            and ".git" not in path.parts
+            and "node_modules" not in path.parts
+            and not path.name.endswith(".dockerignore")
+        )
+    )
+
+
+def _uncovered_tests(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]:
+    uncovered = tuple(
+        relative_path
+        for relative_path in _test_files()
+        if not any(_token_covers(token, relative_path) for token in tokens) and not allowlist.covers_test(relative_path)
+    )
+    directories = tuple(dict.fromkeys(path.rsplit("/", 1)[0] for path in uncovered))
+    return tuple(
+        Finding(
+            subject=directory,
+            detail=_describe(tuple(p for p in uncovered if p.rsplit("/", 1)[0] == directory)),
+        )
+        for directory in directories
+    )
+
+
+def _describe(paths: tuple[str, ...]) -> str:
+    names = ", ".join(path.rsplit("/", 1)[1] for path in paths[:3])
+    suffix = f", +{len(paths) - 3} more" if len(paths) > 3 else ""
+    return f"{len(paths)} test file(s) invoked by no job: {names}{suffix}"
+
+
+def _uncovered_dockerfiles(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]:
+    return tuple(
+        Finding(subject=relative_path, detail="built by no job")
+        for relative_path in _dockerfiles()
+        if relative_path not in tokens and not allowlist.covers_dockerfile(relative_path)
+    )
+
+
+def _parse_entry(item: object, section: str) -> AllowEntry:
+    if not isinstance(item, dict):
+        raise SystemExit(f"{ALLOWLIST_FILE.name}: '{section}' entries must be mappings")
+    paths = item.get("paths")
+    reason = item.get("reason")
+    if (
+        not isinstance(paths, list)
+        or not paths
+        or not all(isinstance(path, str) for path in paths)
+        or not isinstance(reason, str)
+        or not reason.strip()
+    ):
+        raise SystemExit(
+            f"{ALLOWLIST_FILE.name}: every '{section}' entry needs a non-empty 'paths' "
+            "list of strings and a non-empty 'reason'"
+        )
+    return AllowEntry(paths=tuple(paths), reason=reason)
+
+
+def _parse_entries(raw: object, section: str) -> tuple[AllowEntry, ...]:
+    if not isinstance(raw, list):
+        raise SystemExit(f"{ALLOWLIST_FILE.name}: '{section}' must be a list")
+    return tuple(_parse_entry(item, section) for item in raw)
+
+
+def _load_allowlist() -> Allowlist:
+    if not ALLOWLIST_FILE.is_file():
+        return Allowlist(test_paths=(), dockerfiles=())
+    raw = yaml.safe_load(ALLOWLIST_FILE.read_text(encoding="utf-8")) or {}
+    if not isinstance(raw, dict):
+        raise SystemExit(f"{ALLOWLIST_FILE.name}: top level must be a mapping")
+    unknown = sorted(str(key) for key in raw if key not in ALLOWLIST_KEYS)
+    if unknown:
+        raise SystemExit(
+            f"{ALLOWLIST_FILE.name}: unknown top-level key(s) {unknown}; expected only {sorted(ALLOWLIST_KEYS)}"
+        )
+    return Allowlist(
+        test_paths=_parse_entries(raw.get("test_paths", []), "test_paths"),
+        dockerfiles=_parse_entries(raw.get("dockerfiles", []), "dockerfiles"),
+    )
+
+
+def _write(message: str) -> None:
+    sys.stdout.write(f"{message}\n")
+
+
+def _report(title: str, findings: tuple[Finding, ...], remedy: str) -> None:
+    _write(f"ERROR: {title}")
+    for finding in findings:
+        _write(f"  - {finding.subject}: {finding.detail}")
+    _write("")
+    _write(remedy)
+    _write("")
+
+
+def main() -> int:
+    allowlist = _load_allowlist()
+    scalars = _all_scalars()
+
+    test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars))
+    dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
+
+    if test_findings:
+        _report(
+            "test files that no CI job invokes",
+            test_findings,
+            "Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
+        )
+    if dockerfile_findings:
+        _report(
+            "Dockerfiles that no CI job builds",
+            dockerfile_findings,
+            "Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
+        )
+    if test_findings or dockerfile_findings:
+        return 1
+
+    _write(
+        f"OK: {len(_test_files())} test files and {len(_dockerfiles())} Dockerfiles are each "
+        "invoked by at least one job or carry an explicit allowlist entry."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/_test-unit-base.yml

@@ -80,7 +80,7 @@ jobs:
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
 
       - name: Generate Prisma client
         if: steps.changes.outputs.decision != 'skip'
@@ -154,6 +154,19 @@ jobs:
           merge-multiple: true
 
       - name: Upload to Codecov
+        id: codecov-upload
+        continue-on-error: true
+        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
+        with:
+          use_oidc: true
+          directory: coverage-reports
+          root_dir: ${{ github.workspace }}
+          flags: ${{ inputs.artifact-name }}
+          fail_ci_if_error: false
+
+      - name: Upload to Codecov (retry)
+        if: steps.codecov-upload.outcome == 'failure'
+        continue-on-error: true
         uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
         with:
           use_oidc: true

.github/workflows/auto_update_price_and_context_window.yml

@@ -24,9 +24,12 @@ jobs:
       - name: Update JSON Data
         run: |
           uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py"
+      - name: Regenerate JSON Schema
+        run: |
+          uv run --frozen python ci_cd/generate_model_prices_schema.py
       - name: Create Pull Request
         run: |
-          git add model_prices_and_context_window.json 
+          git add model_prices_and_context_window.json model_prices_and_context_window.schema.json
           git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')" 
           gh pr create --title "Update model_prices_and_context_window.json file" \
             --body "Automated update for model_prices_and_context_window.json" \

.github/workflows/check-ui-api-types.yml

@@ -2,18 +2,19 @@ name: Check UI API Types Sync
 
 on:
   pull_request:
-    paths:
-      - "litellm/proxy/**"
-      - "litellm/types/**"
-      - "ui/litellm-dashboard/src/lib/http/schema.d.ts"
-      - "ui/litellm-dashboard/scripts/gen-api-types.mjs"
-      - "ui/litellm-dashboard/package.json"
-      - "ui/litellm-dashboard/package-lock.json"
-      - ".github/workflows/check-ui-api-types.yml"
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
 
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   check-sync:
     name: Verify schema.d.ts matches the proxy OpenAPI spec
@@ -24,18 +25,39 @@ jobs:
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           persist-credentials: false
+          fetch-depth: 2
+
+      - name: Detect changes that can affect the generated types
+        id: changes
+        run: |
+          set -euo pipefail
+          if ! base="$(git rev-parse --verify --quiet HEAD^2 >/dev/null && git rev-parse HEAD^1)"; then
+            echo "Not a pull request merge commit, running the full check."
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          files="$(git diff --name-only "$base" HEAD)"
+          if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/check-ui-api-types\.yml$)' <<< "$files"; then
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No proxy, types or generator changes in this pull request, nothing to verify."
+            echo "relevant=false" >> "$GITHUB_OUTPUT"
+          fi
 
       - name: Set up Python
+        if: steps.changes.outputs.relevant == 'true'
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.12"
 
       - name: Set up uv
+        if: steps.changes.outputs.relevant == 'true'
         uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        if: steps.changes.outputs.relevant == 'true'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -46,31 +68,37 @@ jobs:
             ${{ runner.os }}-uv-
 
       - name: Install backend dependencies
+        if: steps.changes.outputs.relevant == 'true'
         run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
       - name: Generate Prisma client
+        if: steps.changes.outputs.relevant == 'true'
         env:
           PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Set up Node.js
+        if: steps.changes.outputs.relevant == 'true'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
-          node-version: "20"
+          node-version-file: ui/litellm-dashboard/.nvmrc
           cache: "npm"
           cache-dependency-path: ui/litellm-dashboard/package-lock.json
 
       - name: Install dashboard dependencies
+        if: steps.changes.outputs.relevant == 'true'
         working-directory: ui/litellm-dashboard
         run: npm ci
 
       - name: Regenerate types from the live spec
+        if: steps.changes.outputs.relevant == 'true'
         working-directory: ui/litellm-dashboard
         env:
           LITELLM_PYTHON: "uv run --no-sync python"
         run: npm run gen:api
 
       - name: Fail if types are stale
+        if: steps.changes.outputs.relevant == 'true'
         run: |
           if ! git diff --exit-code -- ui/litellm-dashboard/src/lib/http/schema.d.ts; then
             echo "::error file=ui/litellm-dashboard/src/lib/http/schema.d.ts::Generated API types are out of sync with the proxy OpenAPI spec."

.github/workflows/ci-coverage.yml

@@ -0,0 +1,42 @@
+name: "CI Coverage"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  assert-ci-coverage:
+    name: assert-ci-coverage
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Assert every test file and Dockerfile is invoked by a job
+        run: |
+          python -m pip install "pyyaml==6.0.3"
+          python .github/scripts/assert_ci_coverage.py

.github/workflows/codspeed.yml

@@ -5,10 +5,24 @@ on:
     branches:
       - main
       - litellm_internal_staging
+    paths:
+      - "litellm/**"
+      - "tests/benchmarks/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/codspeed.yml"
+      - ".github/actions/setup-uv-with-retries/**"
   pull_request:
     branches:
       - main
       - litellm_internal_staging
+    paths:
+      - "litellm/**"
+      - "tests/benchmarks/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/codspeed.yml"
+      - ".github/actions/setup-uv-with-retries/**"
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:

.github/workflows/create_daily_oss_agent_shin_branch.yml

@@ -13,35 +13,16 @@ jobs:
       contents: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
       - name: Create daily oss-agent-shin branch
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          # Configure Git user
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          # Generate branch name with MM_DD_YYYY format
           BRANCH_NAME="litellm_oss_agent_shin_$(date +'%m_%d_%Y')"
           echo "Creating branch: $BRANCH_NAME"
-
-          # Fetch all branches
-          git fetch --all
-
-          # Check if the branch already exists
-          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+          if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
             echo "Branch $BRANCH_NAME already exists. Skipping creation."
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            # Create the new branch from main
-            git checkout -b $BRANCH_NAME origin/main
-            # Push the new branch
-            git push origin $BRANCH_NAME
-            echo "Successfully created and pushed branch: $BRANCH_NAME"
+            exit 0
           fi
+          MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
+          gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
+          echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"

.github/workflows/create_daily_oss_branch.yml

@@ -1,61 +0,0 @@
-name: Create Daily OSS Branch
-
-on:
-  schedule:
-    - cron: "0 16 * * 1-5" # 9am PT during daylight saving time, weekdays.
-  workflow_dispatch:
-    inputs:
-      date:
-        description: "Branch date in YYYY_MM_DD format. Defaults to today's UTC date."
-        required: false
-        type: string
-
-permissions:
-  contents: write
-
-jobs:
-  create-oss-branch:
-    if: github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - name: Create dated OSS branch
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          REQUESTED_DATE: ${{ inputs.date }}
-        run: |
-          set -euo pipefail
-
-          if [ -n "${REQUESTED_DATE}" ]; then
-            if ! echo "${REQUESTED_DATE}" | grep -Eq '^[0-9]{4}_[0-9]{2}_[0-9]{2}$'; then
-              echo "::error::date must use YYYY_MM_DD format, got '${REQUESTED_DATE}'"
-              exit 1
-            fi
-            BRANCH_DATE="${REQUESTED_DATE}"
-          else
-            BRANCH_DATE="$(date -u +'%Y_%m_%d')"
-          fi
-
-          BRANCH_NAME="litellm_oss_daily_${BRANCH_DATE}"
-          echo "Creating branch: ${BRANCH_NAME}"
-
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          git fetch origin main "${BRANCH_NAME}" || true
-
-          if git show-ref --verify --quiet "refs/remotes/origin/${BRANCH_NAME}"; then
-            echo "Branch ${BRANCH_NAME} already exists. Skipping creation."
-            exit 0
-          fi
-
-          git checkout -b "${BRANCH_NAME}" origin/main
-          git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "${BRANCH_NAME}"
-          echo "Successfully created and pushed branch: ${BRANCH_NAME}"

.github/workflows/create_daily_staging_branch.yml

@@ -13,38 +13,19 @@ jobs:
       contents: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
       - name: Create daily staging branch
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          # Configure Git user
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          # Generate branch name with MM_DD_YYYY format
           BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')"
           echo "Creating branch: $BRANCH_NAME"
-
-          # Fetch all branches
-          git fetch --all
-
-          # Check if the branch already exists
-          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+          if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
             echo "Branch $BRANCH_NAME already exists. Skipping creation."
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            # Create the new branch from main
-            git checkout -b $BRANCH_NAME origin/main
-            # Push the new branch
-            git push origin $BRANCH_NAME
-            echo "Successfully created and pushed branch: $BRANCH_NAME"
+            exit 0
           fi
+          MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
+          gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
+          echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"
 
   create-internal-dev-branch:
     if: github.repository == 'BerriAI/litellm'
@@ -53,35 +34,16 @@ jobs:
       contents: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
       - name: Create internal dev branch
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          # Configure Git user
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          # Generate branch name with MM_DD_YYYY format
           BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
           echo "Creating branch: $BRANCH_NAME"
-
-          # Fetch all branches
-          git fetch --all
-
-          # Check if the branch already exists
-          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+          if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
             echo "Branch $BRANCH_NAME already exists. Skipping creation."
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            # Create the new branch from main
-            git checkout -b $BRANCH_NAME origin/main
-            # Push the new branch
-            git push origin $BRANCH_NAME
-            echo "Successfully created and pushed branch: $BRANCH_NAME"
+            exit 0
           fi
+          MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
+          gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
+          echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"

.github/workflows/guard-main-branch.yml

@@ -31,12 +31,12 @@ jobs:
           echo "PR head repo: $HEAD_REPO"
           echo "PR head branch: $HEAD_REF"
           if [ "$HEAD_REPO" != "$BASE_REPO" ]; then
-            echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the current daily OSS branch (named litellm_oss_daily_YYYY_MM_DD; a fresh one is cut each weekday, so target the most recent) instead."
+            echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against 'litellm_internal_staging' instead."
             exit 1
           fi
           if [ "$HEAD_REF" = "litellm_internal_staging" ] || [[ "$HEAD_REF" == litellm_hotfix_?* ]]; then
             echo "Allowed source branch."
             exit 0
           fi
-          echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against the current daily OSS branch (named litellm_oss_daily_YYYY_MM_DD; a fresh one is cut each weekday, so target the most recent) instead."
+          echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_internal_staging' instead."
           exit 1

.github/workflows/helm_unit_test.yml

@@ -23,21 +23,28 @@ jobs:
         with:
           version: "3.11.1"
 
+      - name: Download and verify Helm Unit Test Plugin
+        run: |
+          curl -fsSLo "$RUNNER_TEMP/helm-unittest.tgz" https://github.com/helm-unittest/helm-unittest/releases/download/v0.8.2/helm-unittest-linux-amd64-0.8.2.tgz
+          echo "56ab3091e6fa52a7c92ee951def9bed957f295d9ce98483aed404e748d7b3a94  $RUNNER_TEMP/helm-unittest.tgz" | sha256sum -c -
+
       - name: Install Helm Unit Test Plugin
         run: |
-          helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
-      - name: Verify Helm Unit Test Plugin integrity
-        run: |
-          EXPECTED_SHA="e251ba198448629678ff2168e1a469249d998155"
           PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest"
-          ACTUAL_SHA="$(git -C "$PLUGIN_DIR" rev-parse HEAD)"
-          if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then
-            echo "::error::Helm unittest plugin checksum mismatch! Expected $EXPECTED_SHA but got $ACTUAL_SHA"
-            exit 1
-          fi
-          echo "Helm unittest plugin integrity verified: $ACTUAL_SHA"
+          mkdir -p "$PLUGIN_DIR"
+          tar -xzf "$RUNNER_TEMP/helm-unittest.tgz" -C "$PLUGIN_DIR"
+          helm plugin list
 
       - name: Run unit tests
         run: |
-          helm unittest -f 'tests/*.yaml' helm/litellm-helm
-          helm unittest -f 'tests/*.yaml' helm/litellm
+          for chart in helm/litellm-helm helm/litellm; do
+            declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')"
+            output="$(mktemp)"
+            helm unittest -f 'tests/*.yaml' "$chart" | tee "$output"
+            executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")"
+            if [ "$declared" != "$executed" ]; then
+              echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute."
+              exit 1
+            fi
+            echo "$chart: all $declared declared test suites ran"
+          done

.github/workflows/image-scan.yml

@@ -8,7 +8,17 @@ on:
       - litellm_oss_branch
       - "litellm_**"
     paths:
+      - Dockerfile
       - docker/Dockerfile.non_root
+      - migrations/Dockerfile
+      - migrations/run.py
+      - gateway/Dockerfile
+      - gateway/main.py
+      - backend/Dockerfile
+      - backend/main.py
+      - docker/component_entrypoint.sh
+      - litellm-proxy-extras/**
+      - tests/proxy_migration_tests/**
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
       - .github/workflows/image-scan.yml
@@ -51,6 +61,23 @@ jobs:
       - name: Build runtime image
         run: docker build -f docker/Dockerfile.non_root -t litellm-image-scan:${{ github.sha }} .
 
+      # The prisma bake must migrate a fresh DB with no egress as an arbitrary
+      # non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyRootFilesystem).
+      # `docker run` as the default uid with network hides a broken bake because
+      # the migration entrypoint exits 0 even when it applied nothing; asserting
+      # the schema was created is what catches it.
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify offline migration as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-image-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+
       # Scans the whole shipped artifact: OS/apk plus every language package
       # baked into the image, including ones no lockfile declares (e.g. prisma's
       # vendored node engine) that osv-scan cannot see. osv-scan stays the fast
@@ -58,8 +85,130 @@ jobs:
       # free OSS, run as a pinned, checksum-verified binary; no GitHub Action
       # dependency and no vendor SaaS callout.
       - name: Scan image for fixable HIGH/CRITICAL CVEs
+        env:
+          GRYPE_MATCH_PYTHON_USING_CPES: "true"
         run: |
           "$RUNNER_TEMP/grype" litellm-image-scan:${{ github.sha }} \
             --only-fixed \
             --fail-on high \
             --output table
+
+  runtime-image:
+    name: runtime-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build runtime image
+        run: docker build -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify offline migration as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+
+  migrations-image:
+    name: migrations-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build migrations image
+        run: docker build -f migrations/Dockerfile -t litellm-migrations-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify offline migration as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-migrations-scan:${{ github.sha }}
+          LITELLM_MIGRATION_INTERPRETER: python3
+          LITELLM_MIGRATION_SCRIPT: /app/run.py
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+
+  gateway-image:
+    name: gateway-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build gateway image
+        run: docker build -f gateway/Dockerfile -t litellm-gateway-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the gateway serves offline as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-gateway-scan:${{ github.sha }}
+          LITELLM_COMPONENT_PORT: "4000"
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
+
+  backend-image:
+    name: backend-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build backend image
+        run: docker build -f backend/Dockerfile -t litellm-backend-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the backend serves offline as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-backend-scan:${{ github.sha }}
+          LITELLM_COMPONENT_PORT: "4001"
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v

.github/workflows/mutation-test.yml

@@ -55,7 +55,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
 
       - name: Generate Prisma client
         env:

.github/workflows/oss_daily_guardrails.yml

@@ -1,50 +0,0 @@
-name: OSS Daily Guardrails
-
-on:
-  push:
-    branches:
-      - "litellm_oss_daily_20*"
-  pull_request:
-    branches:
-      - "litellm_oss_daily_20*"
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  oss-safe-checks:
-    name: Run OSS daily safe checks
-    if: startsWith(github.ref_name, 'litellm_oss_daily_20') || startsWith(github.head_ref, 'litellm_oss_daily_20') || startsWith(github.base_ref, 'litellm_oss_daily_20')
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Run secret scan test
-        run: |
-          uv run --frozen --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v
-
-      - name: Run Ruff
-        run: |
-          uv sync --frozen
-          cd litellm
-          uv run --no-sync ruff check .

.github/workflows/publish-basedpyright-base-counts.yml

@@ -0,0 +1,62 @@
+name: Publish basedpyright base counts
+
+# Every commit on litellm_internal_staging is some branch's future merge-base.
+# Publishing its per-rule basedpyright counts as an artifact lets
+# scripts/type_check_gate.py download them in seconds instead of paying a
+# 60-110s second basedpyright pass on every fresh worktree or moved merge-base.
+# No concurrency group on purpose: runs must never cancel each other, because
+# every sha's artifact matters (any of them can become a merge-base).
+
+on:
+  push:
+    branches:
+      - litellm_internal_staging
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: "Ref to compute and publish base counts for"
+        required: false
+        default: litellm_internal_staging
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+          clean: true
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      # The gate provisions its own measurement env (.venv-typecheck: a frozen
+      # uv sync of its canonical dependency groups plus a generated Prisma
+      # client), so no install step here can drift from what local runs measure.
+      - name: Emit basedpyright counts for HEAD
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts"
+          counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json)
+          echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
+
+      - name: Upload counts artifact
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: ${{ env.COUNTS_ARTIFACT_NAME }}
+          path: ${{ runner.temp }}/basedpyright-counts/
+          if-no-files-found: error

.github/workflows/test-code-quality.yml

@@ -7,13 +7,17 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   code-quality:
@@ -115,6 +119,9 @@ jobs:
       - name: check_fastuuid_usage
         run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
 
+      - name: check_e2e_no_raw_requests
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

.github/workflows/test-linting.yml

@@ -15,6 +15,12 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    # actions: read lets scripts/type_check_gate.py download the base-counts
+    # artifact published by publish-basedpyright-base-counts.yml instead of
+    # re-running basedpyright over the merge-base tree.
+    permissions:
+      contents: read
+      actions: read
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
@@ -23,10 +29,21 @@ jobs:
         # Any-discipline) would otherwise blame on this branch.
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
+          fetch-depth: 1
           clean: true
           persist-credentials: false
 
+      - name: Fetch gate base (merge-base with target branch)
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          MERGE_BASE=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          test -n "$MERGE_BASE"
+          git fetch --no-tags --depth=1 origin "$MERGE_BASE"
+          echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
@@ -60,10 +77,8 @@ jobs:
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Check ruff format
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          git diff --name-only --diff-filter=ACMR "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
+          git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
           if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
             echo "No changed litellm Python files to check with ruff format."
             exit 0
@@ -86,16 +101,12 @@ jobs:
           cd ..
 
       - name: Check strict-rule budget (delta vs base)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          uv run --no-sync python scripts/ruff_strict_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
 
       - name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          uv run --no-sync python scripts/type_discipline_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
 
       - name: Print OpenAI version
         run: |
@@ -103,15 +114,14 @@ jobs:
 
       - name: Check basedpyright budget (delta vs base)
         env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          GH_TOKEN: ${{ github.token }}
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
-          (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA"
 
       - name: Check tests/e2e basedpyright (zero errors)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          if git diff --name-only --diff-filter=ACMRD "$BASE_SHA"...HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
+          if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
             uv run --no-sync basedpyright tests/e2e
           else
             echo "No changed tests/e2e Python files; skipping."
@@ -140,9 +150,15 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
+      - name: Fetch ratchet base
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          git fetch --no-tags --depth=1 origin "$BASE_SHA"
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
@@ -163,7 +179,7 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: Set up Python
@@ -178,13 +194,14 @@ jobs:
 
       - name: Run secret scan test
         run: |
-          uv run --frozen --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v
+          uv run --no-project --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v
 
       - name: Run ggshield secret scan
         env:
           GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
         run: |
           if [ -n "$GITGUARDIAN_API_KEY" ]; then
+            git fetch --no-tags --unshallow origin
             uv tool run --from 'ggshield==1.48.0' ggshield secret scan repo .
           else
             echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"

.github/workflows/test-litellm-ui-build.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
-          node-version: "20"
+          node-version-file: ui/litellm-dashboard/.nvmrc
           cache: "npm"
           cache-dependency-path: ui/litellm-dashboard/package-lock.json
 

.github/workflows/test-litellm-ui-lint.yml

@@ -22,14 +22,25 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: Collect changed files
         id: changed
         env:
+          GH_TOKEN: ${{ github.token }}
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
+          # base.sha is the base branch tip from when the PR was opened, while
+          # actions/checkout leaves HEAD on a merge of the PR into the *current*
+          # base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
+          # landed since, so a PR that touches no UI file still gets linted
+          # against hundreds of other people's files. Diff the PR head against its
+          # own merge base instead, which is exactly what this PR changed.
+          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          test -n "$merge_base"
+          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           : > "$RUNNER_TEMP/prettier_files.txt"
           : > "$RUNNER_TEMP/eslint_files.txt"
           while IFS= read -r f; do
@@ -41,7 +52,7 @@ jobs:
               *.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
                 printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
             esac
-          done < <(git diff --name-only --diff-filter=ACMR --relative "$BASE_SHA"...HEAD -- .)
+          done < <(git diff --name-only --diff-filter=ACMR --relative "$merge_base" "$HEAD_SHA" -- .)
           if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
             echo "has_files=true" >> "$GITHUB_OUTPUT"
           else
@@ -53,7 +64,7 @@ jobs:
         if: steps.changed.outputs.has_files == 'true'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
-          node-version: "20"
+          node-version-file: ui/litellm-dashboard/.nvmrc
           cache: "npm"
           cache-dependency-path: ui/litellm-dashboard/package-lock.json
 

.github/workflows/test-litellm-ui-unit.yml

@@ -0,0 +1,70 @@
+name: UI Unit Tests
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - litellm_internal_staging
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  ui-unit-tests:
+    runs-on: ubuntu-latest-16-cores
+    timeout-minutes: 20
+    defaults:
+      run:
+        working-directory: ui/litellm-dashboard
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
+        with:
+          node-version-file: ui/litellm-dashboard/.nvmrc
+          cache: "npm"
+          cache-dependency-path: ui/litellm-dashboard/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run UI unit tests (Vitest)
+        env:
+          CI: "true"
+          GH_TOKEN: ${{ github.token }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          if [ -n "$BASE_SHA" ]; then
+            merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+            test -n "$merge_base"
+            git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+            changed_files=()
+            while IFS= read -r f; do
+              changed_files+=("$f")
+            done < <(git diff --name-only --relative "$merge_base" "$HEAD_SHA" -- .)
+            if [ ${#changed_files[@]} -eq 0 ]; then
+              echo "No UI files changed in this PR; skipping unit tests."
+              exit 0
+            fi
+            echo "Pull request: running tests related to ${#changed_files[@]} changed UI files"
+            npm run test -- related "${changed_files[@]}" --run --passWithNoTests \
+              --pool forks --poolOptions.forks.maxForks=14
+          else
+            echo "Push to $GITHUB_REF_NAME: running the full suite"
+            npm run test -- --run --pool forks --poolOptions.forks.maxForks=14
+          fi

.github/workflows/test-model-map.yaml

@@ -22,3 +22,12 @@ jobs:
       - name: Validate model_prices_and_context_window.json
         run: |
           jq empty model_prices_and_context_window.json
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Check model_prices_and_context_window.schema.json is in sync
+        run: |
+          uv run --frozen python ci_cd/generate_model_prices_schema.py --check

.github/workflows/test-rust.yml

@@ -61,5 +61,11 @@ jobs:
       - name: Run Clippy
         run: cargo clippy --workspace --all-targets --locked -- -D warnings
 
+      - name: Run Clippy with Bedrock auth
+        run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
+
       - name: Run Rust tests
         run: cargo test --workspace --locked
+
+      - name: Run core tests with Bedrock auth
+        run: cargo test -p litellm-core --features bedrock-auth --locked

.github/workflows/test-unit-core-utils.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   core-utils:

.github/workflows/test-unit-documentation.yml

@@ -7,13 +7,17 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   documentation:

.github/workflows/test-unit-enterprise-routing.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   enterprise-routing:

.github/workflows/test-unit-integrations.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   integrations:

.github/workflows/test-unit-llm-providers.yml

@@ -7,13 +7,17 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   vertex-ai:

.github/workflows/test-unit-misc.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   misc:
@@ -27,6 +31,7 @@ jobs:
         tests/test_litellm/a2a_protocol
         tests/test_litellm/anthropic_interface
         tests/test_litellm/completion_extras
+        tests/test_litellm/compression
         tests/test_litellm/containers
         tests/test_litellm/experimental_mcp_client
         tests/test_litellm/models
@@ -35,7 +40,11 @@ jobs:
         tests/test_litellm/interactions
         tests/test_litellm/ocr
         tests/test_litellm/passthrough
+        tests/test_litellm/rag
+        tests/test_litellm/realtime_api
+        tests/test_litellm/rerank_api
         tests/test_litellm/sandbox
+        tests/test_litellm/test_router
         tests/test_litellm/vector_stores
         tests/test_litellm/videos
         tests/test_litellm/test_*.py

.github/workflows/test-unit-proxy-auth.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   proxy-auth:

.github/workflows/test-unit-proxy-db.yml

@@ -5,13 +5,19 @@ on:
     branches:
       - main
       - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 # Semantic matrix: each shard groups tests by concern (auth, server, logging, …)
 # rather than alphabetical letter ranges. Adding a new test file means adding it

.github/workflows/test-unit-proxy-endpoints.yml

@@ -7,14 +7,18 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
   workflow_dispatch:
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   proxy-endpoints:
@@ -25,7 +29,9 @@ jobs:
     uses: ./.github/workflows/_test-unit-base.yml
     with:
       test-path: >-
+        tests/test_litellm/proxy/analytics_endpoints
         tests/test_litellm/proxy/management_endpoints
+        tests/test_litellm/proxy/memory
         tests/test_litellm/proxy/guardrails
         tests/test_litellm/proxy/management_helpers
         tests/test_litellm/proxy/anthropic_endpoints
@@ -38,6 +44,7 @@ jobs:
         tests/test_litellm/proxy/vector_store_endpoints
         tests/test_litellm/proxy/agent_endpoints
         tests/test_litellm/proxy/a2a
+        tests/test_litellm/proxy/credential_endpoints
         tests/test_litellm/proxy/discovery_endpoints
         tests/test_litellm/proxy/health_endpoints
         tests/test_litellm/proxy/shutdown
@@ -46,6 +53,7 @@ jobs:
         tests/test_litellm/proxy/rag_endpoints
         tests/test_litellm/proxy/realtime_endpoints
         tests/test_litellm/proxy/ui_crud_endpoints
+        tests/test_litellm/proxy/config_resolvers
         tests/test_litellm/proxy/utils
       workers: 2
       reruns: 2

.github/workflows/test-unit-proxy-infra.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   proxy-infra:
@@ -29,6 +33,8 @@ jobs:
         tests/test_litellm/proxy/_experimental
         tests/test_litellm/proxy/experimental
         tests/test_litellm/proxy/common_utils
+        tests/test_litellm/proxy/enterprise_billing
+        tests/test_litellm/proxy/types_utils
         tests/test_litellm/proxy/logging_endpoints
         tests/test_litellm/proxy/test_*.py
       workers: 2

.github/workflows/test-unit-proxy-legacy.yml

@@ -7,13 +7,17 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   test:

.github/workflows/test-unit-responses-caching-types.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   responses-caching-types:

.github/workflows/test_server_root_path.yml

@@ -1,151 +0,0 @@
-name: Test Proxy SERVER_ROOT_PATH Routing
-permissions:
-  contents: read
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-
-jobs:
-  test-server-root-path:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-
-    strategy:
-      fail-fast: false
-      matrix:
-        root_path: ["/api/v1", "/llmproxy"]
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Free up disk space
-        run: |
-          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/share/boost
-          sudo apt-get clean
-          df -h /
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
-
-      - name: Build Docker image
-        uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 # v6.14.0
-        with:
-          context: .
-          file: ./docker/Dockerfile.non_root
-          tags: litellm-test:${{ github.sha }}
-          load: true
-          push: false
-
-      - name: Start LiteLLM container with SERVER_ROOT_PATH
-        run: |
-          docker run -d \
-            --name litellm-test \
-            -p 4000:4000 \
-            -e SERVER_ROOT_PATH="${{ matrix.root_path }}" \
-            -e LITELLM_MASTER_KEY="sk-1234" \
-            litellm-test:${{ github.sha }} \
-            --detailed_debug
-
-      - name: Wait for container to be healthy
-        run: |
-          echo "Waiting for LiteLLM to start..."
-          max_attempts=30
-          attempt=0
-
-          while [ $attempt -lt $max_attempts ]; do
-            if docker logs litellm-test 2>&1 | grep -q "Uvicorn running"; then
-              echo "LiteLLM started successfully"
-              break
-            fi
-            attempt=$((attempt + 1))
-            echo "Attempt $attempt/$max_attempts - waiting for server to start..."
-            sleep 2
-          done
-
-          if [ $attempt -eq $max_attempts ]; then
-            echo "Server failed to start within timeout"
-            docker logs litellm-test
-            exit 1
-          fi
-
-          sleep 5
-
-      - name: Show container logs
-        if: always()
-        run: docker logs litellm-test
-
-      - name: Test UI endpoint with root path
-        run: |
-          ROOT_PATH="${{ matrix.root_path }}"
-          echo "Testing UI at: http://localhost:4000${ROOT_PATH}/ui/"
-
-          for i in 1 2 3; do
-            content=$(curl -sL --max-time 5 -H "Authorization: Bearer sk-1234" "http://localhost:4000${ROOT_PATH}/ui/")
-            if echo "$content" | grep -q -E "(html|<!DOCTYPE|<head|<body)"; then
-              echo "UI page contains valid HTML content"
-              exit 0
-            fi
-            echo "Attempt $i/3 - no valid HTML, retrying in 5s..."
-            sleep 5
-          done
-          echo "UI page does not contain expected HTML content"
-          echo "Response: $content"
-          docker logs litellm-test
-          exit 1
-
-      - name: Setup Node for Playwright
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
-        with:
-          node-version: "20"
-
-      - name: Install UI deps and Chromium
-        working-directory: ui/litellm-dashboard
-        run: |
-          retry() {
-            local attempt=1
-            local max_attempts=4
-            until "$@"; do
-              if [ "$attempt" -ge "$max_attempts" ]; then
-                echo "Command failed after $attempt attempts: $*"
-                return 1
-              fi
-              echo "Attempt $attempt failed: $*. Retrying in $((attempt * 15))s..."
-              sleep $((attempt * 15))
-              attempt=$((attempt + 1))
-            done
-          }
-
-          npm config set fetch-retries 5
-          npm config set fetch-retry-mintimeout 20000
-          npm config set fetch-retry-maxtimeout 120000
-
-          retry npm ci
-          retry npx playwright install --with-deps chromium
-
-      - name: Run SERVER_ROOT_PATH redirect e2e
-        working-directory: ui/litellm-dashboard
-        env:
-          SERVER_ROOT_PATH: ${{ matrix.root_path }}
-        run: npx playwright test --config=e2e_tests/serverRootPath.config.ts
-
-      - name: Upload Playwright artifacts on failure
-        if: failure()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: playwright-trace-${{ strategy.job-index }}
-          path: ui/litellm-dashboard/test-results/
-          retention-days: 7
-
-      - name: Cleanup
-        if: always()
-        run: |
-          docker stop litellm-test || true
-          docker rm litellm-test || true

.github/workflows/weekly_load_anomaly.yml

@@ -0,0 +1,81 @@
+name: "Weekly Load Anomaly Check"
+
+on:
+  schedule:
+    - cron: "0 12 * * 6"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  weekly-load-anomaly:
+    if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-weekly-anomaly-check
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/load/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Run the weekly session anomaly test
+        env:
+          E2E_WEEKLY_ANOMALY: "1"
+        run: |
+          uv run --no-sync pytest tests/e2e/load/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log

.github/workflows/zizmor.yml

@@ -4,7 +4,11 @@ on:
   push:
     branches: [main, litellm_internal_staging]
   pull_request:
-    branches: [main, litellm_internal_staging]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.gitignore

@@ -1,5 +1,6 @@
 .python-version
 .venv
+.venv-typecheck
 .venv_policy_test
 .env
 .claude
@@ -15,6 +16,9 @@ litellm/rust_bridge/_native*.so
 litellm/rust_bridge/_native*.pyd
 litellm-rust/target/
 
+# Python package build output
+dist/
+
 bun.lockb
 **/.DS_Store
 .aider*
@@ -138,3 +142,4 @@ crash.*.log
 .coverage
 
 ui/litellm-dashboard/out/
+litellm.log

CLAUDE.md

@@ -19,7 +19,7 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
 
 End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`
 
-When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose for internal contributors; external / OSS contributions target the current daily OSS branch instead, named `litellm_oss_daily_YYYY_MM_DD` (a fresh one is cut each weekday, so use the most recent)
+When creating PRs, don't set base to `main`. `litellm_internal_staging` is the default base branch and serves that purpose for both internal and external / OSS contributions
 
 When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
 
@@ -29,7 +29,7 @@ Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We pref
 
 If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
 - don't use emojis
-- don't use "—". Instead, reach for ";", ".", etc.
+- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
 - don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
@@ -39,15 +39,13 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
 
 Python max line length is 120, not 88
 
-On a fresh worktree or clone, run `make bootstrap` before anything else. It provisions everything tests, `make pre-commit`, and a local proxy need
-
-Run tests before you commit. Also, run `make pre-commit` right before each commit, which generates types (as needed) and formats/lints your code. Any errors found must be fixed. It only runs when there are staged frontend and/or backend changes and calculates violations, generates types, etc. based on the worktree, so stage what you need or stash/delete unwanted files in litellm/ or ui/ (where backend and frontend lint run, respectively) before running it. If it fails because dashboard api types are stale, it already regenerated them for you. You just need to stage the schema.d.ts, re-run `make pre-commit` to confirm it passes, and commit
-
 When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing
 
+`make pre-commit` saves its complete output to a log file in .git (overwriting previous pre-commit logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
+
 If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
 
-If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
+If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
 
 Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType]  # stubs lack async overload` or `# noqa: TID251  # <reason>`. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing
 
@@ -61,6 +59,8 @@ Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages
 
 When working on a PR, keep the PR description in sync with new commits being made
 
+All GitHub comments must be human-readable and 15-25 words max
+
 Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer dependency-injecting things into classes. That way, at unit test time, you can pass a mocked dependency in
 
 Do not put names of customers or customer company names in code, PR descriptions, issue bodies, etc. This means never mention literally any company name. Especially if you're about to say a sentence mentioning that the reason the PR exists was a feature/model/bug fix/etc. requested by a company. That's the indication that you should replace that company name with "the customer". e.g. not "Model request from Acme (Pylon #1234)" but "Model request from a customer (Pylon #1234)". This is because the codebase is public. The only exception is for publicly known providers or vendors such as OpenAI, Anthropic, AWS Bedrock, etc. only IF we're adding support for that provider/vendor in general and NOT if that PR or whatnot was a request by one of them, and they're actually one of our customers. 
@@ -72,7 +72,8 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - Composition over inheritance
 - Never-nester: early returns over deep nesting
 - Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
-- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), etc.
+- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc.
+  - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>` explaining why
 - Use dependency injection
 - Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
 - Use tagged unions + match

CONTRIBUTING.md

@@ -322,7 +322,7 @@ npm run build
 ## Submitting Your PR
 
 1. **Push your branch**: `git push origin your-feature-branch`
-2. **Create a PR**: Go to GitHub and open a pull request against the current daily OSS branch, named `litellm_oss_daily_YYYY_MM_DD`. A fresh one is cut each weekday, so pick the most recent from the [branch list](https://github.com/BerriAI/litellm/branches/all?query=litellm_oss_daily). Do not target `main`.
+2. **Create a PR**: Go to GitHub and open a pull request against [`litellm_internal_staging`](https://github.com/BerriAI/litellm/tree/litellm_internal_staging), which is the default base branch. Do not target `main`.
 3. **Fill out the PR template**: Provide clear description of changes
 4. **Wait for review**: Maintainers will review and provide feedback
 5. **Address feedback**: Make requested changes and push updates

Dockerfile

@@ -7,7 +7,7 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17
 ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
-ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6
+ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
 
 FROM $UV_IMAGE AS uvbin
 
@@ -64,6 +64,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
 # Copy full source tree
@@ -84,9 +85,12 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -100,7 +104,11 @@ USER root
 RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
 
 WORKDIR /app
-ENV PATH="/app/.venv/bin:${PATH}"
+ENV PATH="/app/.venv/bin:${PATH}" \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
+    PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
+    PRISMA_OFFLINE_MODE=true
 
 # Copy only what runtime needs. The application is installed inside the venv;
 # the rest of the builder's /app is source and build metadata that must not
@@ -114,16 +122,20 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
 # working directory on sys.path; litellm/proxy/hooks resolves
 # enterprise.enterprise_hooks from it)
 COPY --from=builder /app/enterprise /app/enterprise
-# Prisma binaries live in $HOME/.cache (default prisma-python location),
-# which is /root/.cache here. Copy only the Prisma subdirs — copying the
-# whole /root/.cache drags in the uv build cache (~660 MB, includes a
-# setuptools wheel that surfaces as a CVE finding even though it's not
-# on the runtime sys.path).
-COPY --from=builder /root/.cache/prisma /root/.cache/prisma
-COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
+COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
+# Prisma CLI + engines are baked under /opt/prisma, a fixed path every
+# runtime uid can read and that no cache volume mount shadows. The paths are
+# pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and recorded into the
+# generated client at build time, so `prisma migrate deploy` on a fresh
+# database needs no npm and no network access (#33650, #24554).
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 EXPOSE 4000/tcp
 

Makefile

@@ -75,7 +75,7 @@ install-dev:
 bootstrap:
 	$(UV) sync --inexact --frozen --extra proxy --group proxy-dev --group e2e-dev
 	$(UV_RUN) python scripts/prisma_generate_if_needed.py
-	cd ui/litellm-dashboard && npm ci --no-audit --no-fund
+	cd ui/litellm-dashboard && ../../scripts/with_dashboard_node.sh npm install --no-audit --no-fund
 	@main_root=$$(git worktree list --porcelain | head -1 | sed 's/^worktree //'); \
 	if [ "$$main_root" != "$$(git rev-parse --show-toplevel)" ] && [ -f "$$main_root/.env" ] && [ ! -f .env ]; then \
 		cp "$$main_root/.env" .env && echo "bootstrap: copied .env from $$main_root"; \
@@ -99,7 +99,10 @@ install-test-deps: install-proxy-dev
 	$(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma
 
 install-helm-unittest:
-	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
+	@helm plugin list | grep -qE '^unittest[[:space:]]+0\.8\.2([[:space:]]|$$)' || { \
+		helm plugin uninstall unittest >/dev/null 2>&1 || true; \
+		helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.8.2; \
+	}
 
 # Install git hooks that enforce Conventional Commits and Conventional Branches.
 # Opt-in: not chained into install-dev.
@@ -121,10 +124,10 @@ lint-fetch-base:
 	git fetch origin litellm_internal_staging
 
 # Mirror test-linting.yml's lint job environment: the proxy-dev group plus a generated
-# Prisma client, so basedpyright resolves the same modules CI does (without the generated
-# client the DB wrappers typed against it degrade to Unknown, drifting the budget from
-# CI's). --inexact tops up the venv instead of pruning the proxy extras gen:api and the
-# running proxy need.
+# Prisma client, so `basedpyright tests/e2e` resolves the same modules CI does. The
+# budget gate itself no longer measures here (scripts/type_check_gate.py provisions its
+# own .venv-typecheck). --inexact tops up the venv instead of pruning the proxy extras
+# gen:api and the running proxy need.
 lint-install:
 	$(UV) sync --inexact --frozen --group proxy-dev --group e2e-dev
 	$(UV_RUN) python scripts/prisma_generate_if_needed.py
@@ -177,7 +180,7 @@ lint-ruff-FULL-dev: install-dev
 	else echo "No changed .py files to check."; fi
 
 lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
-	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
+	$(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
 
 lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
 	$(UV_RUN) basedpyright tests/e2e
@@ -190,7 +193,7 @@ lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 # --update lowers each limit by what this branch fixed since its branch point, so
 # it needs the base ref fetched to resolve the merge-base.
 lint-basedpyright-budget-update: install-dev lint-fetch-base
-	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update
+	$(UV_RUN) python scripts/type_check_gate.py --update
 
 lint-format: format-check
 
@@ -237,7 +240,7 @@ lint-dev: lint-format-changed check-circular-imports check-import-safety
 # test-linting.yml (Python), test-litellm-ui-build.yml's frontend-lint (dashboard), and
 # check-ui-api-types.yml (API-type drift), skipping any whose files you didn't stage.
 # Not auto-installed as a git hook so it never slows an unrelated human commit.
-pre-commit:
+pre-commit: bootstrap
 	./scripts/pre_commit_lint.sh
 
 # Testing targets

backend/Dockerfile

@@ -59,9 +59,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra semantic-router \
         --python python3
 
-RUN mkdir -p /home/nonroot && \
-    HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
-    chown -R nonroot:nonroot /home/nonroot/.cache
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
+
+RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh
 
 # ---------- Runtime ----------
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
@@ -81,17 +83,20 @@ ENV HOME=/home/nonroot \
     PATH="/app/.venv/bin:${PATH}" \
     PYTHONPATH="/app" \
     PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1
+    PYTHONUNBUFFERED=1 \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
 COPY --from=builder --chown=nonroot:nonroot /app /app
-COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 USER nonroot
 
 EXPOSE 4001/tcp
 
-ENTRYPOINT ["uvicorn", "backend.main:app"]
+ENTRYPOINT ["/app/docker/component_entrypoint.sh", "uvicorn", "backend.main:app"]
 CMD ["--host", "0.0.0.0", "--port", "4001"]

backend/routes/allowlist.py

@@ -18,6 +18,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/team/",
     "/v2/team/",
     "/organization/",
+    "/v2/organization/",
     "/customer/",
     "/end_user/",
     "/sso/",
@@ -43,6 +44,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/router/",
     "/router_settings",
     "/adaptive_router/",
+    "/auto_router/",
     "/fallback",
     "/fallbacks",
     "/cache_settings",
@@ -69,6 +71,10 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/project/",
     "/memory/",
     "/mcp/",
+    # Control plane (see the List Endpoints + Tables standard). Every resource
+    # eventually moves under this prefix, so allowlist it once rather than
+    # per-resource.
+    "/management/v1/",
     # Spend / analytics
     "/spend/",
     "/analytics/",
@@ -76,6 +82,9 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/user_agent",
     "/usage/",
     "/daily/",
+    # Deployment-wide gateway request counts. Scoped to the analytics read rather
+    # than all of /gateway/, which stays free for data-plane routes.
+    "/gateway/daily/",
     # CloudZero cost-export admin (init / settings / export / dry-run / delete)
     "/cloudzero/",
     # Caching admin

basedpyright-code-budget.json

@@ -1,66 +1,66 @@
 {
   "reportAny": {
-    "limit": 37484
+    "limit": 28842
   },
   "reportArgumentType": {
-    "limit": 2704
+    "limit": 2634
   },
   "reportAssignmentType": {
-    "limit": 330
+    "limit": 329
   },
   "reportAttributeAccessIssue": {
-    "limit": 516
+    "limit": 514
   },
   "reportCallIssue": {
-    "limit": 124
+    "limit": 117
   },
   "reportConstantRedefinition": {
-    "limit": 59
+    "limit": 40
   },
   "reportDeprecated": {
-    "limit": 326
+    "limit": 215
   },
   "reportDuplicateImport": {
-    "limit": 42
+    "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 10397
+    "limit": 9103
   },
   "reportFunctionMemberAccess": {
-    "limit": 11
+    "limit": 7
   },
   "reportGeneralTypeIssues": {
-    "limit": 227
+    "limit": 157
   },
   "reportIncompatibleMethodOverride": {
-    "limit": 78
+    "limit": 56
   },
   "reportIncompatibleVariableOverride": {
-    "limit": 12
+    "limit": 8
   },
   "reportInconsistentOverload": {
-    "limit": 18
+    "limit": 12
   },
   "reportIndexIssue": {
-    "limit": 37
+    "limit": 35
   },
   "reportInvalidTypeForm": {
     "limit": 35
   },
   "reportInvalidTypeVarUse": {
-    "limit": 5
+    "limit": 2
   },
   "reportMatchNotExhaustive": {
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5900
+    "limit": 5843
   },
   "reportMissingTypeArgument": {
-    "limit": 15903
+    "limit": 15816
   },
   "reportMissingTypeStubs": {
-    "limit": 41
+    "limit": 40
   },
   "reportOperatorIssue": {
     "limit": 0
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1085
+    "limit": 1078
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -81,16 +81,16 @@
     "limit": 0
   },
   "reportPossiblyUnboundVariable": {
-    "limit": 77
+    "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 2438
+    "limit": 1824
   },
   "reportRedeclaration": {
-    "limit": 12
+    "limit": 8
   },
   "reportReturnType": {
-    "limit": 225
+    "limit": 218
   },
   "reportTypedDictNotRequiredAccess": {
     "limit": 27
@@ -99,48 +99,48 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 45894
+    "limit": 45098
   },
   "reportUnknownLambdaType": {
     "limit": 113
   },
   "reportUnknownMemberType": {
-    "limit": 40539
+    "limit": 39826
   },
   "reportUnknownParameterType": {
-    "limit": 20403
+    "limit": 20237
   },
   "reportUnknownVariableType": {
-    "limit": 32141
+    "limit": 31371
   },
   "reportUnnecessaryCast": {
-    "limit": 177
+    "limit": 122
   },
   "reportUnnecessaryComparison": {
-    "limit": 1025
+    "limit": 701
   },
   "reportUnnecessaryContains": {
-    "limit": 7
+    "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 1209
+    "limit": 864
   },
   "reportUntypedBaseClass": {
-    "limit": 165
+    "limit": 0
   },
   "reportUntypedFunctionDecorator": {
     "limit": 33
   },
   "reportUnusedClass": {
-    "limit": 33
+    "limit": 23
   },
   "reportUnusedFunction": {
-    "limit": 206
+    "limit": 139
   },
   "reportUnusedImport": {
-    "limit": 1005
+    "limit": 555
   },
   "reportUnusedVariable": {
-    "limit": 1297
+    "limit": 146
   }
 }

ci_cd/generate_model_prices_schema.py

@@ -0,0 +1,325 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Optional
+
+import jsonschema
+
+REPO_ROOT = Path(__file__).parent.parent
+PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
+SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
+
+SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
+
+JsonSchema = dict
+
+NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
+NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
+BOOLEAN: JsonSchema = {"type": "boolean"}
+STRING: JsonSchema = {"type": "string"}
+
+EXTRA_BOOLEAN_KEYS = frozenset(
+    {
+        "gemini_native_audio",
+        "gemini_audio_only_live",
+        "uses_embed_content",
+        "use_openai_responses_path",
+        "bedrock_converse_supports_strict_tools",
+    }
+)
+
+OBJECT_KEYS: dict[str, JsonSchema] = {
+    "search_context_cost_per_query": {
+        "type": "object",
+        "description": "USD cost per web search query, keyed by search context size.",
+        "properties": {
+            "search_context_size_low": NONNEG_NUMBER,
+            "search_context_size_medium": NONNEG_NUMBER,
+            "search_context_size_high": NONNEG_NUMBER,
+        },
+        "additionalProperties": False,
+    },
+    "metadata": {
+        "type": "object",
+        "description": "Free-form notes about the entry (e.g. pricing derivation).",
+    },
+    "provider_specific_entry": {
+        "type": "object",
+        "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
+    },
+}
+
+ARRAY_KEYS: dict[str, JsonSchema] = {
+    "supported_endpoints": {
+        "type": "array",
+        "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
+        "items": STRING,
+    },
+    "supported_modalities": {
+        "type": "array",
+        "description": "Input modalities the model accepts.",
+        "items": {"type": "string", "enum": ["text", "image", "audio", "video"]},
+    },
+    "supported_output_modalities": {
+        "type": "array",
+        "description": "Output modalities the model can produce.",
+        "items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
+    },
+    "supported_regions": {
+        "type": "array",
+        "description": "Cloud regions the model is available in ('global' or region ids).",
+        "items": STRING,
+    },
+    "tiered_pricing": {
+        "type": "array",
+        "description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
+        "items": {
+            "type": "object",
+            "properties": {
+                "range": {
+                    "type": "array",
+                    "description": "[min, max] prompt-token span this tier applies to.",
+                    "items": NONNEG_NUMBER,
+                    "minItems": 2,
+                    "maxItems": 2,
+                },
+                "max_results_range": {
+                    "type": "array",
+                    "description": "[min, max] result-count span this tier applies to (search models).",
+                    "items": NONNEG_NUMBER,
+                    "minItems": 2,
+                    "maxItems": 2,
+                },
+                "input_cost_per_token": NONNEG_NUMBER,
+                "output_cost_per_token": NONNEG_NUMBER,
+                "output_cost_per_reasoning_token": NONNEG_NUMBER,
+                "cache_read_input_token_cost": NONNEG_NUMBER,
+                "input_cost_per_query": NONNEG_NUMBER,
+            },
+            "additionalProperties": False,
+        },
+    },
+}
+
+INTEGER_KEYS: dict[str, JsonSchema] = {
+    "max_tokens": {
+        **NONNEG_INTEGER,
+        "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.",
+    },
+    "max_input_tokens": {
+        **NONNEG_INTEGER,
+        "description": "Maximum prompt/context tokens the model accepts.",
+    },
+    "max_output_tokens": {
+        **NONNEG_INTEGER,
+        "description": "Maximum tokens the model can generate in one response.",
+    },
+    "output_vector_size": {
+        **NONNEG_INTEGER,
+        "description": "Embedding dimension for embedding models.",
+    },
+    "prompt_cache_min_tokens": {
+        **NONNEG_INTEGER,
+        "description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
+    },
+    "tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
+    "rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
+}
+
+NUMBER_KEYS: dict[str, JsonSchema] = {
+    "regional_processing_uplift_multiplier_eu": {
+        "type": "number",
+        "minimum": 1,
+        "description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).",
+    },
+    "regional_processing_uplift_multiplier_us": {
+        "type": "number",
+        "minimum": 1,
+        "description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
+    },
+}
+
+COST_DESCRIPTIONS: dict[str, str] = {
+    "input_cost_per_token": "USD per prompt token.",
+    "output_cost_per_token": "USD per generated token.",
+    "output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
+    "cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
+    "cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
+    "input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
+    "output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
+}
+
+
+def cost_description(key: str) -> Optional[str]:
+    if key in COST_DESCRIPTIONS:
+        return COST_DESCRIPTIONS[key]
+    if key.endswith("_flex"):
+        return "Flex service-tier rate for the same-named base field."
+    if key.endswith("_priority"):
+        return "Priority service-tier rate for the same-named base field."
+    if "_above_" in key:
+        return "Rate applied once the prompt exceeds the token threshold in the field name."
+    return None
+
+
+def cost_schema(key: str) -> JsonSchema:
+    description = cost_description(key)
+    return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER)
+
+
+def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
+    return {
+        "litellm_provider": {
+            "type": "string",
+            "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",
+        },
+        "mode": {
+            "type": "string",
+            "description": "Primary API surface / task type of the model.",
+            "enum": list(modes),
+        },
+        "source": {
+            "type": "string",
+            "description": "URL of the provider pricing/model page this entry was taken from.",
+        },
+        "deprecation_date": {
+            "type": "string",
+            "description": "Date the provider deprecates the model, YYYY-MM-DD.",
+            "format": "date",
+            "pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$",
+        },
+        "web_search_billing_unit": {
+            "type": "string",
+            "description": "Whether web search is billed per query or per prompt.",
+            "enum": ["per_query", "per_prompt"],
+        },
+        "bedrock_output_config_effort_ceiling": {
+            "type": "string",
+            "description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
+            "enum": ["low", "medium", "high", "max", "xhigh"],
+        },
+        "comment": STRING,
+        "audio_transcription_config": STRING,
+    }
+
+
+def classify(key: str, modes: tuple) -> Optional[JsonSchema]:
+    curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS}
+    if key in curated:
+        return curated[key]
+    if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS:
+        return BOOLEAN
+    if "cost" in key:
+        return cost_schema(key)
+    return None
+
+
+def build_schema(prices: dict) -> JsonSchema:
+    entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS}
+    all_keys = tuple(sorted({key for entry in entries.values() for key in entry}))
+    modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry}))
+    unclassified = tuple(key for key in all_keys if classify(key, modes) is None)
+    if unclassified:
+        raise SystemExit(
+            f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. "
+            f"Add them to the key tables in {Path(__file__).name} and rerun it."
+        )
+    entry_properties = {key: classify(key, modes) for key in all_keys}
+    return {
+        "$schema": "https://json-schema.org/draft/2020-12/schema",
+        "title": "LiteLLM model_prices_and_context_window.json",
+        "description": (
+            "Schema for LiteLLM's model price and context window registry "
+            "(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
+            "Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
+            "optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
+            "All costs are USD per unit. New optional fields are added regularly, so consumers should "
+            "ignore unknown fields rather than reject them."
+        ),
+        "type": "object",
+        "properties": {
+            "sample_spec": {
+                "type": "object",
+                "description": (
+                    "Documentation placeholder illustrating the entry shape; not a real model and not "
+                    "schema-conformant (several values are prose)."
+                ),
+            },
+            "fallback_generalizations": {
+                "type": "object",
+                "description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
+                "properties": {
+                    "rules": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "name": STRING,
+                                "pattern": STRING,
+                                "description": STRING,
+                            },
+                            "required": ["name", "pattern"],
+                            "additionalProperties": True,
+                        },
+                    }
+                },
+                "additionalProperties": False,
+            },
+        },
+        "additionalProperties": {"$ref": "#/$defs/modelEntry"},
+        "$defs": {
+            "modelEntry": {
+                "type": "object",
+                "description": (
+                    "Pricing, limits, and capability flags for one model. Fields other than litellm_provider "
+                    "are optional; boolean capability flags are simply omitted when unknown or false."
+                ),
+                "required": ["litellm_provider"],
+                "properties": entry_properties,
+                "additionalProperties": True,
+            }
+        },
+    }
+
+
+def render(schema: JsonSchema) -> str:
+    return json.dumps(schema, indent=2) + "\n"
+
+
+def validation_errors(prices: dict, schema: JsonSchema) -> tuple:
+    validator = jsonschema.Draft202012Validator(
+        schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER
+    )
+    return tuple(
+        f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
+        for error in validator.iter_errors(prices)
+    )
+
+
+def main() -> int:
+    check = "--check" in sys.argv[1:]
+    prices = json.loads(PRICES_PATH.read_text())
+    rendered = render(build_schema(prices))
+    errors = validation_errors(prices, json.loads(rendered))
+    if errors:
+        print(f"{PRICES_PATH.name} does not validate against the generated schema:")
+        print("\n".join(errors[:20]))
+        return 1
+    if not check:
+        SCHEMA_PATH.write_text(rendered)
+        print(f"wrote {SCHEMA_PATH}")
+        return 0
+    if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered:
+        print(
+            f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. "
+            f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result."
+        )
+        return 1
+    print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_genai_otel/grafana_dashboard.json

@@ -0,0 +1,523 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "panels": [
+    {
+      "type": "stat",
+      "title": "Requests",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "blue"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "instant": true,
+          "expr": "sum(increase(gen_ai_client_operation_duration_seconds_count{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__range]))"
+        }
+      ],
+      "id": 1
+    },
+    {
+      "type": "stat",
+      "title": "Spend",
+      "description": "LiteLLM's computed cost for the selected window, from gen_ai.usage.cost",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "currencyUSD",
+          "decimals": 4,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "green"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "instant": true,
+          "expr": "sum(increase(gen_ai_usage_cost_USD_sum{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__range]))"
+        }
+      ],
+      "id": 2
+    },
+    {
+      "type": "stat",
+      "title": "Tokens",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "purple"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "instant": true,
+          "expr": "sum(increase(gen_ai_client_token_usage_sum{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__range]))"
+        }
+      ],
+      "id": 3
+    },
+    {
+      "type": "stat",
+      "title": "p95 request duration",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "decimals": 2,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "orange"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "instant": true,
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__range])))"
+        }
+      ],
+      "id": 4
+    },
+    {
+      "type": "timeseries",
+      "title": "Request rate by model",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqpm",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 8,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}}",
+          "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_operation_duration_seconds_count{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])) * 60"
+        }
+      ],
+      "id": 5
+    },
+    {
+      "type": "timeseries",
+      "title": "Spend rate by model",
+      "description": "USD per hour, derived from the gen_ai.usage.cost histogram",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "currencyUSD",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 8,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}}",
+          "expr": "sum by (gen_ai_request_model) (rate(gen_ai_usage_cost_USD_sum{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])) * 3600"
+        }
+      ],
+      "id": 6
+    },
+    {
+      "type": "timeseries",
+      "title": "Tokens per minute by model and type",
+      "description": "gen_ai.client.token.usage split by the gen_ai.token.type attribute",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 8,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}} {{gen_ai_token_type}}",
+          "expr": "sum by (gen_ai_request_model, gen_ai_token_type) (rate(gen_ai_client_token_usage_sum{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])) * 60"
+        }
+      ],
+      "id": 7
+    },
+    {
+      "type": "timeseries",
+      "title": "p95 request duration by model",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 0,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}}",
+          "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])))"
+        }
+      ],
+      "id": 8
+    },
+    {
+      "type": "timeseries",
+      "title": "p95 time to first token (streaming)",
+      "description": "gen_ai.server.time_to_first_token, recorded only for streaming requests",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 20
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 0,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}}",
+          "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_server_time_to_first_token_seconds_bucket{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])))"
+        }
+      ],
+      "id": 9
+    },
+    {
+      "type": "timeseries",
+      "title": "p95 provider generation time",
+      "description": "gen_ai.client.response.duration, upstream generation time excluding LiteLLM overhead",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 20
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 0,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}}",
+          "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_response_duration_seconds_bucket{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])))"
+        }
+      ],
+      "id": 10
+    }
+  ],
+  "preload": false,
+  "refresh": "30s",
+  "schemaVersion": 42,
+  "tags": [
+    "litellm",
+    "genai",
+    "opentelemetry"
+  ],
+  "templating": {
+    "list": [
+      {
+        "name": "datasource",
+        "label": "Prometheus",
+        "type": "datasource",
+        "query": "prometheus",
+        "current": {},
+        "hide": 0
+      },
+      {
+        "name": "service",
+        "label": "Service",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "query": "label_values(gen_ai_client_operation_duration_seconds_count, service_name)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        }
+      },
+      {
+        "name": "model",
+        "label": "Model",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "query": "label_values(gen_ai_client_operation_duration_seconds_count{service_name=~\"$service\"}, gen_ai_request_model)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        }
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "LiteLLM GenAI (OpenTelemetry)",
+  "uid": "litellm-genai-otel",
+  "weekStart": ""
+}

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_genai_otel/readme.md

@@ -0,0 +1,35 @@
+# LiteLLM GenAI dashboard (OpenTelemetry metrics)
+
+Dashboard for the `gen_ai.*` metrics the OpenTelemetry v2 integration emits, as opposed to the `litellm_*` Prometheus metrics the other dashboards in this folder chart.
+
+Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source. Panels: request count, spend, token count, p95 duration, request rate by model, spend rate per hour by model, tokens per minute split by input and output, p95 duration by model, p95 time to first token, and p95 provider generation time. Template variables for data source, service, and model.
+
+## Pre-requisites
+
+Metrics are off by default. In the proxy environment:
+
+```shell
+LITELLM_OTEL_V2=true
+LITELLM_OTEL_INTEGRATION_ENABLE_METRICS=true
+OTEL_EXPORTER="otlp_http"
+OTEL_ENDPOINT="<your OTLP endpoint>"
+```
+
+You also need the metric attribute filter, or the panels will plot flat lines at zero. LiteLLM's default attribute set includes per-request fields, so nearly every request lands in its own time series with a single sample, and `rate()` has nothing to compute over:
+
+```yaml title="config.yaml"
+callback_settings:
+  otel:
+    attributes:
+      include_list:
+        - gen_ai.operation.name
+        - gen_ai.system
+        - gen_ai.request.model
+        - gen_ai.framework
+```
+
+See [Grafana Cloud](https://docs.litellm.ai/docs/observability/grafana_cloud) for the full setup, and [OpenTelemetry v2](https://docs.litellm.ai/docs/observability/opentelemetry_v2#metrics) for the metric reference.
+
+## Note on Grafana's AI Observability integration
+
+Grafana Cloud ships prebuilt GenAI dashboards that query these same metric names, so they look like a drop-in alternative to this one. They are not: twenty of their twenty-two panels filter on `telemetry_sdk_name="openlit"`, a label LiteLLM does not carry and cannot be configured to add, so those panels stay empty.

cookbook/litellm_proxy_server/grafana_dashboard/readme.md

@@ -2,6 +2,10 @@
 
 This folder contains the `json` for creating Grafana Dashboards
 
+## [LiteLLM GenAI Dashboard (OpenTelemetry)](./dashboard_genai_otel)
+
+Charts the `gen_ai.*` metrics from the OpenTelemetry v2 integration: spend, tokens, request rate, and latency percentiles by model. Separate from the dashboards below, which chart the `litellm_*` Prometheus metrics.
+
 ## [LiteLLM v2 Dashboard](./dashboard_v2)
 
 <img width="1316" alt="grafana_1" src="https://github.com/user-attachments/assets/d0df802d-0cb9-4906-a679-941c547789ab">

db_scripts/backfill_daily_tool_spend.sql

@@ -0,0 +1,46 @@
+-- One-shot backfill of the LiteLLM_DailyToolSpend rollup from the per-request
+-- LiteLLM_SpendLogToolIndex x LiteLLM_SpendLogs tables.
+--
+-- This is an opt-in, manual operation. New deployments do not need it: the
+-- rollup is written at request time from the moment the release is deployed.
+-- Run it only if you want the Cost Optimization "Spend by tool" card to show
+-- history from before the deploy, and only once.
+--
+-- IMPORTANT caveats before running:
+--
+-- 1. Pre-deploy index rows may include tools that were merely DECLARED in a
+--    request body but never invoked (the release this ships with stops
+--    recording those). For agentic clients that declare many tools per
+--    request, backfilled history attributes each request's full spend to
+--    every declared tool, overstating per-tool spend. Post-deploy rows do not
+--    have this problem. If your traffic is mostly such clients, consider not
+--    backfilling.
+--
+-- 2. Coverage is bounded by spend-log retention: rows older than
+--    maximum_spend_logs_retention_period are already gone.
+--
+-- 3. Replace the cutover timestamp below with the time you deployed the
+--    release, so backfilled per-request rows cannot double-count on top of
+--    rollup rows the new writer already created. ON CONFLICT DO NOTHING is a
+--    second guard for (date, tool_name) buckets the writer already touched:
+--    such buckets keep the writer's numbers and skip the backfill's.
+--
+-- Usage:
+--   psql "$DATABASE_URL" -v cutover="'2026-07-25T00:00:00Z'" -f db_scripts/backfill_daily_tool_spend.sql
+
+SET TIME ZONE 'UTC';
+
+INSERT INTO "LiteLLM_DailyToolSpend" (date, tool_name, spend, total_tokens, request_count, created_at, updated_at)
+SELECT
+    to_char(ti.start_time, 'YYYY-MM-DD') AS date,
+    ti.tool_name,
+    COALESCE(SUM(sl.spend), 0) AS spend,
+    COALESCE(SUM(sl.total_tokens), 0) AS total_tokens,
+    COUNT(*) AS request_count,
+    now() AS created_at,
+    now() AS updated_at
+FROM "LiteLLM_SpendLogToolIndex" ti
+JOIN "LiteLLM_SpendLogs" sl ON sl.request_id = ti.request_id
+WHERE ti.start_time < :cutover::timestamptz
+GROUP BY 1, 2
+ON CONFLICT (date, tool_name) DO NOTHING;

docker/Dockerfile.database

@@ -7,7 +7,7 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17
 ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
-ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6
+ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
 
 FROM $UV_IMAGE AS uvbin
 
@@ -62,6 +62,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
 # Copy full source tree
@@ -82,9 +83,12 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -97,7 +101,11 @@ USER root
 RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
 
 WORKDIR /app
-ENV PATH="/app/.venv/bin:${PATH}"
+ENV PATH="/app/.venv/bin:${PATH}" \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
+    PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
+    PRISMA_OFFLINE_MODE=true
 
 # Copy only what runtime needs. The application is installed inside the venv;
 # the rest of the builder's /app is source and build metadata that must not
@@ -111,16 +119,22 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
 # working directory on sys.path; litellm/proxy/hooks resolves
 # enterprise.enterprise_hooks from it)
 COPY --from=builder /app/enterprise /app/enterprise
-# Prisma binaries live in $HOME/.cache (default prisma-python location),
-# which is /root/.cache here. Copy them from the builder so they survive
-# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
-# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
-# Only the Prisma subdirs: the whole /root/.cache drags in the uv build cache.
-COPY --from=builder /root/.cache/prisma /root/.cache/prisma
-COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
+COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
+# Prisma CLI + engines are baked under /opt/prisma, a fixed path every
+# runtime uid can read and that no cache volume mount shadows (unlike
+# /app/.cache or $HOME/.cache in readOnlyRootFilesystem + emptyDir setups).
+# The paths are pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and
+# recorded into the generated client at build time, so `prisma migrate
+# deploy` on a fresh database needs no npm and no network access
+# (#33650, #24554).
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 EXPOSE 4000/tcp
 

docker/Dockerfile.non_root

@@ -6,7 +6,7 @@ ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b
 ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
-ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6
+ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
 
 FROM $UV_IMAGE AS uvbin
 
@@ -54,7 +54,6 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
     PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
-    PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
     XDG_CACHE_HOME=/app/.cache
 
 # Copy dependency metadata first for layer caching
@@ -69,6 +68,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
 # Copy full source tree
@@ -95,6 +95,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra saml \
         --python python3 \
         --no-sources-package litellm-proxy-extras; \
     else \
@@ -103,10 +104,13 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra saml \
         --python python3; \
     fi
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -127,8 +131,6 @@ RUN for i in 1 2 3; do \
 # the rest of the builder's /app is source and build metadata that must not
 # ship (manifest-scanning tools attribute everything in it to this image).
 # entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path.
-# Prisma caches live under /app/.cache here (XDG_CACHE_HOME /
-# PRISMA_BINARY_CACHE_DIR) so the runtime prisma generate finds them.
 COPY --from=builder /app/.venv /app/.venv
 COPY --from=builder /app/docker /app/docker
 COPY --from=builder /app/schema.prisma /app/schema.prisma
@@ -137,21 +139,36 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
 # working directory on sys.path; litellm/proxy/hooks resolves
 # enterprise.enterprise_hooks from it)
 COPY --from=builder /app/enterprise /app/enterprise
-COPY --from=builder /app/.cache /app/.cache
+COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
+# Prisma CLI + engines are baked under /opt/prisma, a fixed path every runtime
+# uid can read and that no cache volume mount shadows (unlike /app/.cache or
+# $HOME/.cache under readOnlyRootFilesystem + emptyDir or arbitrary-uid setups).
+# PRISMA_CLI_QUERY_ENGINE_TYPE=binary makes the CLI use the baked binary query
+# engine directly, so `prisma migrate deploy` on a fresh database needs no npm
+# and no network access; without it the CLI looks for the library engine, which
+# prisma stopped baking, and falls back to a download that fails offline or as a
+# non-writable uid (#33650, #24554).
+COPY --from=builder /opt/prisma /opt/prisma
 COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
 COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
 
+# XDG_CACHE_HOME is intentionally left unset so it falls back to $HOME/.cache
+# (/app/.cache, writable by the runtime uid). The prisma bake at the read-only
+# /opt/prisma is anchored by PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH, so
+# nothing needs XDG to point there; pointing it at the read-only bake would
+# deny any XDG-aware library that writes a cache at runtime.
 ENV PATH="/app/.venv/bin:${PATH}" \
-    PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
+    PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
     HOME=/app \
     LITELLM_NON_ROOT=true \
-    XDG_CACHE_HOME=/app/.cache \
     PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
     PRISMA_HIDE_UPDATE_MESSAGE=1 \
     PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
     PRISMA_OFFLINE_MODE=true
 
-RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \
+RUN mkdir -p /nonexistent /app/.cache /var/lib/litellm/assets /var/lib/litellm/ui && \
     chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \
     PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
     chown -R nobody:nogroup "$PRISMA_PATH" && \
@@ -164,12 +181,15 @@ RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \
     [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u "$LITELLM_PROXY_EXTRAS_PATH" || true && \
     chmod -R g+w "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
     [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w "$LITELLM_PROXY_EXTRAS_PATH" || true && \
-    chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets /app/.cache
+    chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
+    chmod -R a+rX /opt/prisma && \
+    test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
+    ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1 && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 USER 65534
 
-RUN prisma generate --schema=./schema.prisma
-
 EXPOSE 4000/tcp
 
 ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]

docker/build_admin_ui.sh

@@ -54,8 +54,9 @@ else
 fi || { echo "nvm checksum verification failed"; exit 1; }
 bash "$NVM_SCRIPT"
 source ~/.nvm/nvm.sh
-nvm install v18.17.0
-nvm use v18.17.0
+NODE_VERSION="$(cat ui/litellm-dashboard/.nvmrc)"
+nvm install "v${NODE_VERSION}"
+nvm use "v${NODE_VERSION}"
 
 
 # cd in to /ui/litellm-dashboard

docker/build_from_pip/Dockerfile.build_from_pip

@@ -36,7 +36,7 @@ RUN uv venv --python python && \
       "opentelemetry-api==1.28.0" \
       "opentelemetry-sdk==1.28.0" \
       "opentelemetry-exporter-otlp==1.28.0" \
-      "ddtrace==2.19.0" \
+      "ddtrace==4.11.0" \
       "sentry-sdk==2.21.0" \
       "mangum==0.17.0" \
       "azure-ai-contentsafety==1.0.0" \
@@ -47,7 +47,13 @@ RUN uv venv --python python && \
       "prisma==0.11.0" \
       "openai==2.24.0"
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma && \
+    chmod -R a+rX /opt/prisma && \
+    python -c "import sys; from prisma.client import BINARY_PATHS; bad = sorted(p for group in BINARY_PATHS.model_dump().values() for p in group.values() if not p.startswith('/opt/prisma/')); sys.exit('prisma engines baked outside /opt/prisma: %r' % bad) if bad else None"
+
+ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
 EXPOSE 4000/tcp
 

docker/component_entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if [ "$USE_DDTRACE" = "true" ]; then
+    export DD_TRACE_OPENAI_ENABLED="False"
+    exec ddtrace-run "$@"
+fi
+
+exec "$@"

enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py

@@ -7,7 +7,8 @@
 #  Thank you users! We ❤️ you! - Krrish & Ishaan
 ## This provides an LLM Guard Integration for content moderation on the proxy
 
-from typing import Literal, Optional
+import asyncio
+from typing import Optional
 
 import aiohttp
 from fastapi import HTTPException
@@ -18,7 +19,6 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.utils import CallTypesLiteral
-from litellm.utils import get_formatted_prompt
 
 
 class _ENTERPRISE_LLMGuard(CustomLogger):
@@ -46,45 +46,44 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
         except Exception:
             pass
 
-    async def moderation_check(self, text: str):
+    async def moderation_check(self, text: str) -> str:
         """
+        Runs the LLM Guard moderation check on ``text``.
+
+        Raises an HTTPException when the content violates the safety policy;
+        otherwise returns the sanitized prompt from LLM Guard, falling back to
+        the original text when the API does not provide one.
+
         [TODO] make this more performant for high-throughput scenario
         """
         try:
-            async with aiohttp.ClientSession() as session:
-                if self.mock_redacted_text is not None:
-                    redacted_text = self.mock_redacted_text
-                else:
-                    # Make the first request to /analyze
-                    analyze_url = f"{self.llm_guard_api_base}analyze/prompt"
-                    verbose_proxy_logger.debug("Making request to: %s", analyze_url)
-                    analyze_payload = {"prompt": text}
-                    redacted_text = None
+            if self.mock_redacted_text is not None:
+                redacted_text = self.mock_redacted_text
+            else:
+                analyze_url = f"{self.llm_guard_api_base}analyze/prompt"
+                verbose_proxy_logger.debug("Making request to: %s", analyze_url)
+                async with aiohttp.ClientSession() as session:
                     async with session.post(
-                        analyze_url, json=analyze_payload
+                        analyze_url, json={"prompt": text}
                     ) as response:
                         redacted_text = await response.json()
-                verbose_proxy_logger.debug(
-                    f"LLM Guard: Received response - {redacted_text}"
+            verbose_proxy_logger.debug(
+                f"LLM Guard: Received response - {redacted_text}"
+            )
+            if redacted_text is None:
+                raise HTTPException(
+                    status_code=500,
+                    detail={
+                        "error": f"Invalid content moderation response: {redacted_text}"
+                    },
                 )
-                if redacted_text is not None:
-                    if (
-                        redacted_text.get("is_valid", None) is not None
-                        and redacted_text["is_valid"] is False
-                    ):
-                        raise HTTPException(
-                            status_code=400,
-                            detail={"error": "Violated content safety policy"},
-                        )
-                    else:
-                        pass
-                else:
-                    raise HTTPException(
-                        status_code=500,
-                        detail={
-                            "error": f"Invalid content moderation response: {redacted_text}"
-                        },
-                    )
+            if redacted_text.get("is_valid", None) is False:
+                raise HTTPException(
+                    status_code=400,
+                    detail={"error": "Violated content safety policy"},
+                )
+            sanitized_prompt = redacted_text.get("sanitized_prompt")
+            return sanitized_prompt if isinstance(sanitized_prompt, str) else text
         except Exception as e:
             verbose_proxy_logger.exception(
                 "litellm.enterprise.enterprise_hooks.llm_guard::moderation_check - Exception occurred - {}".format(
@@ -138,23 +137,75 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
             return
 
         self.print_verbose("Makes LLM Guard Check")
-        try:
-            assert call_type in [
-                "completion",
-                "embeddings",
-                "image_generation",
-                "moderation",
-                "audio_transcription",
-            ]
-        except Exception:
+        if call_type not in [
+            "completion",
+            "embeddings",
+            "image_generation",
+            "moderation",
+            "audio_transcription",
+        ]:
             self.print_verbose(
                 f"Call Type - {call_type}, not in accepted list - ['completion','embeddings','image_generation','moderation','audio_transcription']"
             )
             return data
 
-        formatted_prompt = get_formatted_prompt(data=data, call_type=call_type)  # type: ignore
-        self.print_verbose(f"LLM Guard, formatted_prompt: {formatted_prompt}")
-        return await self.moderation_check(text=formatted_prompt)
+        return await self._moderate_request(data=data)
+
+    async def _moderate_request(self, data: dict) -> dict:
+        """
+        Sanitizes the request in place using the prompt returned by LLM Guard so
+        the provider-bound request carries the redacted content, then returns it.
+        """
+        messages = data.get("messages")
+        if messages is not None:
+            data["messages"] = list(
+                await asyncio.gather(
+                    *(self._moderate_message(message) for message in messages)
+                )
+            )
+            return data
+
+        input_ = data.get("input")
+        if input_ is not None:
+            data["input"] = await self._moderate_input(input_)
+            return data
+
+        prompt = data.get("prompt")
+        if isinstance(prompt, str):
+            data["prompt"] = await self.moderation_check(text=prompt)
+        return data
+
+    async def _moderate_message(self, message: dict) -> dict:
+        content = message.get("content")
+        if isinstance(content, str):
+            return {**message, "content": await self.moderation_check(text=content)}
+        if isinstance(content, list):
+            return {
+                **message,
+                "content": list(
+                    await asyncio.gather(
+                        *(self._moderate_content_part(part) for part in content)
+                    )
+                ),
+            }
+        return message
+
+    async def _moderate_content_part(self, part: dict) -> dict:
+        if part.get("type") == "text" and isinstance(part.get("text"), str):
+            return {**part, "text": await self.moderation_check(text=part["text"])}
+        return part
+
+    async def _moderate_input(self, input_: object) -> object:
+        if isinstance(input_, str):
+            return await self.moderation_check(text=input_)
+        if isinstance(input_, list):
+            return [
+                await self.moderation_check(text=item)
+                if isinstance(item, str)
+                else item
+                for item in input_
+            ]
+        return input_
 
     async def async_post_call_streaming_hook(
         self, user_api_key_dict: UserAPIKeyAuth, response: str

enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py

@@ -113,6 +113,10 @@ class PagerDutyAlerting(SlackAlerting):
                 user_api_key_spend=_meta.get("user_api_key_spend"),
                 user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
                 user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
+                user_api_key_user_spend=_meta.get("user_api_key_user_spend"),
+                user_api_key_user_max_budget=_meta.get("user_api_key_user_max_budget"),
+                user_api_key_team_spend=_meta.get("user_api_key_team_spend"),
+                user_api_key_team_max_budget=_meta.get("user_api_key_team_max_budget"),
                 user_api_key_org_id=_meta.get("user_api_key_org_id"),
                 user_api_key_org_alias=_meta.get("user_api_key_org_alias"),
                 user_api_key_team_id=_meta.get("user_api_key_team_id"),
@@ -196,6 +200,10 @@ class PagerDutyAlerting(SlackAlerting):
                     if user_api_key_dict.budget_reset_at
                     else None
                 ),
+                user_api_key_user_spend=user_api_key_dict.user_spend,
+                user_api_key_user_max_budget=user_api_key_dict.user_max_budget,
+                user_api_key_team_spend=user_api_key_dict.team_spend,
+                user_api_key_team_max_budget=user_api_key_dict.team_max_budget,
                 user_api_key_org_id=user_api_key_dict.org_id,
                 user_api_key_org_alias=user_api_key_dict.organization_alias,
                 user_api_key_team_id=user_api_key_dict.team_id,

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -17,6 +17,7 @@ if TYPE_CHECKING:
     from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
     from litellm.router import Router
+    from litellm.types.router import Deployment
     from litellm.types.utils import LiteLLMBatch
 
 
@@ -281,6 +282,28 @@ class CheckBatchCost:
                 return deployment_id
         return None
 
+    @classmethod
+    def _get_managed_file_model_name(
+        cls,
+        job: "LiteLLM_ManagedObjectTable",
+        deployment_info: "Deployment",
+    ) -> Optional[str]:
+        """
+        Public model group name to encode as ``target_model_names`` on unified output file ids.
+
+        Key model-access checks resolve a managed file id back to a model via its
+        ``target_model_names``, so this must be the model group the caller requested, never the
+        underlying provider model (e.g. ``gpt-5.5``), which no key is allowed to call.
+        """
+        from litellm.proxy.openai_files_endpoints.common_utils import (
+            resolve_managed_output_file_model_name,
+        )
+
+        return resolve_managed_output_file_model_name(
+            unified_input_file_id=cls._get_input_file_id(job),
+            fallback_model_name=deployment_info.model_name or None,
+        )
+
     @staticmethod
     def _get_input_file_id(job: "LiteLLM_ManagedObjectTable") -> Optional[str]:
         import json
@@ -406,6 +429,10 @@ class CheckBatchCost:
         managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
         if managed_files_hook is not None:
             from litellm.proxy._types import UserAPIKeyAuth
+
+            managed_file_model_name = self._get_managed_file_model_name(
+                job=job, deployment_info=deployment_info
+            )
             _minimal_auth = UserAPIKeyAuth(
                 user_id=job.created_by or "default-user-id",
                 team_id=getattr(job, "team_id", None),
@@ -417,7 +444,7 @@ class CheckBatchCost:
                         _unified_file_id = managed_files_hook.get_unified_output_file_id(
                             output_file_id=_raw_file_id,
                             model_id=model_id,
-                            model_name=str(model_name) if model_name else deployment_info.model_name or None,
+                            model_name=managed_file_model_name,
                         )
                         await managed_files_hook.store_unified_file_id(
                             file_id=_unified_file_id,
@@ -471,6 +498,7 @@ class CheckBatchCost:
                 },
                 "metadata": {
                     "user_api_key_user_id": creator_user_id,
+                    "user_api_key_team_id": getattr(job, "team_id", None),
                     **user_info,
                 },
             },
@@ -629,6 +657,20 @@ class CheckBatchCost:
 
             elif response.status in ("failed", "expired", "cancelled"):
                 try:
+                    from litellm.proxy.openai_files_endpoints.common_utils import (
+                        _is_base64_encoded_unified_file_id,
+                        ensure_batch_response_managed_file_ids,
+                    )
+
+                    response.id = job.unified_object_id
+                    await ensure_batch_response_managed_file_ids(
+                        response=response,
+                        managed_files_obj=self.proxy_logging_obj.get_proxy_hook("managed_files"),
+                        prisma_client=self.prisma_client,
+                        verbose_proxy_logger=verbose_proxy_logger,
+                        db_batch_object=job,
+                        unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
+                    )
                     update_data = {
                         "status": response.status,
                         "file_object": response.model_dump_json(),

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -1,10 +1,10 @@
 """
 Polls LiteLLM_ManagedObjectTable to check if the response is complete.
-Cost tracking is handled automatically by litellm.aget_responses().
+Cost tracking is handled automatically by the get-responses call.
 """
 
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Dict, Optional, cast
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -13,11 +13,15 @@ from litellm.constants import (
     MAX_OBJECTS_PER_POLL_CYCLE,
     STALE_OBJECT_CLEANUP_BATCH_SIZE,
 )
+from litellm.responses.utils import ResponsesAPIRequestUtils
+from litellm.types.llms.openai import ResponsesAPIResponse
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
     from litellm.router import Router
 
+TERMINAL_RESPONSE_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete"})
+
 
 class CheckResponsesCost:
     def __init__(
@@ -33,6 +37,28 @@ class CheckResponsesCost:
         self.prisma_client: PrismaClient = prisma_client
         self.llm_router: Router = llm_router
 
+    async def _get_response(
+        self,
+        response_id: str,
+        litellm_metadata: Dict[str, str],
+    ) -> ResponsesAPIResponse:
+        """Fetch the upstream response, using deployment credentials when available.
+
+        LiteLLM-encoded response IDs carry the ``model_id`` of the deployment that
+        served the original request, so routing through ``llm_router`` applies that
+        deployment's ``api_base`` / ``api_key`` / ``api_version``, exactly like
+        ``GET /v1/responses/{id}`` does. ``litellm.aget_responses`` on its own only
+        sees provider env vars, so it fails for every deployment whose credentials
+        live in the config; the row then never leaves ``queued``.
+        """
+        model_id: Optional[str] = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
+        if model_id is None or self.llm_router.get_deployment(model_id=model_id) is None:
+            return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
+        router_response = await self.llm_router.aget_responses(
+            response_id=response_id, litellm_metadata=litellm_metadata
+        )
+        return cast(ResponsesAPIResponse, router_response)
+
     async def _expire_stale_rows(
         self, cutoff: datetime, batch_size: int
     ) -> int:
@@ -87,8 +113,8 @@ class CheckResponsesCost:
         Check if background responses are complete and track their cost.
         - Get all status="queued" or "in_progress" and file_purpose="response" jobs
         - Query the provider to check if response is complete
-        - Cost is automatically tracked by litellm.aget_responses()
-        - Mark completed/failed/cancelled responses as complete in the database
+        - Cost is automatically tracked by the get-responses call
+        - Mark responses in a terminal state as complete in the database
         """
         try:
             await self._cleanup_stale_managed_objects()
@@ -134,7 +160,7 @@ class CheckResponsesCost:
                     litellm_metadata["model"] = model_name
                     litellm_metadata["model_group"] = model_name  # Use same value for model_group
                 
-                response = await litellm.aget_responses(
+                response = await self._get_response(
                     response_id=responses_id_security,
                     litellm_metadata=litellm_metadata,
                 )
@@ -144,21 +170,14 @@ class CheckResponsesCost:
                 )
                 
             except Exception as e:
-                verbose_proxy_logger.info(
+                verbose_proxy_logger.warning(
                     f"Skipping job {unified_object_id} due to error: {e}"
                 )
                 continue
 
-            # Check if response is in a terminal state
-            if response.status == "completed":
+            if response.status in TERMINAL_RESPONSE_STATUSES:
                 verbose_proxy_logger.info(
-                    f"Response {unified_object_id} is complete. Cost automatically tracked by aget_responses."
-                )
-                completed_jobs.append(job)
-                
-            elif response.status in ["failed", "cancelled"]:
-                verbose_proxy_logger.info(
-                    f"Response {unified_object_id} has status {response.status}, marking as complete"
+                    f"Response {unified_object_id} has terminal status {response.status}, marking as complete"
                 )
                 completed_jobs.append(job)
 

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -3,10 +3,13 @@
 
 import base64
 import json
+from collections.abc import Mapping
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Dict, Final, List, Literal, Optional, Union, cast
+from uuid import NAMESPACE_URL, uuid5
 
 from fastapi import HTTPException
+from pydantic import ValidationError
 
 import litellm
 from litellm import Router, verbose_logger
@@ -30,11 +33,14 @@ from litellm.proxy._types import (
 )
 from litellm.proxy.openai_files_endpoints.common_utils import (
     _is_base64_encoded_unified_file_id,
+    apply_unified_file_ids,
+    ensure_batch_response_managed_file_ids,
     get_batch_id_from_unified_batch_id,
     get_content_type_from_file_object,
     get_model_id_from_unified_batch_id,
-    get_models_from_unified_file_id,
+    map_raw_file_ids_to_unified,
     normalize_mime_type_for_provider,
+    resolve_managed_output_file_model_name,
 )
 from litellm.types.llms.openai import (  # pyright: ignore[reportAttributeAccessIssue]
     AllMessageValues,
@@ -60,6 +66,9 @@ if TYPE_CHECKING:
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
+    from prisma.models import (
+        LiteLLM_ManagedObjectTable as PrismaManagedObjectRow,
+    )
 
     from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
     from litellm.proxy.utils import PrismaClient as _PrismaClient
@@ -73,6 +82,44 @@ else:
     PrismaClient = Any
 
 
+def _sanitized_parse_error(e: Exception) -> str:
+    return (
+        str(e.errors(include_input=False, include_url=False, include_context=False))
+        if isinstance(e, ValidationError)
+        else type(e).__name__
+    )
+
+
+def _decode_json_blob(blob: object) -> object:
+    return json.loads(blob) if isinstance(blob, str) else blob
+
+
+def _parse_managed_batch_row(row: "PrismaManagedObjectRow") -> Optional[LiteLLMBatch]:
+    try:
+        batch_obj: Final = LiteLLMBatch.model_validate(_decode_json_blob(row.file_object))
+    except Exception as e:
+        verbose_logger.warning(
+            f"Failed to parse batch object {row.unified_object_id}: {_sanitized_parse_error(e)}"
+        )
+        return None
+    batch_obj.id = row.unified_object_id
+    return batch_obj
+
+
+def _parse_managed_file_object(
+    raw_file_object: object, unified_file_id: str
+) -> Optional[OpenAIFileObject]:
+    if raw_file_object is None:
+        return None
+    try:
+        return OpenAIFileObject.model_validate(raw_file_object)
+    except Exception as e:
+        verbose_logger.warning(
+            f"Failed to parse managed file object {unified_file_id}: {_sanitized_parse_error(e)}"
+        )
+        return None
+
+
 class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     # Class variables or attributes
     def __init__(
@@ -215,7 +262,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         )
 
         if result:
-            return LiteLLM_ManagedFileTable(**result)
+            return LiteLLM_ManagedFileTable.model_validate(result)
 
         ## CHECK DB
         db_object = await self.prisma_client.db.litellm_managedfiletable.find_first(
@@ -223,7 +270,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         )
 
         if db_object:
-            return LiteLLM_ManagedFileTable(**db_object.model_dump())
+            return LiteLLM_ManagedFileTable.model_validate(db_object.model_dump())
         return None
 
     async def delete_unified_file_id(
@@ -316,45 +363,86 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         where_clause: Dict[str, Any] = {"file_purpose": "batch", **owner_filter}
 
         if after:
-            where_clause["id"] = {"gt": after}
+            cursor_row = (
+                await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                    where={**where_clause, "unified_object_id": after}
+                )
+            )
+            if cursor_row is None:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Invalid 'after' cursor: no batch found with id '{after}'.",
+                )
 
-        fetch_limit = limit or 20
-        if target_model_names:
-            # Oversample so post-fetch model-name filtering still has enough rows.
-            fetch_limit = max(fetch_limit * 3, 100)
+        page_size: Final = min(limit or 20, 100)
+        cursor_args: Dict[str, Any] = (
+            {"cursor": {"unified_object_id": after}, "skip": 1} if after else {}
+        )
 
         batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
             where=where_clause,
-            take=fetch_limit,
-            order={"created_at": "desc"},
+            take=page_size + 1,
+            order=[{"created_at": "desc"}, {"unified_object_id": "desc"}],
+            **cursor_args,
         )
 
-        batch_objects: List[LiteLLMBatch] = []
-        for batch in batches:
-            try:
-                # Stop once we have enough after filtering
-                if len(batch_objects) >= (limit or 20):
-                    break
-
-                batch_data = (
-                    json.loads(batch.file_object)
-                    if isinstance(batch.file_object, str)
-                    else batch.file_object
-                )
-                batch_obj = LiteLLMBatch(**batch_data)
-                batch_obj.id = batch.unified_object_id
-                batch_objects.append(batch_obj)
-
-            except Exception as e:
-                verbose_logger.warning(
-                    f"Failed to parse batch object {batch.unified_object_id}: {e}"
-                )
-                continue
+        has_more = len(batches) > page_size
 
+        parsed_rows: Final = tuple(
+            (row, batch_obj)
+            for row in batches[:page_size]
+            if (batch_obj := _parse_managed_batch_row(row)) is not None
+        )
+        unified_id_by_raw_id: Final = await map_raw_file_ids_to_unified(
+            raw_file_ids=frozenset(
+                file_id
+                for _, batch_obj in parsed_rows
+                for file_id in (batch_obj.input_file_id, batch_obj.output_file_id, batch_obj.error_file_id)
+                if file_id and not _is_base64_encoded_unified_file_id(file_id)
+            ),
+            prisma_client=self.prisma_client,
+        )
+        resolved_batches: Final = [
+            await self._resolve_listed_batch(
+                row=row,
+                batch_obj=batch_obj,
+                unified_id_by_raw_id=unified_id_by_raw_id,
+                user_api_key_dict=user_api_key_dict,
+            )
+            for row, batch_obj in parsed_rows
+        ]
         return build_list_page(
-            batch_objects, has_more=len(batch_objects) == (limit or 20)
+            [batch_obj for batch_obj in resolved_batches if batch_obj is not None],
+            has_more=has_more,
         )
 
+    async def _resolve_listed_batch(
+        self,
+        row: "PrismaManagedObjectRow",
+        batch_obj: LiteLLMBatch,
+        unified_id_by_raw_id: Mapping[str, str],
+        user_api_key_dict: UserAPIKeyAuth,
+    ) -> Optional[LiteLLMBatch]:
+        apply_unified_file_ids(batch_obj, unified_id_by_raw_id)
+        try:
+            await ensure_batch_response_managed_file_ids(
+                response=batch_obj,
+                managed_files_obj=self,
+                prisma_client=self.prisma_client,
+                verbose_proxy_logger=verbose_logger,
+                user_api_key_dict=user_api_key_dict,
+                db_batch_object=row,
+                unified_batch_id=_is_base64_encoded_unified_file_id(
+                    row.unified_object_id
+                ),
+            )
+        except Exception as e:
+            verbose_logger.warning(
+                f"Failed to resolve managed file ids for batch {row.unified_object_id}: {e}"
+            )
+            return None
+        return batch_obj
+
     async def get_user_created_file_ids(
         self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
     ) -> List[OpenAIFileObject]:
@@ -376,7 +464,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 "flat_model_file_ids": {"hasSome": model_object_ids},
             }
         )
-        return [OpenAIFileObject(**file_object.file_object) for file_object in file_ids]
+        return [
+            parsed_file_object.model_copy(update={"id": row.unified_file_id})
+            for row in file_ids
+            if (
+                parsed_file_object := _parse_managed_file_object(
+                    row.file_object, row.unified_file_id
+                )
+            )
+            is not None
+        ]
 
     async def check_managed_file_id_access(
         self, data: Dict, user_api_key_dict: UserAPIKeyAuth
@@ -1049,10 +1146,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     def get_unified_output_file_id(
         self, output_file_id: str, model_id: str, model_name: Optional[str]
     ) -> str:
+        deterministic_uuid: Final = uuid5(
+            uuid5(NAMESPACE_URL, model_id), output_file_id
+        )
         unified_output_file_id = (
             SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format(
                 "application/json",
-                str(uuid.uuid4()),
+                str(deterministic_uuid),
                 model_name or "",
                 output_file_id,
                 model_id,
@@ -1088,21 +1188,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             )  # managed batch id
             model_id = cast(Optional[str], response._hidden_params.get("model_id"))
             model_name = cast(Optional[str], response._hidden_params.get("model_name"))
-            resolved_model_name = model_name
 
-            # Some providers (e.g. Vertex batch retrieve) do not set model_name on
-            # the response. In that case, recover target_model_names from the input
-            # managed file metadata so unified output IDs preserve routing metadata.
-            if not resolved_model_name and isinstance(unified_file_id, str):
-                decoded_unified_file_id = (
-                    _is_base64_encoded_unified_file_id(unified_file_id)
-                    or unified_file_id
-                )
-                target_model_names = get_models_from_unified_file_id(
-                    decoded_unified_file_id
-                )
-                if target_model_names:
-                    resolved_model_name = ",".join(target_model_names)
+            resolved_model_name = resolve_managed_output_file_model_name(
+                unified_input_file_id=unified_file_id
+                if isinstance(unified_file_id, str)
+                else response.input_file_id,
+                fallback_model_name=model_name,
+            )
             original_response_id = response.id
 
             if (unified_batch_id or unified_file_id) and model_id:

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -11,13 +11,15 @@ Endpoints for /project operations
 #### PROJECT MANAGEMENT ####
 
 import json
-from typing import List, Optional, Union
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING
 
 from fastapi import APIRouter, Depends, HTTPException, Request
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
 from litellm.proxy._types import *
+from litellm.proxy.auth.auth_checks import delete_cached_project_object
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
 from litellm.proxy.management_helpers.utils import (
@@ -25,15 +27,24 @@ from litellm.proxy.management_helpers.utils import (
 )
 from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
 
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
+    from prisma.actions import LiteLLM_TeamTableActions
+
 router = APIRouter()
 
 
+def _team_table(prisma_client: PrismaClient) -> "LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable]":
+    team_table: LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable] = prisma_client.db.litellm_teamtable
+    return team_table
+
+
 async def _check_user_permission_for_project(
     user_api_key_dict: UserAPIKeyAuth,
-    team_id: Optional[str],
+    team_id: str | None,
     prisma_client: PrismaClient,
     require_admin: bool = False,
-    team_object: Optional[LiteLLM_TeamTable] = None,
+    team_object: LiteLLM_TeamTable | None = None,
 ) -> bool:
     """
     Check if user has permission to manage a project.
@@ -57,9 +68,7 @@ async def _check_user_permission_for_project(
 
     team = team_object
     if team is None:
-        team = await prisma_client.db.litellm_teamtable.find_unique(
-            where={"team_id": team_id}
-        )
+        team = await _team_table(prisma_client).find_unique(where={"team_id": team_id})
 
     if team and team.admins:
         return user_api_key_dict.user_id in team.admins
@@ -70,9 +79,9 @@ async def _check_user_permission_for_project(
 async def _validate_team_exists(
     team_id: str,
     prisma_client: PrismaClient,
-):
+) -> "prisma_models.LiteLLM_TeamTable":
     """Validate that a team exists. Returns the team row."""
-    team = await prisma_client.db.litellm_teamtable.find_unique(
+    team = await _team_table(prisma_client).find_unique(
         where={"team_id": team_id},
     )
 
@@ -89,7 +98,7 @@ async def _validate_team_exists(
 
 def _check_team_project_limits(
     team_object: LiteLLM_TeamTable,
-    data: Union[NewProjectRequest, UpdateProjectRequest],
+    data: NewProjectRequest | UpdateProjectRequest,
 ) -> None:
     """
     Check that project limits respect its parent Team's limits.
@@ -108,16 +117,12 @@ def _check_team_project_limits(
     if data.max_budget is not None and data.max_budget < 0:
         raise HTTPException(
             status_code=400,
-            detail={
-                "error": f"max_budget cannot be negative. Received: {data.max_budget}"
-            },
+            detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"},
         )
     if data.soft_budget is not None and data.soft_budget < 0:
         raise HTTPException(
             status_code=400,
-            detail={
-                "error": f"soft_budget cannot be negative. Received: {data.soft_budget}"
-            },
+            detail={"error": f"soft_budget cannot be negative. Received: {data.soft_budget}"},
         )
 
     # --- soft_budget < max_budget ---
@@ -131,7 +136,7 @@ def _check_team_project_limits(
             )
 
     # --- Validate project models are a subset of team models ---
-    project_models = getattr(data, "models", None)
+    project_models = data.models
     team_models = team_object.models or []
     if project_models and len(team_models) > 0:
         # If team has 'all-proxy-models', skip validation as it allows all models
@@ -148,11 +153,7 @@ def _check_team_project_limits(
     # --- Validate project max_budget <= team max_budget ---
     # Team stores budget fields directly (max_budget, tpm_limit, rpm_limit)
     # unlike Project which uses a separate LiteLLM_BudgetTable relation
-    if (
-        data.max_budget is not None
-        and team_object.max_budget is not None
-        and data.max_budget > team_object.max_budget
-    ):
+    if data.max_budget is not None and team_object.max_budget is not None and data.max_budget > team_object.max_budget:
         raise HTTPException(
             status_code=400,
             detail={
@@ -161,11 +162,7 @@ def _check_team_project_limits(
         )
 
     # --- Validate project tpm_limit <= team tpm_limit ---
-    if (
-        data.tpm_limit is not None
-        and team_object.tpm_limit is not None
-        and data.tpm_limit > team_object.tpm_limit
-    ):
+    if data.tpm_limit is not None and team_object.tpm_limit is not None and data.tpm_limit > team_object.tpm_limit:
         raise HTTPException(
             status_code=400,
             detail={
@@ -174,11 +171,7 @@ def _check_team_project_limits(
         )
 
     # --- Validate project rpm_limit <= team rpm_limit ---
-    if (
-        data.rpm_limit is not None
-        and team_object.rpm_limit is not None
-        and data.rpm_limit > team_object.rpm_limit
-    ):
+    if data.rpm_limit is not None and team_object.rpm_limit is not None and data.rpm_limit > team_object.rpm_limit:
         raise HTTPException(
             status_code=400,
             detail={
@@ -189,19 +182,19 @@ def _check_team_project_limits(
 
 async def _create_budget_for_project(
     data: NewProjectRequest,
-    user_id: Optional[str],
+    user_id: str | None,
     litellm_proxy_admin_name: str,
     prisma_client: PrismaClient,
 ) -> str:
     """Create a budget for the project and return budget_id."""
     budget_params = LiteLLM_BudgetTable.model_fields.keys()
-    _json_data = data.json(exclude_none=True)
+    _json_data: Mapping[str, object] = data.json(exclude_none=True)
     _budget_data = {k: v for k, v in _json_data.items() if k in budget_params}
-    budget_row = LiteLLM_BudgetTable(**_budget_data)
+    budget_row = LiteLLM_BudgetTable.model_validate(_budget_data)
 
     new_budget = prisma_client.jsonify_object(budget_row.json(exclude_none=True))
 
-    _budget = await prisma_client.db.litellm_budgettable.create(
+    _budget: prisma_models.LiteLLM_BudgetTable = await prisma_client.db.litellm_budgettable.create(
         data={
             **new_budget,
             "created_by": user_id or litellm_proxy_admin_name,
@@ -214,8 +207,8 @@ async def _create_budget_for_project(
 
 async def _set_project_object_permission(
     data: NewProjectRequest,
-    prisma_client: Optional[PrismaClient],
-) -> Optional[str]:
+    prisma_client: PrismaClient | None,
+) -> str | None:
     """
     Creates the LiteLLM_ObjectPermissionTable record for the project.
     Returns the object_permission_id if created, otherwise None.
@@ -224,7 +217,7 @@ async def _set_project_object_permission(
         return None
 
     if data.object_permission is not None:
-        created_object_permission = (
+        created_object_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
             await prisma_client.db.litellm_objectpermissiontable.create(
                 data=data.object_permission.model_dump(exclude_none=True),
             )
@@ -344,8 +337,7 @@ async def new_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Only premium users can add tags to projects. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Only premium users can add tags to projects. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -353,8 +345,7 @@ async def new_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Project management is an enterprise feature. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Project management is an enterprise feature. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -375,13 +366,11 @@ async def new_project(
             )
 
         # Validate team exists and get team object with budget
-        team_object = await _validate_team_exists(
-            team_id=data.team_id, prisma_client=prisma_client
-        )
+        team_object = await _validate_team_exists(team_id=data.team_id, prisma_client=prisma_client)
 
         # Validate project limits against team limits
         _check_team_project_limits(
-            team_object=LiteLLM_TeamTable(**team_object.model_dump()),
+            team_object=LiteLLM_TeamTable.model_validate(team_object.model_dump()),
             data=data,
         )
 
@@ -391,7 +380,7 @@ async def new_project(
             user_api_key_dict=user_api_key_dict,
             team_id=data.team_id,
             prisma_client=prisma_client,
-            team_object=LiteLLM_TeamTable(**team_object.model_dump()),
+            team_object=LiteLLM_TeamTable.model_validate(team_object.model_dump()),
         )
 
         if not has_permission:
@@ -449,17 +438,13 @@ async def new_project(
                     value=getattr(data, field),
                 )
 
-        new_project_row = prisma_client.jsonify_object(
-            project_row.json(exclude_none=True)
-        )
+        new_project_row = prisma_client.jsonify_object(project_row.json(exclude_none=True))
 
         # Remove budget fields (following organization_endpoints.py pattern)
         new_project_row = _remove_budget_fields_from_project_data(new_project_row)
 
-        verbose_proxy_logger.info(
-            f"new_project_row: {json.dumps(new_project_row, indent=2)}"
-        )
-        response = await prisma_client.db.litellm_projecttable.create(
+        verbose_proxy_logger.info(f"new_project_row: {json.dumps(new_project_row, indent=2)}")
+        response: prisma_models.LiteLLM_ProjectTable = await prisma_client.db.litellm_projecttable.create(
             data={
                 **new_project_row,  # type: ignore
             },
@@ -469,9 +454,7 @@ async def new_project(
         return response
     except Exception as e:
         verbose_proxy_logger.exception(
-            "litellm.proxy.management_endpoints.project_endpoints.new_project(): Exception occured - {}".format(
-                str(e)
-            )
+            "litellm.proxy.management_endpoints.project_endpoints.new_project(): Exception occured - {}".format(str(e))
         )
         raise handle_exception_on_proxy(e)
 
@@ -532,6 +515,7 @@ async def update_project(
         litellm_proxy_admin_name,
         premium_user,
         prisma_client,
+        user_api_key_cache,
     )
 
     try:
@@ -539,8 +523,7 @@ async def update_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Only premium users can add tags to projects. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Only premium users can add tags to projects. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -548,8 +531,7 @@ async def update_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Project management is an enterprise feature. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Project management is an enterprise feature. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -576,9 +558,9 @@ async def update_project(
             )
 
         # Fetch existing project
-        existing_project = await prisma_client.db.litellm_projecttable.find_unique(
-            where={"project_id": data.project_id}
-        )
+        existing_project: (
+            prisma_models.LiteLLM_ProjectTable | None
+        ) = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": data.project_id})
 
         if existing_project is None:
             raise ProxyException(
@@ -595,9 +577,7 @@ async def update_project(
         target_team_id = data.team_id or existing_project.team_id
         target_team_obj = None
         if target_team_id is not None:
-            target_team_obj = await _validate_team_exists(
-                team_id=target_team_id, prisma_client=prisma_client
-            )
+            target_team_obj = await _validate_team_exists(team_id=target_team_id, prisma_client=prisma_client)
 
         has_permission = await _check_user_permission_for_project(
             user_api_key_dict=user_api_key_dict,
@@ -620,32 +600,26 @@ async def update_project(
                 team_id=data.team_id,
                 prisma_client=prisma_client,
                 team_object=(
-                    LiteLLM_TeamTable(**target_team_obj.model_dump())
-                    if target_team_obj
-                    else None
+                    LiteLLM_TeamTable.model_validate(target_team_obj.model_dump()) if target_team_obj else None
                 ),
             )
             if not can_assign_to_target:
                 raise HTTPException(
                     status_code=403,
-                    detail={
-                        "error": "Cannot reassign project to a team you are not an admin of"
-                    },
+                    detail={"error": "Cannot reassign project to a team you are not an admin of"},
                 )
 
         # Validate project limits against team limits
         if target_team_obj is not None:
             _check_team_project_limits(
-                team_object=LiteLLM_TeamTable(**target_team_obj.model_dump()),
+                team_object=LiteLLM_TeamTable.model_validate(target_team_obj.model_dump()),
                 data=data,
             )
 
         # Prepare update data
         update_data = data.json(exclude_none=True, exclude={"project_id"})
         update_data = prisma_client.jsonify_object(update_data)
-        update_data["updated_by"] = (
-            user_api_key_dict.user_id or litellm_proxy_admin_name
-        )
+        update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
 
         # Handle budget updates
         budget_fields = LiteLLM_BudgetTable.model_fields.keys()
@@ -671,21 +645,17 @@ async def update_project(
                 if existing_project.object_permission_id:
                     # Update existing permission
                     await prisma_client.db.litellm_objectpermissiontable.update(
-                        where={
-                            "object_permission_id": existing_project.object_permission_id
-                        },
+                        where={"object_permission_id": existing_project.object_permission_id},
                         data=object_permission_data,
                     )
                 else:
                     # Create new permission
-                    created_permission = (
+                    created_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
                         await prisma_client.db.litellm_objectpermissiontable.create(
                             data=object_permission_data,
                         )
                     )
-                    update_data["object_permission_id"] = (
-                        created_permission.object_permission_id
-                    )
+                    update_data["object_permission_id"] = created_permission.object_permission_id
 
         # Handle metadata fields
         for field in LiteLLM_ManagementEndpoint_MetadataFields:
@@ -698,12 +668,17 @@ async def update_project(
         update_data = _remove_budget_fields_from_project_data(update_data)
 
         # Update project
-        updated_project = await prisma_client.db.litellm_projecttable.update(
+        updated_project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.update(
             where={"project_id": data.project_id},
             data=update_data,
             include={"litellm_budget_table": True, "object_permission": True},
         )
 
+        await delete_cached_project_object(
+            project_id=data.project_id,
+            user_api_key_cache=user_api_key_cache,
+        )
+
         return updated_project
     except Exception as e:
         verbose_proxy_logger.exception(
@@ -718,7 +693,7 @@ async def update_project(
     "/project/delete",
     tags=["project management"],
     dependencies=[Depends(user_api_key_auth)],
-    response_model=List[LiteLLM_ProjectTable],
+    response_model=list[LiteLLM_ProjectTable],
 )
 @management_endpoint_wrapper
 async def delete_project(
@@ -742,15 +717,14 @@ async def delete_project(
     }'
     ```
     """
-    from litellm.proxy.proxy_server import premium_user, prisma_client
+    from litellm.proxy.proxy_server import premium_user, prisma_client, user_api_key_cache
 
     try:
         if not premium_user:
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Project management is an enterprise feature. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Project management is an enterprise feature. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -778,9 +752,7 @@ async def delete_project(
 
         for project_id in data.project_ids:
             # Check if project exists
-            existing_project = await prisma_client.db.litellm_projecttable.find_unique(
-                where={"project_id": project_id}
-            )
+            existing_project = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": project_id})
 
             if existing_project is None:
                 raise ProxyException(
@@ -791,11 +763,9 @@ async def delete_project(
                 )
 
             # Check if there are any keys associated with this project
-            associated_keys = (
-                await prisma_client.db.litellm_verificationtoken.find_many(
-                    where={"project_id": project_id}
-                )
-            )
+            associated_keys: Sequence[
+                prisma_models.LiteLLM_VerificationToken
+            ] = await prisma_client.db.litellm_verificationtoken.find_many(where={"project_id": project_id})
 
             if len(associated_keys) > 0:
                 raise ProxyException(
@@ -806,8 +776,13 @@ async def delete_project(
                 )
 
             # Delete the project
-            deleted_project = await prisma_client.db.litellm_projecttable.delete(
-                where={"project_id": project_id}
+            deleted_project: (
+                prisma_models.LiteLLM_ProjectTable | None
+            ) = await prisma_client.db.litellm_projecttable.delete(where={"project_id": project_id})
+
+            await delete_cached_project_object(
+                project_id=project_id,
+                user_api_key_cache=user_api_key_cache,
             )
 
             deleted_projects.append(deleted_project)
@@ -854,7 +829,7 @@ async def project_info(
             )
 
         # Fetch project
-        project = await prisma_client.db.litellm_projecttable.find_unique(
+        project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.find_unique(
             where={"project_id": project_id},
             include={"litellm_budget_table": True, "object_permission": True},
         )
@@ -868,21 +843,15 @@ async def project_info(
             )
 
         # Check if user has access to this project (admin or team member)
-        is_admin = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
+        is_admin = user_api_key_has_admin_view(user_api_key_dict)
         is_team_member = False
 
         if project.team_id and user_api_key_dict.user_id:
-            team = await prisma_client.db.litellm_teamtable.find_unique(
-                where={"team_id": project.team_id}
-            )
+            team = await _team_table(prisma_client).find_unique(where={"team_id": project.team_id})
             if team:
                 caller_user_id = user_api_key_dict.user_id
                 for m in team.members_with_roles or []:
-                    m_user_id = (
-                        m.get("user_id")
-                        if isinstance(m, dict)
-                        else getattr(m, "user_id", None)
-                    )
+                    m_user_id = m.get("user_id") if isinstance(m, dict) else getattr(m, "user_id", None)
                     if m_user_id == caller_user_id:
                         is_team_member = True
                         break
@@ -896,9 +865,7 @@ async def project_info(
         return project
     except Exception as e:
         verbose_proxy_logger.exception(
-            "litellm.proxy.management_endpoints.project_endpoints.project_info(): Exception occured - {}".format(
-                str(e)
-            )
+            "litellm.proxy.management_endpoints.project_endpoints.project_info(): Exception occured - {}".format(str(e))
         )
         raise handle_exception_on_proxy(e)
 
@@ -907,7 +874,7 @@ async def project_info(
     "/project/list",
     tags=["project management"],
     dependencies=[Depends(user_api_key_auth)],
-    response_model=List[LiteLLM_ProjectTable],
+    response_model=list[LiteLLM_ProjectTable],
 )
 async def list_projects(
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@@ -931,22 +898,20 @@ async def list_projects(
             )
 
         # If proxy admin, get all projects
-        if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
-            projects = await prisma_client.db.litellm_projecttable.find_many(
+        if user_api_key_has_admin_view(user_api_key_dict):
+            projects: Sequence[
+                prisma_models.LiteLLM_ProjectTable
+            ] = await prisma_client.db.litellm_projecttable.find_many(
                 include={"litellm_budget_table": True, "object_permission": True}
             )
         else:
             # Look up the user's team memberships via the reverse-index on
             # LiteLLM_UserTable.teams (maintained by team_member_add alongside
             # members_with_roles). This avoids a full scan of all team rows.
-            user_record = await prisma_client.db.litellm_usertable.find_unique(
+            user_record: prisma_models.LiteLLM_UserTable | None = await prisma_client.db.litellm_usertable.find_unique(
                 where={"user_id": user_api_key_dict.user_id},
             )
-            user_team_ids = (
-                user_record.teams
-                if user_record is not None and user_record.teams
-                else []
-            )
+            user_team_ids: Sequence[str] = user_record.teams if user_record is not None and user_record.teams else []
 
             projects = await prisma_client.db.litellm_projecttable.find_many(
                 where={"team_id": {"in": user_team_ids}},

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.50"
+version = "0.1.54"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.50"
+version = "0.1.54"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -46,6 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra bedrock-realtime \
         --python python3
 
 # Stage 2 — copy source and install the project + workspace members.
@@ -57,11 +58,14 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra bedrock-realtime \
         --python python3
 
-RUN mkdir -p /home/nonroot && \
-    HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
-    chown -R nonroot:nonroot /home/nonroot/.cache
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
+
+RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh
 
 # ---------- Runtime ----------
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
@@ -81,17 +85,20 @@ ENV HOME=/home/nonroot \
     PATH="/app/.venv/bin:${PATH}" \
     PYTHONPATH="/app" \
     PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1
+    PYTHONUNBUFFERED=1 \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
 COPY --from=builder --chown=nonroot:nonroot /app /app
-COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 USER nonroot
 
 EXPOSE 4000/tcp
 
-ENTRYPOINT ["sh", "-c", "exec uvicorn gateway.main:app --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
+ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh uvicorn gateway.main:app --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
 CMD ["--host", "0.0.0.0", "--port", "4000"]

gateway/routes/allowlist.py

@@ -54,6 +54,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/messages",
     "/v1/skills",
     "/v1/a2a/",
+    "/a2a/",
     # LiteLLM-native LLM surface
     "/v1/rerank",
     "/v2/rerank",

helm/litellm-helm/Chart.lock

@@ -5,5 +5,5 @@ dependencies:
 - name: redis
   repository: oci://registry-1.docker.io/bitnamicharts
   version: 18.19.1
-digest: sha256:8660fe6287f9941d08c0902f3f13731079b8cecd2a5da2fbc54e5b7aae4a6f62
-generated: "2024-03-10T02:28:52.275022+05:30"
+digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
+generated: "2026-07-28T10:21:22.511401-07:00"

helm/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.1.0
+version: 1.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -32,10 +32,10 @@ annotations:
 
 dependencies:
   - name: "postgresql"
-    version: ">=13.3.0"
+    version: "14.3.1"
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: db.deployStandalone
   - name: redis
-    version: ">=18.0.0"
+    version: "18.19.1"
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: redis.enabled

helm/litellm-helm/README.md

@@ -39,7 +39,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `livenessProbe.*`           | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                  | See `values.yaml`         |
 | `readinessProbe.*`          | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                 | See `values.yaml`         |
 | `startupProbe.*`            | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                   | See `values.yaml`         |
-| `resources.*`               | CPU/memory requests and limits for the LiteLLM container.                                                                                                                                                                                       | `{}`                      |
+| `resources.*`               | CPU/memory requests and limits for the LiteLLM container. Unset by default; production deployments should set 1 CPU and 4Gi of memory per worker.                                                                                             | `{}`                      |
 | `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                                                                  | `""`                      |
 | `ingress.labels`            | Additional labels for the Ingress resource                                                                                                                                                                                                    | `{}`                      |
 | `ingress.*`                 | See [values.yaml](./values.yaml) for example settings                                                                                                                                                                                         | N/A                       |
@@ -54,6 +54,12 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `pdb.annotations`           | Extra metadata annotations to add to the PDB                                                                                                                                                                                                  | `{}`                      |
 | `pdb.labels`                | Extra metadata labels to add to the PDB                                                                                                                                                                                                       | `{}`                      |
 
+| `billingMetrics.enabled`    | Enable enterprise billable-request metering. Requires an enterprise license.                                                                                                                                                                  | `false`                   |
+| `billingMetrics.endpoint`   | Collector that the billable-request counter is pushed to.                                                                                                                                                                                    | `https://telemetry.litellm.ai` |
+| `billingMetrics.secretName` | Name of an existing Secret holding the mTLS client certificate, under the keys `tls.crt` and `tls.key`.                                                                                                                                       | `litellm-billing-metrics-mtls` |
+| `billingMetrics.caSecretName` | Name of an existing Secret holding a CA bundle under the key `ca.crt`. Only needed for a private or test collector whose server certificate is not on the public web PKI.                                                                    | `""`                      |
+| `billingMetrics.exportIntervalMs` | How often the counter is pushed, in milliseconds. The proxy defaults to `60000` when unset.                                                                                                                                             | `""`                      |
+
 #### Example `proxy_config` ConfigMap from values (default):
 
 ```
@@ -94,6 +100,21 @@ data:
 type: Opaque
 ```
 
+#### Enterprise billable-request metering
+
+Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector over mutual TLS. The chart does not create the client certificate; it mounts one you already hold, read-only, so the private key is never exposed through the environment. Create the Secret under the name the chart expects, then turn the block on:
+
+```
+kubectl create secret tls litellm-billing-metrics-mtls --cert=client.crt --key=client.key
+```
+
+```
+billingMetrics:
+  enabled: true
+```
+
+Set `billingMetrics.caSecretName` only when the collector is a private or test one whose server certificate is not on the public web PKI; the production collector needs no CA override. The chart fails the render rather than deploying a proxy that silently never exports, so a missing `secretName` or an emptied `endpoint` surfaces at `helm install` time.
+
 ### Database Settings
 
 | Name                      | Description                                                                                                                                                                                                                                                                               | Value                                                                                      |
@@ -109,6 +130,16 @@ type: Opaque
 | `db.deployStandalone`     | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups.                                                                                                  | `true`                                                                                     |
 | `postgresql.*`            | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml)                                                           |
 | `postgresql.auth.*`       | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used.                                                                                                                                                | `NoTaGrEaTpAsSwOrD`                                                                        |
+| `postgresql.image.*`      | If `db.deployStandalone` is `true`, the image for the bundled Postgres. Pinned to a `docker.io/bitnamilegacy` build because Bitnami retired the versioned tags under `docker.io/bitnami`.                                                                                                 | `bitnamilegacy/postgresql:16.2.0-debian-12-r6`                                             |
+| `redis.image.*`           | If `redis.enabled` is `true`, the image for the bundled Redis. Pinned to a `docker.io/bitnamilegacy` build for the same reason.                                                                                                                                                            | `bitnamilegacy/redis:7.2.4-debian-12-r9`                                                   |
+
+#### Bundled Postgres image
+
+Bitnami removed the versioned tags from `docker.io/bitnami` and republished the archived builds under `docker.io/bitnamilegacy`, so the image defaults that ship inside the `postgresql` and `redis` subcharts no longer pull. The chart pins both to the `bitnamilegacy` copies of the exact builds those subchart versions were released with, which keeps the on-disk data directory layout unchanged for existing installs.
+
+Keep `postgresql.image.tag` pinned. `docker.io/bitnami/postgresql` still publishes a floating `latest`, and pointing the bundled Postgres at a different major version starts the server against a data directory it cannot read (`database files are incompatible with server`). There is no in-place way back, so crossing a major version means dumping the database with the old image and restoring it into the new one. The chart refuses to render when the tag is empty or `latest`.
+
+Those images no longer receive updates. For anything beyond getting started, run Postgres outside the chart and point at it with `db.useExisting`.
 
 #### Example Postgres `db.useExisting` Secret
 

helm/litellm-helm/templates/_helpers.tpl

@@ -50,6 +50,53 @@ app.kubernetes.io/name: {{ include "litellm.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
+{{/*
+Enterprise billable-request metering. The client certificate identifies the
+deployment to LiteLLM's collector, so it is mounted read-only from an existing
+Secret rather than passed through the environment.
+*/}}
+{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}}
+{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}}
+
+{{- define "litellm.billingMetricsEnv" -}}
+- name: LITELLM_BILLING_METRICS_ENDPOINT
+  value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_CERT
+  value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_KEY
+  value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: LITELLM_BILLING_METRICS_CA_CERT
+  value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }}
+{{- end }}
+{{- with .Values.billingMetrics.exportIntervalMs }}
+- name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+  value: {{ . | quote }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumes" -}}
+- name: billing-metrics-mtls
+  secret:
+    secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  secret:
+    secretName: {{ .Values.billingMetrics.caSecretName }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumeMounts" -}}
+- name: billing-metrics-mtls
+  mountPath: {{ include "litellm.billingMetrics.certDir" . }}
+  readOnly: true
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  mountPath: {{ include "litellm.billingMetrics.caDir" . }}
+  readOnly: true
+{{- end }}
+{{- end -}}
+
 {{/*
 Create the name of the service account to use
 */}}
@@ -99,3 +146,18 @@ Get redis service port
 {{ .Values.redis.master.service.ports.redis }}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Reject an unpinned image tag for the bundled PostgreSQL.
+A floating tag lets a chart upgrade start a newer PostgreSQL major against the
+existing PersistentVolumeClaim. The server then refuses to start on a data
+directory written by another major version, and the only way back is a dump
+taken before the change, which by that point no longer exists.
+*/}}
+{{- define "litellm.validateBundledPostgresImageTag" -}}
+{{- $tag := .Values.postgresql.image.tag | default "" | toString -}}
+{{- $digest := .Values.postgresql.image.digest | default "" | toString -}}
+{{- if and (eq $digest "") (or (eq $tag "") (eq $tag "latest")) -}}
+{{- fail (printf "postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got %q). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore." $tag) -}}
+{{- end -}}
+{{- end -}}

helm/litellm-helm/templates/deployment.yaml

@@ -142,6 +142,9 @@ spec:
             {{- with .Values.extraEnvVars }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
+            {{- end }}
             {{- if .Values.migrationJob.enabled }}
             # Schema updates are owned by the dedicated migrations Job; skip
             # the proxy's startup `prisma db push` so N replicas don't race
@@ -220,6 +223,9 @@ spec:
             - name: npm
               mountPath: /.npm
           {{- end }}
+          {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
+          {{- end }}
           {{- with .Values.volumeMounts }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
@@ -252,6 +258,9 @@ spec:
             items:
               - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
                 path: "config.yaml"
+        {{- if .Values.billingMetrics.enabled }}
+        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
+        {{- end }}
       {{- with .Values.volumes }}
         {{- toYaml . | nindent 8 }}
       {{- end }}

helm/litellm-helm/templates/migrations-job.yaml

@@ -35,6 +35,8 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
       {{- with .Values.migrationJob.extraInitContainers }}
       initContainers:
         {{- tpl (toYaml .) $ | nindent 8 }}

helm/litellm-helm/templates/secret-dbcredentials.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.db.deployStandalone -}}
+{{- include "litellm.validateBundledPostgresImageTag" . -}}
 apiVersion: v1
 kind: Secret
 metadata:

helm/litellm-helm/templates/tests/test-servicemonitor.yaml

@@ -10,7 +10,7 @@ metadata:
 spec:
   containers:
     - name: test
-      image: bitnami/kubectl:latest
+      image: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3
       command: ['sh', '-c']
       args:
         - |

helm/litellm-helm/tests/billing_metrics_tests.yaml

@@ -0,0 +1,297 @@
+suite: test billingMetrics wiring on the proxy deployment
+templates:
+  - deployment.yaml
+  - configmap-litellm.yaml
+  - migrations-job.yaml
+tests:
+  - it: is off by default, adding no env, volume, or mount
+    template: deployment.yaml
+    asserts:
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+      - notContains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+
+  - it: renders the endpoint and the mounted cert paths when enabled
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CLIENT_CERT
+            value: /etc/litellm/billing-mtls/tls.crt
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CLIENT_KEY
+            value: /etc/litellm/billing-mtls/tls.key
+
+  # The conventional Secret name is the default, so enabling the block is enough.
+  - it: mounts the default cert secret read-only alongside the config volume
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+
+  - it: honours a secretName override
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: my-billing-mtls
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: my-billing-mtls
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+
+  - it: honours an endpoint override
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        endpoint: https://collector.internal:4318
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://collector.internal:4318
+
+  # The production collector presents a public web-PKI certificate, so the CA
+  # override must stay absent unless a private collector is configured.
+  - it: omits the CA env, volume, and mount when no caSecretName is set
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls-ca
+            secret:
+              secretName: billing-ca
+      - notContains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls-ca
+            mountPath: /etc/litellm/billing-mtls-ca
+            readOnly: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CA_CERT
+            value: /etc/litellm/billing-mtls-ca/ca.crt
+
+  - it: mounts the CA secret when caSecretName is set
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        caSecretName: billing-ca
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CA_CERT
+            value: /etc/litellm/billing-mtls-ca/ca.crt
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls-ca
+            secret:
+              secretName: billing-ca
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls-ca
+            mountPath: /etc/litellm/billing-mtls-ca
+            readOnly: true
+
+  - it: passes the export interval through only when set
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        exportIntervalMs: 5000
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+            value: "5000"
+
+  - it: omits the export interval when unset
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+            value: "60000"
+
+  # Kubernetes resolves duplicate env names last-wins, so the chart-owned billing
+  # entries must render after .Values.envVars or a user could silently redirect
+  # the metering export. The three billing entries are the last ones emitted here
+  # (migrationJob, which appends DISABLE_SCHEMA_UPDATE, is off for this case).
+  - it: renders the billing endpoint after envVars so it cannot be shadowed
+    template: deployment.yaml
+    set:
+      migrationJob:
+        enabled: false
+      billingMetrics:
+        enabled: true
+      envVars:
+        LITELLM_BILLING_METRICS_ENDPOINT: https://shadowed.example
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://shadowed.example
+      - equal:
+          path: spec.template.spec.containers[0].env[-3]
+          value:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - equal:
+          path: spec.template.spec.containers[0].env[-2].name
+          value: LITELLM_BILLING_METRICS_CLIENT_CERT
+      - equal:
+          path: spec.template.spec.containers[0].env[-1].name
+          value: LITELLM_BILLING_METRICS_CLIENT_KEY
+
+  - it: keeps user-supplied volumes and mounts alongside the billing secret
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+      volumes:
+        - name: custom-callbacks
+          configMap:
+            name: my-callbacks
+      volumeMounts:
+        - name: custom-callbacks
+          mountPath: /app/callbacks
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: custom-callbacks
+            configMap:
+              name: my-callbacks
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: custom-callbacks
+            mountPath: /app/callbacks
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+
+  - it: still mounts the proxy config when enabled
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: litellm-config
+            mountPath: /etc/litellm/config.yaml
+            subPath: config.yaml
+
+  # Only the proxy serves billable traffic. The migrations Job must never mount
+  # the client certificate, and it renders its own env and volumes, so nothing
+  # stops a future edit from wiring the billing include into it by mistake.
+  - it: does not touch the migrations job when enabled
+    template: migrations-job.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - notExists:
+          path: spec.template.spec.containers[0].volumeMounts
+      - notExists:
+          path: spec.template.spec.volumes
+
+  - it: fails loudly when enabled with an emptied secretName
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)
+
+  - it: fails loudly when enabled without an endpoint
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        endpoint: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true

helm/litellm-helm/tests/bundled_db_images_tests.yaml

@@ -0,0 +1,94 @@
+suite: test bundled database images
+templates:
+  - charts/postgresql/templates/primary/statefulset.yaml
+  - charts/redis/templates/master/application.yaml
+  - charts/redis/templates/configmap.yaml
+  - charts/redis/templates/health-configmap.yaml
+  - charts/redis/templates/scripts-configmap.yaml
+  - charts/redis/templates/secret.yaml
+  - secret-dbcredentials.yaml
+  - templates/tests/test-servicemonitor.yaml
+tests:
+  - it: should pull the bundled postgres from a repository that still publishes the pinned tag
+    template: charts/postgresql/templates/primary/statefulset.yaml
+    set:
+      db.deployStandalone: true
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: docker.io/bitnamilegacy/postgresql:16.2.0-debian-12-r6
+
+  - it: should pull the bundled postgres metrics exporter from the same repository
+    template: charts/postgresql/templates/primary/statefulset.yaml
+    set:
+      db.deployStandalone: true
+      postgresql.metrics.enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[1].image
+          value: docker.io/bitnamilegacy/postgres-exporter:0.15.0-debian-12-r14
+
+  - it: should run the bundled postgres init container from the same repository
+    template: charts/postgresql/templates/primary/statefulset.yaml
+    set:
+      db.deployStandalone: true
+      postgresql.volumePermissions.enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.initContainers[0].image
+          value: docker.io/bitnamilegacy/os-shell:12-debian-12-r16
+
+  - it: should pull the bundled redis from a repository that still publishes the pinned tag
+    template: charts/redis/templates/master/application.yaml
+    set:
+      redis.enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: docker.io/bitnamilegacy/redis:7.2.4-debian-12-r9
+
+  - it: should reject a floating postgres tag that could cross a major version on an existing volume
+    template: secret-dbcredentials.yaml
+    set:
+      db.deployStandalone: true
+      postgresql.image.tag: latest
+    asserts:
+      - failedTemplate:
+          errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got "latest"). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.'
+
+  - it: should reject an empty postgres tag
+    template: secret-dbcredentials.yaml
+    set:
+      db.deployStandalone: true
+      postgresql.image.tag: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got ""). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.'
+
+  - it: should accept an empty postgres tag when the image is pinned by digest
+    template: secret-dbcredentials.yaml
+    set:
+      db.deployStandalone: true
+      postgresql.image.tag: ""
+      postgresql.image.digest: sha256:0d0e2f1a5b3c4d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6
+    asserts:
+      - hasDocuments:
+          count: 1
+
+  - it: should run the servicemonitor test pod from a pinned image
+    template: templates/tests/test-servicemonitor.yaml
+    set:
+      serviceMonitor.enabled: true
+    asserts:
+      - equal:
+          path: spec.containers[0].image
+          value: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3
+
+  - it: should not constrain the postgres tag when the bundled database is not deployed
+    template: secret-dbcredentials.yaml
+    set:
+      db.deployStandalone: false
+      postgresql.image.tag: latest
+    asserts:
+      - hasDocuments:
+          count: 0

helm/litellm-helm/tests/migrations-job_tests.yaml

@@ -254,3 +254,39 @@ tests:
           content:
             name: sidecar-tpl
             image: "ghcr.io/berriai/litellm-database:test"
+  - it: should render the pod-level securityContext from podSecurityContext
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+      podSecurityContext:
+        fsGroup: 10000
+        runAsUser: 10000
+        runAsNonRoot: true
+    asserts:
+      - equal:
+          path: spec.template.spec.securityContext
+          value:
+            fsGroup: 10000
+            runAsUser: 10000
+            runAsNonRoot: true
+  - it: should keep the pod-level and container-level securityContext separate
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+      podSecurityContext:
+        fsGroup: 10000
+      securityContext:
+        allowPrivilegeEscalation: false
+        readOnlyRootFilesystem: true
+    asserts:
+      - equal:
+          path: spec.template.spec.securityContext
+          value:
+            fsGroup: 10000
+      - equal:
+          path: spec.template.spec.containers[0].securityContext
+          value:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true

helm/litellm-helm/values.yaml

@@ -139,6 +139,20 @@ masterkeySecretName: ""
 # if set, use this secret key for the master key; otherwise, use the default key
 masterkeySecretKey: ""
 
+# Optional: enterprise billable-request metering. When enabled, the proxy counts
+# successful requests to inference, MCP, and A2A endpoints and pushes them to
+# LiteLLM's collector over mutual TLS. Requires an enterprise license.
+# The client certificate identifies the deployment, so it is mounted read-only
+# from an existing Secret and never passed through the environment.
+billingMetrics:
+  enabled: false
+  endpoint: https://telemetry.litellm.ai   # collector to push the counter to
+  secretName: litellm-billing-metrics-mtls # existing Secret holding tls.crt and tls.key
+  # Only for private or test collectors whose server certificate is not on the
+  # public web PKI. The production collector needs no CA override.
+  caSecretName: ""                         # existing Secret holding ca.crt
+  exportIntervalMs: ""                     # push cadence; the proxy defaults to 60000
+
 proxyConfigMap:
   # when true, creates a new configmap
   create: true
@@ -167,16 +181,19 @@ proxy_config:
 
 resources:
   {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
+  # Unset by default so the chart installs on small clusters such as Minikube, and so an
+  # upgrade never leaves a running pod Pending. Production deployments should set these.
+  # A proxy at DB-connected steady state needs about 1 CPU and 4Gi of memory per worker;
+  # sizing below that gets the pod OOMKilled once traffic and DB connections ramp up.
+  # Scale both figures with --num_workers, then uncomment the lines below and remove the
+  # curly braces after 'resources:'. See "Recommended Machine Specifications" in
+  # https://docs.litellm.ai/docs/proxy/prod.
   # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
+  #   cpu: "1"
+  #   memory: 4Gi
+  # limits:
+  #   cpu: "1"
+  #   memory: 4Gi
 
 autoscaling:
   enabled: false
@@ -314,8 +331,32 @@ lifecycle: {}
 
 # Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored
 #  otherwise)
+#
+# Bitnami retired the versioned tags under docker.io/bitnami and republished the
+# archived builds under docker.io/bitnamilegacy, so the subchart's own image
+# defaults no longer resolve. The repository below points at the same build the
+# subchart was released with, which keeps the on-disk data directory layout
+# identical for existing installs.
+#
+# Keep the tag pinned. docker.io/bitnami still publishes a floating `latest`,
+# and starting a newer PostgreSQL major against an existing data directory
+# leaves the server refusing to boot ("database files are incompatible with
+# server") with no way back other than a dump taken beforehand. Crossing a major
+# version is a dump-and-restore, not an image bump. The chart refuses to render
+# an unpinned tag for this reason
 postgresql:
   architecture: standalone
+  image:
+    repository: bitnamilegacy/postgresql
+    tag: 16.2.0-debian-12-r6
+  volumePermissions:
+    image:
+      repository: bitnamilegacy/os-shell
+      tag: 12-debian-12-r16
+  metrics:
+    image:
+      repository: bitnamilegacy/postgres-exporter
+      tag: 0.15.0-debian-12-r14
   auth:
     username: litellm
     database: litellm
@@ -345,9 +386,36 @@ postgresql:
 # When `redis.sentinel.enabled` is set, the coordination block is rendered with
 # `sentinel_nodes` and `service_name` (from `redis.sentinel.masterSet`) instead
 # of host/port, because a plain Redis client cannot talk to the sentinel port
+#
+# The image repositories carry the same bitnamilegacy repoint as postgresql
+# above; the versioned tags the subchart ships with are gone from
+# docker.io/bitnami
 redis:
   enabled: false
   architecture: standalone
+  image:
+    repository: bitnamilegacy/redis
+    tag: 7.2.4-debian-12-r9
+  sentinel:
+    image:
+      repository: bitnamilegacy/redis-sentinel
+      tag: 7.2.4-debian-12-r7
+  metrics:
+    image:
+      repository: bitnamilegacy/redis-exporter
+      tag: 1.58.0-debian-12-r4
+  volumePermissions:
+    image:
+      repository: bitnamilegacy/os-shell
+      tag: 12-debian-12-r16
+  sysctl:
+    image:
+      repository: bitnamilegacy/os-shell
+      tag: 12-debian-12-r16
+  kubectl:
+    image:
+      repository: bitnamilegacy/kubectl
+      tag: 1.29.2-debian-12-r3
   coordination:
     # Set to false to keep the bundled Redis for response caching only and leave
     # `general_settings.coordination_redis` out of the rendered config. A
@@ -367,9 +435,9 @@ migrationJob:
   annotations: {}
   ttlSecondsAfterFinished: 120
   resources: {}
-  #  requests:
-  #    cpu: 100m
-  #    memory: 100Mi
+  #  Unset by default. This job runs the database migration and exits, so it does not
+  #  need the steady-state headroom the proxy does; size it from your own migration
+  #  runs rather than from the proxy figures above.
   extraContainers: []
   extraInitContainers: []
 

helm/litellm/templates/NOTES.txt

@@ -46,4 +46,9 @@ Reminders:
       - gateway.config.proxy_config                            (rendered into a ConfigMap and mounted at
                                                                 /app/config/config.yaml; gateway reads it via
                                                                 CONFIG_FILE_PATH)
+      - {component}.pdb.{enabled,minAvailable,maxUnavailable}  (per-component PodDisruptionBudget; disabled by
+                                                                default — with hpa.minReplicas of 1, minAvailable: 1
+                                                                would block node drains)
+      - {component}.topologySpreadConstraints                  (standard k8s list, e.g. spread replicas across
+                                                                topology.kubernetes.io/zone)
   - Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend.

helm/litellm/templates/_helpers.tpl

@@ -34,6 +34,57 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
 {{- end -}}
 
+{{/*
+Enterprise billable-request metering. Wired into gateway and backend, not the
+migrations job. The gateway serves nearly all billable traffic, but the backend
+keeps the named-server MCP transport (/{mcp_server_name}/mcp), which writes a
+SpendLogs row, so metering only the gateway would silently drop that traffic.
+The client certificate identifies the deployment to LiteLLM's collector, so it is
+mounted read-only from an existing Secret rather than passed through the
+environment.
+*/}}
+{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}}
+{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}}
+
+{{- define "litellm.billingMetricsEnv" -}}
+- name: LITELLM_BILLING_METRICS_ENDPOINT
+  value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_CERT
+  value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_KEY
+  value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: LITELLM_BILLING_METRICS_CA_CERT
+  value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }}
+{{- end }}
+{{- with .Values.billingMetrics.exportIntervalMs }}
+- name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+  value: {{ . | quote }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumes" -}}
+- name: billing-metrics-mtls
+  secret:
+    secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  secret:
+    secretName: {{ .Values.billingMetrics.caSecretName }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumeMounts" -}}
+- name: billing-metrics-mtls
+  mountPath: {{ include "litellm.billingMetrics.certDir" . }}
+  readOnly: true
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  mountPath: {{ include "litellm.billingMetrics.caDir" . }}
+  readOnly: true
+{{- end }}
+{{- end -}}
+
 {{/*
 Per-component selector labels — used in both Service selectors and Deployment matchLabels.
 */}}
@@ -87,6 +138,59 @@ is false the chart uses the provided name, or the namespace `default` SA.
 {{- end -}}
 {{- end -}}
 
+{{/*
+ServiceAccount name for the migrations Job.
+
+The Job is a pre-install / pre-upgrade hook, so it is created before the
+chart's ordinary resources. A ServiceAccount the chart creates is one of
+those ordinary resources, which makes borrowing the backend name a cycle:
+the hook pod is rejected because the account does not exist yet. So when
+`serviceAccounts.backend.create` is true the Job falls back to the namespace
+`default` account unless the operator names one that already exists. With
+`create` false the backend name is either an operator-supplied existing
+account or `default`, both of which are safe for the hook, so the Job keeps
+sharing it.
+
+`migrationJob.serviceAccountName` always wins when set, which is how a Job
+that needs credentials of its own (IRSA / Workload Identity for IAM database
+auth) gets them.
+*/}}
+{{- define "litellm.migrations.serviceAccountName" -}}
+{{- if .Values.migrationJob.serviceAccountName -}}
+{{ .Values.migrationJob.serviceAccountName }}
+{{- else if .Values.serviceAccounts.backend.create -}}
+default
+{{- else -}}
+{{ include "litellm.backend.serviceAccountName" . }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Extra pod labels for a component's Deployment, validated against its selector.
+
+Invoke with a dict:
+  (dict "podLabels" .Values.gateway.podLabels "componentName" "gateway")
+
+The three selector keys are also emitted on the pod template, so a podLabels
+entry reusing one renders a duplicate YAML key whose later value wins. That
+leaves the pod template no longer matching the (immutable) selector and the
+apiserver rejects the Deployment. Fail at template time naming the key
+instead, so the operator gets the reason here rather than an opaque
+`selector does not match template labels` from the apiserver.
+
+The migrations Job takes podLabels unvalidated: a Job's selector is generated
+by the controller rather than declared, so nothing there can collide.
+*/}}
+{{- define "litellm.podLabels" -}}
+{{- $componentName := .componentName -}}
+{{- range $key, $value := .podLabels }}
+{{- if has $key (list "app.kubernetes.io/name" "app.kubernetes.io/instance" "app.kubernetes.io/component") }}
+{{- fail (printf "%s.podLabels cannot set %s: it is part of the Deployment's immutable selector" $componentName $key) }}
+{{- end }}
+{{- end }}
+{{- toYaml .podLabels }}
+{{- end -}}
+
 {{/*
 Master-key + database + redis env block — shared by gateway, backend, and the
 migrations Job.
@@ -244,6 +348,52 @@ harmless no-op for the Job and authoritative for the app pods.
 {{- end }}
 {{- end -}}
 
+{{/*
+PodDisruptionBudget shared by gateway, backend, and ui.
+
+Invoke with a dict:
+  (dict "root" $ "component" .Values.gateway "componentName" "gateway"
+        "fullname" (include "litellm.gateway.fullname" .)
+        "selectorLabels" (include "litellm.gateway.selectorLabels" .))
+
+Renders nothing unless both the component and its `pdb.enabled` are on.
+Only one of minAvailable / maxUnavailable should be set; if both are,
+minAvailable wins. If neither is set, falls back to `maxUnavailable: 1` so
+an enabled-but-unconfigured PDB still permits node drains.
+
+"Set" means non-nil and non-empty-string, so an explicit 0 (e.g.
+`maxUnavailable: 0` to forbid all voluntary disruptions) is honored rather
+than silently replaced by the fallback.
+*/}}
+{{- define "litellm.pdb" -}}
+{{- $root := .root -}}
+{{- $component := .component -}}
+{{- $min := $component.pdb.minAvailable -}}
+{{- $max := $component.pdb.maxUnavailable -}}
+{{- $minSet := not (or (kindIs "invalid" $min) (eq (printf "%v" $min) "")) -}}
+{{- $maxSet := not (or (kindIs "invalid" $max) (eq (printf "%v" $max) "")) -}}
+{{- if and $component.enabled $component.pdb $component.pdb.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ .fullname }}
+  labels:
+    {{- include "litellm.commonLabels" $root | nindent 4 }}
+    app.kubernetes.io/component: {{ .componentName }}
+spec:
+  selector:
+    matchLabels:
+      {{- .selectorLabels | nindent 6 }}
+  {{- if $minSet }}
+  minAvailable: {{ $min }}
+  {{- else if $maxSet }}
+  maxUnavailable: {{ $max }}
+  {{- else }}
+  maxUnavailable: 1
+  {{- end }}
+{{- end }}
+{{- end -}}
+
 {{/*
 Renders `envFrom:` block for a component's `envConfigMaps` / `envSecrets`
 lists. Each entry is a resource name; the chart wires the whole ConfigMap /

helm/litellm/templates/backend/deployment.yaml

@@ -23,9 +23,16 @@ spec:
       {{- end }}
       labels:
         {{- include "litellm.backend.selectorLabels" . | nindent 8 }}
+        {{- with .Values.backend.podLabels }}
+        {{- include "litellm.podLabels" (dict "podLabels" . "componentName" "backend") | nindent 8 }}
+        {{- end }}
     spec:
       serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }}
       automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }}
+      {{- with .Values.backend.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
@@ -34,6 +41,10 @@ spec:
         - name: backend
           image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.backend.image.pullPolicy }}
+          {{- with .Values.backend.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           ports:
             - name: http
               containerPort: 4001
@@ -44,14 +55,20 @@ spec:
             - name: CONFIG_FILE_PATH
               value: /app/config/config.yaml
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
+            {{- end }}
           {{- include "litellm.envFrom" .Values.backend | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.backend.volumeMounts }}
+          {{- if or .Values.gateway.config.create .Values.backend.volumeMounts .Values.billingMetrics.enabled }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
               mountPath: /app/config/config.yaml
               subPath: config.yaml
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
+            {{- end }}
             {{- with .Values.backend.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -64,15 +81,25 @@ spec:
           readinessProbe:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.backend.lifecycle }}
+          lifecycle:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           resources:
             {{- toYaml .Values.backend.resources | nindent 12 }}
-      {{- if or .Values.gateway.config.create .Values.backend.volumes }}
+      {{- with .Values.backend.extraContainers }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- if or .Values.gateway.config.create .Values.backend.volumes .Values.billingMetrics.enabled }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
           configMap:
             name: {{ include "litellm.gateway.fullname" . }}-config
         {{- end }}
+        {{- if .Values.billingMetrics.enabled }}
+        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
+        {{- end }}
         {{- with .Values.backend.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -89,4 +116,12 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.backend.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- $gracePeriod := .Values.backend.terminationGracePeriodSeconds }}
+      {{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }}
+      terminationGracePeriodSeconds: {{ $gracePeriod }}
+      {{- end }}
 {{- end }}

helm/litellm/templates/backend/poddisruptionbudget.yaml

@@ -0,0 +1,6 @@
+{{- include "litellm.pdb" (dict
+  "root" $
+  "component" .Values.backend
+  "componentName" "backend"
+  "fullname" (include "litellm.backend.fullname" .)
+  "selectorLabels" (include "litellm.backend.selectorLabels" .)) }}