Merge branch 'litellm_internal_staging' into fix/vllm-passthrough-spend-logging-33210

.circleci/config.yml

@@ -88,6 +88,59 @@ commands:
             rm -f /tmp/uv-install.sh
             echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
             export PATH="$HOME/.local/bin:$PATH"
+  install_node:
+    description: "Install the Node.js version pinned in ui/litellm-dashboard/.nvmrc (24.19.0, which bundles npm 11.17.0) with checksum verification, and prepend it to PATH. Run this on any executor whose image does not already ship that version, or `npm ci` in ui/litellm-dashboard fails EBADENGINE against the engines floor. Installs into /opt/node rather than over /usr/local on purpose: cimg/python:*-browsers ships its own node there, and unpacking the tarball on top of it leaves npm 11.17 files merged with the image's npm 11.9 tree, which reports the new version and then exits 1 on `npm ci` with no error text at all. Requires checkout, which the .nvmrc drift check reads."
+    steps:
+      - run:
+          name: Install Node.js 24.19.0
+          command: |
+            NODE_VERSION="24.19.0"
+            NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz"
+            NODE_EXPECTED_SHA="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647"
+            NVMRC_VERSION="$(tr -d '[:space:]' < ui/litellm-dashboard/.nvmrc)"
+            if [ "$NVMRC_VERSION" != "$NODE_VERSION" ]; then
+              echo "install_node: ui/litellm-dashboard/.nvmrc pins ${NVMRC_VERSION} but this command pins ${NODE_VERSION}; update NODE_VERSION and NODE_EXPECTED_SHA together" >&2
+              exit 1
+            fi
+            curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}"
+            echo "${NODE_EXPECTED_SHA}  /tmp/${NODE_TARBALL}" | sha256sum -c -
+            sudo mkdir -p /opt/node
+            sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /opt/node --strip-components=1
+            rm -f "/tmp/${NODE_TARBALL}"
+            echo 'export PATH="/opt/node/bin:$PATH"' >> "$BASH_ENV"
+            export PATH="/opt/node/bin:$PATH"
+            node --version
+            npm --version
+  install_rust:
+    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.97.1) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
+    steps:
+      - run:
+          name: Install Rust (rustup 1.28.2, toolchain 1.97.1)
+          command: |
+            case "$(uname -m)" in
+              x86_64)
+                RUSTUP_TRIPLE=x86_64-unknown-linux-gnu
+                RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c
+                ;;
+              aarch64)
+                RUSTUP_TRIPLE=aarch64-unknown-linux-gnu
+                RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c
+                ;;
+              *)
+                echo "install_rust: unsupported architecture $(uname -m)" >&2
+                exit 1
+                ;;
+            esac
+            curl -sSLf -o /tmp/rustup-init \
+              "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init"
+            echo "${RUSTUP_SHA256}  /tmp/rustup-init" | sha256sum -c -
+            chmod +x /tmp/rustup-init
+            /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.97.1
+            rm -f /tmp/rustup-init
+            echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
+            export PATH="$HOME/.cargo/bin:$PATH"
+            rustc --version
+            cargo --version
   start_postgres:
     description: "Start a postgres-db container on port 5432 and wait until it accepts connections."
     parameters:
@@ -163,6 +216,26 @@ commands:
             done
             echo "fake OpenAI endpoint did not become ready" >&2
             exit 1
+  start_cost_center_service:
+    description: "Start the stand-in cost center validation service (tests/store_model_in_db_tests/cost_center_service.py) on host port 9414 and wait until healthy. The proxy's team-metadata validator (team_metadata_validator_e2e.py, impl 'http') reaches it via TEAM_METADATA_VALIDATION_SERVICE_URL=http://host.docker.internal:9414/validate. Run after uv deps are synced."
+    steps:
+      - run:
+          name: Start cost center validation service
+          background: true
+          command: |
+            uv run --no-sync python tests/store_model_in_db_tests/cost_center_service.py --host 0.0.0.0 --port 9414
+      - run:
+          name: Wait for cost center validation service
+          command: |
+            for i in $(seq 1 30); do
+              if curl -sf http://localhost:9414/health >/dev/null 2>&1; then
+                echo "cost center validation service is up"
+                exit 0
+              fi
+              sleep 1
+            done
+            echo "cost center validation service did not become ready" >&2
+            exit 1
   setup_litellm_enterprise_pip:
     steps:
       - run:
@@ -178,6 +251,7 @@ commands:
       - checkout
       - setup_google_dns
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -281,6 +355,33 @@ jobs:
             uv build --wheel --out-dir dist
             uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py
 
+  base_sdk_install:
+    docker:
+      - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+    steps:
+      - checkout
+      - setup_google_dns
+      - install_uv
+      - install_rust
+      - run:
+          name: Build the wheel
+          environment:
+            UV_HTTP_TIMEOUT: "300"
+          command: |
+            uv build --wheel --out-dir dist
+      - run:
+          name: Install the wheel with no extras and smoke-check it
+          environment:
+            UV_HTTP_TIMEOUT: "300"
+          command: |
+            uv venv /tmp/base-sdk --python 3.12
+            VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl
+            /tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py
+
   local_testing_part1:
     docker:
       - &python312_image
@@ -298,6 +399,7 @@ jobs:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -371,6 +473,7 @@ jobs:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -445,6 +548,7 @@ jobs:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -496,6 +600,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -562,6 +667,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -602,6 +708,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -643,6 +750,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -676,6 +784,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -726,6 +835,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -777,6 +887,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -810,6 +921,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -856,6 +968,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -902,6 +1015,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -944,6 +1058,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -990,6 +1105,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1037,6 +1153,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -1077,6 +1194,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1122,6 +1240,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1166,6 +1285,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1198,6 +1318,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1241,6 +1362,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1285,6 +1407,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1329,6 +1452,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1360,6 +1484,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1406,6 +1531,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1451,6 +1577,7 @@ jobs:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1501,6 +1628,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1525,6 +1653,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1551,6 +1680,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1652,6 +1782,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1747,6 +1878,7 @@ jobs:
           at: ~/project
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1835,6 +1967,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -1918,6 +2051,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2050,6 +2184,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2136,6 +2271,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2232,12 +2368,14 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
             uv sync --frozen --all-groups --all-extras --python 3.12
       - start_postgres
       - start_fake_openai_endpoint
+      - start_cost_center_service
       - attach_workspace:
           at: ~/project
       - run:
@@ -2257,11 +2395,13 @@ jobs:
               -e STORE_MODEL_IN_DB="True" \
               -e LITELLM_MASTER_KEY="sk-1234" \
               -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
+              -e TEAM_METADATA_VALIDATION_SERVICE_URL=http://host.docker.internal:9414/validate \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
               -e LITELLM_LOG=ERROR \
               --add-host host.docker.internal:host-gateway \
               --name my-app \
               -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
+              -v $(pwd)/litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py:/app/team_metadata_validator_e2e.py \
               litellm-docker-database:ci \
               --config /app/config.yaml \
               --port 4000
@@ -2307,6 +2447,7 @@ jobs:
       - setup_google_dns
       # Remove Docker CLI installation since it's already available in machine executor
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2388,6 +2529,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2473,20 +2615,9 @@ jobs:
             bundle exec rspec
           no_output_timeout: 30m
       # Install Node.js directly from nodejs.org with SHA256 verification,
-      # instead of piping NodeSource's setup_18.x apt-repo installer into
+      # instead of piping NodeSource's setup_24.x apt-repo installer into
       # sudo bash (which runs a mutable upstream script unattended).
-      - run:
-          name: Install Node.js 18.20.8
-          command: |
-            NODE_VERSION="18.20.8"
-            NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz"
-            NODE_EXPECTED_SHA="5467ee62d6af1411d46b6a10e3fb5cacc92734dbcef465fea14e7b90993001c9"
-            curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}"
-            echo "${NODE_EXPECTED_SHA}  /tmp/${NODE_TARBALL}" | sha256sum -c -
-            sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /usr/local --strip-components=1
-            rm -f "/tmp/${NODE_TARBALL}"
-            node --version
-            npm --version
+      - install_node
 
       - run:
           name: Install Node.js test dependencies
@@ -2527,6 +2658,7 @@ jobs:
       - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
+      - install_rust
       - run:
           name: Install Dependencies
           command: |
@@ -2614,7 +2746,7 @@ jobs:
 
   ui_build:
     docker:
-      - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81
+      - image: cimg/node:24.19@sha256:8966565f07189a67d64d6808a2b127f31dafae566508e3547f55640e1070bfad
         auth:
           username: ${DOCKERHUB_USERNAME}
           password: ${DOCKERHUB_PASSWORD}
@@ -2658,7 +2790,7 @@ jobs:
 
   ui_unit_tests:
     docker:
-      - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81
+      - image: cimg/node:24.19@sha256:8966565f07189a67d64d6808a2b127f31dafae566508e3547f55640e1070bfad
         auth:
           username: ${DOCKERHUB_USERNAME}
           password: ${DOCKERHUB_PASSWORD}
@@ -2716,7 +2848,9 @@ jobs:
       - skip_if_unrelated_changes:
           category: client
       - setup_google_dns
+      - install_node
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -2731,7 +2865,7 @@ jobs:
             - ~/.cache/uv
       - restore_cache:
           keys:
-            - ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
+            - ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
       - run:
           name: Install Node dependencies and Playwright
           # The cimg/python:3.12-browsers image already ships the Chromium system
@@ -2746,7 +2880,7 @@ jobs:
             npm ci
             npx playwright install chromium
       - save_cache:
-          key: ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
+          key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
           paths:
             - ui/litellm-dashboard/node_modules
             - tests/e2e/ui/node_modules
@@ -2858,7 +2992,9 @@ jobs:
       - skip_if_unrelated_changes:
           category: client
       - setup_google_dns
+      - install_node
       - install_uv
+      - install_rust
       - restore_cache:
           keys:
             - v1-uv-cache-{{ checksum "uv.lock" }}
@@ -2873,7 +3009,7 @@ jobs:
             - ~/.cache/uv
       - restore_cache:
           keys:
-            - ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
+            - ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
       - run:
           name: Install Node dependencies and Playwright
           command: |
@@ -2883,7 +3019,7 @@ jobs:
             npm ci
             npx playwright install chromium
       - save_cache:
-          key: ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
+          key: ui-e2e-node-deps-v4-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
           paths:
             - ui/litellm-dashboard/node_modules
             - tests/e2e/ui/node_modules
@@ -3031,6 +3167,8 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - base_sdk_install:
+          filters: *main_branches
       - local_testing_part1:
           filters: *main_branches
       - local_testing_part2:

.flake8

@@ -1,46 +0,0 @@
-[flake8]
-ignore =
-    # The following ignores can be removed when formatting using black
-    W191,W291,W292,W293,W391,W504
-    E101,E111,E114,E116,E117,E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,
-    E201,E202,E221,E222,E225,E226,E231,E241,E251,E252,E261,E265,E271,E272,E275,
-    E301,E302,E303,E305,E306,
-    # line break before binary operator
-    W503,
-    # inline comment should start with '# '
-    E262,
-    # too many leading '#' for block comment
-    E266,
-    # multiple imports on one line
-    E401,
-    # module level import not at top of file
-    E402,
-    # Line too long (82 > 79 characters)
-    E501,
-    # comparison to None should be 'if cond is None:'
-    E711,
-    # comparison to True should be 'if cond is True:' or 'if cond:'
-    E712,
-    # do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()`
-    E721,
-    # do not use bare 'except'
-    E722,
-    # x is imported but unused
-    F401,
-    # 'from . import *' used; unable to detect undefined names
-    F403,
-    # x may be undefined, or defined from star imports:
-    F405,
-    # f-string is missing placeholders
-    F541,
-    # dictionary key '' repeated with different values
-    F601,
-    # redefinition of unused x from line 123
-    F811,
-    # undefined name x
-    F821,
-    # local variable x is assigned to but never used
-    F841,
-
-# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8
-extend-ignore = E203

.git-blame-ignore-revs

@@ -17,3 +17,24 @@
 
 # style: unify ruff format width on 120 (#31518)
 48b5a5a0cc5a694a11219416ee0b6eb6e620e74e
+
+# refactor(imports): move collections.abc names out of typing (#35495)
+397e8e4918777e4e60a7f5e88699e0a9a7dabb3d
+
+# refactor(lint): apply every safe ruff autofix and zero 28 strict-rule budgets (#35495)
+b604e2b20c6db2099085a2f0e59b7e99e87eed6f
+
+# refactor(logging): drop redundant !s conversion flags from f-strings (#35546)
+7b2d3440cba3160277470f7a0180098ae9b87864
+
+# perf: build log messages lazily so filtered-out log records cost nothing (#35703)
+c9887a1f94bc1e7e4bdfe64d640f0509a0bc19dd
+
+# feat(lint): enforce Final on locals and freeze function parameters (#35807)
+2708620d6a599cc73c1950a942d26ac26a7ed3d4
+
+# chore(lint): remove litellm/types from the ruff lint exclusion (#35926)
+4e32a8bf6a1e1af1e04b67c759841ccef44b2235
+
+# chore(lint): strip inert type: ignore comments and zero LIT009/LIT010/LIT011 headroom (#35928)
+338e411103ad5d7003e97f34f04fa36bca542dbe

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -23,30 +23,56 @@ body:
       label: What happened?
       description: Also tell us, what did you expect to happen?
       placeholder: Tell us what you see!
-      value: "A bug happened!"
     validations:
       required: true
   - type: textarea
-    id: steps-to-reproduce
+    id: user-flow
     attributes:
-      label: Steps to Reproduce
-      description: Please provide a numbered list of the exact steps to reproduce this bug (include a curl/python snippet to reproduce it). Number each step (1., 2., 3., ...) in the order you performed them.
+      label: User Flow
+      description: |
+        Two ordered lists, "Before a (hypothetical) fix" and "After a (hypothetical) fix", walking the same end user through the same task, written strictly from that user's seat. Every rule below applies.
+
+        - Describe the real application and the routes its users actually hit, not a generic scenario
+        - Lead each list with one plain sentence saying where the flow fails (before) or would succeed (after), then number the steps
+        - Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen
+        - No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. "The upload hands back an ID that looks like OpenAI's own `file-abc123` instead of the scrambled one the gateway returned" is right, "no managed-file row was registered" is wrong
+        - Keep the two lists step-for-step identical until they diverge, so the broken step is obvious
+        - If the bug has a security or authorization consequence, end each list with what another user can do that they shouldn't be able to, and what they could no longer do after a fix
       placeholder: |
-        1. config.yaml file/ .env file/ etc.
-        2. Run the following code...
-        3. Observe the error...
-      value: |
-        1. 
-        2. 
-        3. 
+        Before a (hypothetical) fix: a developer whose app streams chat completions gets no token counts back, so their cost dashboard reads zero
+
+        1. They send POST https://litellm-domain/v1/chat/completions with "stream": true and no stream_options
+        2. The last SSE chunk arrives with "usage": null, so their app records 0 prompt and 0 completion tokens
+        3. They open https://litellm-domain/ui/?page=logs and see the request logged at $0 spend
+
+        After a (hypothetical) fix: the same request comes back with real token counts, so the dashboard shows real spend
+
+        1. The proxy admin sets always_include_stream_usage: true and restarts the proxy
+        2. The developer sends the same POST https://litellm-domain/v1/chat/completions with "stream": true and no stream_options
+        3. The last SSE chunk now carries a usage object with real prompt and completion token counts
+        4. https://litellm-domain/ui/?page=logs shows that request at non-zero spend
     validations:
       required: true
   - type: textarea
-    id: logs
+    id: proof-of-bug
     attributes:
-      label: Relevant log output
-      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
-      render: shell
+      label: Proof the bug occurs
+      description: |
+        The commands (e.g., curl) and their full output, screenshots, or a screen recording demonstrating that the bug happens. Every rule below applies.
+
+        - The proof must be completely e2e with no mocks, against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), hitting real LLM provider APIs, costing real $ if needed, where the bug involves a provider call. `pytest` commands are not enough
+        - Show exactly what the end user sees or does, matching the User Flow above step for step
+        - Start with the config.yaml (or SDK setup) and any env vars the proxy ran with, then the exact version or commit hash the proof was captured at, so a maintainer can stand up the same proxy before running your commands. Keep the real values for env vars that aren't sensitive, they are often the reason the bug happens, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue
+        - If the bug applies to more than one of the LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every one of them, not just one
+        - For UI bugs: include screenshots and the page URLs you were on. Scrub keys and tokens out of screenshots too (for example, the virtual key is briefly shown in the panel right after you create a virtual key)
+      placeholder: |
+        Config / setup the proxy ran with:
+
+        Version or commit:
+
+        Commands and their full output:
+    validations:
+      required: true
   - type: dropdown
     id: component
     attributes:

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -24,10 +24,53 @@ body:
     validations:
       required: true
   - type: textarea
-    id: motivation
+    id: user-flow
     attributes:
-      label: Motivation, pitch
-      description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too.
+      label: User Flow
+      description: |
+        Two ordered lists, "Before this feature (today)" and "After this feature (ideal user flow)", walking the same end user through the same task, written strictly from that user's seat. Every rule below applies.
+
+        - Describe the real application and the routes its users actually hit, not a generic scenario. Link any related GitHub issue or provider API docs
+        - Lead each list with one plain sentence saying where the flow dead-ends today and what it would let them do instead, then number the steps
+        - Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen
+        - No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. Ask for the behavior you need, not the implementation you imagine
+        - Keep the two lists step-for-step identical until they diverge, so the missing capability is obvious
+        - "Before this feature" is also where you show the workaround you're living with, which is what tells us how badly this is needed
+      placeholder: |
+        Before this feature (today): a developer batching nightly summaries has no way to mark those calls as low priority, so they compete with live traffic for the same rate limit
+
+        1. They send POST https://litellm-domain/v1/chat/completions for 500 documents in a loop
+        2. Around document 120 they start getting 429s naming the rpm limit, and their user-facing chat app starts getting them too
+        3. Their workaround is a hand-rolled sleep between calls, which stretches the batch to 3 hours and still collides at peak
+
+        After this feature (ideal user flow): the same batch runs as background work that yields to live traffic
+
+        1. The developer sends the same POST with "service_tier": "flex"
+        2. Batch calls queue behind interactive ones instead of 429ing, and the response comes back with the tier it was served at
+        3. The live chat app keeps returning 200s throughout the batch
+        4. https://litellm-domain/ui/?page=logs shows the batch requests tagged with that tier
+    validations:
+      required: true
+  - type: textarea
+    id: how-far-you-got
+    attributes:
+      label: How far you got
+      description: |
+        Run as many steps of the "After this feature (ideal user flow)" list as you can against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), then paste the commands (e.g., curl) and their full output, ending at the step that dead-ends. Every rule below applies.
+
+        - Say plainly what stopped you there, in user terms: the option you passed came back ignored, the response 400'd naming an unsupported field, there is no button on the page for it. This is what proves the feature is genuinely missing rather than undocumented
+        - No mocks. Where the flow involves a provider call, hit the real provider API, even if it costs real $. `pytest` commands are not enough
+        - Include the config.yaml (or SDK setup) and env vars the proxy ran with, plus the version or commit you were on. Keep the real values for env vars that aren't sensitive, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue
+        - If the provider already supports this, link their API docs and paste a direct call to them succeeding, so we can see the shape LiteLLM should be sending
+        - For UI asks: include screenshots of the page you got stuck on and its URL. Scrub keys and tokens out of screenshots too (for example, the virtual key is briefly shown in the panel right after you create a virtual key)
+      placeholder: |
+        Config / setup the proxy ran with:
+
+        Version or commit:
+
+        Commands and their full output, up to the step that dead-ends:
+
+        What stopped me there:
     validations:
       required: true
   - type: dropdown

.github/actions/cache-prisma-binaries/action.yml

@@ -0,0 +1,40 @@
+name: "Cache Prisma binaries"
+description: >-
+  Cache the Prisma CLI and engine binaries that `prisma generate` downloads, so
+  only the first job on a given prisma-client-py version pays for the download.
+
+  prisma-client-py shells out to `npm install prisma@<version>` whenever its
+  binary cache directory has no CLI entrypoint, which pulls ~85 MB of query and
+  schema engines over the network. That normally takes a few seconds, but it is
+  unbounded: one shard of a proxy-db run took 5m18s on that single step versus
+  3.8s on its eleven siblings, which pushed the job past its timeout and got a
+  fully passing test run cancelled.
+
+  Callers must not set PRISMA_BINARY_CACHE_DIR. The prisma-client-py default
+  (~/.cache/prisma-python/binaries/<prisma-version>/<engine-version>) is already
+  keyed by both versions, so a cache entry can never be served to a run that
+  expects different binaries.
+
+runs:
+  using: composite
+  steps:
+    - name: Resolve prisma-client-py version
+      id: version
+      shell: bash
+      run: |
+        version="$(grep -A1 '^name = "prisma"$' uv.lock | sed -n 's/^version = "\(.*\)"$/\1/p' | head -1)"
+        if [ -z "${version}" ]; then
+          echo "could not resolve the prisma package version from uv.lock" >&2
+          exit 1
+        fi
+        echo "version=${version}" >> "$GITHUB_OUTPUT"
+
+    - name: Restore Prisma binaries
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        # ~/.cache/prisma-python holds the npm install tree prisma-client-py
+        # drives; ~/.cache/prisma is where @prisma/engines stages its downloads.
+        path: |
+          ~/.cache/prisma-python
+          ~/.cache/prisma
+        key: ${{ runner.os }}-prisma-binaries-${{ steps.version.outputs.version }}

.github/ci-coverage-allowlist.yml

@@ -0,0 +1,152 @@
+description: >-
+  Paths deliberately outside CI coverage, each with the reason it is exempt.
+  assert_ci_coverage.py fails when a test file or Dockerfile is neither invoked
+  by a job nor listed here, so every entry below is a decision on the record.
+
+test_paths:
+  - reason: >-
+      The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
+      from a pull request; it needs a live gateway and provider credentials no PR job holds
+    paths:
+      - tests/e2e
+  - reason: >-
+      The documentation and code-quality workflows execute four files in this directory by name as
+      scripts and pytest never collects the directory, so these six run nowhere; listed individually
+      so a seventh cannot inherit the exemption
+    paths:
+      - tests/documentation_tests/test_exception_types.py
+      - tests/documentation_tests/test_general_setting_keys.py
+      - tests/documentation_tests/test_optional_params.py
+      - tests/documentation_tests/test_readme_providers.py
+      - tests/documentation_tests/test_requests_lib_usage.py
+      - tests/documentation_tests/test_standard_logging_payload.py
+  - reason: >-
+      Sibling files here are executed by name from the code-quality workflow; this one is referenced
+      by no job
+    paths:
+      - tests/code_coverage_tests/test_aio_http_image_conversion.py
+  - reason: >-
+      A second mirror of the package tree living beside tests/test_litellm, which is the mirror the
+      repo convention names; only test_no_hardcoded_secrets.py is invoked, from the linting
+      workflow, and whether this directory should exist at all is unresolved
+    paths:
+      - tests/litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_headers.py
+      - tests/litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_transformation.py
+      - tests/litellm/integrations/helicone/test_helicone_gemini.py
+      - tests/litellm/litellm_core_utils/test_json_schema_validation.py
+      - tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py
+      - tests/litellm/llms/anthropic/test_anthropic_schema_filter.py
+      - tests/litellm/llms/azure/test_azure_embedding.py
+      - tests/litellm/llms/bedrock/embed/test_embedding.py
+      - tests/litellm/llms/bedrock/test_nova_imported_models.py
+      - tests/litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py
+      - tests/litellm/llms/gradient_ai/chat/test_gradient_ai_chat_transformation.py
+      - tests/litellm/llms/oci/chat/test_oci_chat_transformation.py
+      - tests/litellm/llms/openai_like/test_abliteration_provider.py
+      - tests/litellm/llms/openai_like/test_assemblyai_provider.py
+      - tests/litellm/llms/openai_like/test_empiriolabs_provider.py
+      - tests/litellm/llms/vertex_ai/agent_engine/test_transformation.py
+      - tests/litellm/llms/vertex_ai/gemini/test_transformation.py
+      - tests/litellm/llms/vertex_ai/text_to_speech/test_transformation.py
+      - tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
+      - tests/litellm/proxy/agent_endpoints/test_agent_rbac.py
+      - tests/litellm/proxy/common_utils/test_rbac_utils.py
+      - tests/litellm/proxy/management_endpoints/test_common_utils.py
+      - tests/litellm/proxy/management_endpoints/test_cost_estimate_endpoint.py
+      - tests/litellm/proxy/test_claude_code_marketplace.py
+      - tests/litellm/proxy/test_init_litellm_callbacks.py
+      - tests/litellm/proxy/test_prisma_engine_watchdog.py
+      - tests/litellm/proxy/vector_store_endpoints/test_vector_store_rbac.py
+      - tests/litellm/test_bedrock_extended_beta_models.py
+      - tests/litellm/test_bedrock_nemotron_super.py
+      - tests/litellm/test_proxy_auth.py
+      - tests/litellm/test_router_retry_backoff_headers.py
+      - tests/litellm/test_sambanova_model_metadata.py
+      - tests/litellm/test_stream_chunk_builder_images.py
+  - reason: >-
+      Legacy proxy suite superseded by the proxy shards; no job invokes it and whether it still
+      describes supported behaviour is unresolved
+    paths:
+      - tests/old_proxy_tests/tests/test_anthropic_context_caching.py
+      - tests/old_proxy_tests/tests/test_anthropic_sdk.py
+      - tests/old_proxy_tests/tests/test_async.py
+      - tests/old_proxy_tests/tests/test_gemini_context_caching.py
+      - tests/old_proxy_tests/tests/test_langchain_embedding.py
+      - tests/old_proxy_tests/tests/test_langchain_request.py
+      - tests/old_proxy_tests/tests/test_llamaindex.py
+      - tests/old_proxy_tests/tests/test_mistral_sdk.py
+      - tests/old_proxy_tests/tests/test_openai_embedding.py
+      - tests/old_proxy_tests/tests/test_openai_exception_request.py
+      - tests/old_proxy_tests/tests/test_openai_request.py
+      - tests/old_proxy_tests/tests/test_openai_request_with_traceparent.py
+      - tests/old_proxy_tests/tests/test_openai_simple_embedding.py
+      - tests/old_proxy_tests/tests/test_openai_tts_request.py
+      - tests/old_proxy_tests/tests/test_pass_through_langfuse.py
+      - tests/old_proxy_tests/tests/test_q.py
+      - tests/old_proxy_tests/tests/test_simple_traceparent_openai.py
+      - tests/old_proxy_tests/tests/test_vertex_sdk_forward_headers.py
+      - tests/old_proxy_tests/tests/test_vtx_embedding.py
+      - tests/old_proxy_tests/tests/test_vtx_sdk_embedding.py
+  - reason: >-
+      No job invokes this suite and its files mix pure transformation tests with ones driving live
+      vendor vector stores, so assigning them needs a per-file decision
+    paths:
+      - tests/vector_store_tests/rag/test_rag_bedrock.py
+      - tests/vector_store_tests/rag/test_rag_openai.py
+      - tests/vector_store_tests/rag/test_rag_s3_vectors.py
+      - tests/vector_store_tests/rag/test_rag_vertex_ai.py
+      - tests/vector_store_tests/test_azure_ai_vector_store.py
+      - tests/vector_store_tests/test_azure_vector_store.py
+      - tests/vector_store_tests/test_bedrock_vector_store.py
+      - tests/vector_store_tests/test_gemini_vector_store.py
+      - tests/vector_store_tests/test_milvus_vector_store.py
+      - tests/vector_store_tests/test_openai_vector_store.py
+      - tests/vector_store_tests/test_ragflow_vector_store.py
+      - tests/vector_store_tests/test_s3_vectors_vector_store.py
+      - tests/vector_store_tests/test_vertex_ai_search_api_vector_store.py
+      - tests/vector_store_tests/test_vertex_ai_vector_store.py
+  - reason: >-
+      Throughput and memory-growth measurements whose runtime and variance make them unsuitable for
+      a per-pull-request job
+    paths:
+      - tests/load_tests/test_datadog_load_test.py
+      - tests/load_tests/test_langsmith_load_test.py
+      - tests/load_tests/test_linear_memory_growth.py
+      - tests/load_tests/test_memory_usage.py
+      - tests/load_tests/test_otel_load_test.py
+      - tests/load_tests/test_vertex_embeddings_load_test.py
+      - tests/load_tests/test_vertex_load_tests.py
+  - reason: >-
+      Third-party integration tests that skip themselves without OCI configuration or sandbox
+      credentials, neither of which a pull request job holds
+    paths:
+      - tests/integration/sandbox/test_e2b_sandbox.py
+      - tests/integration/test_oci_integration.py
+      - tests/integration/test_oci_proxy_integration.py
+  - reason: >-
+      Two prompt-factory tests sitting at the top level of tests/ instead of under the
+      tests/test_litellm mirror the shards enumerate; they need moving rather than a shard entry
+    paths:
+      - tests/litellm_core_utils/test_anthropic_dedup_factory.py
+      - tests/litellm_core_utils/test_bedrock_converse_dedup_factory.py
+  - reason: >-
+      A unit test for the proxy-extras package that no job invokes, while the package's other tests
+      live under tests/proxy_migration_tests
+    paths:
+      - tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
+
+dockerfiles:
+  - reason: >-
+      The dashboard container is a static Next.js export served by nginx, and the dashboard build
+      and lint workflows already exercise that output, so building the image adds no signal about it
+    paths:
+      - ui/Dockerfile
+  - reason: >-
+      The Rust gateway ships as its own chart and package with a separate release pipeline, so its
+      image is not part of this repo's Python image set
+    paths:
+      - litellm-rust/crates/ai-gateway/Dockerfile
+  - reason: >-
+      An example image under cookbook/ that is documentation rather than a shipped artifact
+    paths:
+      - cookbook/litellm-ollama-docker-image/Dockerfile

.github/pull_request_template.md

@@ -13,6 +13,33 @@ How it solves it:
 - <blah>
 - ...
 
+## User Flow
+
+<!-- Two ordered lists, Before and After, walking the same end user through the same task, written strictly from that user's seat
+     Read the linked issue, ticket, or customer thread first so the flow reflects the real application and the routes its users actually hit; don't invent a generic scenario
+     Lead each list with one plain sentence saying where the flow fails (Before) or succeeds (After), then number the steps
+     Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen
+     No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. "The upload hands back an ID that looks like OpenAI's own `file-abc123` instead of the scrambled one the gateway returned" is right, "no managed-file row was registered" is wrong
+     Keep the two lists step-for-step identical until they diverge, so the changed step is obvious
+     If the bug had a security or authorization consequence, end each list with what another user could or could no longer do
+     Regenerate this section whenever new commits change the PR's behavior, so it never describes an older revision
+
+Example:
+
+Before: a developer whose app streams chat completions gets no token counts back, so their cost dashboard reads zero
+
+1. They send POST https://litellm-domain/v1/chat/completions with `"stream": true` and no `stream_options`
+2. The last SSE chunk arrives with `"usage": null`, so their app records 0 prompt and 0 completion tokens
+3. They open https://litellm-domain/ui/?page=logs and see the request logged at $0 spend
+
+After: the same request comes back with real token counts, so the dashboard shows real spend
+
+1. The proxy admin sets `always_include_stream_usage: true` and restarts the proxy
+2. The developer sends the same POST https://litellm-domain/v1/chat/completions with `"stream": true` and no `stream_options`
+3. The last SSE chunk now carries a `usage` object with real prompt and completion token counts
+4. https://litellm-domain/ui/?page=logs shows that request at non-zero spend
+-->
+
 ## Relevant issues
 
 <!-- e.g., "Fixes #000" -->
@@ -40,6 +67,7 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
      The proof must be completely e2e with no mocks, using, for example, actual LLM calls costing real $. `pytest` commands are not enough
      For bug fixes: show reproduction before the fix and passing behavior after
      Include the commit hash each proof was captured at, for both the before and the after runs
+     If the change applies to all three LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every single one of them, not just one
      For new features: show the feature working end-to-end
      For UI changes: include before/after screenshots -->
 
@@ -55,7 +83,11 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 🚄 Infrastructure
 ✅ Test
 
-## Changes
+## Caveats (if any)
+
+<!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
+     Call out known limitations, follow-up work, or anything a reviewer should watch out for
+     Leave this section empty if there are none -->
 
 ## QA runbook
 

.github/scripts/assert_ci_coverage.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+import pathlib
+import re
+import sys
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+
+import yaml
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[2]
+WORKFLOW_DIR = REPO_ROOT / ".github" / "workflows"
+CIRCLECI_CONFIG = REPO_ROOT / ".circleci" / "config.yml"
+ALLOWLIST_FILE = REPO_ROOT / ".github" / "ci-coverage-allowlist.yml"
+TESTS_ROOT = REPO_ROOT / "tests"
+
+ALLOWLIST_KEYS = frozenset({"description", "test_paths", "dockerfiles"})
+PATH_FILTER_KEYS = frozenset({"paths", "paths-ignore"})
+TEST_PATH_KEYS = frozenset({"test-path", "test-paths"})
+DOCKERFILE_INPUT_KEYS = frozenset({"file", "dockerfile"})
+TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bcircleci tests\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s")
+IMAGE_BUILD_RE = re.compile(r"\bdocker\s+(?:buildx\s+)?build\b")
+TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?-]+")
+DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
+COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
+GLOB_CHARS = frozenset("*?")
+
+
+@dataclass(frozen=True, slots=True)
+class AllowEntry:
+    paths: tuple[str, ...]
+    reason: str
+
+
+@dataclass(frozen=True, slots=True)
+class Allowlist:
+    test_paths: tuple[AllowEntry, ...]
+    dockerfiles: tuple[AllowEntry, ...]
+
+    def covers_test(self, relative_path: str) -> bool:
+        return any(_token_covers(path, relative_path) for entry in self.test_paths for path in entry.paths)
+
+    def covers_dockerfile(self, relative_path: str) -> bool:
+        return any(relative_path == path for entry in self.dockerfiles for path in entry.paths)
+
+
+@dataclass(frozen=True, slots=True)
+class Scalar:
+    key: str
+    value: str
+
+
+@dataclass(frozen=True, slots=True)
+class Finding:
+    subject: str
+    detail: str
+
+
+def _scalars(node: object, key: str) -> tuple[Scalar, ...]:
+    if isinstance(node, str):
+        return (Scalar(key=key, value=node),)
+    if isinstance(node, Mapping):
+        return tuple(
+            scalar
+            for child_key, value in node.items()
+            if child_key not in PATH_FILTER_KEYS
+            for scalar in _scalars(value, str(child_key))
+        )
+    if isinstance(node, Sequence):
+        return tuple(scalar for item in node for scalar in _scalars(item, key))
+    return ()
+
+
+def _config_files() -> tuple[pathlib.Path, ...]:
+    workflows = tuple(sorted(path for path in WORKFLOW_DIR.iterdir() if path.suffix in (".yml", ".yaml")))
+    circleci = (CIRCLECI_CONFIG,) if CIRCLECI_CONFIG.is_file() else ()
+    return workflows + circleci
+
+
+def _all_scalars() -> tuple[Scalar, ...]:
+    return tuple(
+        scalar
+        for path in _config_files()
+        for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
+    )
+
+
+def _uncommented(value: str) -> str:
+    return COMMENT_RE.sub("", value)
+
+
+def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
+    return frozenset(
+        match.group(0).rstrip("/")
+        for scalar in scalars
+        if scalar.key in TEST_PATH_KEYS or TEST_RUNNER_RE.search(scalar.value)
+        for match in TEST_TOKEN_RE.finditer(_uncommented(scalar.value))
+    )
+
+
+def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
+    return frozenset(
+        match.group(0)
+        for scalar in scalars
+        if scalar.key in DOCKERFILE_INPUT_KEYS or IMAGE_BUILD_RE.search(scalar.value)
+        for match in DOCKERFILE_TOKEN_RE.finditer(_uncommented(scalar.value))
+    )
+
+
+def _glob_to_regex(token: str) -> re.Pattern[str]:
+    parts = re.split(r"(\*\*/|\*\*|\*|\?)", token)
+    translated = "".join(
+        {"**/": r"(?:.*/)?", "**": r".*", "*": r"[^/]*", "?": r"[^/]"}.get(part, re.escape(part)) for part in parts
+    )
+    return re.compile(rf"{translated}(?:/.*)?$")
+
+
+def _token_covers(token: str, relative_path: str) -> bool:
+    if GLOB_CHARS & set(token):
+        return _glob_to_regex(token).match(relative_path) is not None
+    return relative_path == token or relative_path.startswith(f"{token}/")
+
+
+def _test_files() -> tuple[str, ...]:
+    return tuple(
+        sorted(
+            path.relative_to(REPO_ROOT).as_posix()
+            for path in TESTS_ROOT.rglob("test_*.py")
+            if path.is_file() and "node_modules" not in path.parts
+        )
+    )
+
+
+def _dockerfiles() -> tuple[str, ...]:
+    return tuple(
+        sorted(
+            path.relative_to(REPO_ROOT).as_posix()
+            for path in REPO_ROOT.rglob("Dockerfile*")
+            if path.is_file()
+            and ".git" not in path.parts
+            and "node_modules" not in path.parts
+            and not path.name.endswith(".dockerignore")
+        )
+    )
+
+
+def _uncovered_tests(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]:
+    uncovered = tuple(
+        relative_path
+        for relative_path in _test_files()
+        if not any(_token_covers(token, relative_path) for token in tokens) and not allowlist.covers_test(relative_path)
+    )
+    directories = tuple(dict.fromkeys(path.rsplit("/", 1)[0] for path in uncovered))
+    return tuple(
+        Finding(
+            subject=directory,
+            detail=_describe(tuple(p for p in uncovered if p.rsplit("/", 1)[0] == directory)),
+        )
+        for directory in directories
+    )
+
+
+def _describe(paths: tuple[str, ...]) -> str:
+    names = ", ".join(path.rsplit("/", 1)[1] for path in paths[:3])
+    suffix = f", +{len(paths) - 3} more" if len(paths) > 3 else ""
+    return f"{len(paths)} test file(s) invoked by no job: {names}{suffix}"
+
+
+def _uncovered_dockerfiles(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]:
+    return tuple(
+        Finding(subject=relative_path, detail="built by no job")
+        for relative_path in _dockerfiles()
+        if relative_path not in tokens and not allowlist.covers_dockerfile(relative_path)
+    )
+
+
+def _parse_entry(item: object, section: str) -> AllowEntry:
+    if not isinstance(item, dict):
+        raise SystemExit(f"{ALLOWLIST_FILE.name}: '{section}' entries must be mappings")
+    paths = item.get("paths")
+    reason = item.get("reason")
+    if (
+        not isinstance(paths, list)
+        or not paths
+        or not all(isinstance(path, str) for path in paths)
+        or not isinstance(reason, str)
+        or not reason.strip()
+    ):
+        raise SystemExit(
+            f"{ALLOWLIST_FILE.name}: every '{section}' entry needs a non-empty 'paths' "
+            "list of strings and a non-empty 'reason'"
+        )
+    return AllowEntry(paths=tuple(paths), reason=reason)
+
+
+def _parse_entries(raw: object, section: str) -> tuple[AllowEntry, ...]:
+    if not isinstance(raw, list):
+        raise SystemExit(f"{ALLOWLIST_FILE.name}: '{section}' must be a list")
+    return tuple(_parse_entry(item, section) for item in raw)
+
+
+def _load_allowlist() -> Allowlist:
+    if not ALLOWLIST_FILE.is_file():
+        return Allowlist(test_paths=(), dockerfiles=())
+    raw = yaml.safe_load(ALLOWLIST_FILE.read_text(encoding="utf-8")) or {}
+    if not isinstance(raw, dict):
+        raise SystemExit(f"{ALLOWLIST_FILE.name}: top level must be a mapping")
+    unknown = sorted(str(key) for key in raw if key not in ALLOWLIST_KEYS)
+    if unknown:
+        raise SystemExit(
+            f"{ALLOWLIST_FILE.name}: unknown top-level key(s) {unknown}; expected only {sorted(ALLOWLIST_KEYS)}"
+        )
+    return Allowlist(
+        test_paths=_parse_entries(raw.get("test_paths", []), "test_paths"),
+        dockerfiles=_parse_entries(raw.get("dockerfiles", []), "dockerfiles"),
+    )
+
+
+def _write(message: str) -> None:
+    sys.stdout.write(f"{message}\n")
+
+
+def _report(title: str, findings: tuple[Finding, ...], remedy: str) -> None:
+    _write(f"ERROR: {title}")
+    for finding in findings:
+        _write(f"  - {finding.subject}: {finding.detail}")
+    _write("")
+    _write(remedy)
+    _write("")
+
+
+def main() -> int:
+    allowlist = _load_allowlist()
+    scalars = _all_scalars()
+
+    test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars))
+    dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
+
+    if test_findings:
+        _report(
+            "test files that no CI job invokes",
+            test_findings,
+            "Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
+        )
+    if dockerfile_findings:
+        _report(
+            "Dockerfiles that no CI job builds",
+            dockerfile_findings,
+            "Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
+        )
+    if test_findings or dockerfile_findings:
+        return 1
+
+    _write(
+        f"OK: {len(_test_files())} test files and {len(_dockerfiles())} Dockerfiles are each "
+        "invoked by at least one job or carry an explicit allowlist entry."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/triage_with_llm.py

@@ -582,7 +582,9 @@ def build_issue_prompt(*, title: str, body: str) -> str:
                   Commands whose external dependencies (LLM provider, DB,
                   network) are mocked or stubbed do NOT count.
               Prose-only "steps to reproduce" with no run output, video, or
-              screenshot do NOT satisfy (1).
+              screenshot do NOT satisfy (1). An unfilled template scaffold
+              (bare headings such as "Version or commit:" with nothing under
+              them, empty numbered lists) counts as absent, not as evidence.
           (2) Expected vs. actual behavior (`has_expected_vs_actual`).
 
         FAIL the bug report if either (1) or (2) is missing. Do not bias
@@ -595,6 +597,13 @@ def build_issue_prompt(*, title: str, body: str) -> str:
             that it does not today).
           - Motivation / use case with a concrete example (config, API call,
             UI flow, or scenario showing what's blocked today).
+          - END-TO-END EVIDENCE OF THE DEAD-END (set
+            `has_dead_end_evidence=true` only when this is present): a video,
+            a screenshot, or the exact command(s) actually run paired with
+            their real output, showing the point where the flow stops today.
+            Mocked or stubbed dependencies do NOT count, and an unfilled
+            template scaffold (bare headings, empty numbered lists) counts as
+            absent.
 
         For an issue that is neither a bug report nor a feature request (a
         question, support request, or discussion), PASS as long as it has a
@@ -608,6 +617,7 @@ def build_issue_prompt(*, title: str, body: str) -> str:
           "has_repro": boolean,
           "has_expected_vs_actual": boolean,
           "has_motivation_example": boolean,
+          "has_dead_end_evidence": boolean,
           "missing": ["plain-english strings naming what is missing"],
           "explanation": "1-2 sentence reasoning for the team to skim"
         }}
@@ -705,6 +715,10 @@ _ISSUE_BUG_LABELS: tuple[tuple[str, str], ...] = (
 )
 _ISSUE_FEATURE_LABELS: tuple[tuple[str, str], ...] = (
     ("has_motivation_example", "Motivation and concrete example"),
+    (
+        "has_dead_end_evidence",
+        "End-to-end evidence of the dead-end (video, screenshot, or command + real output)",
+    ),
 )
 
 
@@ -836,8 +850,11 @@ def format_issue_close_comment(verdict: dict) -> str:
         "video, a screenshot, or the exact commands you ran with their real output / "
         "traceback) plus expected vs. actual behavior. Written steps with no run output, "
         "video, or screenshot don't count, and mocked or stubbed runs don't count.\n"
-        "   - For **feature requests**: a concrete description of what should change, plus a "
-        "use case and example (config / API call / UI flow).\n"
+        "   - For **feature requests**: a concrete description of what should change, a "
+        "use case and example (config / API call / UI flow), plus end-to-end evidence of "
+        "the dead-end (a video, a screenshot, or the exact commands you ran with their "
+        "real output showing where the flow stops today). Mocked or stubbed runs don't "
+        "count.\n"
         "2. Comment `@agent-shin reconsider`. I'll re-run triage and reopen the issue if it "
         "now meets the bar. (GitHub doesn't let external authors reopen an issue a maintainer "
         "or bot closed, so the comment-based reconsider is the reliable path.)\n"
@@ -943,8 +960,10 @@ def format_grace_warning_issue_comment(verdict: dict) -> str:
         "screenshot, or the exact commands you ran with their real output / traceback) plus "
         "expected vs. actual behavior. Written steps with no run output don't count, and "
         "mocked or stubbed runs don't count.\n"
-        "- For **feature requests**: a concrete description of what should change, plus a use "
-        "case and example (config / API call / UI flow).\n"
+        "- For **feature requests**: a concrete description of what should change, a use "
+        "case and example (config / API call / UI flow), plus end-to-end evidence of the "
+        "dead-end (a video, a screenshot, or the exact commands you ran with their real "
+        "output showing where the flow stops today). Mocked or stubbed runs don't count.\n"
         "\n"
         "**If the issue does get auto-closed in 2 hours**, comment `@agent-shin reconsider` "
         "and I'll re-evaluate. If it now meets the bar, I'll reopen the issue.\n"

.github/workflows/_test-unit-base.yml

@@ -18,10 +18,25 @@ on:
         type: number
         default: 2
       timeout-minutes:
-        description: "Job timeout in minutes"
+        description: >-
+          Timeout for the test step alone. Setup (checkout, dependency install,
+          Prisma client generation) gets its own allowance on top, so a slow
+          runner or a cold binary download can never cancel passing tests.
         required: false
         type: number
         default: 20
+      job-timeout-minutes:
+        description: >-
+          Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for
+          the per-step ceilings on the setup steps below, and 5 for the runner
+          overhead the job clock charges but no step owns (job init, step
+          transitions, post-job cleanup). That headroom is what makes the test
+          budget a floor rather than a hope, since setup cannot overrun into it
+          without failing its own step first. GitHub expressions have no
+          arithmetic, so the sum is passed in rather than computed.
+        required: false
+        type: number
+        default: 55
       max-failures:
         description: "Stop after this many failures"
         required: false
@@ -44,30 +59,35 @@ jobs:
   run:
     name: Run tests
     runs-on: ubuntu-latest
-    timeout-minutes: ${{ inputs.timeout-minutes }}
+    timeout-minutes: ${{ inputs.job-timeout-minutes }}
     outputs:
       decision: ${{ steps.changes.outputs.decision }}
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        timeout-minutes: 3
         with:
           persist-credentials: false
 
       - name: Detect backend-relevant changes
         id: changes
+        timeout-minutes: 2
         uses: ./.github/actions/detect-backend-changes
 
       - name: Set up Python
+        timeout-minutes: 3
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.12"
 
       - name: Set up uv
+        timeout-minutes: 3
         uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        timeout-minutes: 5
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -79,18 +99,24 @@ jobs:
 
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 8
         run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
+
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/cache-prisma-binaries
 
       - name: Generate Prisma client
         if: steps.changes.outputs.decision != 'skip'
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        timeout-minutes: 3
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Run tests
         if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: ${{ inputs.timeout-minutes }}
         env:
           TEST_PATH: ${{ inputs.test-path }}
           MAX_FAILURES: ${{ inputs.max-failures }}
@@ -154,6 +180,19 @@ jobs:
           merge-multiple: true
 
       - name: Upload to Codecov
+        id: codecov-upload
+        continue-on-error: true
+        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
+        with:
+          use_oidc: true
+          directory: coverage-reports
+          root_dir: ${{ github.workspace }}
+          flags: ${{ inputs.artifact-name }}
+          fail_ci_if_error: false
+
+      - name: Upload to Codecov (retry)
+        if: steps.codecov-upload.outcome == 'failure'
+        continue-on-error: true
         uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
         with:
           use_oidc: true

.github/workflows/auto_update_price_and_context_window.yml

@@ -24,9 +24,12 @@ jobs:
       - name: Update JSON Data
         run: |
           uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py"
+      - name: Regenerate JSON Schema
+        run: |
+          uv run --frozen python ci_cd/generate_model_prices_schema.py
       - name: Create Pull Request
         run: |
-          git add model_prices_and_context_window.json 
+          git add model_prices_and_context_window.json model_prices_and_context_window.schema.json
           git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')" 
           gh pr create --title "Update model_prices_and_context_window.json file" \
             --body "Automated update for model_prices_and_context_window.json" \

.github/workflows/check-schema-sync.yml

@@ -10,6 +10,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   check-sync:
     name: Verify schema.prisma copies match root

.github/workflows/check-ui-api-types.yml

@@ -2,18 +2,19 @@ name: Check UI API Types Sync
 
 on:
   pull_request:
-    paths:
-      - "litellm/proxy/**"
-      - "litellm/types/**"
-      - "ui/litellm-dashboard/src/lib/http/schema.d.ts"
-      - "ui/litellm-dashboard/scripts/gen-api-types.mjs"
-      - "ui/litellm-dashboard/package.json"
-      - "ui/litellm-dashboard/package-lock.json"
-      - ".github/workflows/check-ui-api-types.yml"
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
 
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   check-sync:
     name: Verify schema.d.ts matches the proxy OpenAPI spec
@@ -24,18 +25,39 @@ jobs:
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           persist-credentials: false
+          fetch-depth: 2
+
+      - name: Detect changes that can affect the generated types
+        id: changes
+        run: |
+          set -euo pipefail
+          if ! base="$(git rev-parse --verify --quiet HEAD^2 >/dev/null && git rev-parse HEAD^1)"; then
+            echo "Not a pull request merge commit, running the full check."
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          files="$(git diff --name-only "$base" HEAD)"
+          if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/check-ui-api-types\.yml$)' <<< "$files"; then
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No proxy, types or generator changes in this pull request, nothing to verify."
+            echo "relevant=false" >> "$GITHUB_OUTPUT"
+          fi
 
       - name: Set up Python
+        if: steps.changes.outputs.relevant == 'true'
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.12"
 
       - name: Set up uv
+        if: steps.changes.outputs.relevant == 'true'
         uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        if: steps.changes.outputs.relevant == 'true'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -46,31 +68,39 @@ jobs:
             ${{ runner.os }}-uv-
 
       - name: Install backend dependencies
+        if: steps.changes.outputs.relevant == 'true'
         run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.relevant == 'true'
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        if: steps.changes.outputs.relevant == 'true'
         run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Set up Node.js
+        if: steps.changes.outputs.relevant == 'true'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
-          node-version: "20"
+          node-version-file: ui/litellm-dashboard/.nvmrc
           cache: "npm"
           cache-dependency-path: ui/litellm-dashboard/package-lock.json
 
       - name: Install dashboard dependencies
+        if: steps.changes.outputs.relevant == 'true'
         working-directory: ui/litellm-dashboard
         run: npm ci
 
       - name: Regenerate types from the live spec
+        if: steps.changes.outputs.relevant == 'true'
         working-directory: ui/litellm-dashboard
         env:
           LITELLM_PYTHON: "uv run --no-sync python"
         run: npm run gen:api
 
       - name: Fail if types are stale
+        if: steps.changes.outputs.relevant == 'true'
         run: |
           if ! git diff --exit-code -- ui/litellm-dashboard/src/lib/http/schema.d.ts; then
             echo "::error file=ui/litellm-dashboard/src/lib/http/schema.d.ts::Generated API types are out of sync with the proxy OpenAPI spec."

.github/workflows/ci-coverage.yml

@@ -0,0 +1,42 @@
+name: "CI Coverage"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  assert-ci-coverage:
+    name: assert-ci-coverage
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Assert every test file and Dockerfile is invoked by a job
+        run: |
+          python -m pip install "pyyaml==6.0.3"
+          python .github/scripts/assert_ci_coverage.py

.github/workflows/codspeed.yml

@@ -5,10 +5,24 @@ on:
     branches:
       - main
       - litellm_internal_staging
+    paths:
+      - "litellm/**"
+      - "tests/benchmarks/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/codspeed.yml"
+      - ".github/actions/setup-uv-with-retries/**"
   pull_request:
     branches:
       - main
       - litellm_internal_staging
+    paths:
+      - "litellm/**"
+      - "tests/benchmarks/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/codspeed.yml"
+      - ".github/actions/setup-uv-with-retries/**"
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:

.github/workflows/conventional-commits.yml

@@ -14,6 +14,10 @@ on:
 permissions:
   pull-requests: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   lint-pr-title:
     name: Validate PR title

.github/workflows/create_daily_oss_agent_shin_branch.yml

@@ -13,35 +13,16 @@ jobs:
       contents: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
       - name: Create daily oss-agent-shin branch
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          # Configure Git user
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          # Generate branch name with MM_DD_YYYY format
           BRANCH_NAME="litellm_oss_agent_shin_$(date +'%m_%d_%Y')"
           echo "Creating branch: $BRANCH_NAME"
-
-          # Fetch all branches
-          git fetch --all
-
-          # Check if the branch already exists
-          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+          if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
             echo "Branch $BRANCH_NAME already exists. Skipping creation."
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            # Create the new branch from main
-            git checkout -b $BRANCH_NAME origin/main
-            # Push the new branch
-            git push origin $BRANCH_NAME
-            echo "Successfully created and pushed branch: $BRANCH_NAME"
+            exit 0
           fi
+          MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
+          gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
+          echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"

.github/workflows/create_daily_staging_branch.yml

@@ -13,38 +13,19 @@ jobs:
       contents: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
       - name: Create daily staging branch
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          # Configure Git user
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          # Generate branch name with MM_DD_YYYY format
           BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')"
           echo "Creating branch: $BRANCH_NAME"
-
-          # Fetch all branches
-          git fetch --all
-
-          # Check if the branch already exists
-          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+          if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
             echo "Branch $BRANCH_NAME already exists. Skipping creation."
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            # Create the new branch from main
-            git checkout -b $BRANCH_NAME origin/main
-            # Push the new branch
-            git push origin $BRANCH_NAME
-            echo "Successfully created and pushed branch: $BRANCH_NAME"
+            exit 0
           fi
+          MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
+          gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
+          echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"
 
   create-internal-dev-branch:
     if: github.repository == 'BerriAI/litellm'
@@ -53,35 +34,16 @@ jobs:
       contents: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
       - name: Create internal dev branch
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          # Configure Git user
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          # Generate branch name with MM_DD_YYYY format
           BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
           echo "Creating branch: $BRANCH_NAME"
-
-          # Fetch all branches
-          git fetch --all
-
-          # Check if the branch already exists
-          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+          if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
             echo "Branch $BRANCH_NAME already exists. Skipping creation."
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            # Create the new branch from main
-            git checkout -b $BRANCH_NAME origin/main
-            # Push the new branch
-            git push origin $BRANCH_NAME
-            echo "Successfully created and pushed branch: $BRANCH_NAME"
+            exit 0
           fi
+          MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
+          gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
+          echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"

.github/workflows/guard-fork-dependencies.yml

@@ -15,6 +15,10 @@ on:
 
 permissions: {}
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   guard:
     name: Block fork dependency changes

.github/workflows/helm_unit_test.yml

@@ -9,6 +9,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   unit-test:
     runs-on: ubuntu-latest
@@ -23,21 +27,28 @@ jobs:
         with:
           version: "3.11.1"
 
+      - name: Download and verify Helm Unit Test Plugin
+        run: |
+          curl -fsSLo "$RUNNER_TEMP/helm-unittest.tgz" https://github.com/helm-unittest/helm-unittest/releases/download/v0.8.2/helm-unittest-linux-amd64-0.8.2.tgz
+          echo "56ab3091e6fa52a7c92ee951def9bed957f295d9ce98483aed404e748d7b3a94  $RUNNER_TEMP/helm-unittest.tgz" | sha256sum -c -
+
       - name: Install Helm Unit Test Plugin
         run: |
-          helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
-      - name: Verify Helm Unit Test Plugin integrity
-        run: |
-          EXPECTED_SHA="e251ba198448629678ff2168e1a469249d998155"
           PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest"
-          ACTUAL_SHA="$(git -C "$PLUGIN_DIR" rev-parse HEAD)"
-          if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then
-            echo "::error::Helm unittest plugin checksum mismatch! Expected $EXPECTED_SHA but got $ACTUAL_SHA"
-            exit 1
-          fi
-          echo "Helm unittest plugin integrity verified: $ACTUAL_SHA"
+          mkdir -p "$PLUGIN_DIR"
+          tar -xzf "$RUNNER_TEMP/helm-unittest.tgz" -C "$PLUGIN_DIR"
+          helm plugin list
 
       - name: Run unit tests
         run: |
-          helm unittest -f 'tests/*.yaml' helm/litellm-helm
-          helm unittest -f 'tests/*.yaml' helm/litellm
+          for chart in helm/litellm-helm helm/litellm; do
+            declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')"
+            output="$(mktemp)"
+            helm unittest -f 'tests/*.yaml' "$chart" | tee "$output"
+            executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")"
+            if [ "$declared" != "$executed" ]; then
+              echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute."
+              exit 1
+            fi
+            echo "$chart: all $declared declared test suites ran"
+          done

.github/workflows/image-scan.yml

@@ -8,8 +8,17 @@ on:
       - litellm_oss_branch
       - "litellm_**"
     paths:
+      - Dockerfile
       - docker/Dockerfile.non_root
-      - tests/proxy_migration_tests/test_offline_image_migration.py
+      - migrations/Dockerfile
+      - migrations/run.py
+      - gateway/Dockerfile
+      - gateway/main.py
+      - backend/Dockerfile
+      - backend/main.py
+      - docker/component_entrypoint.sh
+      - litellm-proxy-extras/**
+      - tests/proxy_migration_tests/**
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
       - .github/workflows/image-scan.yml
@@ -83,3 +92,123 @@ jobs:
             --only-fixed \
             --fail-on high \
             --output table
+
+  runtime-image:
+    name: runtime-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build runtime image
+        run: docker build -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify offline migration as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+
+  migrations-image:
+    name: migrations-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build migrations image
+        run: docker build -f migrations/Dockerfile -t litellm-migrations-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify offline migration as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-migrations-scan:${{ github.sha }}
+          LITELLM_MIGRATION_INTERPRETER: python3
+          LITELLM_MIGRATION_SCRIPT: /app/run.py
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+
+  gateway-image:
+    name: gateway-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build gateway image
+        run: docker build -f gateway/Dockerfile -t litellm-gateway-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the gateway serves offline as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-gateway-scan:${{ github.sha }}
+          LITELLM_COMPONENT_PORT: "4000"
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
+
+  backend-image:
+    name: backend-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build backend image
+        run: docker build -f backend/Dockerfile -t litellm-backend-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the backend serves offline as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-backend-scan:${{ github.sha }}
+          LITELLM_COMPONENT_PORT: "4001"
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v

.github/workflows/mutation-test.yml

@@ -55,11 +55,12 @@ jobs:
 
       - name: Install dependencies
         run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
 
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.github/workflows/publish-basedpyright-base-counts.yml

@@ -0,0 +1,63 @@
+name: Publish basedpyright base counts
+
+# Every commit on litellm_internal_staging is some branch's future merge-base.
+# Publishing its per-rule basedpyright counts as an artifact lets
+# scripts/type_check_gate.py download them in seconds instead of paying a
+# 60-110s second basedpyright pass on every fresh worktree or moved merge-base.
+# No concurrency group on purpose: runs must never cancel each other, because
+# every sha's artifact matters (any of them can become a merge-base).
+
+on:
+  push:
+    branches:
+      - litellm_internal_staging
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: "Ref to compute and publish base counts for"
+        required: false
+        default: litellm_internal_staging
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+          clean: true
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      # The gate provisions its own measurement env (.venv-typecheck: a frozen
+      # uv sync of its canonical dependency groups plus a generated Prisma
+      # client), so no install step here can drift from what local runs measure.
+      - name: Emit basedpyright counts for HEAD
+        run: |
+          python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts"
+          counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json)
+          echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
+
+      - name: Upload counts artifact
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: ${{ env.COUNTS_ARTIFACT_NAME }}
+          path: ${{ runner.temp }}/basedpyright-counts/
+          if-no-files-found: error

.github/workflows/test-code-quality.yml

@@ -7,13 +7,17 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   code-quality:
@@ -61,6 +65,12 @@ jobs:
       - name: check_provider_folders_documented
         run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
 
+      - name: check_prisma_binary_cache
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_prisma_binary_cache.py
+
+      - name: check_workflow_startup_safety
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
+
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 

.github/workflows/test-linting.yml

@@ -11,10 +11,20 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   lint:
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    # actions: read lets scripts/type_check_gate.py download the base-counts
+    # artifact published by publish-basedpyright-base-counts.yml instead of
+    # re-running basedpyright over the merge-base tree.
+    permissions:
+      contents: read
+      actions: read
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
@@ -23,10 +33,22 @@ jobs:
         # Any-discipline) would otherwise blame on this branch.
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
+          fetch-depth: 1
           clean: true
           persist-credentials: false
 
+      - name: Fetch gate base (merge-base with target branch)
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
+          MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          test -n "$MERGE_BASE"
+          retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
+          echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
@@ -50,20 +72,19 @@ jobs:
         run: |
           uv sync --frozen --group proxy-dev --group e2e-dev
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       # basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma)
       # only after `prisma generate` writes prisma/client.py et al. Without this the
       # DB wrappers typed against the generated client would degrade to Unknown.
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Check ruff format
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          git diff --name-only --diff-filter=ACMR "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
+          git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
           if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
             echo "No changed litellm Python files to check with ruff format."
             exit 0
@@ -86,16 +107,12 @@ jobs:
           cd ..
 
       - name: Check strict-rule budget (delta vs base)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          uv run --no-sync python scripts/ruff_strict_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
 
       - name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          uv run --no-sync python scripts/type_discipline_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
 
       - name: Print OpenAI version
         run: |
@@ -103,15 +120,13 @@ jobs:
 
       - name: Check basedpyright budget (delta vs base)
         env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          GH_TOKEN: ${{ github.token }}
         run: |
-          (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA"
 
       - name: Check tests/e2e basedpyright (zero errors)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          if git diff --name-only --diff-filter=ACMRD "$BASE_SHA"...HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
+          if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
             uv run --no-sync basedpyright tests/e2e
           else
             echo "No changed tests/e2e Python files; skipping."
@@ -140,9 +155,16 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
+      - name: Fetch ratchet base
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
+          retry git fetch --no-tags --depth=1 origin "$BASE_SHA"
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
@@ -163,7 +185,7 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: Set up Python
@@ -178,13 +200,15 @@ jobs:
 
       - name: Run secret scan test
         run: |
-          uv run --frozen --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v
+          uv run --no-project --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v
 
       - name: Run ggshield secret scan
         env:
           GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
         run: |
           if [ -n "$GITGUARDIAN_API_KEY" ]; then
+            retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
+            retry git fetch --no-tags --unshallow origin
             uv tool run --from 'ggshield==1.48.0' ggshield secret scan repo .
           else
             echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"

.github/workflows/test-litellm-ui-build.yml

@@ -10,6 +10,10 @@ on:
       - litellm_oss_staging
       - "litellm_**"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   build-ui:
     runs-on: ubuntu-latest
@@ -27,7 +31,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
-          node-version: "20"
+          node-version-file: ui/litellm-dashboard/.nvmrc
           cache: "npm"
           cache-dependency-path: ui/litellm-dashboard/package-lock.json
 

.github/workflows/test-litellm-ui-lint.yml

@@ -10,6 +10,10 @@ on:
       - litellm_oss_staging
       - "litellm_**"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   frontend-lint:
     runs-on: ubuntu-latest
@@ -22,14 +26,25 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: Collect changed files
         id: changed
         env:
+          GH_TOKEN: ${{ github.token }}
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
+          # base.sha is the base branch tip from when the PR was opened, while
+          # actions/checkout leaves HEAD on a merge of the PR into the *current*
+          # base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
+          # landed since, so a PR that touches no UI file still gets linted
+          # against hundreds of other people's files. Diff the PR head against its
+          # own merge base instead, which is exactly what this PR changed.
+          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          test -n "$merge_base"
+          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           : > "$RUNNER_TEMP/prettier_files.txt"
           : > "$RUNNER_TEMP/eslint_files.txt"
           while IFS= read -r f; do
@@ -41,7 +56,7 @@ jobs:
               *.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
                 printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
             esac
-          done < <(git diff --name-only --diff-filter=ACMR --relative "$BASE_SHA"...HEAD -- .)
+          done < <(git diff --name-only --diff-filter=ACMR --relative "$merge_base" "$HEAD_SHA" -- .)
           if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
             echo "has_files=true" >> "$GITHUB_OUTPUT"
           else
@@ -53,7 +68,7 @@ jobs:
         if: steps.changed.outputs.has_files == 'true'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
-          node-version: "20"
+          node-version-file: ui/litellm-dashboard/.nvmrc
           cache: "npm"
           cache-dependency-path: ui/litellm-dashboard/package-lock.json
 

.github/workflows/test-litellm-ui-unit.yml

@@ -29,27 +29,45 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: Setup Node.js
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
-          node-version: "20"
+          node-version-file: ui/litellm-dashboard/.nvmrc
           cache: "npm"
           cache-dependency-path: ui/litellm-dashboard/package-lock.json
 
       - name: Install dependencies
         run: npm ci
 
+      - name: Run UI type tests (Vitest)
+        env:
+          CI: "true"
+        run: npm run test:types
+
       - name: Run UI unit tests (Vitest)
         env:
           CI: "true"
+          GH_TOKEN: ${{ github.token }}
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
           if [ -n "$BASE_SHA" ]; then
-            echo "Pull request: running only tests related to changes since $BASE_SHA"
-            npm run test -- --run --changed "$BASE_SHA" --passWithNoTests \
+            merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+            test -n "$merge_base"
+            git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+            changed_files=()
+            while IFS= read -r f; do
+              changed_files+=("$f")
+            done < <(git diff --name-only --relative "$merge_base" "$HEAD_SHA" -- .)
+            if [ ${#changed_files[@]} -eq 0 ]; then
+              echo "No UI files changed in this PR; skipping unit tests."
+              exit 0
+            fi
+            echo "Pull request: running tests related to ${#changed_files[@]} changed UI files"
+            npm run test -- related "${changed_files[@]}" --run --passWithNoTests \
               --pool forks --poolOptions.forks.maxForks=14
           else
             echo "Push to $GITHUB_REF_NAME: running the full suite"

.github/workflows/test-mcp.yml

@@ -11,6 +11,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   test:
     runs-on: ubuntu-latest

.github/workflows/test-model-map.yaml

@@ -11,6 +11,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   validate-model-prices-json:
     runs-on: ubuntu-latest
@@ -22,3 +26,12 @@ jobs:
       - name: Validate model_prices_and_context_window.json
         run: |
           jq empty model_prices_and_context_window.json
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Check model_prices_and_context_window.schema.json is in sync
+        run: |
+          uv run --frozen python ci_cd/generate_model_prices_schema.py --check

.github/workflows/test-terraform-modules.yml

@@ -0,0 +1,54 @@
+name: Terraform Modules
+
+on:
+  push:
+    paths:
+      - "terraform/litellm/aws/**"
+      - ".github/workflows/test-terraform-modules.yml"
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+    paths:
+      - "terraform/litellm/aws/**"
+      - ".github/workflows/test-terraform-modules.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  aws-module:
+    name: fmt, validate, test (aws)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: terraform/litellm/aws
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
+        with:
+          terraform_version: 1.13.3
+          terraform_wrapper: false
+
+      - name: fmt
+        run: terraform fmt -recursive -check -diff
+
+      - name: init
+        run: terraform init -backend=false -input=false
+
+      - name: validate
+        run: terraform validate
+
+      # Plan-only, mock_provider-backed: no AWS credentials, no API calls.
+      - name: test
+        run: terraform test

.github/workflows/test-terraform-provider.yml

@@ -92,9 +92,10 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.github/workflows/test-unit-core-utils.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   core-utils:

.github/workflows/test-unit-documentation.yml

@@ -7,13 +7,17 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   documentation:
@@ -61,10 +65,12 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
         if: steps.changes.outputs.decision != 'skip'
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.github/workflows/test-unit-enterprise-routing.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   enterprise-routing:

.github/workflows/test-unit-integrations.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   integrations:

.github/workflows/test-unit-llm-providers.yml

@@ -7,13 +7,17 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   vertex-ai:

.github/workflows/test-unit-misc.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   misc:
@@ -27,6 +31,7 @@ jobs:
         tests/test_litellm/a2a_protocol
         tests/test_litellm/anthropic_interface
         tests/test_litellm/completion_extras
+        tests/test_litellm/compression
         tests/test_litellm/containers
         tests/test_litellm/experimental_mcp_client
         tests/test_litellm/models
@@ -35,7 +40,11 @@ jobs:
         tests/test_litellm/interactions
         tests/test_litellm/ocr
         tests/test_litellm/passthrough
+        tests/test_litellm/rag
+        tests/test_litellm/realtime_api
+        tests/test_litellm/rerank_api
         tests/test_litellm/sandbox
+        tests/test_litellm/test_router
         tests/test_litellm/vector_stores
         tests/test_litellm/videos
         tests/test_litellm/test_*.py

.github/workflows/test-unit-proxy-auth.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   proxy-auth:

.github/workflows/test-unit-proxy-db.yml

@@ -7,13 +7,17 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 # Semantic matrix: each shard groups tests by concern (auth, server, logging, …)
 # rather than alphabetical letter ranges. Adding a new test file means adding it
@@ -24,6 +28,10 @@ concurrency:
 #     Most of a shard's time is pytest plugin load + xdist worker imports +
 #     pytest-cov instrumentation, not the tests themselves. Keeping per-shard
 #     work low and matching worker count to runner cores is what controls it.
+#   * `timeout` bounds the pytest step only. Checkout, dependency install, and
+#     Prisma client generation draw on a separate allowance in the base
+#     workflow, so slow setup shows up as a slow job rather than as a
+#     cancelled shard whose tests were passing.
 #   * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores
 #     oversubscribes 2x and workers fight for CPU during their cold-start
 #     imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective).
@@ -127,8 +135,6 @@ jobs:
             test-path: >-
               tests/proxy_unit_tests/test_proxy_server.py
               tests/proxy_unit_tests/test_proxy_server_keys.py
-              tests/proxy_unit_tests/test_proxy_server_caching.py
-              tests/proxy_unit_tests/test_proxy_server_langfuse.py
               tests/proxy_unit_tests/test_proxy_server_spend.py
               tests/proxy_unit_tests/test_aproxy_startup.py
             workers: 4

.github/workflows/test-unit-proxy-endpoints.yml

@@ -7,14 +7,18 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
   workflow_dispatch:
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   proxy-endpoints:
@@ -25,19 +29,24 @@ jobs:
     uses: ./.github/workflows/_test-unit-base.yml
     with:
       test-path: >-
+        tests/test_litellm/proxy/analytics_endpoints
         tests/test_litellm/proxy/management_endpoints
+        tests/test_litellm/proxy/memory
         tests/test_litellm/proxy/guardrails
         tests/test_litellm/proxy/management_helpers
         tests/test_litellm/proxy/anthropic_endpoints
         tests/test_litellm/proxy/google_endpoints
         tests/test_litellm/proxy/openai_files_endpoint
         tests/test_litellm/proxy/batches_endpoints
+        tests/test_litellm/proxy/fine_tuning_endpoints
+        tests/test_litellm/proxy/vector_store_files_endpoints
         tests/test_litellm/proxy/video_endpoints
         tests/test_litellm/proxy/response_api_endpoints
         tests/test_litellm/proxy/image_endpoints
         tests/test_litellm/proxy/vector_store_endpoints
         tests/test_litellm/proxy/agent_endpoints
         tests/test_litellm/proxy/a2a
+        tests/test_litellm/proxy/credential_endpoints
         tests/test_litellm/proxy/discovery_endpoints
         tests/test_litellm/proxy/health_endpoints
         tests/test_litellm/proxy/shutdown
@@ -67,4 +76,5 @@ jobs:
       workers: 4
       reruns: 2
       timeout-minutes: 60
+      job-timeout-minutes: 95
       artifact-name: proxy-server

.github/workflows/test-unit-proxy-infra.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   proxy-infra:
@@ -29,6 +33,8 @@ jobs:
         tests/test_litellm/proxy/_experimental
         tests/test_litellm/proxy/experimental
         tests/test_litellm/proxy/common_utils
+        tests/test_litellm/proxy/enterprise_billing
+        tests/test_litellm/proxy/types_utils
         tests/test_litellm/proxy/logging_endpoints
         tests/test_litellm/proxy/test_*.py
       workers: 2

.github/workflows/test-unit-proxy-legacy.yml

@@ -7,13 +7,17 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   test:
@@ -78,10 +82,12 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
         if: steps.changes.outputs.decision != 'skip'
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.github/workflows/test-unit-responses-caching-types.yml

@@ -7,6 +7,10 @@ on:
       - litellm_internal_staging
       - litellm_oss_staging
       - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
 
 permissions:
   contents: read
@@ -14,8 +18,8 @@ permissions:
   pull-requests: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   responses-caching-types:

.github/workflows/test_server_root_path.yml

@@ -1,151 +0,0 @@
-name: Test Proxy SERVER_ROOT_PATH Routing
-permissions:
-  contents: read
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-
-jobs:
-  test-server-root-path:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-
-    strategy:
-      fail-fast: false
-      matrix:
-        root_path: ["/api/v1", "/llmproxy"]
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Free up disk space
-        run: |
-          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/share/boost
-          sudo apt-get clean
-          df -h /
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
-
-      - name: Build Docker image
-        uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 # v6.14.0
-        with:
-          context: .
-          file: ./docker/Dockerfile.non_root
-          tags: litellm-test:${{ github.sha }}
-          load: true
-          push: false
-
-      - name: Start LiteLLM container with SERVER_ROOT_PATH
-        run: |
-          docker run -d \
-            --name litellm-test \
-            -p 4000:4000 \
-            -e SERVER_ROOT_PATH="${{ matrix.root_path }}" \
-            -e LITELLM_MASTER_KEY="sk-1234" \
-            litellm-test:${{ github.sha }} \
-            --detailed_debug
-
-      - name: Wait for container to be healthy
-        run: |
-          echo "Waiting for LiteLLM to start..."
-          max_attempts=30
-          attempt=0
-
-          while [ $attempt -lt $max_attempts ]; do
-            if docker logs litellm-test 2>&1 | grep -q "Uvicorn running"; then
-              echo "LiteLLM started successfully"
-              break
-            fi
-            attempt=$((attempt + 1))
-            echo "Attempt $attempt/$max_attempts - waiting for server to start..."
-            sleep 2
-          done
-
-          if [ $attempt -eq $max_attempts ]; then
-            echo "Server failed to start within timeout"
-            docker logs litellm-test
-            exit 1
-          fi
-
-          sleep 5
-
-      - name: Show container logs
-        if: always()
-        run: docker logs litellm-test
-
-      - name: Test UI endpoint with root path
-        run: |
-          ROOT_PATH="${{ matrix.root_path }}"
-          echo "Testing UI at: http://localhost:4000${ROOT_PATH}/ui/"
-
-          for i in 1 2 3; do
-            content=$(curl -sL --max-time 5 -H "Authorization: Bearer sk-1234" "http://localhost:4000${ROOT_PATH}/ui/")
-            if echo "$content" | grep -q -E "(html|<!DOCTYPE|<head|<body)"; then
-              echo "UI page contains valid HTML content"
-              exit 0
-            fi
-            echo "Attempt $i/3 - no valid HTML, retrying in 5s..."
-            sleep 5
-          done
-          echo "UI page does not contain expected HTML content"
-          echo "Response: $content"
-          docker logs litellm-test
-          exit 1
-
-      - name: Setup Node for Playwright
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
-        with:
-          node-version: "20"
-
-      - name: Install e2e deps and Chromium
-        working-directory: tests/e2e/ui
-        run: |
-          retry() {
-            local attempt=1
-            local max_attempts=4
-            until "$@"; do
-              if [ "$attempt" -ge "$max_attempts" ]; then
-                echo "Command failed after $attempt attempts: $*"
-                return 1
-              fi
-              echo "Attempt $attempt failed: $*. Retrying in $((attempt * 15))s..."
-              sleep $((attempt * 15))
-              attempt=$((attempt + 1))
-            done
-          }
-
-          npm config set fetch-retries 5
-          npm config set fetch-retry-mintimeout 20000
-          npm config set fetch-retry-maxtimeout 120000
-
-          retry npm ci
-          retry npx playwright install --with-deps chromium
-
-      - name: Run SERVER_ROOT_PATH redirect e2e
-        working-directory: tests/e2e/ui
-        env:
-          SERVER_ROOT_PATH: ${{ matrix.root_path }}
-        run: npx playwright test --config=serverRootPath.config.ts
-
-      - name: Upload Playwright artifacts on failure
-        if: failure()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: playwright-trace-${{ strategy.job-index }}
-          path: tests/e2e/ui/test-results/
-          retention-days: 7
-
-      - name: Cleanup
-        if: always()
-        run: |
-          docker stop litellm-test || true
-          docker rm litellm-test || true

.github/workflows/weekly_load_anomaly.yml

@@ -51,9 +51,10 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.gitignore

@@ -1,5 +1,6 @@
 .python-version
 .venv
+.venv-typecheck
 .venv_policy_test
 .env
 .claude
@@ -141,3 +142,4 @@ crash.*.log
 .coverage
 
 ui/litellm-dashboard/out/
+litellm.log

CLAUDE.md

@@ -1,4 +1,12 @@
-Do not write any comments (existing comments can stay) unless explicitly asked to in a user (not system) prompt
+Do not write comments unless they are any of:
+- absolutely necessary to explain some very complex business logic (in which case, keep it concise and clear)
+- used as an input for tools to read and act on. For example:
+  - entries in `.git-blame-ignore-revs` saying which commit is excluded from git blame
+  - a lint or type checker suppression like `# mutable-ok` or `# pyright: ignore[reportArgumentType]  # <reason>` when introducing a truly unavoidable violation
+- a TODO or FIXME
+  - Not great to have those, but if it's unavoidable, make sure to include a strong, concise reason for why it's there or, better yet, link to a GitHub issue for the follow-up work
+
+Explanation: The point of this rule is to keep out AI slop comments. AI writes way too many and way too verbose comments. Code comments are, in a way, a violation of DRY code. You must update logic in two locations to change the code, and "hard to change" is literally the definition of tech debt. We should instead aim to write code that is intuitive and clear, even at a glance, to the reader, being both easy to maintain and high performance
 
 Don't assume that the existing code is correct or the right way of doing things / good coding patterns. In fact, there are a lot of bad coding practices, overly complex code, code smells, etc. If something doesn't look right, speak up. Feel free to break existing patterns or question weird existing code to make new code high quality, as in:
 
@@ -9,7 +17,7 @@ Don't assume that the existing code is correct or the right way of doing things
 - easy to maintain/change
 - modern
 
-In that order of importance
+In descending order of importance
 
 When adding new features, add meaningful tests. Don't add tests that don't check anything substantial and is there just to make the code coverage pass. Yes, code coverage is important, but I'd rather have no signal whether the code is working than tests that don't fail when code is broken. The goal is to have tests that would fail before the feature was added/if the code was mutated in a way that breaks the feature and succeed only when the feature is fully working. I should run mutation testing and see > 90% kill rate
 
@@ -21,7 +29,9 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions
 
 When creating PRs, don't set base to `main`. `litellm_internal_staging` is the default base branch and serves that purpose for both internal and external / OSS contributions
 
-When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
+When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
+
+Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
 
@@ -29,7 +39,7 @@ Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We pref
 
 If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
 - don't use emojis
-- don't use "—". Instead, reach for ";", ".", etc.
+- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
 - don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
@@ -39,15 +49,13 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
 
 Python max line length is 120, not 88
 
-On a fresh worktree or clone, run `make bootstrap` before anything else. It provisions everything tests, `make pre-commit`, and a local proxy need
-
-Run tests before you commit. Also, run `make pre-commit` right before each commit, which generates types (as needed) and formats/lints your code. Any errors found must be fixed. It only runs when there are staged frontend and/or backend changes and calculates violations, generates types, etc. based on the worktree, so stage what you need or stash/delete unwanted files in litellm/ or ui/ (where backend and frontend lint run, respectively) before running it. If it fails because dashboard api types are stale, it already regenerated them for you. You just need to stage the schema.d.ts, re-run `make pre-commit` to confirm it passes, and commit
-
 When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing
 
+`make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
+
 If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
 
-If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
+If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
 
 Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType]  # stubs lack async overload` or `# noqa: TID251  # <reason>`. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing
 
@@ -61,6 +69,8 @@ Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages
 
 When working on a PR, keep the PR description in sync with new commits being made
 
+All GitHub comments must be human-readable and 15-25 words max
+
 Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer dependency-injecting things into classes. That way, at unit test time, you can pass a mocked dependency in
 
 Do not put names of customers or customer company names in code, PR descriptions, issue bodies, etc. This means never mention literally any company name. Especially if you're about to say a sentence mentioning that the reason the PR exists was a feature/model/bug fix/etc. requested by a company. That's the indication that you should replace that company name with "the customer". e.g. not "Model request from Acme (Pylon #1234)" but "Model request from a customer (Pylon #1234)". This is because the codebase is public. The only exception is for publicly known providers or vendors such as OpenAI, Anthropic, AWS Bedrock, etc. only IF we're adding support for that provider/vendor in general and NOT if that PR or whatnot was a request by one of them, and they're actually one of our customers. 
@@ -72,7 +82,8 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - Composition over inheritance
 - Never-nester: early returns over deep nesting
 - Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
-- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), etc.
+- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc.
+  - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>` explaining why
 - Use dependency injection
 - Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
 - Use tagged unions + match

Dockerfile

@@ -7,7 +7,7 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17
 ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
-ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6
+ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
 
 FROM $UV_IMAGE AS uvbin
 
@@ -64,6 +64,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
 # Copy full source tree
@@ -84,6 +85,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
@@ -132,7 +134,8 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete && \
     chmod -R a+rX /opt/prisma && \
     test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
-    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 EXPOSE 4000/tcp
 

Makefile

@@ -8,7 +8,7 @@
 	lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
 	lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
 	install-dev install-proxy-dev install-test-deps install-hooks \
-	install-helm-unittest check-circular-imports check-import-safety pre-commit \
+	install-helm-unittest check-circular-imports check-import-safety check pre-commit \
 	lint-install lint-fetch-base bootstrap
 
 # Default target
@@ -22,7 +22,8 @@ help:
 	@echo "  make install-test-deps  - Install the full local test environment"
 	@echo "  make install-helm-unittest - Install helm unittest plugin"
 	@echo "  make install-hooks      - Install git hooks (Conventional Commits + Branches)"
-	@echo "  make pre-commit         - Run CI-equivalent lint on staged files (run before committing)"
+	@echo "  make check              - Run CI-equivalent lint on staged files, or on the diff vs the base branch when nothing is staged"
+	@echo "  make pre-commit         - Legacy alias for make check"
 	@echo "  make format             - Apply ruff format code formatting"
 	@echo "  make format-check       - Check ruff format code formatting (matches CI)"
 	@echo "  make lint               - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)"
@@ -75,7 +76,7 @@ install-dev:
 bootstrap:
 	$(UV) sync --inexact --frozen --extra proxy --group proxy-dev --group e2e-dev
 	$(UV_RUN) python scripts/prisma_generate_if_needed.py
-	cd ui/litellm-dashboard && npm ci --no-audit --no-fund
+	cd ui/litellm-dashboard && ../../scripts/with_dashboard_node.sh npm install --no-audit --no-fund
 	@main_root=$$(git worktree list --porcelain | head -1 | sed 's/^worktree //'); \
 	if [ "$$main_root" != "$$(git rev-parse --show-toplevel)" ] && [ -f "$$main_root/.env" ] && [ ! -f .env ]; then \
 		cp "$$main_root/.env" .env && echo "bootstrap: copied .env from $$main_root"; \
@@ -99,7 +100,10 @@ install-test-deps: install-proxy-dev
 	$(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma
 
 install-helm-unittest:
-	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
+	@helm plugin list | grep -qE '^unittest[[:space:]]+0\.8\.2([[:space:]]|$$)' || { \
+		helm plugin uninstall unittest >/dev/null 2>&1 || true; \
+		helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.8.2; \
+	}
 
 # Install git hooks that enforce Conventional Commits and Conventional Branches.
 # Opt-in: not chained into install-dev.
@@ -121,10 +125,10 @@ lint-fetch-base:
 	git fetch origin litellm_internal_staging
 
 # Mirror test-linting.yml's lint job environment: the proxy-dev group plus a generated
-# Prisma client, so basedpyright resolves the same modules CI does (without the generated
-# client the DB wrappers typed against it degrade to Unknown, drifting the budget from
-# CI's). --inexact tops up the venv instead of pruning the proxy extras gen:api and the
-# running proxy need.
+# Prisma client, so `basedpyright tests/e2e` resolves the same modules CI does. The
+# budget gate itself no longer measures here (scripts/type_check_gate.py provisions its
+# own .venv-typecheck). --inexact tops up the venv instead of pruning the proxy extras
+# gen:api and the running proxy need.
 lint-install:
 	$(UV) sync --inexact --frozen --group proxy-dev --group e2e-dev
 	$(UV_RUN) python scripts/prisma_generate_if_needed.py
@@ -177,7 +181,7 @@ lint-ruff-FULL-dev: install-dev
 	else echo "No changed .py files to check."; fi
 
 lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
-	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
+	$(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
 
 lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
 	$(UV_RUN) basedpyright tests/e2e
@@ -190,7 +194,7 @@ lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 # --update lowers each limit by what this branch fixed since its branch point, so
 # it needs the base ref fetched to resolve the merge-base.
 lint-basedpyright-budget-update: install-dev lint-fetch-base
-	($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update
+	$(UV_RUN) python scripts/type_check_gate.py --update
 
 lint-format: format-check
 
@@ -233,13 +237,20 @@ lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline
 # Faster linting for local development (only checks changed code)
 lint-dev: lint-format-changed check-circular-imports check-import-safety
 
-# Run the gating CI checks against your staged files right before committing. Mirrors
+# Run the gating CI checks against your changes. Scopes to staged files when anything
+# is staged (warning about changed files left unstaged); with nothing staged it falls
+# back to the working tree's diff against the merge base with the base branch, so a
+# fresh merge commit or an unstaged working tree still gets checked. Mirrors
 # test-linting.yml (Python), test-litellm-ui-build.yml's frontend-lint (dashboard), and
-# check-ui-api-types.yml (API-type drift), skipping any whose files you didn't stage.
+# check-ui-api-types.yml (API-type drift), skipping any whose files aren't in scope.
 # Not auto-installed as a git hook so it never slows an unrelated human commit.
-pre-commit:
+check: bootstrap
 	./scripts/pre_commit_lint.sh
 
+pre-commit:
+	@echo "make pre-commit is a legacy alias; use make check" >&2
+	@$(MAKE) check
+
 # Testing targets
 test: install-test-deps
 	$(UV_RUN) pytest tests/

backend/Dockerfile

@@ -59,9 +59,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra semantic-router \
         --python python3
 
-RUN mkdir -p /home/nonroot && \
-    HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
-    chown -R nonroot:nonroot /home/nonroot/.cache
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
+
+RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh
 
 # ---------- Runtime ----------
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
@@ -81,17 +83,20 @@ ENV HOME=/home/nonroot \
     PATH="/app/.venv/bin:${PATH}" \
     PYTHONPATH="/app" \
     PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1
+    PYTHONUNBUFFERED=1 \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
 COPY --from=builder --chown=nonroot:nonroot /app /app
-COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 USER nonroot
 
 EXPOSE 4001/tcp
 
-ENTRYPOINT ["uvicorn", "backend.main:app"]
+ENTRYPOINT ["/app/docker/component_entrypoint.sh", "uvicorn", "backend.main:app"]
 CMD ["--host", "0.0.0.0", "--port", "4001"]

backend/routes/allowlist.py

@@ -44,6 +44,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/router/",
     "/router_settings",
     "/adaptive_router/",
+    "/auto_router/",
     "/fallback",
     "/fallbacks",
     "/cache_settings",
@@ -70,6 +71,10 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/project/",
     "/memory/",
     "/mcp/",
+    # Control plane (see the List Endpoints + Tables standard). Every resource
+    # eventually moves under this prefix, so allowlist it once rather than
+    # per-resource.
+    "/management/v1/",
     # Spend / analytics
     "/spend/",
     "/analytics/",
@@ -77,6 +82,9 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/user_agent",
     "/usage/",
     "/daily/",
+    # Deployment-wide gateway request counts. Scoped to the analytics read rather
+    # than all of /gateway/, which stays free for data-plane routes.
+    "/gateway/daily/",
     # CloudZero cost-export admin (init / settings / export / dry-run / delete)
     "/cloudzero/",
     # Caching admin

basedpyright-code-budget.json

@@ -1,66 +1,66 @@
 {
   "reportAny": {
-    "limit": 37484
+    "limit": 23914
   },
   "reportArgumentType": {
-    "limit": 2704
+    "limit": 2580
   },
   "reportAssignmentType": {
-    "limit": 330
+    "limit": 323
   },
   "reportAttributeAccessIssue": {
-    "limit": 516
+    "limit": 488
   },
   "reportCallIssue": {
-    "limit": 124
+    "limit": 114
   },
   "reportConstantRedefinition": {
-    "limit": 59
+    "limit": 40
   },
   "reportDeprecated": {
-    "limit": 326
+    "limit": 213
   },
   "reportDuplicateImport": {
-    "limit": 42
+    "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 10389
+    "limit": 7573
   },
   "reportFunctionMemberAccess": {
-    "limit": 11
+    "limit": 7
   },
   "reportGeneralTypeIssues": {
-    "limit": 227
+    "limit": 157
   },
   "reportIncompatibleMethodOverride": {
-    "limit": 78
+    "limit": 56
   },
   "reportIncompatibleVariableOverride": {
-    "limit": 12
+    "limit": 8
   },
   "reportInconsistentOverload": {
-    "limit": 18
+    "limit": 12
   },
   "reportIndexIssue": {
-    "limit": 37
+    "limit": 35
   },
   "reportInvalidTypeForm": {
     "limit": 35
   },
   "reportInvalidTypeVarUse": {
-    "limit": 5
+    "limit": 2
   },
   "reportMatchNotExhaustive": {
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5900
+    "limit": 5719
   },
   "reportMissingTypeArgument": {
-    "limit": 15903
+    "limit": 15657
   },
   "reportMissingTypeStubs": {
-    "limit": 41
+    "limit": 40
   },
   "reportOperatorIssue": {
     "limit": 0
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1085
+    "limit": 1069
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -81,66 +81,66 @@
     "limit": 0
   },
   "reportPossiblyUnboundVariable": {
-    "limit": 77
+    "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 2438
+    "limit": 1824
   },
   "reportRedeclaration": {
-    "limit": 12
+    "limit": 8
   },
   "reportReturnType": {
-    "limit": 225
+    "limit": 213
   },
   "reportTypedDictNotRequiredAccess": {
-    "limit": 27
+    "limit": 26
   },
   "reportUndefinedVariable": {
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 45894
+    "limit": 44832
   },
   "reportUnknownLambdaType": {
     "limit": 113
   },
   "reportUnknownMemberType": {
-    "limit": 40539
+    "limit": 39269
   },
   "reportUnknownParameterType": {
-    "limit": 20403
+    "limit": 19988
   },
   "reportUnknownVariableType": {
-    "limit": 32141
+    "limit": 30923
   },
   "reportUnnecessaryCast": {
-    "limit": 177
+    "limit": 118
   },
   "reportUnnecessaryComparison": {
-    "limit": 1025
+    "limit": 699
   },
   "reportUnnecessaryContains": {
-    "limit": 7
+    "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 1209
+    "limit": 853
   },
   "reportUntypedBaseClass": {
-    "limit": 165
+    "limit": 0
   },
   "reportUntypedFunctionDecorator": {
-    "limit": 33
+    "limit": 27
   },
   "reportUnusedClass": {
-    "limit": 33
+    "limit": 23
   },
   "reportUnusedFunction": {
-    "limit": 206
+    "limit": 139
   },
   "reportUnusedImport": {
-    "limit": 1005
+    "limit": 545
   },
   "reportUnusedVariable": {
-    "limit": 1297
+    "limit": 146
   }
 }

ci_cd/generate_model_prices_schema.py

@@ -0,0 +1,325 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Optional
+
+import jsonschema
+
+REPO_ROOT = Path(__file__).parent.parent
+PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
+SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
+
+SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
+
+JsonSchema = dict
+
+NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
+NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
+BOOLEAN: JsonSchema = {"type": "boolean"}
+STRING: JsonSchema = {"type": "string"}
+
+EXTRA_BOOLEAN_KEYS = frozenset(
+    {
+        "gemini_native_audio",
+        "gemini_audio_only_live",
+        "uses_embed_content",
+        "use_openai_responses_path",
+        "bedrock_converse_supports_strict_tools",
+    }
+)
+
+OBJECT_KEYS: dict[str, JsonSchema] = {
+    "search_context_cost_per_query": {
+        "type": "object",
+        "description": "USD cost per web search query, keyed by search context size.",
+        "properties": {
+            "search_context_size_low": NONNEG_NUMBER,
+            "search_context_size_medium": NONNEG_NUMBER,
+            "search_context_size_high": NONNEG_NUMBER,
+        },
+        "additionalProperties": False,
+    },
+    "metadata": {
+        "type": "object",
+        "description": "Free-form notes about the entry (e.g. pricing derivation).",
+    },
+    "provider_specific_entry": {
+        "type": "object",
+        "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
+    },
+}
+
+ARRAY_KEYS: dict[str, JsonSchema] = {
+    "supported_endpoints": {
+        "type": "array",
+        "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
+        "items": STRING,
+    },
+    "supported_modalities": {
+        "type": "array",
+        "description": "Input modalities the model accepts.",
+        "items": {"type": "string", "enum": ["text", "image", "audio", "video"]},
+    },
+    "supported_output_modalities": {
+        "type": "array",
+        "description": "Output modalities the model can produce.",
+        "items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
+    },
+    "supported_regions": {
+        "type": "array",
+        "description": "Cloud regions the model is available in ('global' or region ids).",
+        "items": STRING,
+    },
+    "tiered_pricing": {
+        "type": "array",
+        "description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
+        "items": {
+            "type": "object",
+            "properties": {
+                "range": {
+                    "type": "array",
+                    "description": "[min, max] prompt-token span this tier applies to.",
+                    "items": NONNEG_NUMBER,
+                    "minItems": 2,
+                    "maxItems": 2,
+                },
+                "max_results_range": {
+                    "type": "array",
+                    "description": "[min, max] result-count span this tier applies to (search models).",
+                    "items": NONNEG_NUMBER,
+                    "minItems": 2,
+                    "maxItems": 2,
+                },
+                "input_cost_per_token": NONNEG_NUMBER,
+                "output_cost_per_token": NONNEG_NUMBER,
+                "output_cost_per_reasoning_token": NONNEG_NUMBER,
+                "cache_read_input_token_cost": NONNEG_NUMBER,
+                "input_cost_per_query": NONNEG_NUMBER,
+            },
+            "additionalProperties": False,
+        },
+    },
+}
+
+INTEGER_KEYS: dict[str, JsonSchema] = {
+    "max_tokens": {
+        **NONNEG_INTEGER,
+        "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.",
+    },
+    "max_input_tokens": {
+        **NONNEG_INTEGER,
+        "description": "Maximum prompt/context tokens the model accepts.",
+    },
+    "max_output_tokens": {
+        **NONNEG_INTEGER,
+        "description": "Maximum tokens the model can generate in one response.",
+    },
+    "output_vector_size": {
+        **NONNEG_INTEGER,
+        "description": "Embedding dimension for embedding models.",
+    },
+    "prompt_cache_min_tokens": {
+        **NONNEG_INTEGER,
+        "description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
+    },
+    "tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
+    "rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
+}
+
+NUMBER_KEYS: dict[str, JsonSchema] = {
+    "regional_processing_uplift_multiplier_eu": {
+        "type": "number",
+        "minimum": 1,
+        "description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).",
+    },
+    "regional_processing_uplift_multiplier_us": {
+        "type": "number",
+        "minimum": 1,
+        "description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
+    },
+}
+
+COST_DESCRIPTIONS: dict[str, str] = {
+    "input_cost_per_token": "USD per prompt token.",
+    "output_cost_per_token": "USD per generated token.",
+    "output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
+    "cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
+    "cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
+    "input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
+    "output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
+}
+
+
+def cost_description(key: str) -> Optional[str]:
+    if key in COST_DESCRIPTIONS:
+        return COST_DESCRIPTIONS[key]
+    if key.endswith("_flex"):
+        return "Flex service-tier rate for the same-named base field."
+    if key.endswith("_priority"):
+        return "Priority service-tier rate for the same-named base field."
+    if "_above_" in key:
+        return "Rate applied once the prompt exceeds the token threshold in the field name."
+    return None
+
+
+def cost_schema(key: str) -> JsonSchema:
+    description = cost_description(key)
+    return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER)
+
+
+def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
+    return {
+        "litellm_provider": {
+            "type": "string",
+            "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",
+        },
+        "mode": {
+            "type": "string",
+            "description": "Primary API surface / task type of the model.",
+            "enum": list(modes),
+        },
+        "source": {
+            "type": "string",
+            "description": "URL of the provider pricing/model page this entry was taken from.",
+        },
+        "deprecation_date": {
+            "type": "string",
+            "description": "Date the provider deprecates the model, YYYY-MM-DD.",
+            "format": "date",
+            "pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$",
+        },
+        "web_search_billing_unit": {
+            "type": "string",
+            "description": "Whether web search is billed per query or per prompt.",
+            "enum": ["per_query", "per_prompt"],
+        },
+        "bedrock_output_config_effort_ceiling": {
+            "type": "string",
+            "description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
+            "enum": ["low", "medium", "high", "max", "xhigh"],
+        },
+        "comment": STRING,
+        "audio_transcription_config": STRING,
+    }
+
+
+def classify(key: str, modes: tuple) -> Optional[JsonSchema]:
+    curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS}
+    if key in curated:
+        return curated[key]
+    if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS:
+        return BOOLEAN
+    if "cost" in key:
+        return cost_schema(key)
+    return None
+
+
+def build_schema(prices: dict) -> JsonSchema:
+    entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS}
+    all_keys = tuple(sorted({key for entry in entries.values() for key in entry}))
+    modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry}))
+    unclassified = tuple(key for key in all_keys if classify(key, modes) is None)
+    if unclassified:
+        raise SystemExit(
+            f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. "
+            f"Add them to the key tables in {Path(__file__).name} and rerun it."
+        )
+    entry_properties = {key: classify(key, modes) for key in all_keys}
+    return {
+        "$schema": "https://json-schema.org/draft/2020-12/schema",
+        "title": "LiteLLM model_prices_and_context_window.json",
+        "description": (
+            "Schema for LiteLLM's model price and context window registry "
+            "(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
+            "Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
+            "optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
+            "All costs are USD per unit. New optional fields are added regularly, so consumers should "
+            "ignore unknown fields rather than reject them."
+        ),
+        "type": "object",
+        "properties": {
+            "sample_spec": {
+                "type": "object",
+                "description": (
+                    "Documentation placeholder illustrating the entry shape; not a real model and not "
+                    "schema-conformant (several values are prose)."
+                ),
+            },
+            "fallback_generalizations": {
+                "type": "object",
+                "description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
+                "properties": {
+                    "rules": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "name": STRING,
+                                "pattern": STRING,
+                                "description": STRING,
+                            },
+                            "required": ["name", "pattern"],
+                            "additionalProperties": True,
+                        },
+                    }
+                },
+                "additionalProperties": False,
+            },
+        },
+        "additionalProperties": {"$ref": "#/$defs/modelEntry"},
+        "$defs": {
+            "modelEntry": {
+                "type": "object",
+                "description": (
+                    "Pricing, limits, and capability flags for one model. Fields other than litellm_provider "
+                    "are optional; boolean capability flags are simply omitted when unknown or false."
+                ),
+                "required": ["litellm_provider"],
+                "properties": entry_properties,
+                "additionalProperties": True,
+            }
+        },
+    }
+
+
+def render(schema: JsonSchema) -> str:
+    return json.dumps(schema, indent=2) + "\n"
+
+
+def validation_errors(prices: dict, schema: JsonSchema) -> tuple:
+    validator = jsonschema.Draft202012Validator(
+        schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER
+    )
+    return tuple(
+        f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
+        for error in validator.iter_errors(prices)
+    )
+
+
+def main() -> int:
+    check = "--check" in sys.argv[1:]
+    prices = json.loads(PRICES_PATH.read_text())
+    rendered = render(build_schema(prices))
+    errors = validation_errors(prices, json.loads(rendered))
+    if errors:
+        print(f"{PRICES_PATH.name} does not validate against the generated schema:")
+        print("\n".join(errors[:20]))
+        return 1
+    if not check:
+        SCHEMA_PATH.write_text(rendered)
+        print(f"wrote {SCHEMA_PATH}")
+        return 0
+    if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered:
+        print(
+            f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. "
+            f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result."
+        )
+        return 1
+    print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_genai_otel/grafana_dashboard.json

@@ -0,0 +1,523 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "panels": [
+    {
+      "type": "stat",
+      "title": "Requests",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "blue"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "instant": true,
+          "expr": "sum(increase(gen_ai_client_operation_duration_seconds_count{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__range]))"
+        }
+      ],
+      "id": 1
+    },
+    {
+      "type": "stat",
+      "title": "Spend",
+      "description": "LiteLLM's computed cost for the selected window, from gen_ai.usage.cost",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "currencyUSD",
+          "decimals": 4,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "green"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "instant": true,
+          "expr": "sum(increase(gen_ai_usage_cost_USD_sum{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__range]))"
+        }
+      ],
+      "id": 2
+    },
+    {
+      "type": "stat",
+      "title": "Tokens",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "purple"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "instant": true,
+          "expr": "sum(increase(gen_ai_client_token_usage_sum{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__range]))"
+        }
+      ],
+      "id": 3
+    },
+    {
+      "type": "stat",
+      "title": "p95 request duration",
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "decimals": 2,
+          "color": {
+            "mode": "fixed",
+            "fixedColor": "orange"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "instant": true,
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__range])))"
+        }
+      ],
+      "id": 4
+    },
+    {
+      "type": "timeseries",
+      "title": "Request rate by model",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqpm",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 8,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}}",
+          "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_operation_duration_seconds_count{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])) * 60"
+        }
+      ],
+      "id": 5
+    },
+    {
+      "type": "timeseries",
+      "title": "Spend rate by model",
+      "description": "USD per hour, derived from the gen_ai.usage.cost histogram",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "currencyUSD",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 8,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}}",
+          "expr": "sum by (gen_ai_request_model) (rate(gen_ai_usage_cost_USD_sum{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])) * 3600"
+        }
+      ],
+      "id": 6
+    },
+    {
+      "type": "timeseries",
+      "title": "Tokens per minute by model and type",
+      "description": "gen_ai.client.token.usage split by the gen_ai.token.type attribute",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 8,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}} {{gen_ai_token_type}}",
+          "expr": "sum by (gen_ai_request_model, gen_ai_token_type) (rate(gen_ai_client_token_usage_sum{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])) * 60"
+        }
+      ],
+      "id": 7
+    },
+    {
+      "type": "timeseries",
+      "title": "p95 request duration by model",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 0,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}}",
+          "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])))"
+        }
+      ],
+      "id": 8
+    },
+    {
+      "type": "timeseries",
+      "title": "p95 time to first token (streaming)",
+      "description": "gen_ai.server.time_to_first_token, recorded only for streaming requests",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 20
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 0,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}}",
+          "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_server_time_to_first_token_seconds_bucket{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])))"
+        }
+      ],
+      "id": 9
+    },
+    {
+      "type": "timeseries",
+      "title": "p95 provider generation time",
+      "description": "gen_ai.client.response.duration, upstream generation time excluding LiteLLM overhead",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 20
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "custom": {
+            "lineWidth": 2,
+            "fillOpacity": 0,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "editorMode": "code",
+          "legendFormat": "{{gen_ai_request_model}}",
+          "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_response_duration_seconds_bucket{service_name=~\"$service\", gen_ai_request_model=~\"$model\"}[$__rate_interval])))"
+        }
+      ],
+      "id": 10
+    }
+  ],
+  "preload": false,
+  "refresh": "30s",
+  "schemaVersion": 42,
+  "tags": [
+    "litellm",
+    "genai",
+    "opentelemetry"
+  ],
+  "templating": {
+    "list": [
+      {
+        "name": "datasource",
+        "label": "Prometheus",
+        "type": "datasource",
+        "query": "prometheus",
+        "current": {},
+        "hide": 0
+      },
+      {
+        "name": "service",
+        "label": "Service",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "query": "label_values(gen_ai_client_operation_duration_seconds_count, service_name)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        }
+      },
+      {
+        "name": "model",
+        "label": "Model",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "query": "label_values(gen_ai_client_operation_duration_seconds_count{service_name=~\"$service\"}, gen_ai_request_model)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        }
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "LiteLLM GenAI (OpenTelemetry)",
+  "uid": "litellm-genai-otel",
+  "weekStart": ""
+}

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_genai_otel/readme.md

@@ -0,0 +1,35 @@
+# LiteLLM GenAI dashboard (OpenTelemetry metrics)
+
+Dashboard for the `gen_ai.*` metrics the OpenTelemetry v2 integration emits, as opposed to the `litellm_*` Prometheus metrics the other dashboards in this folder chart.
+
+Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source. Panels: request count, spend, token count, p95 duration, request rate by model, spend rate per hour by model, tokens per minute split by input and output, p95 duration by model, p95 time to first token, and p95 provider generation time. Template variables for data source, service, and model.
+
+## Pre-requisites
+
+Metrics are off by default. In the proxy environment:
+
+```shell
+LITELLM_OTEL_V2=true
+LITELLM_OTEL_INTEGRATION_ENABLE_METRICS=true
+OTEL_EXPORTER="otlp_http"
+OTEL_ENDPOINT="<your OTLP endpoint>"
+```
+
+You also need the metric attribute filter, or the panels will plot flat lines at zero. LiteLLM's default attribute set includes per-request fields, so nearly every request lands in its own time series with a single sample, and `rate()` has nothing to compute over:
+
+```yaml title="config.yaml"
+callback_settings:
+  otel:
+    attributes:
+      include_list:
+        - gen_ai.operation.name
+        - gen_ai.system
+        - gen_ai.request.model
+        - gen_ai.framework
+```
+
+See [Grafana Cloud](https://docs.litellm.ai/docs/observability/grafana_cloud) for the full setup, and [OpenTelemetry v2](https://docs.litellm.ai/docs/observability/opentelemetry_v2#metrics) for the metric reference.
+
+## Note on Grafana's AI Observability integration
+
+Grafana Cloud ships prebuilt GenAI dashboards that query these same metric names, so they look like a drop-in alternative to this one. They are not: twenty of their twenty-two panels filter on `telemetry_sdk_name="openlit"`, a label LiteLLM does not carry and cannot be configured to add, so those panels stay empty.

cookbook/litellm_proxy_server/grafana_dashboard/readme.md

@@ -2,6 +2,10 @@
 
 This folder contains the `json` for creating Grafana Dashboards
 
+## [LiteLLM GenAI Dashboard (OpenTelemetry)](./dashboard_genai_otel)
+
+Charts the `gen_ai.*` metrics from the OpenTelemetry v2 integration: spend, tokens, request rate, and latency percentiles by model. Separate from the dashboards below, which chart the `litellm_*` Prometheus metrics.
+
 ## [LiteLLM v2 Dashboard](./dashboard_v2)
 
 <img width="1316" alt="grafana_1" src="https://github.com/user-attachments/assets/d0df802d-0cb9-4906-a679-941c547789ab">

db_scripts/backfill_daily_tool_spend.sql

@@ -0,0 +1,46 @@
+-- One-shot backfill of the LiteLLM_DailyToolSpend rollup from the per-request
+-- LiteLLM_SpendLogToolIndex x LiteLLM_SpendLogs tables.
+--
+-- This is an opt-in, manual operation. New deployments do not need it: the
+-- rollup is written at request time from the moment the release is deployed.
+-- Run it only if you want the Cost Optimization "Spend by tool" card to show
+-- history from before the deploy, and only once.
+--
+-- IMPORTANT caveats before running:
+--
+-- 1. Pre-deploy index rows may include tools that were merely DECLARED in a
+--    request body but never invoked (the release this ships with stops
+--    recording those). For agentic clients that declare many tools per
+--    request, backfilled history attributes each request's full spend to
+--    every declared tool, overstating per-tool spend. Post-deploy rows do not
+--    have this problem. If your traffic is mostly such clients, consider not
+--    backfilling.
+--
+-- 2. Coverage is bounded by spend-log retention: rows older than
+--    maximum_spend_logs_retention_period are already gone.
+--
+-- 3. Replace the cutover timestamp below with the time you deployed the
+--    release, so backfilled per-request rows cannot double-count on top of
+--    rollup rows the new writer already created. ON CONFLICT DO NOTHING is a
+--    second guard for (date, tool_name) buckets the writer already touched:
+--    such buckets keep the writer's numbers and skip the backfill's.
+--
+-- Usage:
+--   psql "$DATABASE_URL" -v cutover="'2026-07-25T00:00:00Z'" -f db_scripts/backfill_daily_tool_spend.sql
+
+SET TIME ZONE 'UTC';
+
+INSERT INTO "LiteLLM_DailyToolSpend" (date, tool_name, spend, total_tokens, request_count, created_at, updated_at)
+SELECT
+    to_char(ti.start_time, 'YYYY-MM-DD') AS date,
+    ti.tool_name,
+    COALESCE(SUM(sl.spend), 0) AS spend,
+    COALESCE(SUM(sl.total_tokens), 0) AS total_tokens,
+    COUNT(*) AS request_count,
+    now() AS created_at,
+    now() AS updated_at
+FROM "LiteLLM_SpendLogToolIndex" ti
+JOIN "LiteLLM_SpendLogs" sl ON sl.request_id = ti.request_id
+WHERE ti.start_time < :cutover::timestamptz
+GROUP BY 1, 2
+ON CONFLICT (date, tool_name) DO NOTHING;

docker/Dockerfile.database

@@ -7,7 +7,7 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17
 ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
-ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6
+ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
 
 FROM $UV_IMAGE AS uvbin
 
@@ -62,6 +62,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
 # Copy full source tree
@@ -82,6 +83,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
@@ -131,7 +133,8 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete && \
     chmod -R a+rX /opt/prisma && \
     test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
-    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 EXPOSE 4000/tcp
 

docker/Dockerfile.non_root

@@ -6,7 +6,7 @@ ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b
 ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
-ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6
+ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
 
 FROM $UV_IMAGE AS uvbin
 
@@ -68,6 +68,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
 # Copy full source tree
@@ -94,6 +95,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra saml \
         --python python3 \
         --no-sources-package litellm-proxy-extras; \
     else \
@@ -102,6 +104,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra saml \
         --python python3; \
     fi
 
@@ -182,7 +185,8 @@ RUN mkdir -p /nonexistent /app/.cache /var/lib/litellm/assets /var/lib/litellm/u
     chmod -R a+rX /opt/prisma && \
     test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
     test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
-    ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1
+    ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1 && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 USER 65534
 

docker/build_admin_ui.sh

@@ -54,8 +54,9 @@ else
 fi || { echo "nvm checksum verification failed"; exit 1; }
 bash "$NVM_SCRIPT"
 source ~/.nvm/nvm.sh
-nvm install v18.17.0
-nvm use v18.17.0
+NODE_VERSION="$(cat ui/litellm-dashboard/.nvmrc)"
+nvm install "v${NODE_VERSION}"
+nvm use "v${NODE_VERSION}"
 
 
 # cd in to /ui/litellm-dashboard

docker/build_from_pip/Dockerfile.build_from_pip

@@ -47,7 +47,13 @@ RUN uv venv --python python && \
       "prisma==0.11.0" \
       "openai==2.24.0"
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma && \
+    chmod -R a+rX /opt/prisma && \
+    python -c "import sys; from prisma.client import BINARY_PATHS; bad = sorted(p for group in BINARY_PATHS.model_dump().values() for p in group.values() if not p.startswith('/opt/prisma/')); sys.exit('prisma engines baked outside /opt/prisma: %r' % bad) if bad else None"
+
+ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
 EXPOSE 4000/tcp
 

docker/component_entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if [ "$USE_DDTRACE" = "true" ]; then
+    export DD_TRACE_OPENAI_ENABLED="False"
+    exec ddtrace-run "$@"
+fi
+
+exec "$@"

enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py

@@ -99,6 +99,7 @@ class BaseEmailLogger(CustomLogger):
         email_html_content = USER_INVITATION_EMAIL_TEMPLATE.format(
             email_logo_url=email_params.logo_url,
             recipient_email=email_params.recipient_email,
+            invitation_link=email_params.base_url,
             base_url=email_params.base_url,
             email_support_contact=email_params.support_contact,
             email_footer=email_params.signature,
@@ -826,10 +827,15 @@ class BaseEmailLogger(CustomLogger):
         """
         # Early validation
         if not user_id:
-            verbose_proxy_logger.debug("No user_id provided for invitation link")
+            verbose_proxy_logger.warning(
+                "No user_id provided for invitation link. Email will link to base URL instead of onboarding page"
+            )
             return base_url
 
         if not await self._is_prisma_client_available():
+            verbose_proxy_logger.warning(
+                "Prisma client not available. Email will link to base URL instead of onboarding page"
+            )
             return base_url
 
         # Wait for any concurrent invitation creation to complete
@@ -839,11 +845,15 @@ class BaseEmailLogger(CustomLogger):
         invitation = await self._get_or_create_invitation(user_id)
         if not invitation:
             verbose_proxy_logger.warning(
-                f"Failed to get/create invitation for user_id: {user_id}"
+                f"Failed to get/create invitation for user_id: {user_id}. Email will link to base URL instead of onboarding page"
             )
             return base_url
 
-        return self._construct_invitation_link(invitation.id, base_url)
+        invitation_link = self._construct_invitation_link(invitation.id, base_url)
+        verbose_proxy_logger.info(
+            f"Successfully created invitation link for user_id: {user_id}"
+        )
+        return invitation_link
 
     async def _is_prisma_client_available(self) -> bool:
         """Check if Prisma client is available"""
@@ -921,7 +931,9 @@ class BaseEmailLogger(CustomLogger):
 
         # http://localhost:4000/ui/onboarding?invitation_id=7a096b3a-37c6-440f-9dd1-ba22e8043f6b
         """
-        return f"{base_url}/ui/onboarding?invitation_id={invitation_id}"
+        base_url = base_url.rstrip("/")
+        invitation_link = f"{base_url}/ui/onboarding?invitation_id={invitation_id}"
+        return invitation_link
 
     async def send_email(
         self,

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -3,7 +3,7 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
 """
 
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
     from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
     from litellm.router import Router
+    from litellm.types.router import Deployment
     from litellm.types.utils import LiteLLMBatch
 
 
@@ -42,11 +43,15 @@ class CheckBatchCost:
         # the guaranteed-failing primary query on every subsequent cycle.
         self._has_batch_processed_column: bool = True
 
-    async def _get_user_info(self, batch_id, user_id) -> dict:
+    async def _get_user_info(self, batch_id: str, user_id: Optional[str]) -> Dict[str, Any]:
         """
         Look up user email and key alias by user_id for enriching the S3 callback metadata.
         Returns a dict with user_api_key_user_email and user_api_key_alias (both may be None).
+        Returns an empty dict when user_id is None: batches created by a team or service
+        account key carry no user id, and find_unique(where={"user_id": None}) raises.
         """
+        if not user_id:
+            return {}
         try:
             user_row = await self.prisma_client.db.litellm_usertable.find_unique(
                 where={"user_id": user_id}
@@ -61,6 +66,66 @@ class CheckBatchCost:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up user {user_id} for batch {batch_id}: {e}")
             return {}
 
+    async def _get_key_alias(self, batch_id: str, api_key: str | None) -> str | None:
+        """Resolve the creating virtual key's alias from its hashed token."""
+        if not api_key:
+            return None
+        try:
+            key_row = await self.prisma_client.db.litellm_verificationtoken.find_unique(
+                where={"token": api_key}
+            )
+            return getattr(key_row, "key_alias", None) if key_row is not None else None
+        except Exception as e:
+            verbose_proxy_logger.error(f"CheckBatchCost: could not look up key alias for batch {batch_id}: {e}")
+            return None
+
+    async def _get_team_alias(self, team_id: str | None) -> str | None:
+        """Resolve a team's alias from its id."""
+        if not team_id:
+            return None
+        try:
+            team_row = await self.prisma_client.db.litellm_teamtable.find_unique(
+                where={"team_id": team_id}
+            )
+            return getattr(team_row, "team_alias", None) if team_row is not None else None
+        except Exception as e:
+            verbose_proxy_logger.error(f"CheckBatchCost: could not look up team alias for team {team_id}: {e}")
+            return None
+
+    async def _build_creator_attribution_metadata(
+        self, job: "LiteLLM_ManagedObjectTable", batch_id: str
+    ) -> Dict[str, Any]:
+        """
+        Rebuild the spend-tracking metadata for the key, team, and tags that created the
+        batch so the batch-cost spend log is attributed the same way a non-batch request
+        is. Rows created before api_key and request_tags were persisted carry only
+        created_by and team_id, and fall back to those. A named creating key owns
+        user_api_key_alias; when it has no alias, or the key has since been rotated or
+        deleted, the field keeps the creating user's alias that _get_user_info filled in,
+        because a resolvable name is more useful on the spend row than a null.
+        """
+        api_key = getattr(job, "api_key", None)
+        team_id = getattr(job, "team_id", None)
+        request_tags = getattr(job, "request_tags", None)
+
+        metadata: Dict[str, Any] = {
+            "user_api_key_user_id": job.created_by,
+            "user_api_key": api_key,
+            "user_api_key_team_id": team_id,
+            **(await self._get_user_info(batch_id, job.created_by)),
+        }
+
+        key_alias = await self._get_key_alias(batch_id, api_key)
+        if key_alias is not None:
+            metadata["user_api_key_alias"] = key_alias
+        team_alias = await self._get_team_alias(team_id)
+        if team_alias is not None:
+            metadata["user_api_key_team_alias"] = team_alias
+        if isinstance(request_tags, list) and request_tags:
+            metadata["tags"] = [tag for tag in request_tags if isinstance(tag, str)]
+
+        return metadata
+
     async def _cleanup_stale_managed_objects(self) -> None:
         """
         Mark managed objects older than MANAGED_OBJECT_STALENESS_CUTOFF_DAYS days
@@ -281,6 +346,28 @@ class CheckBatchCost:
                 return deployment_id
         return None
 
+    @classmethod
+    def _get_managed_file_model_name(
+        cls,
+        job: "LiteLLM_ManagedObjectTable",
+        deployment_info: "Deployment",
+    ) -> Optional[str]:
+        """
+        Public model group name to encode as ``target_model_names`` on unified output file ids.
+
+        Key model-access checks resolve a managed file id back to a model via its
+        ``target_model_names``, so this must be the model group the caller requested, never the
+        underlying provider model (e.g. ``gpt-5.5``), which no key is allowed to call.
+        """
+        from litellm.proxy.openai_files_endpoints.common_utils import (
+            resolve_managed_output_file_model_name,
+        )
+
+        return resolve_managed_output_file_model_name(
+            unified_input_file_id=cls._get_input_file_id(job),
+            fallback_model_name=deployment_info.model_name or None,
+        )
+
     @staticmethod
     def _get_input_file_id(job: "LiteLLM_ManagedObjectTable") -> Optional[str]:
         import json
@@ -406,6 +493,10 @@ class CheckBatchCost:
         managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
         if managed_files_hook is not None:
             from litellm.proxy._types import UserAPIKeyAuth
+
+            managed_file_model_name = self._get_managed_file_model_name(
+                job=job, deployment_info=deployment_info
+            )
             _minimal_auth = UserAPIKeyAuth(
                 user_id=job.created_by or "default-user-id",
                 team_id=getattr(job, "team_id", None),
@@ -417,7 +508,7 @@ class CheckBatchCost:
                         _unified_file_id = managed_files_hook.get_unified_output_file_id(
                             output_file_id=_raw_file_id,
                             model_id=model_id,
-                            model_name=str(model_name) if model_name else deployment_info.model_name or None,
+                            model_name=managed_file_model_name,
                         )
                         await managed_files_hook.store_unified_file_id(
                             file_id=_unified_file_id,
@@ -458,9 +549,6 @@ class CheckBatchCost:
             function_id=str(uuid.uuid4()),
         )
 
-        creator_user_id = job.created_by
-        user_info = await self._get_user_info(batch_id, job.created_by)
-
         logging_obj.update_environment_variables(
             litellm_params={
                 # set the user-agent header so that S3 callback consumers can easily identify CheckBatchCost callbacks
@@ -469,10 +557,7 @@ class CheckBatchCost:
                         "user-agent": CHECK_BATCH_COST_USER_AGENT,
                     }
                 },
-                "metadata": {
-                    "user_api_key_user_id": creator_user_id,
-                    **user_info,
-                },
+                "metadata": await self._build_creator_attribution_metadata(job, batch_id),
             },
             optional_params={},
         )
@@ -629,6 +714,20 @@ class CheckBatchCost:
 
             elif response.status in ("failed", "expired", "cancelled"):
                 try:
+                    from litellm.proxy.openai_files_endpoints.common_utils import (
+                        _is_base64_encoded_unified_file_id,
+                        ensure_batch_response_managed_file_ids,
+                    )
+
+                    response.id = job.unified_object_id
+                    await ensure_batch_response_managed_file_ids(
+                        response=response,
+                        managed_files_obj=self.proxy_logging_obj.get_proxy_hook("managed_files"),
+                        prisma_client=self.prisma_client,
+                        verbose_proxy_logger=verbose_proxy_logger,
+                        db_batch_object=job,
+                        unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
+                    )
                     update_data = {
                         "status": response.status,
                         "file_object": response.model_dump_json(),

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -1,10 +1,10 @@
 """
 Polls LiteLLM_ManagedObjectTable to check if the response is complete.
-Cost tracking is handled automatically by litellm.aget_responses().
+Cost tracking is handled automatically by the get-responses call.
 """
 
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Dict, Optional, cast
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -13,11 +13,15 @@ from litellm.constants import (
     MAX_OBJECTS_PER_POLL_CYCLE,
     STALE_OBJECT_CLEANUP_BATCH_SIZE,
 )
+from litellm.responses.utils import ResponsesAPIRequestUtils
+from litellm.types.llms.openai import ResponsesAPIResponse
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
     from litellm.router import Router
 
+TERMINAL_RESPONSE_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete"})
+
 
 class CheckResponsesCost:
     def __init__(
@@ -33,6 +37,28 @@ class CheckResponsesCost:
         self.prisma_client: PrismaClient = prisma_client
         self.llm_router: Router = llm_router
 
+    async def _get_response(
+        self,
+        response_id: str,
+        litellm_metadata: Dict[str, str],
+    ) -> ResponsesAPIResponse:
+        """Fetch the upstream response, using deployment credentials when available.
+
+        LiteLLM-encoded response IDs carry the ``model_id`` of the deployment that
+        served the original request, so routing through ``llm_router`` applies that
+        deployment's ``api_base`` / ``api_key`` / ``api_version``, exactly like
+        ``GET /v1/responses/{id}`` does. ``litellm.aget_responses`` on its own only
+        sees provider env vars, so it fails for every deployment whose credentials
+        live in the config; the row then never leaves ``queued``.
+        """
+        model_id: Optional[str] = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
+        if model_id is None or self.llm_router.get_deployment(model_id=model_id) is None:
+            return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
+        router_response = await self.llm_router.aget_responses(
+            response_id=response_id, litellm_metadata=litellm_metadata
+        )
+        return cast(ResponsesAPIResponse, router_response)
+
     async def _expire_stale_rows(
         self, cutoff: datetime, batch_size: int
     ) -> int:
@@ -87,8 +113,8 @@ class CheckResponsesCost:
         Check if background responses are complete and track their cost.
         - Get all status="queued" or "in_progress" and file_purpose="response" jobs
         - Query the provider to check if response is complete
-        - Cost is automatically tracked by litellm.aget_responses()
-        - Mark completed/failed/cancelled responses as complete in the database
+        - Cost is automatically tracked by the get-responses call
+        - Mark responses in a terminal state as complete in the database
         """
         try:
             await self._cleanup_stale_managed_objects()
@@ -134,7 +160,7 @@ class CheckResponsesCost:
                     litellm_metadata["model"] = model_name
                     litellm_metadata["model_group"] = model_name  # Use same value for model_group
                 
-                response = await litellm.aget_responses(
+                response = await self._get_response(
                     response_id=responses_id_security,
                     litellm_metadata=litellm_metadata,
                 )
@@ -144,21 +170,14 @@ class CheckResponsesCost:
                 )
                 
             except Exception as e:
-                verbose_proxy_logger.info(
+                verbose_proxy_logger.warning(
                     f"Skipping job {unified_object_id} due to error: {e}"
                 )
                 continue
 
-            # Check if response is in a terminal state
-            if response.status == "completed":
+            if response.status in TERMINAL_RESPONSE_STATUSES:
                 verbose_proxy_logger.info(
-                    f"Response {unified_object_id} is complete. Cost automatically tracked by aget_responses."
-                )
-                completed_jobs.append(job)
-                
-            elif response.status in ["failed", "cancelled"]:
-                verbose_proxy_logger.info(
-                    f"Response {unified_object_id} has status {response.status}, marking as complete"
+                    f"Response {unified_object_id} has terminal status {response.status}, marking as complete"
                 )
                 completed_jobs.append(job)
 

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -11,13 +11,15 @@ Endpoints for /project operations
 #### PROJECT MANAGEMENT ####
 
 import json
-from typing import List, Optional, Union
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING
 
 from fastapi import APIRouter, Depends, HTTPException, Request
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
 from litellm.proxy._types import *
+from litellm.proxy.auth.auth_checks import delete_cached_project_object
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
 from litellm.proxy.management_helpers.utils import (
@@ -25,15 +27,24 @@ from litellm.proxy.management_helpers.utils import (
 )
 from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
 
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
+    from prisma.actions import LiteLLM_TeamTableActions
+
 router = APIRouter()
 
 
+def _team_table(prisma_client: PrismaClient) -> "LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable]":
+    team_table: LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable] = prisma_client.db.litellm_teamtable
+    return team_table
+
+
 async def _check_user_permission_for_project(
     user_api_key_dict: UserAPIKeyAuth,
-    team_id: Optional[str],
+    team_id: str | None,
     prisma_client: PrismaClient,
     require_admin: bool = False,
-    team_object: Optional[LiteLLM_TeamTable] = None,
+    team_object: LiteLLM_TeamTable | None = None,
 ) -> bool:
     """
     Check if user has permission to manage a project.
@@ -57,9 +68,7 @@ async def _check_user_permission_for_project(
 
     team = team_object
     if team is None:
-        team = await prisma_client.db.litellm_teamtable.find_unique(
-            where={"team_id": team_id}
-        )
+        team = await _team_table(prisma_client).find_unique(where={"team_id": team_id})
 
     if team and team.admins:
         return user_api_key_dict.user_id in team.admins
@@ -70,9 +79,9 @@ async def _check_user_permission_for_project(
 async def _validate_team_exists(
     team_id: str,
     prisma_client: PrismaClient,
-):
+) -> "prisma_models.LiteLLM_TeamTable":
     """Validate that a team exists. Returns the team row."""
-    team = await prisma_client.db.litellm_teamtable.find_unique(
+    team = await _team_table(prisma_client).find_unique(
         where={"team_id": team_id},
     )
 
@@ -89,7 +98,7 @@ async def _validate_team_exists(
 
 def _check_team_project_limits(
     team_object: LiteLLM_TeamTable,
-    data: Union[NewProjectRequest, UpdateProjectRequest],
+    data: NewProjectRequest | UpdateProjectRequest,
 ) -> None:
     """
     Check that project limits respect its parent Team's limits.
@@ -108,16 +117,12 @@ def _check_team_project_limits(
     if data.max_budget is not None and data.max_budget < 0:
         raise HTTPException(
             status_code=400,
-            detail={
-                "error": f"max_budget cannot be negative. Received: {data.max_budget}"
-            },
+            detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"},
         )
     if data.soft_budget is not None and data.soft_budget < 0:
         raise HTTPException(
             status_code=400,
-            detail={
-                "error": f"soft_budget cannot be negative. Received: {data.soft_budget}"
-            },
+            detail={"error": f"soft_budget cannot be negative. Received: {data.soft_budget}"},
         )
 
     # --- soft_budget < max_budget ---
@@ -131,7 +136,7 @@ def _check_team_project_limits(
             )
 
     # --- Validate project models are a subset of team models ---
-    project_models = getattr(data, "models", None)
+    project_models = data.models
     team_models = team_object.models or []
     if project_models and len(team_models) > 0:
         # If team has 'all-proxy-models', skip validation as it allows all models
@@ -148,11 +153,7 @@ def _check_team_project_limits(
     # --- Validate project max_budget <= team max_budget ---
     # Team stores budget fields directly (max_budget, tpm_limit, rpm_limit)
     # unlike Project which uses a separate LiteLLM_BudgetTable relation
-    if (
-        data.max_budget is not None
-        and team_object.max_budget is not None
-        and data.max_budget > team_object.max_budget
-    ):
+    if data.max_budget is not None and team_object.max_budget is not None and data.max_budget > team_object.max_budget:
         raise HTTPException(
             status_code=400,
             detail={
@@ -161,11 +162,7 @@ def _check_team_project_limits(
         )
 
     # --- Validate project tpm_limit <= team tpm_limit ---
-    if (
-        data.tpm_limit is not None
-        and team_object.tpm_limit is not None
-        and data.tpm_limit > team_object.tpm_limit
-    ):
+    if data.tpm_limit is not None and team_object.tpm_limit is not None and data.tpm_limit > team_object.tpm_limit:
         raise HTTPException(
             status_code=400,
             detail={
@@ -174,11 +171,7 @@ def _check_team_project_limits(
         )
 
     # --- Validate project rpm_limit <= team rpm_limit ---
-    if (
-        data.rpm_limit is not None
-        and team_object.rpm_limit is not None
-        and data.rpm_limit > team_object.rpm_limit
-    ):
+    if data.rpm_limit is not None and team_object.rpm_limit is not None and data.rpm_limit > team_object.rpm_limit:
         raise HTTPException(
             status_code=400,
             detail={
@@ -189,19 +182,19 @@ def _check_team_project_limits(
 
 async def _create_budget_for_project(
     data: NewProjectRequest,
-    user_id: Optional[str],
+    user_id: str | None,
     litellm_proxy_admin_name: str,
     prisma_client: PrismaClient,
 ) -> str:
     """Create a budget for the project and return budget_id."""
     budget_params = LiteLLM_BudgetTable.model_fields.keys()
-    _json_data = data.json(exclude_none=True)
+    _json_data: Mapping[str, object] = data.json(exclude_none=True)
     _budget_data = {k: v for k, v in _json_data.items() if k in budget_params}
-    budget_row = LiteLLM_BudgetTable(**_budget_data)
+    budget_row = LiteLLM_BudgetTable.model_validate(_budget_data)
 
     new_budget = prisma_client.jsonify_object(budget_row.json(exclude_none=True))
 
-    _budget = await prisma_client.db.litellm_budgettable.create(
+    _budget: prisma_models.LiteLLM_BudgetTable = await prisma_client.db.litellm_budgettable.create(
         data={
             **new_budget,
             "created_by": user_id or litellm_proxy_admin_name,
@@ -214,8 +207,8 @@ async def _create_budget_for_project(
 
 async def _set_project_object_permission(
     data: NewProjectRequest,
-    prisma_client: Optional[PrismaClient],
-) -> Optional[str]:
+    prisma_client: PrismaClient | None,
+) -> str | None:
     """
     Creates the LiteLLM_ObjectPermissionTable record for the project.
     Returns the object_permission_id if created, otherwise None.
@@ -224,7 +217,7 @@ async def _set_project_object_permission(
         return None
 
     if data.object_permission is not None:
-        created_object_permission = (
+        created_object_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
             await prisma_client.db.litellm_objectpermissiontable.create(
                 data=data.object_permission.model_dump(exclude_none=True),
             )
@@ -344,8 +337,7 @@ async def new_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Only premium users can add tags to projects. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Only premium users can add tags to projects. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -353,8 +345,7 @@ async def new_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Project management is an enterprise feature. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Project management is an enterprise feature. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -375,13 +366,11 @@ async def new_project(
             )
 
         # Validate team exists and get team object with budget
-        team_object = await _validate_team_exists(
-            team_id=data.team_id, prisma_client=prisma_client
-        )
+        team_object = await _validate_team_exists(team_id=data.team_id, prisma_client=prisma_client)
 
         # Validate project limits against team limits
         _check_team_project_limits(
-            team_object=LiteLLM_TeamTable(**team_object.model_dump()),
+            team_object=LiteLLM_TeamTable.model_validate(team_object.model_dump()),
             data=data,
         )
 
@@ -391,7 +380,7 @@ async def new_project(
             user_api_key_dict=user_api_key_dict,
             team_id=data.team_id,
             prisma_client=prisma_client,
-            team_object=LiteLLM_TeamTable(**team_object.model_dump()),
+            team_object=LiteLLM_TeamTable.model_validate(team_object.model_dump()),
         )
 
         if not has_permission:
@@ -449,17 +438,13 @@ async def new_project(
                     value=getattr(data, field),
                 )
 
-        new_project_row = prisma_client.jsonify_object(
-            project_row.json(exclude_none=True)
-        )
+        new_project_row = prisma_client.jsonify_object(project_row.json(exclude_none=True))
 
         # Remove budget fields (following organization_endpoints.py pattern)
         new_project_row = _remove_budget_fields_from_project_data(new_project_row)
 
-        verbose_proxy_logger.info(
-            f"new_project_row: {json.dumps(new_project_row, indent=2)}"
-        )
-        response = await prisma_client.db.litellm_projecttable.create(
+        verbose_proxy_logger.info(f"new_project_row: {json.dumps(new_project_row, indent=2)}")
+        response: prisma_models.LiteLLM_ProjectTable = await prisma_client.db.litellm_projecttable.create(
             data={
                 **new_project_row,  # type: ignore
             },
@@ -469,9 +454,7 @@ async def new_project(
         return response
     except Exception as e:
         verbose_proxy_logger.exception(
-            "litellm.proxy.management_endpoints.project_endpoints.new_project(): Exception occured - {}".format(
-                str(e)
-            )
+            "litellm.proxy.management_endpoints.project_endpoints.new_project(): Exception occured - {}".format(str(e))
         )
         raise handle_exception_on_proxy(e)
 
@@ -532,6 +515,7 @@ async def update_project(
         litellm_proxy_admin_name,
         premium_user,
         prisma_client,
+        user_api_key_cache,
     )
 
     try:
@@ -539,8 +523,7 @@ async def update_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Only premium users can add tags to projects. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Only premium users can add tags to projects. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -548,8 +531,7 @@ async def update_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Project management is an enterprise feature. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Project management is an enterprise feature. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -576,9 +558,9 @@ async def update_project(
             )
 
         # Fetch existing project
-        existing_project = await prisma_client.db.litellm_projecttable.find_unique(
-            where={"project_id": data.project_id}
-        )
+        existing_project: (
+            prisma_models.LiteLLM_ProjectTable | None
+        ) = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": data.project_id})
 
         if existing_project is None:
             raise ProxyException(
@@ -595,9 +577,7 @@ async def update_project(
         target_team_id = data.team_id or existing_project.team_id
         target_team_obj = None
         if target_team_id is not None:
-            target_team_obj = await _validate_team_exists(
-                team_id=target_team_id, prisma_client=prisma_client
-            )
+            target_team_obj = await _validate_team_exists(team_id=target_team_id, prisma_client=prisma_client)
 
         has_permission = await _check_user_permission_for_project(
             user_api_key_dict=user_api_key_dict,
@@ -620,32 +600,26 @@ async def update_project(
                 team_id=data.team_id,
                 prisma_client=prisma_client,
                 team_object=(
-                    LiteLLM_TeamTable(**target_team_obj.model_dump())
-                    if target_team_obj
-                    else None
+                    LiteLLM_TeamTable.model_validate(target_team_obj.model_dump()) if target_team_obj else None
                 ),
             )
             if not can_assign_to_target:
                 raise HTTPException(
                     status_code=403,
-                    detail={
-                        "error": "Cannot reassign project to a team you are not an admin of"
-                    },
+                    detail={"error": "Cannot reassign project to a team you are not an admin of"},
                 )
 
         # Validate project limits against team limits
         if target_team_obj is not None:
             _check_team_project_limits(
-                team_object=LiteLLM_TeamTable(**target_team_obj.model_dump()),
+                team_object=LiteLLM_TeamTable.model_validate(target_team_obj.model_dump()),
                 data=data,
             )
 
         # Prepare update data
         update_data = data.json(exclude_none=True, exclude={"project_id"})
         update_data = prisma_client.jsonify_object(update_data)
-        update_data["updated_by"] = (
-            user_api_key_dict.user_id or litellm_proxy_admin_name
-        )
+        update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
 
         # Handle budget updates
         budget_fields = LiteLLM_BudgetTable.model_fields.keys()
@@ -671,21 +645,17 @@ async def update_project(
                 if existing_project.object_permission_id:
                     # Update existing permission
                     await prisma_client.db.litellm_objectpermissiontable.update(
-                        where={
-                            "object_permission_id": existing_project.object_permission_id
-                        },
+                        where={"object_permission_id": existing_project.object_permission_id},
                         data=object_permission_data,
                     )
                 else:
                     # Create new permission
-                    created_permission = (
+                    created_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
                         await prisma_client.db.litellm_objectpermissiontable.create(
                             data=object_permission_data,
                         )
                     )
-                    update_data["object_permission_id"] = (
-                        created_permission.object_permission_id
-                    )
+                    update_data["object_permission_id"] = created_permission.object_permission_id
 
         # Handle metadata fields
         for field in LiteLLM_ManagementEndpoint_MetadataFields:
@@ -698,12 +668,17 @@ async def update_project(
         update_data = _remove_budget_fields_from_project_data(update_data)
 
         # Update project
-        updated_project = await prisma_client.db.litellm_projecttable.update(
+        updated_project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.update(
             where={"project_id": data.project_id},
             data=update_data,
             include={"litellm_budget_table": True, "object_permission": True},
         )
 
+        await delete_cached_project_object(
+            project_id=data.project_id,
+            user_api_key_cache=user_api_key_cache,
+        )
+
         return updated_project
     except Exception as e:
         verbose_proxy_logger.exception(
@@ -718,7 +693,7 @@ async def update_project(
     "/project/delete",
     tags=["project management"],
     dependencies=[Depends(user_api_key_auth)],
-    response_model=List[LiteLLM_ProjectTable],
+    response_model=list[LiteLLM_ProjectTable],
 )
 @management_endpoint_wrapper
 async def delete_project(
@@ -742,15 +717,14 @@ async def delete_project(
     }'
     ```
     """
-    from litellm.proxy.proxy_server import premium_user, prisma_client
+    from litellm.proxy.proxy_server import premium_user, prisma_client, user_api_key_cache
 
     try:
         if not premium_user:
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Project management is an enterprise feature. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Project management is an enterprise feature. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -778,9 +752,7 @@ async def delete_project(
 
         for project_id in data.project_ids:
             # Check if project exists
-            existing_project = await prisma_client.db.litellm_projecttable.find_unique(
-                where={"project_id": project_id}
-            )
+            existing_project = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": project_id})
 
             if existing_project is None:
                 raise ProxyException(
@@ -791,11 +763,9 @@ async def delete_project(
                 )
 
             # Check if there are any keys associated with this project
-            associated_keys = (
-                await prisma_client.db.litellm_verificationtoken.find_many(
-                    where={"project_id": project_id}
-                )
-            )
+            associated_keys: Sequence[
+                prisma_models.LiteLLM_VerificationToken
+            ] = await prisma_client.db.litellm_verificationtoken.find_many(where={"project_id": project_id})
 
             if len(associated_keys) > 0:
                 raise ProxyException(
@@ -806,8 +776,13 @@ async def delete_project(
                 )
 
             # Delete the project
-            deleted_project = await prisma_client.db.litellm_projecttable.delete(
-                where={"project_id": project_id}
+            deleted_project: (
+                prisma_models.LiteLLM_ProjectTable | None
+            ) = await prisma_client.db.litellm_projecttable.delete(where={"project_id": project_id})
+
+            await delete_cached_project_object(
+                project_id=project_id,
+                user_api_key_cache=user_api_key_cache,
             )
 
             deleted_projects.append(deleted_project)
@@ -854,7 +829,7 @@ async def project_info(
             )
 
         # Fetch project
-        project = await prisma_client.db.litellm_projecttable.find_unique(
+        project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.find_unique(
             where={"project_id": project_id},
             include={"litellm_budget_table": True, "object_permission": True},
         )
@@ -868,21 +843,15 @@ async def project_info(
             )
 
         # Check if user has access to this project (admin or team member)
-        is_admin = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
+        is_admin = user_api_key_has_admin_view(user_api_key_dict)
         is_team_member = False
 
         if project.team_id and user_api_key_dict.user_id:
-            team = await prisma_client.db.litellm_teamtable.find_unique(
-                where={"team_id": project.team_id}
-            )
+            team = await _team_table(prisma_client).find_unique(where={"team_id": project.team_id})
             if team:
                 caller_user_id = user_api_key_dict.user_id
                 for m in team.members_with_roles or []:
-                    m_user_id = (
-                        m.get("user_id")
-                        if isinstance(m, dict)
-                        else getattr(m, "user_id", None)
-                    )
+                    m_user_id = m.get("user_id") if isinstance(m, dict) else getattr(m, "user_id", None)
                     if m_user_id == caller_user_id:
                         is_team_member = True
                         break
@@ -896,9 +865,7 @@ async def project_info(
         return project
     except Exception as e:
         verbose_proxy_logger.exception(
-            "litellm.proxy.management_endpoints.project_endpoints.project_info(): Exception occured - {}".format(
-                str(e)
-            )
+            "litellm.proxy.management_endpoints.project_endpoints.project_info(): Exception occured - {}".format(str(e))
         )
         raise handle_exception_on_proxy(e)
 
@@ -907,7 +874,7 @@ async def project_info(
     "/project/list",
     tags=["project management"],
     dependencies=[Depends(user_api_key_auth)],
-    response_model=List[LiteLLM_ProjectTable],
+    response_model=list[LiteLLM_ProjectTable],
 )
 async def list_projects(
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@@ -931,22 +898,20 @@ async def list_projects(
             )
 
         # If proxy admin, get all projects
-        if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
-            projects = await prisma_client.db.litellm_projecttable.find_many(
+        if user_api_key_has_admin_view(user_api_key_dict):
+            projects: Sequence[
+                prisma_models.LiteLLM_ProjectTable
+            ] = await prisma_client.db.litellm_projecttable.find_many(
                 include={"litellm_budget_table": True, "object_permission": True}
             )
         else:
             # Look up the user's team memberships via the reverse-index on
             # LiteLLM_UserTable.teams (maintained by team_member_add alongside
             # members_with_roles). This avoids a full scan of all team rows.
-            user_record = await prisma_client.db.litellm_usertable.find_unique(
+            user_record: prisma_models.LiteLLM_UserTable | None = await prisma_client.db.litellm_usertable.find_unique(
                 where={"user_id": user_api_key_dict.user_id},
             )
-            user_team_ids = (
-                user_record.teams
-                if user_record is not None and user_record.teams
-                else []
-            )
+            user_team_ids: Sequence[str] = user_record.teams if user_record is not None and user_record.teams else []
 
             projects = await prisma_client.db.litellm_projecttable.find_many(
                 where={"team_id": {"in": user_team_ids}},

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.51"
+version = "0.1.55"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.51"
+version = "0.1.55"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -46,6 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra bedrock-realtime \
         --python python3
 
 # Stage 2 — copy source and install the project + workspace members.
@@ -57,11 +58,14 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra bedrock-realtime \
         --python python3
 
-RUN mkdir -p /home/nonroot && \
-    HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
-    chown -R nonroot:nonroot /home/nonroot/.cache
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
+
+RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh
 
 # ---------- Runtime ----------
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
@@ -81,17 +85,20 @@ ENV HOME=/home/nonroot \
     PATH="/app/.venv/bin:${PATH}" \
     PYTHONPATH="/app" \
     PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1
+    PYTHONUNBUFFERED=1 \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
 COPY --from=builder --chown=nonroot:nonroot /app /app
-COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 USER nonroot
 
 EXPOSE 4000/tcp
 
-ENTRYPOINT ["sh", "-c", "exec uvicorn gateway.main:app --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
+ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh uvicorn gateway.main:app --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
 CMD ["--host", "0.0.0.0", "--port", "4000"]

gateway/routes/allowlist.py

@@ -54,6 +54,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/messages",
     "/v1/skills",
     "/v1/a2a/",
+    "/a2a/",
     # LiteLLM-native LLM surface
     "/v1/rerank",
     "/v2/rerank",

helm/litellm-helm/Chart.lock

@@ -5,5 +5,5 @@ dependencies:
 - name: redis
   repository: oci://registry-1.docker.io/bitnamicharts
   version: 18.19.1
-digest: sha256:8660fe6287f9941d08c0902f3f13731079b8cecd2a5da2fbc54e5b7aae4a6f62
-generated: "2024-03-10T02:28:52.275022+05:30"
+digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
+generated: "2026-07-28T10:21:22.511401-07:00"

helm/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.1.0
+version: 1.1.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -32,10 +32,10 @@ annotations:
 
 dependencies:
   - name: "postgresql"
-    version: ">=13.3.0"
+    version: "14.3.1"
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: db.deployStandalone
   - name: redis
-    version: ">=18.0.0"
+    version: "18.19.1"
     repository: oci://registry-1.docker.io/bitnamicharts
     condition: redis.enabled

helm/litellm-helm/README.md

@@ -39,7 +39,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `livenessProbe.*`           | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                  | See `values.yaml`         |
 | `readinessProbe.*`          | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                 | See `values.yaml`         |
 | `startupProbe.*`            | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                   | See `values.yaml`         |
-| `resources.*`               | CPU/memory requests and limits for the LiteLLM container.                                                                                                                                                                                       | `{}`                      |
+| `resources.*`               | CPU/memory requests and limits for the LiteLLM container. Unset by default; production deployments should set 1 CPU and 4Gi of memory per worker.                                                                                             | `{}`                      |
 | `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                                                                  | `""`                      |
 | `ingress.labels`            | Additional labels for the Ingress resource                                                                                                                                                                                                    | `{}`                      |
 | `ingress.*`                 | See [values.yaml](./values.yaml) for example settings                                                                                                                                                                                         | N/A                       |
@@ -130,6 +130,16 @@ Set `billingMetrics.caSecretName` only when the collector is a private or test o
 | `db.deployStandalone`     | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups.                                                                                                  | `true`                                                                                     |
 | `postgresql.*`            | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml)                                                           |
 | `postgresql.auth.*`       | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used.                                                                                                                                                | `NoTaGrEaTpAsSwOrD`                                                                        |
+| `postgresql.image.*`      | If `db.deployStandalone` is `true`, the image for the bundled Postgres. Pinned to a `docker.io/bitnamilegacy` build because Bitnami retired the versioned tags under `docker.io/bitnami`.                                                                                                 | `bitnamilegacy/postgresql:16.2.0-debian-12-r6`                                             |
+| `redis.image.*`           | If `redis.enabled` is `true`, the image for the bundled Redis. Pinned to a `docker.io/bitnamilegacy` build for the same reason.                                                                                                                                                            | `bitnamilegacy/redis:7.2.4-debian-12-r9`                                                   |
+
+#### Bundled Postgres image
+
+Bitnami removed the versioned tags from `docker.io/bitnami` and republished the archived builds under `docker.io/bitnamilegacy`, so the image defaults that ship inside the `postgresql` and `redis` subcharts no longer pull. The chart pins both to the `bitnamilegacy` copies of the exact builds those subchart versions were released with, which keeps the on-disk data directory layout unchanged for existing installs.
+
+Keep `postgresql.image.tag` pinned. `docker.io/bitnami/postgresql` still publishes a floating `latest`, and pointing the bundled Postgres at a different major version starts the server against a data directory it cannot read (`database files are incompatible with server`). There is no in-place way back, so crossing a major version means dumping the database with the old image and restoring it into the new one. The chart refuses to render when the tag is empty or `latest`.
+
+Those images no longer receive updates. For anything beyond getting started, run Postgres outside the chart and point at it with `db.useExisting`.
 
 #### Example Postgres `db.useExisting` Secret
 

helm/litellm-helm/templates/_helpers.tpl

@@ -146,3 +146,18 @@ Get redis service port
 {{ .Values.redis.master.service.ports.redis }}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Reject an unpinned image tag for the bundled PostgreSQL.
+A floating tag lets a chart upgrade start a newer PostgreSQL major against the
+existing PersistentVolumeClaim. The server then refuses to start on a data
+directory written by another major version, and the only way back is a dump
+taken before the change, which by that point no longer exists.
+*/}}
+{{- define "litellm.validateBundledPostgresImageTag" -}}
+{{- $tag := .Values.postgresql.image.tag | default "" | toString -}}
+{{- $digest := .Values.postgresql.image.digest | default "" | toString -}}
+{{- if and (eq $digest "") (or (eq $tag "") (eq $tag "latest")) -}}
+{{- fail (printf "postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got %q). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore." $tag) -}}
+{{- end -}}
+{{- end -}}

helm/litellm-helm/templates/migrations-job.yaml

@@ -35,6 +35,8 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
       {{- with .Values.migrationJob.extraInitContainers }}
       initContainers:
         {{- tpl (toYaml .) $ | nindent 8 }}
@@ -103,6 +105,10 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       restartPolicy: OnFailure
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

helm/litellm-helm/templates/secret-dbcredentials.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.db.deployStandalone -}}
+{{- include "litellm.validateBundledPostgresImageTag" . -}}
 apiVersion: v1
 kind: Secret
 metadata:

helm/litellm-helm/templates/tests/test-servicemonitor.yaml

@@ -10,7 +10,7 @@ metadata:
 spec:
   containers:
     - name: test
-      image: bitnami/kubectl:latest
+      image: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3
       command: ['sh', '-c']
       args:
         - |

helm/litellm-helm/tests/bundled_db_images_tests.yaml

@@ -0,0 +1,94 @@
+suite: test bundled database images
+templates:
+  - charts/postgresql/templates/primary/statefulset.yaml
+  - charts/redis/templates/master/application.yaml
+  - charts/redis/templates/configmap.yaml
+  - charts/redis/templates/health-configmap.yaml
+  - charts/redis/templates/scripts-configmap.yaml
+  - charts/redis/templates/secret.yaml
+  - secret-dbcredentials.yaml
+  - templates/tests/test-servicemonitor.yaml
+tests:
+  - it: should pull the bundled postgres from a repository that still publishes the pinned tag
+    template: charts/postgresql/templates/primary/statefulset.yaml
+    set:
+      db.deployStandalone: true
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: docker.io/bitnamilegacy/postgresql:16.2.0-debian-12-r6
+
+  - it: should pull the bundled postgres metrics exporter from the same repository
+    template: charts/postgresql/templates/primary/statefulset.yaml
+    set:
+      db.deployStandalone: true
+      postgresql.metrics.enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[1].image
+          value: docker.io/bitnamilegacy/postgres-exporter:0.15.0-debian-12-r14
+
+  - it: should run the bundled postgres init container from the same repository
+    template: charts/postgresql/templates/primary/statefulset.yaml
+    set:
+      db.deployStandalone: true
+      postgresql.volumePermissions.enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.initContainers[0].image
+          value: docker.io/bitnamilegacy/os-shell:12-debian-12-r16
+
+  - it: should pull the bundled redis from a repository that still publishes the pinned tag
+    template: charts/redis/templates/master/application.yaml
+    set:
+      redis.enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: docker.io/bitnamilegacy/redis:7.2.4-debian-12-r9
+
+  - it: should reject a floating postgres tag that could cross a major version on an existing volume
+    template: secret-dbcredentials.yaml
+    set:
+      db.deployStandalone: true
+      postgresql.image.tag: latest
+    asserts:
+      - failedTemplate:
+          errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got "latest"). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.'
+
+  - it: should reject an empty postgres tag
+    template: secret-dbcredentials.yaml
+    set:
+      db.deployStandalone: true
+      postgresql.image.tag: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got ""). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.'
+
+  - it: should accept an empty postgres tag when the image is pinned by digest
+    template: secret-dbcredentials.yaml
+    set:
+      db.deployStandalone: true
+      postgresql.image.tag: ""
+      postgresql.image.digest: sha256:0d0e2f1a5b3c4d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6
+    asserts:
+      - hasDocuments:
+          count: 1
+
+  - it: should run the servicemonitor test pod from a pinned image
+    template: templates/tests/test-servicemonitor.yaml
+    set:
+      serviceMonitor.enabled: true
+    asserts:
+      - equal:
+          path: spec.containers[0].image
+          value: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3
+
+  - it: should not constrain the postgres tag when the bundled database is not deployed
+    template: secret-dbcredentials.yaml
+    set:
+      db.deployStandalone: false
+      postgresql.image.tag: latest
+    asserts:
+      - hasDocuments:
+          count: 0

helm/litellm-helm/tests/migrations-job_tests.yaml

@@ -254,3 +254,63 @@ tests:
           content:
             name: sidecar-tpl
             image: "ghcr.io/berriai/litellm-database:test"
+  - it: should render the pod-level securityContext from podSecurityContext
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+      podSecurityContext:
+        fsGroup: 10000
+        runAsUser: 10000
+        runAsNonRoot: true
+    asserts:
+      - equal:
+          path: spec.template.spec.securityContext
+          value:
+            fsGroup: 10000
+            runAsUser: 10000
+            runAsNonRoot: true
+  - it: should keep the pod-level and container-level securityContext separate
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+      podSecurityContext:
+        fsGroup: 10000
+      securityContext:
+        allowPrivilegeEscalation: false
+        readOnlyRootFilesystem: true
+    asserts:
+      - equal:
+          path: spec.template.spec.securityContext
+          value:
+            fsGroup: 10000
+      - equal:
+          path: spec.template.spec.containers[0].securityContext
+          value:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+  - it: should schedule onto the same nodes as the gateway
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+      nodeSelector:
+        karpenter.sh/nodepool: litellm-e2e
+      tolerations:
+        - key: workload
+          operator: Equal
+          value: litellm-e2e
+          effect: NoSchedule
+    asserts:
+      - equal:
+          path: spec.template.spec.nodeSelector
+          value:
+            karpenter.sh/nodepool: litellm-e2e
+      - equal:
+          path: spec.template.spec.tolerations
+          value:
+            - key: workload
+              operator: Equal
+              value: litellm-e2e
+              effect: NoSchedule

helm/litellm-helm/values.yaml

@@ -181,16 +181,19 @@ proxy_config:
 
 resources:
   {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
+  # Unset by default so the chart installs on small clusters such as Minikube, and so an
+  # upgrade never leaves a running pod Pending. Production deployments should set these.
+  # A proxy at DB-connected steady state needs about 1 CPU and 4Gi of memory per worker;
+  # sizing below that gets the pod OOMKilled once traffic and DB connections ramp up.
+  # Scale both figures with --num_workers, then uncomment the lines below and remove the
+  # curly braces after 'resources:'. See "Recommended Machine Specifications" in
+  # https://docs.litellm.ai/docs/proxy/prod.
   # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
+  #   cpu: "1"
+  #   memory: 4Gi
+  # limits:
+  #   cpu: "1"
+  #   memory: 4Gi
 
 autoscaling:
   enabled: false
@@ -328,8 +331,32 @@ lifecycle: {}
 
 # Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored
 #  otherwise)
+#
+# Bitnami retired the versioned tags under docker.io/bitnami and republished the
+# archived builds under docker.io/bitnamilegacy, so the subchart's own image
+# defaults no longer resolve. The repository below points at the same build the
+# subchart was released with, which keeps the on-disk data directory layout
+# identical for existing installs.
+#
+# Keep the tag pinned. docker.io/bitnami still publishes a floating `latest`,
+# and starting a newer PostgreSQL major against an existing data directory
+# leaves the server refusing to boot ("database files are incompatible with
+# server") with no way back other than a dump taken beforehand. Crossing a major
+# version is a dump-and-restore, not an image bump. The chart refuses to render
+# an unpinned tag for this reason
 postgresql:
   architecture: standalone
+  image:
+    repository: bitnamilegacy/postgresql
+    tag: 16.2.0-debian-12-r6
+  volumePermissions:
+    image:
+      repository: bitnamilegacy/os-shell
+      tag: 12-debian-12-r16
+  metrics:
+    image:
+      repository: bitnamilegacy/postgres-exporter
+      tag: 0.15.0-debian-12-r14
   auth:
     username: litellm
     database: litellm
@@ -359,9 +386,36 @@ postgresql:
 # When `redis.sentinel.enabled` is set, the coordination block is rendered with
 # `sentinel_nodes` and `service_name` (from `redis.sentinel.masterSet`) instead
 # of host/port, because a plain Redis client cannot talk to the sentinel port
+#
+# The image repositories carry the same bitnamilegacy repoint as postgresql
+# above; the versioned tags the subchart ships with are gone from
+# docker.io/bitnami
 redis:
   enabled: false
   architecture: standalone
+  image:
+    repository: bitnamilegacy/redis
+    tag: 7.2.4-debian-12-r9
+  sentinel:
+    image:
+      repository: bitnamilegacy/redis-sentinel
+      tag: 7.2.4-debian-12-r7
+  metrics:
+    image:
+      repository: bitnamilegacy/redis-exporter
+      tag: 1.58.0-debian-12-r4
+  volumePermissions:
+    image:
+      repository: bitnamilegacy/os-shell
+      tag: 12-debian-12-r16
+  sysctl:
+    image:
+      repository: bitnamilegacy/os-shell
+      tag: 12-debian-12-r16
+  kubectl:
+    image:
+      repository: bitnamilegacy/kubectl
+      tag: 1.29.2-debian-12-r3
   coordination:
     # Set to false to keep the bundled Redis for response caching only and leave
     # `general_settings.coordination_redis` out of the rendered config. A
@@ -381,9 +435,9 @@ migrationJob:
   annotations: {}
   ttlSecondsAfterFinished: 120
   resources: {}
-  #  requests:
-  #    cpu: 100m
-  #    memory: 100Mi
+  #  Unset by default. This job runs the database migration and exits, so it does not
+  #  need the steady-state headroom the proxy does; size it from your own migration
+  #  runs rather than from the proxy figures above.
   extraContainers: []
   extraInitContainers: []
 

helm/litellm/templates/_helpers.tpl

@@ -138,6 +138,59 @@ is false the chart uses the provided name, or the namespace `default` SA.
 {{- end -}}
 {{- end -}}
 
+{{/*
+ServiceAccount name for the migrations Job.
+
+The Job is a pre-install / pre-upgrade hook, so it is created before the
+chart's ordinary resources. A ServiceAccount the chart creates is one of
+those ordinary resources, which makes borrowing the backend name a cycle:
+the hook pod is rejected because the account does not exist yet. So when
+`serviceAccounts.backend.create` is true the Job falls back to the namespace
+`default` account unless the operator names one that already exists. With
+`create` false the backend name is either an operator-supplied existing
+account or `default`, both of which are safe for the hook, so the Job keeps
+sharing it.
+
+`migrationJob.serviceAccountName` always wins when set, which is how a Job
+that needs credentials of its own (IRSA / Workload Identity for IAM database
+auth) gets them.
+*/}}
+{{- define "litellm.migrations.serviceAccountName" -}}
+{{- if .Values.migrationJob.serviceAccountName -}}
+{{ .Values.migrationJob.serviceAccountName }}
+{{- else if .Values.serviceAccounts.backend.create -}}
+default
+{{- else -}}
+{{ include "litellm.backend.serviceAccountName" . }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Extra pod labels for a component's Deployment, validated against its selector.
+
+Invoke with a dict:
+  (dict "podLabels" .Values.gateway.podLabels "componentName" "gateway")
+
+The three selector keys are also emitted on the pod template, so a podLabels
+entry reusing one renders a duplicate YAML key whose later value wins. That
+leaves the pod template no longer matching the (immutable) selector and the
+apiserver rejects the Deployment. Fail at template time naming the key
+instead, so the operator gets the reason here rather than an opaque
+`selector does not match template labels` from the apiserver.
+
+The migrations Job takes podLabels unvalidated: a Job's selector is generated
+by the controller rather than declared, so nothing there can collide.
+*/}}
+{{- define "litellm.podLabels" -}}
+{{- $componentName := .componentName -}}
+{{- range $key, $value := .podLabels }}
+{{- if has $key (list "app.kubernetes.io/name" "app.kubernetes.io/instance" "app.kubernetes.io/component") }}
+{{- fail (printf "%s.podLabels cannot set %s: it is part of the Deployment's immutable selector" $componentName $key) }}
+{{- end }}
+{{- end }}
+{{- toYaml .podLabels }}
+{{- end -}}
+
 {{/*
 Master-key + database + redis env block — shared by gateway, backend, and the
 migrations Job.

helm/litellm/templates/backend/deployment.yaml

@@ -23,9 +23,16 @@ spec:
       {{- end }}
       labels:
         {{- include "litellm.backend.selectorLabels" . | nindent 8 }}
+        {{- with .Values.backend.podLabels }}
+        {{- include "litellm.podLabels" (dict "podLabels" . "componentName" "backend") | nindent 8 }}
+        {{- end }}
     spec:
       serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }}
       automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }}
+      {{- with .Values.backend.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
@@ -34,6 +41,10 @@ spec:
         - name: backend
           image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.backend.image.pullPolicy }}
+          {{- with .Values.backend.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           ports:
             - name: http
               containerPort: 4001
@@ -70,8 +81,15 @@ spec:
           readinessProbe:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.backend.lifecycle }}
+          lifecycle:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           resources:
             {{- toYaml .Values.backend.resources | nindent 12 }}
+      {{- with .Values.backend.extraContainers }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
       {{- if or .Values.gateway.config.create .Values.backend.volumes .Values.billingMetrics.enabled }}
       volumes:
         {{- if .Values.gateway.config.create }}
@@ -102,4 +120,8 @@ spec:
       topologySpreadConstraints:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- $gracePeriod := .Values.backend.terminationGracePeriodSeconds }}
+      {{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }}
+      terminationGracePeriodSeconds: {{ $gracePeriod }}
+      {{- end }}
 {{- end }}

helm/litellm/templates/gateway/deployment.yaml

@@ -21,9 +21,16 @@ spec:
         {{- end }}
       labels:
         {{- include "litellm.gateway.selectorLabels" . | nindent 8 }}
+        {{- with .Values.gateway.podLabels }}
+        {{- include "litellm.podLabels" (dict "podLabels" . "componentName" "gateway") | nindent 8 }}
+        {{- end }}
     spec:
       serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }}
       automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }}
+      {{- with .Values.gateway.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
@@ -32,6 +39,10 @@ spec:
         - name: gateway
           image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
+          {{- with .Values.gateway.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           ports:
             - name: http
               containerPort: 4000
@@ -72,8 +83,15 @@ spec:
           readinessProbe:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.gateway.lifecycle }}
+          lifecycle:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           resources:
             {{- toYaml .Values.gateway.resources | nindent 12 }}
+      {{- with .Values.gateway.extraContainers }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
       {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled }}
       volumes:
         {{- if .Values.gateway.config.create }}
@@ -104,4 +122,8 @@ spec:
       topologySpreadConstraints:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }}
+      {{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }}
+      terminationGracePeriodSeconds: {{ $gracePeriod }}
+      {{- end }}
 {{- end }}

helm/litellm/templates/ingress.yaml

@@ -19,7 +19,7 @@
     "/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads"
     "/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes"
     "/v1/models" "/models" "/openai" "/engines"
-    "/v1/messages" "/messages" "/v1/skills" "/v1/a2a"
+    "/v1/messages" "/messages" "/v1/skills" "/v1/a2a" "/a2a"
     "/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag"
     "/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
     "/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"

helm/litellm/templates/migrations-job.yaml

@@ -23,12 +23,21 @@ spec:
   ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
   template:
     metadata:
+      {{- /* The Job's selector is generated by the controller rather than
+             declared, so podLabels may override a chart label here. Merge
+             instead of appending so an override replaces the key rather than
+             rendering it twice. */}}
+      {{- $chartLabels := merge (dict "app.kubernetes.io/component" "migrations") (fromYaml (include "litellm.commonLabels" .)) }}
       labels:
-        {{- include "litellm.commonLabels" . | nindent 8 }}
-        app.kubernetes.io/component: migrations
+        {{- toYaml (merge (deepCopy .Values.migrationJob.podLabels) $chartLabels) | nindent 8 }}
     spec:
       restartPolicy: Never
-      serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }}
+      serviceAccountName: {{ include "litellm.migrations.serviceAccountName" . }}
+      automountServiceAccountToken: {{ .Values.migrationJob.automountServiceAccountToken }}
+      {{- with .Values.migrationJob.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
@@ -37,10 +46,22 @@ spec:
         - name: prisma-migrations
           image: "{{ .Values.migrationJob.image.repository }}:{{ .Values.migrationJob.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.migrationJob.image.pullPolicy }}
+          {{- with .Values.migrationJob.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           env:
             {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.migrationJob) | nindent 12 }}
+          {{- with .Values.migrationJob.volumeMounts }}
+          volumeMounts:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           {{- with .Values.migrationJob.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+      {{- with .Values.migrationJob.volumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

helm/litellm/templates/ui/deployment.yaml

@@ -18,9 +18,16 @@ spec:
       {{- end }}
       labels:
         {{- include "litellm.ui.selectorLabels" . | nindent 8 }}
+        {{- with .Values.ui.podLabels }}
+        {{- include "litellm.podLabels" (dict "podLabels" . "componentName" "ui") | nindent 8 }}
+        {{- end }}
     spec:
       serviceAccountName: {{ include "litellm.ui.serviceAccountName" . }}
       automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }}
+      {{- with .Values.ui.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
@@ -29,6 +36,10 @@ spec:
         - name: ui
           image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.ui.image.pullPolicy }}
+          {{- with .Values.ui.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           ports:
             - name: http
               containerPort: 3000
@@ -58,8 +69,15 @@ spec:
           readinessProbe:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.ui.lifecycle }}
+          lifecycle:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           resources:
             {{- toYaml .Values.ui.resources | nindent 12 }}
+      {{- with .Values.ui.extraContainers }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
       {{- with .Values.ui.volumes }}
       volumes:
         {{- toYaml . | nindent 8 }}
@@ -80,4 +98,8 @@ spec:
       topologySpreadConstraints:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- $gracePeriod := .Values.ui.terminationGracePeriodSeconds }}
+      {{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }}
+      terminationGracePeriodSeconds: {{ $gracePeriod }}
+      {{- end }}
 {{- end }}

helm/litellm/tests/migration_job_tests.yaml

@@ -0,0 +1,169 @@
+suite: test migrations Job ServiceAccount resolution and pod hardening
+templates:
+  - migrations-job.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: borrows the namespace default account when no ServiceAccount is configured
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: default
+
+  - it: falls back to the namespace default account when the chart creates the backend ServiceAccount
+    set:
+      serviceAccounts.backend.create: true
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: default
+      - notEqual:
+          path: spec.template.spec.serviceAccountName
+          value: RELEASE-NAME-litellm-backend
+
+  - it: keeps sharing an existing backend ServiceAccount the chart does not create
+    set:
+      serviceAccounts.backend.create: false
+      serviceAccounts.backend.name: existing-backend-sa
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: existing-backend-sa
+
+  - it: prefers an explicit migration ServiceAccount over the created backend one
+    set:
+      serviceAccounts.backend.create: true
+      migrationJob.serviceAccountName: migrations-sa
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: migrations-sa
+
+  - it: prefers an explicit migration ServiceAccount over an existing backend one
+    set:
+      serviceAccounts.backend.create: false
+      serviceAccounts.backend.name: existing-backend-sa
+      migrationJob.serviceAccountName: migrations-sa
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: migrations-sa
+
+  - it: mounts no ServiceAccount token by default
+    asserts:
+      - equal:
+          path: spec.template.spec.automountServiceAccountToken
+          value: false
+
+  - it: mounts a ServiceAccount token when the operator asks for one
+    set:
+      migrationJob.automountServiceAccountToken: true
+    asserts:
+      - equal:
+          path: spec.template.spec.automountServiceAccountToken
+          value: true
+
+  - it: keeps the token off the Job when the backend disables automounting
+    set:
+      serviceAccounts.backend.create: true
+      serviceAccounts.backend.automount: false
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: default
+      - equal:
+          path: spec.template.spec.automountServiceAccountToken
+          value: false
+
+  - it: renders no hardening fields by default
+    asserts:
+      - isNull:
+          path: spec.template.spec.securityContext
+      - isNull:
+          path: spec.template.spec.containers[0].securityContext
+      - isNull:
+          path: spec.template.spec.volumes
+      - isNull:
+          path: spec.template.spec.containers[0].volumeMounts
+      - equal:
+          path: spec.template.metadata.labels
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+            app.kubernetes.io/managed-by: Helm
+            helm.sh/chart: litellm-0.1.0
+            app.kubernetes.io/component: migrations
+
+  - it: renders pod-level and container-level securityContext in their own scopes
+    set:
+      migrationJob.podSecurityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      migrationJob.securityContext:
+        allowPrivilegeEscalation: false
+        readOnlyRootFilesystem: true
+        capabilities:
+          drop:
+            - ALL
+    asserts:
+      - equal:
+          path: spec.template.spec.securityContext
+          value:
+            runAsNonRoot: true
+            runAsUser: 65532
+            seccompProfile:
+              type: RuntimeDefault
+      - equal:
+          path: spec.template.spec.containers[0].securityContext
+          value:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop:
+                - ALL
+
+  - it: renders volumes on the pod and volumeMounts on the migration container
+    set:
+      migrationJob.volumes:
+        - name: tmp
+          emptyDir:
+            sizeLimit: 64Mi
+      migrationJob.volumeMounts:
+        - name: tmp
+          mountPath: /tmp
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes
+          value:
+            - name: tmp
+              emptyDir:
+                sizeLimit: 64Mi
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts
+          value:
+            - name: tmp
+              mountPath: /tmp
+
+  - it: merges podLabels with the chart labels on the Job pod
+    set:
+      migrationJob.podLabels:
+        egress-policy: restricted
+    asserts:
+      - equal:
+          path: spec.template.metadata.labels['egress-policy']
+          value: restricted
+      - equal:
+          path: spec.template.metadata.labels['app.kubernetes.io/component']
+          value: migrations
+
+  - it: accepts a podLabel that reuses a chart label, since the Job selector is controller-generated
+    set:
+      migrationJob.podLabels:
+        app.kubernetes.io/component: batch-migrations
+    asserts:
+      - notFailedTemplate: {}
+      - equal:
+          path: spec.template.metadata.labels['app.kubernetes.io/component']
+          value: batch-migrations