Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_lit6552_fix_build_base_response_empty_choices

.circleci/config.yml

@@ -112,10 +112,10 @@ commands:
             node --version
             npm --version
   install_rust:
-    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.97.1) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
+    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
     steps:
       - run:
-          name: Install Rust (rustup 1.28.2, toolchain 1.97.1)
+          name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
           command: |
             case "$(uname -m)" in
               x86_64)
@@ -135,7 +135,7 @@ commands:
               "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init"
             echo "${RUSTUP_SHA256}  /tmp/rustup-init" | sha256sum -c -
             chmod +x /tmp/rustup-init
-            /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.97.1
+            /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
             rm -f /tmp/rustup-init
             echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
             export PATH="$HOME/.cargo/bin:$PATH"
@@ -300,7 +300,7 @@ jobs:
             if ($rustupActual -ne $rustupExpected) {
               throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual"
             }
-            & $rustupInit -y --profile minimal --default-toolchain stable
+            & $rustupInit -y --profile minimal --default-toolchain 1.98.0
             if ($LASTEXITCODE -ne 0) {
               exit $LASTEXITCODE
             }
@@ -430,7 +430,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise \
                 --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
@@ -504,7 +504,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise \
                 --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
@@ -575,7 +575,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
+                -v \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -k \"langfuse\""
@@ -630,8 +630,8 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
-                --cov=./litellm --cov-report=xml \
+                -v \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -651,126 +651,6 @@ jobs:
             - auth_ui_unit_tests_coverage.xml
             - auth_ui_unit_tests_coverage
 
-  proxy_behavior_tests:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Seed DB schema via prisma db push
-          command: |
-            uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Run proxy management behavior tests
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_behavior \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
-  proxy_security_tests:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Seed DB schema via prisma db push
-          command: |
-            uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Run proxy security tests
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_security_tests \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
-  schema_migration_check:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      # An empty database; the test applies every committed migration itself.
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Check schema.prisma is in sync with committed migrations
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_migration_tests \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
   litellm_router_testing: # Runs all tests with the "router" keyword
     docker:
       - *python312_image
@@ -857,8 +737,8 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
-                --cov=./litellm --cov-report=xml \
+                -v \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -902,7 +782,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
+                -v \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -k \"assistants\""
@@ -985,7 +865,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
                 -n 4 \
@@ -1029,8 +909,8 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                -vv -s \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -1074,7 +954,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2 \
@@ -1119,8 +999,8 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                -vv -s \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 --retries 3 --retry-delay 5"
@@ -1174,7 +1054,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
+                -v \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 8 \
@@ -1210,8 +1090,8 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x \
-                --cov=./litellm --cov-report=xml \
+                -vv \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1254,8 +1134,8 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x \
-                --cov=./litellm --cov-report=xml \
+                -vv \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1274,40 +1154,6 @@ jobs:
           paths:
             - search_coverage.xml
             - search_coverage
-  litellm_mapped_enterprise_tests:
-    docker:
-      - *python312_image
-    working_directory: ~/project
-    resource_class: large
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - setup_litellm_enterprise_pip
-      - run:
-          name: Run enterprise tests
-          command: |
-            uv run --no-sync python -m prisma generate
-            mkdir -p test-results
-            TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
-            echo "$TEST_FILES" | circleci tests run \
-              --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
-                --junitxml=test-results/junit-enterprise.xml \
-                --durations=10 \
-                -n 4"
-          no_output_timeout: 15m
-      # Store test results
-      - store_test_results:
-          path: test-results
   batches_testing:
     docker:
       - *python312_image
@@ -1332,8 +1178,8 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                -vv -s \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -1376,8 +1222,8 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                -vv -s \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -1421,8 +1267,8 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x \
-                --cov=./litellm --cov-report=xml \
+                -vv \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1466,7 +1312,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
+                -v \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1501,7 +1347,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 -n 4 \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
@@ -1545,8 +1391,8 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                -vv -s \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -1598,8 +1444,8 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                -vv -s \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 -n 2 \
                 --reruns 2 --reruns-delay 1"
@@ -1859,7 +1705,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
+                -v \
                 --junitxml=test-results/junit-2.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -1948,7 +1794,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -s -v -x \
+                -s -v \
                 --junitxml=test-results/junit.xml \
                 -n 4 \
                 --durations=5"
@@ -2166,7 +2012,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
+                -v \
                 --junitxml=test-results/junit-2.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -2246,7 +2092,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x \
+                -vv \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -2349,7 +2195,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x \
+                -vv \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -2420,7 +2266,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x \
+                -vv \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -2504,7 +2350,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x \
+                -vv \
                 --junitxml=test-results/junit-2.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -2575,45 +2421,6 @@ jobs:
       - wait_for_service:
           url: http://localhost:4000
           timeout: "300"
-      # Add Ruby installation and testing before the existing Node.js and Python tests
-      - run:
-          name: Install Ruby and Bundler
-          command: |
-            # Clone RVM at pinned tag and verify the commit SHA matches the
-            # published tag before running its install script.
-            RVM_VERSION="1.29.12"
-            RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81"
-            git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm
-            RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)"
-            if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then
-              echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2
-              exit 1
-            fi
-
-            # Import RVM signing keys (used by `rvm install` to verify Ruby tarballs)
-            gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-
-            # Install RVM from the verified checkout. The install script
-            # sources `scripts/functions/installer` using paths relative to
-            # its own working directory, so it must be run from /tmp/rvm.
-            (cd /tmp/rvm && ./install --path "$HOME/.rvm")
-            source "$HOME/.rvm/scripts/rvm"
-
-            # Install Ruby 3.2.2 (RVM verifies the tarball PGP signature)
-            rvm install 3.2.2
-            rvm use 3.2.2 --default
-
-            # Install latest Bundler
-            gem install bundler
-
-      - run:
-          name: Run Ruby tests
-          command: |
-            source $HOME/.rvm/scripts/rvm
-            cd tests/pass_through_tests/ruby_passthrough_tests
-            bundle install
-            bundle exec rspec
-          no_output_timeout: 30m
       # Install Node.js directly from nodejs.org with SHA256 verification,
       # instead of piping NodeSource's setup_24.x apt-repo installer into
       # sudo bash (which runs a mutable upstream script unattended).
@@ -2639,7 +2446,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
+                -v \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -2709,7 +2516,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -vv -x -s \
+                -vv -s \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -2841,6 +2648,19 @@ jobs:
           name: Start mock LLM server
           command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
           background: true
+      - run:
+          name: Start mock Presidio server
+          command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py
+          background: true
+      - run:
+          name: Wait for mock Presidio server
+          command: |
+            for i in $(seq 1 30); do
+              if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi
+              sleep 1
+            done
+            echo "Mock Presidio server never answered /health on port 8091" >&2
+            exit 1
       - run:
           name: Start LiteLLM proxy
           environment:
@@ -2971,6 +2791,19 @@ jobs:
           name: Start mock LLM server
           command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
           background: true
+      - run:
+          name: Start mock Presidio server
+          command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py
+          background: true
+      - run:
+          name: Wait for mock Presidio server
+          command: |
+            for i in $(seq 1 30); do
+              if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi
+              sleep 1
+            done
+            echo "Mock Presidio server never answered /health on port 8091" >&2
+            exit 1
       - run:
           name: Start LiteLLM proxy under a server root path
           environment:
@@ -3105,12 +2938,6 @@ workflows:
           filters: *main_branches
       - auth_ui_unit_tests:
           filters: *main_branches
-      - proxy_behavior_tests:
-          filters: *main_branches
-      - proxy_security_tests:
-          filters: *main_branches
-      - schema_migration_check:
-          filters: *main_branches
       - build_docker_database_image:
           filters: *main_branches
       - e2e_ui_testing:
@@ -3167,8 +2994,6 @@ workflows:
           filters: *main_branches
       - search_testing:
           filters: *main_branches
-      - litellm_mapped_enterprise_tests:
-          filters: *main_branches
       - batches_testing:
           filters: *main_branches
       - litellm_utils_testing:
@@ -3191,7 +3016,6 @@ workflows:
             - guardrails_testing
             - ocr_testing
             - search_testing
-            - litellm_mapped_enterprise_tests
             - batches_testing
             - litellm_utils_testing
             - pass_through_unit_testing

.github/CODEOWNERS

@@ -1,5 +1,10 @@
-/ui/ @yuneng-jiang @ryan-crabbe-berri
-/litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri
+/ui/ @yuneng-berri @ryan-crabbe-berri
+/litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
+/ui/Dockerfile
+/ui/nginx.conf
 /ui/litellm-dashboard/src/lib/http/schema.d.ts
+/ui/litellm-dashboard/tsconfig.tsbuildinfo
 /model_prices_and_context_window.json @mateo-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri
+/litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri
+/.github/CODEOWNERS @yuneng-berri

.github/actions/cache-cargo-build/action.yml

@@ -0,0 +1,30 @@
+name: "Cache the Rust build"
+description: >-
+  Cache the Cargo registry and target directory the root package's build needs,
+  so only the first job on a given Cargo.lock compiles the bridge from scratch.
+
+  litellm builds through maturin, which compiles litellm-rust/crates/python-bridge
+  in the dev profile for editable installs. `uv sync` therefore pays a full build
+  in every job that installs the workspace. Nothing caught it, because the uv cache
+  holds wheels uv downloads rather than wheels it builds, and a path dependency
+  whose source moves every commit could never hit that cache anyway. Cargo rebuilds
+  only what changed when its target directory survives, so a warm job pays for the
+  bridge crate alone.
+
+  The key namespace is separate from test-rust.yml's check and release caches. They
+  cache the same directory for different workloads, and a shared key would let
+  whichever ran first deny the others a save.
+
+runs:
+  using: composite
+  steps:
+    - name: Restore the Cargo registry and target directory
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          litellm-rust/target
+        key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-maturin-dev-

.github/actions/setup-uv-with-retries/action.yml

@@ -1,11 +1,7 @@
 name: "Set up uv with retries"
 description: >-
-  Install uv via astral-sh/setup-uv, retrying on transient failures. Even with
-  an exact pinned version, the action resolves the artifact URL by fetching
-  https://raw.githubusercontent.com/astral-sh/versions/main/v1/uv.ndjson in a
-  single request with no retry, timeout, or fallback, so one connection-level
-  network error ("fetch failed") fails the whole job before any test runs.
-  Retrying the full step covers the manifest fetch and the binary download.
+  Install uv via astral-sh/setup-uv, retrying the full setup step so manifest
+  resolution and binary downloads get fresh attempts after transient failures.
 
 inputs:
   version:
@@ -18,7 +14,7 @@ runs:
     - name: Set up uv (attempt 1)
       id: attempt-1
       continue-on-error: true
-      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
 
@@ -31,7 +27,7 @@ runs:
       id: attempt-2
       if: steps.attempt-1.outcome == 'failure'
       continue-on-error: true
-      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}
 
@@ -42,6 +38,6 @@ runs:
 
     - name: Set up uv (attempt 3)
       if: steps.attempt-2.outcome == 'failure'
-      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       with:
         version: ${{ inputs.version }}

.github/ci-coverage-allowlist.yml

@@ -5,24 +5,26 @@ description: >-
 
 test_paths:
   - reason: >-
-      The caching suite in tests/local_testing, which runs nowhere. Every job that globs that
-      directory either deselects it (local_testing_part1 and part2 carry `-k "... and not caching
-      and not cache"`) or keeps only another keyword (langfuse, router, assistants), and no job
-      names these files the way redis_caching_unit_tests names test_dual_cache.py. Measured
-      2026-08-20 by collecting the directory under each job's own selector: 118 tests across
-      these eight files are selected by none of them. Listed so the gap is a decision rather
-      than an accident, and so the --slices guard has a baseline to ratchet down from. Revisit
-      when tests/local_testing is ported off CircleCI, where the keyless part of this suite
-      belongs in a real job
+      The Rust/Python parity harness is run manually through its local CLI. Recorded replay,
+      fixture generation, and harness checks are intentionally outside pull request CI
+    paths:
+      - tests/rust-python-harness
+  - reason: >-
+      What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
+      globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
+      not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
+      and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
+      The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
+      run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
+      credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
+      test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
+      test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
+      split that porting tests/local_testing off CircleCI will force, not a job that is red by
+      construction
     paths:
-      - tests/local_testing/test_cache_preset_key.py
       - tests/local_testing/test_caching.py
-      - tests/local_testing/test_caching_handler.py
       - tests/local_testing/test_disk_cache_unit_tests.py
       - tests/local_testing/test_gcs_cache_unit_tests.py
-      - tests/local_testing/test_prompt_caching.py
-      - tests/local_testing/test_responses_stream_cache_keys.py
-      - tests/local_testing/test_unit_test_caching.py
   - reason: >-
       The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
       from a pull request; it needs a live gateway and provider credentials no PR job holds
@@ -48,16 +50,6 @@ test_paths:
       choice it informed is settled
     paths:
       - tests/code_coverage_tests/test_aio_http_image_conversion.py
-  - reason: >-
-      The last file of a second mirror that sat beside tests/test_litellm and ran nowhere. Its
-      other 33 files landed in the real mirror during August 2026, 30 as moves and 3 by merging
-      their bodies into the live file of the same name. This one cannot follow either route yet:
-      its live twin was rewritten from 1268 lines to 9434, and of the 19 tests here 5 have no
-      counterpart while 25 assertions fail against today's code, so what survives that rewrite
-      is a judgement about the endpoints, not a merge. Revisit by deciding which of the five
-      behaviours still hold
-    paths:
-      - tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
   - reason: >-
       No job invokes this suite and its files mix pure transformation tests with ones driving live
       vendor vector stores, so assigning them needs a per-file decision
@@ -87,6 +79,11 @@ test_paths:
       - tests/load_tests/test_otel_load_test.py
       - tests/load_tests/test_vertex_embeddings_load_test.py
       - tests/load_tests/test_vertex_load_tests.py
+  - reason: >-
+      Env-gated saturation benchmark requires a live proxy and provider credentials, so it is run
+      locally rather than in pull-request jobs
+    paths:
+      - tests/load_tests/test_granian_admission_saturation.py
   - reason: >-
       A local-only agent rig: test_a2a_completion_bridge.py needs a LangGraph server on
       localhost:2024 and test_a2a.py drives a live A2A endpoint, so neither can run in a
@@ -102,17 +99,6 @@ test_paths:
       - tests/integration/sandbox/test_e2b_sandbox.py
       - tests/integration/test_oci_integration.py
       - tests/integration/test_oci_proxy_integration.py
-  - reason: >-
-      A unit test for the proxy-extras package that no job invokes, while the package's other
-      tests live under tests/proxy_migration_tests. Measured 2026-08-20: 24 of its 28 tests pass
-      and the 4 in TestMigrationSQLIdempotency fail, because 13 migrations from 2026-03 onward use
-      bare CREATE TABLE, ADD COLUMN, CREATE INDEX and ADD CONSTRAINT rather than the guarded forms
-      this file requires. It also matches those keywords inside SQL comments, so two further
-      migrations are reported that are in fact fine. Wiring it up means deciding what to do about
-      the 13 first, and they cannot simply be edited: Prisma checksums an applied migration, so a
-      changed one breaks migrate deploy for existing installs
-    paths:
-      - tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
 
 dockerfiles:
   - reason: >-

.github/e2e-stack/assert_tests_ran.py

@@ -0,0 +1,38 @@
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Final
+
+
+def main() -> int:
+    selected: Final = tuple(sys.argv[2:])
+    try:
+        report: Final = ET.parse(Path(sys.argv[1])).getroot()
+    except (ET.ParseError, OSError):
+        _ = sys.stdout.write("::error::could not read the test execution report\n")
+        return 1
+    cases: Final = tuple(report.iter("testcase"))
+    passed: Final = frozenset(
+        case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
+    )
+    missing: Final = tuple(path for path in selected if path not in passed)
+    for path in selected:
+        collected: Final = sum(case.get("file") == path for case in cases)
+        skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
+        _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
+        for case in cases:
+            if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error")):
+                continue
+            _ = sys.stdout.write(f"  failed: {case.get('classname', '')}::{case.get('name', '')}\n")
+    if (
+        selected
+        and not missing
+        and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
+    ):
+        return 0
+    _ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/down.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+
+for pid_file in "${STACK_DIR}"/pids/*.pid; do
+  [[ -f "${pid_file}" ]] || continue
+  pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null
+  kill -TERM "$(cat "${pid_file}")" 2>/dev/null
+  rm -f "${pid_file}"
+done
+
+for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do
+  docker rm -f "${container}" >/dev/null 2>&1
+done
+
+exit 0

.github/e2e-stack/secrets_to_env.py

@@ -0,0 +1,49 @@
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
+ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+MIN_MASKED_LENGTH: Final = 8
+
+
+def main() -> int:
+    env_path: Final = Path(sys.argv[1])
+    try:
+        secrets: Final = {
+            key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()
+        }
+    except (ValidationError, UnicodeError):
+        _ = sys.stderr.write("expected a JSON object containing string environment values\n")
+        return 1
+    unusable: Final = tuple(
+        key
+        for key, value in secrets.items()
+        if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0")
+    )
+    if unusable:
+        _ = sys.stderr.write(
+            f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
+        )
+        return 1
+    for value in secrets.values():
+        if len(value) >= MIN_MASKED_LENGTH:
+            _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
+    sys.stdout.flush()
+    lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
+    try:
+        with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:
+            os.fchmod(handle.fileno(), 0o600)
+            _ = handle.write("\n".join(lines) + "\n")
+    except OSError:
+        _ = sys.stderr.write("could not write the environment file\n")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/select_tests.py

@@ -0,0 +1,44 @@
+import re
+import sys
+from typing import Final
+
+SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
+UNSUPPORTED: Final = re.compile(
+    r"^tests/e2e/(ui|claude_code|load)/"
+    r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
+    r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
+    r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
+)
+HARNESS: Final = re.compile(
+    r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
+    r"|^tests/e2e/gateway/"
+    r"|^\.github/e2e-stack/"
+    r"|^\.github/workflows/test-e2e-changed\.yml$"
+)
+UNEXPANDED: Final = re.compile(r"[*?\[]")
+
+
+def is_selectable(path: str) -> bool:
+    return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None
+
+
+def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
+    direct: Final = frozenset(path for path in changed if is_selectable(path))
+    harness_changed: Final = any(HARNESS.match(path) for path in changed)
+    canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path))
+    return tuple(sorted(direct | canary_tests))
+
+
+def main() -> int:
+    canary: Final = tuple(sys.argv[1:])
+    unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path))
+    if unexpanded:
+        _ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n")
+        return 1
+    changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
+    _ = sys.stdout.write(" ".join(select(changed, canary)) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-stack/up.sh

@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+set -euo pipefail
+umask 077
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+CERTS_DIR="${STACK_DIR}/certs"
+LOGS_DIR="${STACK_DIR}/logs"
+PIDS_DIR="${STACK_DIR}/pids"
+
+POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
+VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}"
+JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}"
+NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}"
+
+LB_PORT="${E2E_LB_PORT:-4000}"
+GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}"
+GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}"
+BACKEND_PORT="${E2E_BACKEND_PORT:-4001}"
+REDIS_PORT="${E2E_REDIS_PORT:-6379}"
+DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}"
+DATABASE_PORT="${E2E_DATABASE_PORT:-5432}"
+DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
+DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
+DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
+JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
+JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
+
+MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
+
+mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 755 "${CERTS_DIR}"
+
+log() { printf 'e2e-stack: %s\n' "$*"; }
+
+port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
+
+wait_for() {
+  local label="$1" check="$2" deadline=$((SECONDS + ${3:-120}))
+  until eval "${check}"; do
+    if ((SECONDS >= deadline)); then
+      log "timed out waiting for ${label}"
+      exit 1
+    fi
+    sleep 2
+  done
+  log "${label} is up"
+}
+
+if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then
+  set -a
+  source "${REPO_ROOT}/tests/e2e/.env"
+  set +a
+fi
+
+if [[ -z "${DD_API_KEY:-}" ]]; then
+  log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env"
+  exit 1
+fi
+export DD_SITE="${DD_SITE:-datadoghq.com}"
+
+if ! port_open "${DATABASE_PORT}"; then
+  docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \
+    -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \
+    "${POSTGRES_IMAGE}" >/dev/null
+fi
+wait_for "postgres" "port_open ${DATABASE_PORT}"
+
+if ! port_open "${JAEGER_QUERY_PORT}"; then
+  docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \
+    "${JAEGER_IMAGE}" >/dev/null
+fi
+wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null"
+
+openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null
+openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \
+  -subj "/CN=litellm-e2e-ca" \
+  -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \
+  -out "${CERTS_DIR}/ca.crt" 2>/dev/null
+openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null
+openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null
+openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \
+  -CAcreateserial -days 7 -sha256 \
+  -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \
+  -out "${CERTS_DIR}/server.crt" 2>/dev/null
+chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt
+
+CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')"
+cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem"
+
+docker rm -f e2e-valkey >/dev/null 2>&1 || true
+docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \
+  "${VALKEY_IMAGE}" valkey-server \
+  --cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \
+  --tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \
+  --tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null
+VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}"
+wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG"
+${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null
+wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok"
+
+CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml"
+CONFIG_PATH="${CONFIG_SOURCE}"
+if [[ "${REDIS_PORT}" != "6379" ]]; then
+  CONFIG_PATH="${STACK_DIR}/litellm-config.yml"
+  sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}"
+fi
+
+SERVER_ENV=(
+  "LITELLM_MASTER_KEY=${MASTER_KEY}"
+  "DATABASE_HOST=${DATABASE_HOST}"
+  "DATABASE_PORT=${DATABASE_PORT}"
+  "DATABASE_USER=${DATABASE_USER}"
+  "DATABASE_PASSWORD=${DATABASE_PASSWORD}"
+  "DATABASE_NAME=${DATABASE_NAME}"
+  "DISABLE_SCHEMA_UPDATE=true"
+  "REDIS_HOST=127.0.0.1"
+  "REDIS_PORT=${REDIS_PORT}"
+  "REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]"
+  "CONFIG_FILE_PATH=${CONFIG_PATH}"
+  "STORE_MODEL_IN_DB=True"
+  "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
+  "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
+  "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
+  "PYTHONPATH=${REPO_ROOT}"
+)
+if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then
+  printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json"
+  SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json")
+fi
+
+cd "${REPO_ROOT}"
+
+log "running migrations"
+env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1
+
+start_server() {
+  local name="$1"; shift
+  env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
+  echo $! > "${PIDS_DIR}/${name}.pid"
+}
+
+start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
+start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
+start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
+
+if [[ "$(uname)" == "Linux" ]]; then
+  NGINX_UPSTREAM_HOST=127.0.0.1
+  NGINX_DOCKER_ARGS=(--network host)
+else
+  NGINX_UPSTREAM_HOST=host.docker.internal
+  NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}")
+fi
+
+cat > "${STACK_DIR}/nginx.conf" <<EOF
+events {}
+http {
+  map \$http_upgrade \$connection_upgrade {
+    default upgrade;
+    '' close;
+  }
+  upstream litellm_gateways {
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_1};
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_2};
+  }
+  server {
+    listen ${LB_PORT};
+    client_max_body_size 100m;
+    location / {
+      proxy_pass http://litellm_gateways;
+      proxy_http_version 1.1;
+      proxy_set_header Host \$host;
+      proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto \$scheme;
+      proxy_set_header Upgrade \$http_upgrade;
+      proxy_set_header Connection \$connection_upgrade;
+      proxy_buffering off;
+      proxy_read_timeout 600s;
+      proxy_send_timeout 600s;
+    }
+  }
+}
+EOF
+
+docker rm -f e2e-nginx >/dev/null 2>&1 || true
+docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
+  -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null
+
+wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
+wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
+wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300
+wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60
+
+cat > "${STACK_DIR}/stack.env" <<EOF
+LITELLM_PROXY_URL=http://127.0.0.1:${LB_PORT}
+LITELLM_CONTROL_PLANE_URL=http://127.0.0.1:${BACKEND_PORT}
+LITELLM_PROXY_REPLICA_URLS=http://127.0.0.1:${GATEWAY_PORT_1},http://127.0.0.1:${GATEWAY_PORT_2}
+LITELLM_MASTER_KEY=${MASTER_KEY}
+REDIS_HOST=127.0.0.1
+REDIS_PORT=${REDIS_PORT}
+E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
+SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
+DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
+EOF
+
+log "stack is up; pytest env written to ${STACK_DIR}/stack.env"

.github/mutmut-coverage.rc

@@ -0,0 +1,5 @@
+# mutmut's gather_coverage() looks covered lines up by absolute path, so the
+# repo's `relative_files = true` makes every lookup miss and mutmut generates
+# zero mutants. Point COVERAGE_RCFILE here for mutation runs only.
+[run]
+relative_files = false

.github/pull_request_template.md

@@ -1,7 +1,10 @@
+<!-- The whole description's target audience is humans, not AI agents: write it in plain, simple,
+     everyday engineering language, extremely parsable and readable at a glance. This goes double for
+     the TLDR, User Flow, and Caveats sections -->
+
 ## TLDR
 
-<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
-     This section must be extremely human parsable, comprehensible, and readable: its target audience is humans, not AI agents -->
+<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max -->
 
 Problem this solves:
 
@@ -110,8 +113,20 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 
 ## Caveats (if any)
 
-<!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
+<!-- Group caveats under severity subheadings (### Severe, ### High, ### Medium, ### Low), with
+     short bullet points inside each, just like the TLDR: one line per bullet, roughly 10 words max
      Call out known limitations, follow-up work, or anything a reviewer should watch out for
+     Include only the tiers that have caveats; drop the empty ones
+     - Severe: inherent to what the PR deliberately ships, there even when the code works as intended:
+       it can degrade or take down a running deployment (e.g. a slow or table-locking boot migration),
+       rewrite data by design, break an existing workflow on purpose, or change auth behavior. An
+       operator must plan around it before rollout
+     - High: an unintended hole: a correctness, security, data-loss, or backward-compatibility bug,
+       unsafe to ship as is
+     - Medium: a real gap someone can hit, but with a workaround or a narrow blast radius
+     - Low: anything else worth noting: naming, cleanup, an edge case nobody hits
+     Nest bullets as deep as helps: hierarchy beats one long line when it makes things clearer to a
+     human reader
      Leave this section empty if there are none -->
 
 ## QA runbook
@@ -134,6 +149,6 @@ Example checklists:
   - [ ] Sanity check: this test makes sense to add and is not hand-wavey (e.g., assert actual expected spend instead of just spend > 0) or potentially flaky
 -->
 
-### Final Attestation
+## Final Attestation
 
 - [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR

.github/scripts/assert_ci_coverage.py

@@ -312,8 +312,23 @@ def _matchable_names(relative_path: str) -> frozenset[str]:
     )
 
 
+def _workflow_named_tokens() -> frozenset[str]:
+    """Test tokens a GitHub Actions job names directly.
+
+    A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a
+    workflow names it, so the slice check has to credit those the same way the census does.
+    """
+    return _invoked_test_tokens(
+        scalar
+        for path in _config_files()
+        if path != CIRCLECI_CONFIG
+        for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
+    )
+
+
 def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
     slices: Final = _slices()
+    named_by_workflow: Final = _workflow_named_tokens()
     globbed: Final = tuple(
         path
         for path in _test_files()
@@ -326,6 +341,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
         )
         for path in globbed
         if not allowlist.covers_test(path)
+        and not any(_token_covers(token, path) for token in named_by_workflow)
         and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices)
     )
 

.github/scripts/close_duplicate_issues.py

@@ -1,230 +0,0 @@
-#!/usr/bin/env python3
-"""
-Detect and close duplicate GitHub issues using title similarity.
-
-Modes:
-  --scan           Compare all open issues against each other (batch)
-  --issue-number N Check a single issue against older open issues
-
-Requires the `gh` CLI to be authenticated.
-"""
-
-import argparse
-import difflib
-import json
-import re
-import subprocess
-import sys
-
-
-def normalize_title(title: str) -> str:
-    """Strip common prefixes, lowercase, and collapse whitespace."""
-    title = re.sub(
-        r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
-        "",
-        title,
-        flags=re.IGNORECASE,
-    )
-    return " ".join(title.lower().split())
-
-
-def gh(*args: str) -> str:
-    """Run a gh CLI command and return stdout."""
-    result = subprocess.run(
-        ["gh", *args],
-        capture_output=True,
-        text=True,
-        check=True,
-    )
-    return result.stdout
-
-
-def fetch_open_issues(repo: str | None) -> list[dict]:
-    """Fetch all open issues (excluding PRs) via gh api --paginate."""
-    if repo:
-        endpoint = (
-            f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
-        )
-    else:
-        endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
-    cmd = ["api", "--paginate", endpoint]
-
-    raw = gh(*cmd)
-    # gh --paginate concatenates JSON arrays, so we may get multiple arrays
-    issues = []
-    for line in raw.strip().splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        parsed = json.loads(line)
-        if isinstance(parsed, list):
-            issues.extend(parsed)
-        else:
-            issues.append(parsed)
-
-    # Filter out pull requests (they also appear in the issues endpoint)
-    return [i for i in issues if "pull_request" not in i]
-
-
-def close_as_duplicate(
-    issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
-) -> None:
-    """Close an issue as duplicate of another, adding a comment and label."""
-    repo_args = ["--repo", repo] if repo else []
-
-    if dry_run:
-        print(
-            f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}"
-        )
-        return
-
-    # Add comment
-    comment_body = (
-        f"Closing as duplicate of #{duplicate_of}.\n\n"
-        "If you believe this is not a duplicate, please reopen and add context "
-        "explaining how this differs."
-    )
-    gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
-
-    # Add label
-    gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
-
-    # Close with not_planned reason
-    gh(
-        "api",
-        f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
-        "-X",
-        "PATCH",
-        "-f",
-        "state=closed",
-        "-f",
-        "state_reason=not_planned",
-    )
-
-    print(f"  Closed #{issue_number} as duplicate of #{duplicate_of}")
-
-
-def find_duplicate(
-    issue: dict, candidates: list[dict], threshold: float
-) -> dict | None:
-    """Return the first candidate whose normalized title is above threshold."""
-    norm = normalize_title(issue["title"])
-    for candidate in candidates:
-        if candidate["number"] == issue["number"]:
-            continue
-        cand_norm = normalize_title(candidate["title"])
-        ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
-        if ratio >= threshold:
-            return candidate
-    return None
-
-
-def scan_all(
-    issues: list[dict], threshold: float, repo: str | None, dry_run: bool
-) -> int:
-    """Compare every issue against all older issues. Returns count of duplicates found."""
-    # Sort oldest first
-    issues.sort(key=lambda i: i["number"])
-    closed_count = 0
-
-    for idx, issue in enumerate(issues):
-        older = issues[:idx]
-        if not older:
-            continue
-        dup = find_duplicate(issue, older, threshold)
-        if dup:
-            ratio = difflib.SequenceMatcher(
-                None,
-                normalize_title(issue["title"]),
-                normalize_title(dup["title"]),
-            ).ratio()
-            print(
-                f"#{issue['number']}: \"{issue['title']}\"\n"
-                f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
-                f"({ratio:.0%} similar)"
-            )
-            close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
-            closed_count += 1
-
-    return closed_count
-
-
-def check_single(
-    issue_number: int,
-    issues: list[dict],
-    threshold: float,
-    repo: str | None,
-    dry_run: bool,
-) -> bool:
-    """Check a single issue against all older open issues. Returns True if duplicate found."""
-    target = None
-    for i in issues:
-        if i["number"] == issue_number:
-            target = i
-            break
-
-    if target is None:
-        print(f"Issue #{issue_number} not found among open issues.")
-        return False
-
-    older = [i for i in issues if i["number"] < issue_number]
-    dup = find_duplicate(target, older, threshold)
-    if dup:
-        ratio = difflib.SequenceMatcher(
-            None,
-            normalize_title(target["title"]),
-            normalize_title(dup["title"]),
-        ).ratio()
-        print(
-            f"#{target['number']}: \"{target['title']}\"\n"
-            f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
-            f"({ratio:.0%} similar)"
-        )
-        close_as_duplicate(issue_number, dup["number"], repo, dry_run)
-        return True
-
-    print(f"#{issue_number}: no duplicate found above threshold {threshold}")
-    return False
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Detect and close duplicate GitHub issues"
-    )
-    mode = parser.add_mutually_exclusive_group(required=True)
-    mode.add_argument("--scan", action="store_true", help="Scan all open issues")
-    mode.add_argument("--issue-number", type=int, help="Check a single issue number")
-    parser.add_argument(
-        "--threshold", type=float, default=0.85, help="Similarity threshold (0-1)"
-    )
-    parser.add_argument(
-        "--close",
-        action="store_true",
-        help="Actually close duplicates (default is dry-run)",
-    )
-    parser.add_argument(
-        "--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted."
-    )
-    args = parser.parse_args()
-
-    dry_run = not args.close
-
-    if dry_run:
-        print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
-
-    print("Fetching open issues...")
-    issues = fetch_open_issues(args.repo)
-    print(f"Found {len(issues)} open issues.\n")
-
-    if args.scan:
-        count = scan_all(issues, args.threshold, args.repo, dry_run)
-        print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
-    else:
-        found = check_single(
-            args.issue_number, issues, args.threshold, args.repo, dry_run
-        )
-        sys.exit(0 if found else 0)  # Always exit 0; finding no dup is not an error
-
-
-if __name__ == "__main__":
-    main()

.github/scripts/e2e_egress_sentinel.py

@@ -0,0 +1,198 @@
+"""Prove an e2e replay run makes zero outbound provider calls, by counting them.
+
+`serve` pins each provider host (`--host`) to a local sink address in the hosts
+file and binds a counting listener on that address, so any connection the proxy
+or the record/replay edge opens to a real provider is redirected to the sink,
+recorded as one line in `--hits-file`, and never leaves the box. The record and
+replay edge only ever dials `127.0.0.1:<edge-port>` (a different host than the
+pinned provider names), so in a clean replay the sink sees nothing; a single hit
+means a provider call escaped the bundle. `assert-empty` turns that hit file into
+the pass/fail check.
+
+Stdlib only, so CI runs it under the system interpreter as root (binding :443 and
+editing the hosts file both need root); `--sink-address`, `--port`, and
+`--hosts-file` are injectable so it runs unprivileged against a temp hosts file on
+a high port under test.
+"""
+
+# ruff: noqa: T201  # CLI script: its stdout/stderr progress and results are the interface
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import signal
+import socket
+import sys
+import threading
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from types import FrameType
+from typing import Final
+
+_BLOCK_BEGIN: Final = "# BEGIN e2e-egress-sentinel"
+_BLOCK_END: Final = "# END e2e-egress-sentinel"
+
+
+@dataclass(frozen=True, slots=True)
+class ServeConfig:
+    hosts: tuple[str, ...]
+    sink_address: str
+    ports: tuple[int, ...]
+    hits_file: Path
+    hosts_file: Path
+    ready_file: Path | None
+    pid_file: Path | None
+
+
+def _pin_block(sink_address: str, hosts: tuple[str, ...]) -> str:
+    lines = "\n".join(f"{sink_address}\t{host}" for host in hosts)
+    return f"\n{_BLOCK_BEGIN}\n{lines}\n{_BLOCK_END}\n"
+
+
+def _install_pins(hosts_file: Path, sink_address: str, hosts: tuple[str, ...]) -> bytes:
+    original = hosts_file.read_bytes() if hosts_file.exists() else b""
+    hosts_file.write_bytes(original + _pin_block(sink_address, hosts).encode())
+    return original
+
+
+def _restore_pins(hosts_file: Path, original: bytes) -> None:
+    hosts_file.write_bytes(original)
+
+
+def _bind(sink_address: str, port: int) -> socket.socket:
+    listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    listener.bind((sink_address, port))
+    listener.listen(128)
+    return listener
+
+
+@dataclass(frozen=True, slots=True)
+class _HitLog:
+    path: Path
+    _lock: threading.Lock
+
+    def record(self, *, port: int, peer: tuple[str, int]) -> None:
+        entry = json.dumps({"ts": time.time(), "port": port, "peer": list(peer)})
+        with self._lock:
+            with self.path.open("a", encoding="utf-8") as handle:
+                handle.write(entry + "\n")
+
+
+def _serve_socket(listener: socket.socket, port: int, hits: _HitLog, stop: threading.Event) -> None:
+    while not stop.is_set():
+        try:
+            conn, peer = listener.accept()
+        except OSError:
+            return
+        hits.record(port=port, peer=(peer[0], peer[1]))
+        try:
+            conn.close()
+        except OSError:
+            pass
+
+
+def serve(config: ServeConfig) -> int:
+    config.hits_file.write_text("", encoding="utf-8")
+    original_hosts = _install_pins(config.hosts_file, config.sink_address, config.hosts)
+    try:
+        listeners = tuple(_bind(config.sink_address, port) for port in config.ports)
+    except OSError as exc:
+        _restore_pins(config.hosts_file, original_hosts)
+        print(f"egress sentinel could not bind a sink: {exc}", file=sys.stderr)
+        return 1
+
+    stop = threading.Event()
+    hits = _HitLog(path=config.hits_file, _lock=threading.Lock())
+    threads = tuple(
+        threading.Thread(target=_serve_socket, args=(listener, port, hits, stop), daemon=True)
+        for listener, port in zip(listeners, config.ports)
+    )
+    for thread in threads:
+        thread.start()
+
+    def _handle(_signum: int, _frame: FrameType | None) -> None:
+        stop.set()
+        for listener in listeners:
+            try:
+                listener.close()
+            except OSError:
+                pass
+
+    signal.signal(signal.SIGTERM, _handle)
+    signal.signal(signal.SIGINT, _handle)
+
+    if config.pid_file is not None:
+        config.pid_file.write_text(str(os.getpid()), encoding="utf-8")
+    if config.ready_file is not None:
+        config.ready_file.write_text("ready", encoding="utf-8")
+    print(
+        f"egress sentinel up: pinned {', '.join(config.hosts)} to {config.sink_address} "
+        f"on port(s) {', '.join(str(p) for p in config.ports)}",
+        flush=True,
+    )
+
+    stop.wait()
+    _restore_pins(config.hosts_file, original_hosts)
+    if config.ready_file is not None and config.ready_file.exists():
+        config.ready_file.unlink()
+    if config.pid_file is not None and config.pid_file.exists():
+        config.pid_file.unlink()
+    return 0
+
+
+def assert_empty(hits_file: Path) -> int:
+    if not hits_file.exists():
+        print(f"egress sentinel recorded no provider calls ({hits_file} absent): zero egress")
+        return 0
+    hits = [line for line in hits_file.read_text(encoding="utf-8").splitlines() if line.strip()]
+    if not hits:
+        print("egress sentinel recorded no provider calls: zero egress")
+        return 0
+    print(f"egress sentinel recorded {len(hits)} provider call(s); replay was not hermetic:", file=sys.stderr)
+    for line in hits:
+        print(f"  {line}", file=sys.stderr)
+    return 1
+
+
+def _serve_from_args(args: argparse.Namespace) -> int:
+    config = ServeConfig(
+        hosts=tuple(args.host),
+        sink_address=args.sink_address,
+        ports=tuple(args.port),
+        hits_file=Path(args.hits_file),
+        hosts_file=Path(args.hosts_file),
+        ready_file=Path(args.ready_file) if args.ready_file else None,
+        pid_file=Path(args.pid_file) if args.pid_file else None,
+    )
+    return serve(config)
+
+
+def main(argv: tuple[str, ...]) -> int:
+    parser = argparse.ArgumentParser(description="count outbound provider calls during an e2e replay")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    serve_parser = sub.add_parser("serve", help="pin provider hosts and count connection attempts")
+    serve_parser.add_argument("--host", action="append", required=True, help="provider host to pin and watch")
+    serve_parser.add_argument("--sink-address", default="127.0.0.1")
+    serve_parser.add_argument("--port", action="append", type=int, default=None)
+    serve_parser.add_argument("--hits-file", required=True)
+    serve_parser.add_argument("--hosts-file", default="/etc/hosts")
+    serve_parser.add_argument("--ready-file", default=None)
+    serve_parser.add_argument("--pid-file", default=None)
+
+    assert_parser = sub.add_parser("assert-empty", help="exit non-zero if any provider call was recorded")
+    assert_parser.add_argument("--hits-file", required=True)
+
+    args = parser.parse_args(argv)
+    if args.command == "serve":
+        if args.port is None:
+            args.port = [443]
+        return _serve_from_args(args)
+    return assert_empty(Path(args.hits_file))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(tuple(sys.argv[1:])))

.github/scripts/e2e_fetch_fixture_bundle.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="${1:-${GITHUB_REPOSITORY:?REPO required}}"
+ARTIFACT_NAME="${2:-e2e-fixtures-bundle}"
+BASE_BRANCH="${3:?base branch required}"
+DEST_DIR="${4:?destination bundle dir required}"
+
+: "${GH_TOKEN:?GH_TOKEN required to query and download artifacts}"
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "${WORKDIR}"' EXIT
+
+echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}"
+
+SELECTED="$(
+  gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \
+    --jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \
+    | jq -s 'sort_by(.created_at) | reverse | .[0] // empty'
+)"
+
+if [[ -z "${SELECTED}" ]]; then
+  echo "no usable '${ARTIFACT_NAME}' artifact on ${BASE_BRANCH}: the last record run produced none (a red Saturday), so there is nothing fresh to replay; failing loudly instead of replaying a stale bundle" >&2
+  exit 1
+fi
+
+RUN_ID="$(echo "${SELECTED}" | jq -r '.run_id')"
+RUN_NUMBER="$(echo "${SELECTED}" | jq -r '.run_number')"
+ARTIFACT_ID="$(echo "${SELECTED}" | jq -r '.id')"
+GH_DIGEST="$(echo "${SELECTED}" | jq -r '.digest // "unknown"')"
+CREATED_AT="$(echo "${SELECTED}" | jq -r '.created_at')"
+
+echo "pinned bundle: run #${RUN_NUMBER} (run_id=${RUN_ID}, artifact_id=${ARTIFACT_ID}), recorded ${CREATED_AT}, github digest ${GH_DIGEST}"
+
+gh run download "${RUN_ID}" --repo "${REPO}" -n "${ARTIFACT_NAME}" -D "${WORKDIR}"
+
+TARBALL="$(find "${WORKDIR}" -name '*.tar.gz' -type f | head -n 1)"
+if [[ -z "${TARBALL}" ]]; then
+  echo "downloaded artifact contained no tarball" >&2
+  exit 1
+fi
+SIDECAR="${TARBALL}.sha256"
+if [[ ! -f "${SIDECAR}" ]]; then
+  echo "downloaded artifact has no ${SIDECAR}: cannot verify the bundle digest" >&2
+  exit 1
+fi
+
+echo "verifying bundle against its recorded sha256 digest"
+( cd "$(dirname "${TARBALL}")" && sha256sum -c "$(basename "${SIDECAR}")" )
+
+mkdir -p "${DEST_DIR}"
+tar xzf "${TARBALL}" -C "${DEST_DIR}"
+
+echo "extracted bundle into ${DEST_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  recorded_at', m['recorded_at'], 'harness', m['harness_version'], 'format_version', m['format_version'])" "${DEST_DIR}/manifest.json"

.github/scripts/e2e_pack_fixture_bundle.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 2 ]]; then
+  echo "usage: $0 <bundle-dir> <out-tarball>" >&2
+  exit 2
+fi
+
+BUNDLE_DIR="$1"
+OUT_TARBALL="$2"
+
+MANIFEST="${BUNDLE_DIR}/manifest.json"
+if [[ ! -f "${MANIFEST}" ]]; then
+  echo "no ${MANIFEST}: refusing to publish a bundle with no manifest (record produced nothing)" >&2
+  exit 1
+fi
+
+echo "packing fixture bundle from ${BUNDLE_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  format_version', m['format_version'], 'recorded_at', m['recorded_at'], 'harness', m['harness_version'])" "${MANIFEST}"
+
+TEST_DIRS=$(find "${BUNDLE_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')
+if [[ "${TEST_DIRS}" -eq 0 ]]; then
+  echo "bundle at ${BUNDLE_DIR} has a manifest but no recorded interactions; refusing to publish an empty bundle" >&2
+  exit 1
+fi
+echo "  ${TEST_DIRS} recorded test director(ies)"
+
+mkdir -p "$(dirname "${OUT_TARBALL}")"
+tar czf "${OUT_TARBALL}" -C "${BUNDLE_DIR}" .
+
+OUT_DIR="$(cd "$(dirname "${OUT_TARBALL}")" && pwd)"
+OUT_BASE="$(basename "${OUT_TARBALL}")"
+( cd "${OUT_DIR}" && sha256sum "${OUT_BASE}" > "${OUT_BASE}.sha256" )
+
+echo "wrote ${OUT_TARBALL} ($(du -h "${OUT_TARBALL}" | cut -f1)) and ${OUT_BASE}.sha256"
+cat "${OUT_DIR}/${OUT_BASE}.sha256"

.github/scripts/smoke_test_native_wheel.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import Final
+
+CHILD_SCRIPT: Final = """
+from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
+import sys
+
+native_path = Path(sys.argv[1])
+spec = spec_from_file_location("litellm.rust_bridge._native", native_path)
+if spec is None or spec.loader is None:
+    raise RuntimeError("cannot create native extension import specification")
+module = module_from_spec(spec)
+spec.loader.exec_module(module)
+
+before = module.gil_stats()
+if not isinstance(before.get("releases"), int):
+    raise AssertionError(f"unexpected gil_stats result: {before!r}")
+
+try:
+    module._panic_for_test()
+except BaseException as error:
+    if type(error).__name__ != "PanicException":
+        raise AssertionError(f"expected PanicException, got {type(error).__name__}") from error
+else:
+    raise AssertionError("Rust panic returned without raising")
+
+after = module.gil_stats()
+if not isinstance(after.get("releases"), int):
+    raise AssertionError(f"native module unusable after panic: {after!r}")
+"""
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n")
+        return 2
+
+    wheel: Final = Path(sys.argv[1])
+    with tempfile.TemporaryDirectory() as temporary_directory, zipfile.ZipFile(wheel) as archive:
+        native_members: Final = tuple(
+            member
+            for member in archive.infolist()
+            if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so")
+        )
+        if len(native_members) != 1:
+            sys.stderr.write(f"expected one native extension, found {len(native_members)}\n")
+            return 1
+
+        native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name
+        native_path.write_bytes(archive.read(native_members[0]))
+        result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False)
+
+    if result.returncode != 0:
+        sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n")
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/verify_linux_native_wheel.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import importlib.util
+import os
+import re
+import subprocess
+import sys
+import zipfile
+from collections.abc import Callable, Mapping, Sequence
+from itertools import product
+from pathlib import Path, PurePosixPath
+from types import MappingProxyType, ModuleType
+from typing import Final, Protocol
+
+EXPECTED_PYTHON_TAG: Final = "cp310"
+EXPECTED_ABI_TAG: Final = "abi3"
+EXPECTED_PLATFORM_TAG: Final = "linux_x86_64"
+
+
+class CommandRunner(Protocol):
+    def __call__(
+        self,
+        command: tuple[str, ...],
+        *,
+        check: bool,
+        capture_output: bool,
+        text: bool,
+    ) -> subprocess.CompletedProcess[str]: ...
+
+
+def _run_command(
+    command: tuple[str, ...],
+    *,
+    check: bool,
+    capture_output: bool,
+    text: bool,
+) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(command, check=check, capture_output=capture_output, text=text)
+
+
+def _dist_info_directory(member: zipfile.ZipInfo) -> str | None:
+    parts: Final = PurePosixPath(member.filename).parts
+    if not parts or not parts[0].endswith(".dist-info"):
+        return None
+    return parts[0]
+
+
+def _wheel_metadata_tags(archive: zipfile.ZipFile, members: tuple[zipfile.ZipInfo, ...]) -> tuple[str, ...]:
+    if len(members) != 1:
+        return ()
+    lines: Final = archive.read(members[0]).splitlines()
+    return tuple(line.removeprefix(b"Tag:").strip().decode("ascii") for line in lines if line.startswith(b"Tag:"))
+
+
+def _load_native_module(native_path: Path) -> ModuleType | None:
+    module_spec: Final = importlib.util.spec_from_file_location("litellm.rust_bridge._native", native_path)
+    if module_spec is None or module_spec.loader is None:
+        return None
+    try:
+        native_module: Final = importlib.util.module_from_spec(module_spec)
+        module_spec.loader.exec_module(native_module)
+    except Exception as error:  # noqa: BLE001  # native module initialization can raise arbitrary exceptions
+        sys.stderr.write(f"native module load failed: {error}\n")
+        return None
+    return native_module
+
+
+def main(
+    argv: Sequence[str] | None = None,
+    environment: Mapping[str, str] | None = None,
+    load_native_module: Callable[[Path], ModuleType | None] = _load_native_module,
+    run_command: CommandRunner = _run_command,
+) -> int:
+    arguments: Final = tuple(sys.argv if argv is None else argv)
+    resolved_environment: Final = os.environ if environment is None else environment
+    if len(arguments) != 2:
+        sys.stderr.write(f"usage: {Path(arguments[0]).name} WHEEL\n")
+        return 2
+
+    wheel: Final = Path(arguments[1])
+    wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3)
+    if len(wheel_tags) != 4:
+        sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n")
+        return 1
+
+    wheel_identity: Final = wheel_tags[0].split("-")
+    if len(wheel_identity) != 2 or wheel_identity[0] != "litellm" or not wheel_identity[1]:
+        sys.stderr.write(f"unexpected wheel identity: {wheel_tags[0]}\n")
+        return 1
+
+    expected_dist_info_directory: Final = f"{wheel_tags[0]}.dist-info"
+    expected_dist_info_directories: Final = frozenset((expected_dist_info_directory,))
+    python_tag: Final = wheel_tags[1]
+    abi_tag: Final = wheel_tags[2]
+    platform_tag: Final = wheel_tags[3]
+    expanded_filename_tags: Final = frozenset(
+        "-".join(tag) for tag in product(python_tag.split("."), abi_tag.split("."), platform_tag.split("."))
+    )
+
+    with zipfile.ZipFile(wheel) as archive:
+        wheel_members: Final = archive.infolist()
+        dist_info_directories: Final = frozenset(
+            directory for member in wheel_members if (directory := _dist_info_directory(member)) is not None
+        )
+        required_dist_info_files: Final = ("METADATA", "RECORD", "WHEEL")
+        dist_info_file_counts: Final = MappingProxyType(
+            {
+                filename: sum(
+                    member.filename == f"{expected_dist_info_directory}/{filename}" for member in wheel_members
+                )
+                for filename in required_dist_info_files
+            }
+        )
+        wheel_metadata_members: Final = tuple(
+            member for member in wheel_members if member.filename == f"{expected_dist_info_directory}/WHEEL"
+        )
+        wheel_metadata_tags: Final = _wheel_metadata_tags(archive, wheel_metadata_members)
+        native_members: Final = tuple(
+            member
+            for member in wheel_members
+            if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so")
+        )
+        if len(native_members) != 1:
+            sys.stderr.write(f"expected one native extension, found {len(native_members)}\n")
+            return 1
+
+        unexpected_members: Final = tuple(
+            member.filename
+            for member in wheel_members
+            if member.filename.endswith((".pdb", ".dwp", ".rlib", ".rmeta", "Cargo.toml", "Cargo.lock"))
+            or any(part.endswith(".dSYM") for part in PurePosixPath(member.filename).parts)
+        )
+        native_member: Final = native_members[0]
+        uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members)
+        native_path: Final = wheel.parent / "native" / Path(native_member.filename).name
+        native_path.parent.mkdir(parents=True, exist_ok=True)
+        native_path.write_bytes(archive.read(native_member))
+
+    wheel_metadata_tags_match: Final = (
+        len(wheel_metadata_tags) == len(expanded_filename_tags)
+        and frozenset(wheel_metadata_tags) == expanded_filename_tags
+    )
+    commit_sha: Final = resolved_environment.get(
+        "RELEASE_WHEEL_COMMIT_SHA", resolved_environment.get("GITHUB_SHA", "unknown")
+    )
+    rustc_version: Final = run_command(
+        ("rustc", "--version"),
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text()
+    maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject)
+    if maturin_match is None:
+        sys.stderr.write("build-system does not pin an exact Maturin version\n")
+        return 1
+
+    maturin_version: Final = maturin_match.group(1)
+    native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100
+    size_report: Final = "\n".join(
+        (
+            "## Native wheel build report",
+            "",
+            "| Build | Value |",
+            "| --- | --- |",
+            f"| Commit | `{commit_sha}` |",
+            f"| Platform | `{platform_tag}` |",
+            f"| Python ABI | `{python_tag}-{abi_tag}` |",
+            f"| Rust compiler | `{rustc_version}` |",
+            f"| Maturin | `{maturin_version}` |",
+            "| Cargo profile | `release` |",
+            "",
+            "| Artifact | Size |",
+            "| --- | ---: |",
+            f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |",
+            f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |",
+            f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |",
+            f"| Native share | {native_percentage:.2f}% |",
+            "",
+        )
+    )
+    summary_path: Final = resolved_environment.get("GITHUB_STEP_SUMMARY")
+    if summary_path is None:
+        sys.stdout.write(size_report)
+    else:
+        Path(summary_path).write_text(size_report)
+
+    sections: Final = run_command(
+        ("readelf", "--sections", "--wide", str(native_path)),
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout
+    debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections)
+    debug_sections_absent: Final = not debug_sections
+    static_symbol_table_absent: Final = ".symtab" not in sections
+
+    dynamic_symbols: Final = run_command(
+        ("readelf", "--dyn-syms", "--wide", str(native_path)),
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout
+    extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols
+    native_module: Final = load_native_module(native_path)
+    native_module_loads: Final = native_module is not None
+    panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
+    native_size_limit: Final = 20_000_000
+    native_size_within_limit: Final = native_member.file_size <= native_size_limit
+    validations: Final = (
+        (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
+        (f"ABI tag is {EXPECTED_ABI_TAG}", abi_tag == EXPECTED_ABI_TAG),
+        (f"Platform tag is {EXPECTED_PLATFORM_TAG}", platform_tag == EXPECTED_PLATFORM_TAG),
+        ("Wheel dist-info directory matches the filename", dist_info_directories == expected_dist_info_directories),
+        (
+            "Required dist-info files are present exactly once",
+            all(count == 1 for count in dist_info_file_counts.values()),
+        ),
+        ("Wheel metadata tags match the filename", wheel_metadata_tags_match),
+        ("Debug sections are absent", debug_sections_absent),
+        ("Static symbol table is absent", static_symbol_table_absent),
+        ("Python extension entry point is present", extension_entry_point_present),
+        ("Native module loads", native_module_loads),
+        ("Production module omits the panic test hook", panic_test_hook_absent),
+        ("Native extension does not exceed 20 MB", native_size_within_limit),
+        ("Wheel contents are valid", not unexpected_members),
+    )
+
+    verified_report: Final = size_report + "\n".join(
+        ("", "| Validation | Expected | Result |", "| --- | --- | :---: |")
+        + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations)
+        + ("",)
+    )
+    if summary_path is not None:
+        Path(summary_path).write_text(verified_report)
+
+    invalid_dist_info_files: Final = any(count != 1 for count in dist_info_file_counts.values())
+    validation_errors: Final = tuple(
+        message
+        for failed, message in (
+            (bool(debug_sections), f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}"),
+            (not static_symbol_table_absent, f"{native_member.filename} contains a static symbol table"),
+            (not extension_entry_point_present, "native extension does not export PyInit__native"),
+            (
+                python_tag != EXPECTED_PYTHON_TAG,
+                f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}",
+            ),
+            (abi_tag != EXPECTED_ABI_TAG, f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}"),
+            (
+                platform_tag != EXPECTED_PLATFORM_TAG,
+                f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}",
+            ),
+            (
+                dist_info_directories != expected_dist_info_directories,
+                f"unexpected dist-info directories: expected {expected_dist_info_directory}, "
+                f"found {', '.join(sorted(dist_info_directories))}",
+            ),
+            (invalid_dist_info_files, f"required dist-info file counts are invalid: {dist_info_file_counts}"),
+            (
+                not invalid_dist_info_files and not wheel_metadata_tags_match,
+                f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, "
+                f"found {', '.join(sorted(wheel_metadata_tags))}",
+            ),
+            (
+                native_module is not None and not panic_test_hook_absent,
+                "production native module exposes _panic_for_test",
+            ),
+            (
+                not native_size_within_limit,
+                f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB",
+            ),
+            (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
+        )
+        if failed
+    )
+    sys.stderr.write("".join(f"{message}\n" for message in validation_errors))
+
+    return 0 if all(passed for _, passed in validations) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/_test-unit-base.yml

@@ -27,7 +27,7 @@ on:
         default: 20
       job-timeout-minutes:
         description: >-
-          Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for
+          Backstop for the whole job. Keep it >= `timeout-minutes` plus 40: 35 for
           the per-step ceilings on the setup steps below, and 5 for the runner
           overhead the job clock charges but no step owns (job init, step
           transitions, post-job cleanup). That headroom is what makes the test
@@ -36,7 +36,7 @@ on:
           arithmetic, so the sum is passed in rather than computed.
         required: false
         type: number
-        default: 55
+        default: 60
       max-failures:
         description: "Stop after this many failures"
         required: false
@@ -55,6 +55,9 @@ on:
 permissions:
   contents: read
 
+env:
+  UV_PYTHON: "3.12"
+
 jobs:
   run:
     name: Run tests
@@ -82,7 +85,7 @@ jobs:
         timeout-minutes: 3
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: "3.12"
+          python-version: ${{ env.UV_PYTHON }}
 
       - name: Set up uv
         if: steps.changes.outputs.decision != 'skip'
@@ -96,18 +99,22 @@ jobs:
         timeout-minutes: 5
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
-          path: |
-            ~/.cache/uv
-            .venv
-          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          path: ${{ env.UV_CACHE_DIR }}
+          key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }}
           restore-keys: |
-            ${{ runner.os }}-uv-
+            ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-
+
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: ./.github/actions/cache-cargo-build
 
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 8
         run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra mongodb
+          uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
 
       - name: Cache Prisma binaries
         if: steps.changes.outputs.decision != 'skip'
@@ -129,12 +136,6 @@ jobs:
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
           DIST: ${{ inputs.dist }}
-          # coverage.py's sys.monitoring backend (PEP 669), the cheapest core it has.
-          # It is only the default from Python 3.14, and these shards run 3.12, so it
-          # has to be asked for. Coverage refuses it when branch measurement is on
-          # (`branch_right_left` needs > 3.14.0a5) and falls back to the slow core with
-          # a `no-sysmon` warning, so turning on `branch = true` here means giving this
-          # back until the runners move to 3.14.
           COVERAGE_CORE: sysmon
         run: |
           if [ "${WORKERS}" = "0" ]; then
@@ -144,7 +145,7 @@ jobs:
               --reruns "${RERUNS}" \
               --reruns-delay 1 \
               --durations=20 \
-              --cov=./litellm \
+              --cov=./litellm --cov=./enterprise/litellm_enterprise \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           else
@@ -156,7 +157,7 @@ jobs:
               --reruns-delay 1 \
               --dist="${DIST}" \
               --durations=20 \
-              --cov=./litellm \
+              --cov=./litellm --cov=./enterprise/litellm_enterprise \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           fi

.github/workflows/auto-close-duplicates.yml

@@ -0,0 +1,69 @@
+name: Auto-close duplicate issues
+
+on:
+  schedule:
+    - cron: "0 9 * * *"
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: Log which issues would close without closing anything
+        type: boolean
+        default: true
+      grace_period_days:
+        description: Days a duplicate notice must go unanswered before the close
+        type: number
+        default: 3
+  pull_request:
+    paths:
+      - .github/workflows/auto-close-duplicates.yml
+      - scripts/auto-close-duplicates.ts
+      - scripts/auto-close-duplicates.test.ts
+
+permissions: {}
+
+jobs:
+  sweep-tests:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          bun-version: "1.4.0"
+
+      - name: Test the sweep
+        run: bun test scripts/auto-close-duplicates.test.ts
+
+  sweep:
+    if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          # Exact version, never latest: the next step holds an issues: write token
+          bun-version: "1.4.0"
+
+      - name: Close unanswered duplicates, reopen ones the reporter answered
+        run: bun run scripts/auto-close-duplicates.ts
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DRY_RUN: ${{ inputs.dry_run == true }}
+          GRACE_PERIOD_DAYS: ${{ inputs.grace_period_days }}

.github/workflows/check-ui-api-types.yml

@@ -67,6 +67,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.relevant == 'true'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install backend dependencies
         if: steps.changes.outputs.relevant == 'true'
         run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
@@ -79,6 +83,24 @@ jobs:
         if: steps.changes.outputs.relevant == 'true'
         run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
+      - name: Regenerate the lazy OpenAPI snapshot
+        if: steps.changes.outputs.relevant == 'true'
+        run: uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
+
+      - name: Fail if the lazy OpenAPI snapshot is stale
+        if: steps.changes.outputs.relevant == 'true'
+        run: |
+          if ! git diff --exit-code -- litellm/proxy/_lazy_openapi_snapshot.json; then
+            echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::The lazy OpenAPI snapshot is out of sync with the lazily loaded routes."
+            echo ""
+            echo "A lazily loaded route or model changed without regenerating the snapshot that /openapi.json serves for unloaded features."
+            echo "To fix, run from the repo root:"
+            echo "  uv run python -m litellm.proxy._lazy_openapi_snapshot"
+            echo "then run npm run gen:api from ui/litellm-dashboard and commit both files."
+            exit 1
+          fi
+          echo "_lazy_openapi_snapshot.json is in sync with the lazily loaded routes."
+
       - name: Set up Node.js
         if: steps.changes.outputs.relevant == 'true'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0

.github/workflows/check_duplicate_issues.yml

@@ -1,12 +1,19 @@
 name: Check Duplicate Issues
 
+# Flagging only. "Auto-close duplicate issues" closes a flagged issue 3 days later,
+# and only when its title is identical to an older open issue and nobody replied.
+# The HTML marker below is the handshake between the two, so keep it in the template.
+
 on:
   issues:
     types: [opened, edited]
 
+permissions: {}
+
 jobs:
   check-duplicate:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     permissions:
       issues: write
       contents: read
@@ -19,35 +26,12 @@ jobs:
           threshold: 0.6
           reaction: eyes
           comment: |
-            **⚠️ Potential duplicate detected**
+            <!-- litellm:potential-duplicate candidates={{#issues}}{{number}},{{/issues}} -->
+            **Potential duplicate detected**
 
-            This issue appears similar to existing issue(s):
+            This looks similar to:
             {{#issues}}
-            - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
+            - #{{number}} - {{title}}
             {{/issues}}
 
-            Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
-
-      - name: Checkout close script
-        if: github.event.action == 'opened'
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          sparse-checkout: .github/scripts
-          persist-credentials: false
-
-      - name: Set up Python
-        if: github.event.action == 'opened'
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Auto-close if high-confidence duplicate
-        if: github.event.action == 'opened'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python3 .github/scripts/close_duplicate_issues.py \
-            --issue-number ${{ github.event.issue.number }} \
-            --repo ${{ github.repository }} \
-            --threshold 0.85 \
-            --close
+            If this is a duplicate, add a thumbs-up reaction to the existing issue and follow along there. When the title is identical to an older open issue, this issue closes automatically in 3 days unless someone responds. If it is not a duplicate, comment here or add a thumbs-down reaction to this comment and it stays open.

.github/workflows/codspeed.yml

@@ -12,6 +12,7 @@ on:
       - "uv.lock"
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
+      - ".github/actions/cache-cargo-build/**"
   pull_request:
     branches:
       - main
@@ -23,6 +24,7 @@ on:
       - "uv.lock"
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
+      - ".github/actions/cache-cargo-build/**"
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:
@@ -55,6 +57,26 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      # Build the wheel and resolve every dependency outside the CodSpeed
+      # runner: the same maturin build took 42 minutes inside `codspeed run`
+      # versus under 3 minutes as a plain step (LIT-6183)
+      - name: Build environment
+        run: >
+          env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
+          uv run --frozen --no-default-groups
+          --with pytest==8.3.5
+          --with pytest-codspeed==4.3.0
+          --with "mcp>=1.26.0,<2.0"
+          --with "a2a-sdk>=1.1.0,<2.0"
+          pytest
+          -p pytest_codspeed.plugin
+          tests/benchmarks/
+          --codspeed
+          --collect-only -q
+
       - name: Run benchmarks
         uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
         with:

.github/workflows/cost-map-guard.yml

@@ -0,0 +1,45 @@
+name: Cost map guard
+
+on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's cost map files are read as data and never executed
+  pull_request_target:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  cost-map-guard:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+      - name: Fetch the pull request head and its merge base
+        id: revisions
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
+          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+          echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+      - name: Run the guard
+        env:
+          MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          HEAD_REF: ${{ github.event.pull_request.head.ref }}
+        run: |
+          uv run --frozen python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"

.github/workflows/e2e_record_replay.yml

@@ -0,0 +1,237 @@
+name: "E2E Record and Replay"
+
+on:
+  schedule:
+    - cron: "0 8 * * 6"
+    - cron: "0 8 * * 1-5"
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "record (hits real providers and publishes a fresh bundle) or replay (bundle only, zero provider egress)"
+        type: choice
+        options:
+          - record
+          - replay
+        default: record
+
+permissions:
+  contents: read
+
+jobs:
+  record:
+    name: "Record the e2e suite against real providers"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 6' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'record'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Record the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: record
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Pack the fixture bundle
+        run: |
+          .github/scripts/e2e_pack_fixture_bundle.sh tests/e2e/.fixtures "${RUNNER_TEMP}/bundle/e2e-fixtures.tar.gz"
+
+      - name: Publish the fixture bundle
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: e2e-fixtures-bundle
+          path: |
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz.sha256
+          if-no-files-found: error
+          retention-days: 30
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log
+
+  replay:
+    name: "Replay the e2e suite from the pinned bundle with zero egress"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 1-5' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'replay'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      contents: read
+      actions: read
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      GH_TOKEN: ${{ github.token }}
+      OPENAI_API_KEY: sk-replay-must-never-reach-a-provider
+      ANTHROPIC_API_KEY: sk-ant-replay-must-never-reach-a-provider
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Fetch the pinned fixture bundle by digest
+        env:
+          BASE_BRANCH: ${{ github.ref_name }}
+        run: |
+          .github/scripts/e2e_fetch_fixture_bundle.sh \
+            "${GITHUB_REPOSITORY}" \
+            e2e-fixtures-bundle \
+            "${BASE_BRANCH}" \
+            tests/e2e/.fixtures
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Start the egress sentinel
+        run: |
+          # shellcheck disable=SC2024  # the log redirect is deliberately the runner user's, so a later non-sudo cat can read it
+          sudo python3 .github/scripts/e2e_egress_sentinel.py serve \
+            --host api.openai.com \
+            --host api.anthropic.com \
+            --hits-file "${RUNNER_TEMP}/egress-hits.jsonl" \
+            --ready-file "${RUNNER_TEMP}/egress-ready" \
+            --pid-file "${RUNNER_TEMP}/egress.pid" \
+            > "${RUNNER_TEMP}/egress-sentinel.log" 2>&1 &
+          for _ in $(seq 1 30); do
+            if [[ -f "${RUNNER_TEMP}/egress-ready" ]]; then
+              cat "${RUNNER_TEMP}/egress-sentinel.log"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "egress sentinel never became ready"
+          cat "${RUNNER_TEMP}/egress-sentinel.log"
+          exit 1
+
+      - name: Replay the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: replay
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Stop the egress sentinel and assert zero provider egress
+        if: always()
+        run: |
+          if [[ -f "${RUNNER_TEMP}/egress.pid" ]]; then
+            sudo kill -TERM "$(cat "${RUNNER_TEMP}/egress.pid")" 2>/dev/null || true
+            sleep 2
+          fi
+          python3 .github/scripts/e2e_egress_sentinel.py assert-empty --hits-file "${RUNNER_TEMP}/egress-hits.jsonl"
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log

.github/workflows/image-scan.yml

@@ -17,10 +17,14 @@ on:
       - backend/Dockerfile
       - backend/main.py
       - docker/component_entrypoint.sh
+      - docker/entrypoint.sh
+      - litellm/proxy/prisma_migration.py
       - litellm-proxy-extras/**
       - tests/proxy_migration_tests/**
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
+      - ui/Dockerfile
+      - ui/nginx.conf
       - .github/workflows/image-scan.yml
   schedule:
     - cron: "41 6 * * *"
@@ -76,7 +80,7 @@ jobs:
           LITELLM_IMAGE: litellm-image-scan:${{ github.sha }}
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
 
       # Scans the whole shipped artifact: OS/apk plus every language package
       # baked into the image, including ones no lockfile declares (e.g. prisma's
@@ -120,7 +124,7 @@ jobs:
           LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
 
   migrations-image:
     name: migrations-image
@@ -181,7 +185,36 @@ jobs:
           LITELLM_COMPONENT_PORT: "4000"
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
+
+  ui-image:
+    name: ui-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build UI image
+        run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
+        env:
+          LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
 
   backend-image:
     name: backend-image

.github/workflows/mutation-test.yml

@@ -53,6 +53,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
@@ -84,11 +87,20 @@ jobs:
         run: |
           uv pip uninstall pytest-retry || true
 
+      # Ends before the job's own deadline so a run that outlasts the budget is
+      # still followed by the report and upload steps. mutmut saves after every
+      # mutant result, to mutants/<source path>.meta, so an interrupted run
+      # still scores the mutants it finished and export-cicd-stats can read
+      # them; a cancelled job skips those steps and publishes nothing at all.
       - name: Run mutmut
+        timeout-minutes: 300
         env:
           # Make the mutants/ sandbox win over site-packages on sys.path so the
           # trampolined files are imported instead of the installed copy.
           PYTHONPATH: ${{ github.workspace }}/mutants
+          # Without this mutmut finds no covered lines and generates 0 mutants.
+          # See the file itself for why.
+          COVERAGE_RCFILE: ${{ github.workspace }}/.github/mutmut-coverage.rc
         run: |
           set -o pipefail
           mkdir -p mutants
@@ -127,6 +139,7 @@ jobs:
             mutmut-run.log
             mutants/mutmut-stats.json
             mutants/mutmut-cicd-stats.json
+            mutants/**/*.meta
             mutants/litellm/proxy/management_endpoints/**/*.py
           if-no-files-found: warn
           retention-days: 14

.github/workflows/publish-basedpyright-base-counts.yml

@@ -1,6 +1,6 @@
 name: Publish basedpyright base counts
 
-# Every commit on litellm_internal_staging is some branch's future merge-base.
+# Every commit on main or litellm_internal_staging can become a future merge-base.
 # Publishing its per-rule basedpyright counts as an artifact lets
 # scripts/type_check_gate.py download them in seconds instead of paying a
 # 60-110s second basedpyright pass on every fresh worktree or moved merge-base.
@@ -10,13 +10,13 @@ name: Publish basedpyright base counts
 on:
   push:
     branches:
+      - main
       - litellm_internal_staging
   workflow_dispatch:
     inputs:
       ref:
-        description: "Ref to compute and publish base counts for"
+        description: "Ref to compute and publish base counts for (defaults to the workflow run's commit)"
         required: false
-        default: litellm_internal_staging
 
 permissions:
   contents: read
@@ -43,6 +43,9 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Cache Prisma binaries
         uses: ./.github/actions/cache-prisma-binaries
 

.github/workflows/sync-together-ai-models.yml

@@ -0,0 +1,70 @@
+name: Sync Together AI model registry
+
+on:
+  schedule:
+    - cron: "30 6 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  sync_together_ai_models:
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    env:
+      BASE_BRANCH: ${{ github.event.repository.default_branch }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          ref: ${{ env.BASE_BRANCH }}
+          persist-credentials: false
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+      - name: Look for an already-open sync PR
+        id: existing
+        run: |
+          open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \
+            --jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')"
+          echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT"
+          if [ -n "$open_pr" ]; then
+            echo "An open sync PR already exists on branch $open_pr; skipping this run."
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
+      - name: Run the sync
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
+        env:
+          TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+      - name: Regenerate the JSON schema
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          uv run --frozen python ci_cd/generate_model_prices_schema.py
+      - name: Create a pull request when the registry changed
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          if git diff --quiet; then
+            echo "Registry already in sync; no PR needed."
+            exit 0
+          fi
+          branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$branch"
+          git add model_prices_and_context_window.json \
+            litellm/model_prices_and_context_window_backup.json \
+            model_prices_and_context_window.schema.json
+          git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
+          gh auth setup-git
+          git push origin "$branch"
+          gh pr create --title "feat(models): sync together_ai model registry" \
+            --body-file "$RUNNER_TEMP/pr_body.md" \
+            --head "$branch" \
+            --base "$BASE_BRANCH"
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}

.github/workflows/test-code-quality.yml

@@ -56,6 +56,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: uv sync --frozen --all-groups --all-extras
 
@@ -71,6 +74,15 @@ jobs:
       - name: check_workflow_startup_safety
         run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
 
+      - name: check_workflow_job_name_collisions
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_job_name_collisions.py
+
+      - name: test_workflow_job_name_collisions
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py
+
+      - name: test_e2e_changed_gate
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py
+
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 
@@ -125,9 +137,15 @@ jobs:
       - name: check_fastuuid_usage
         run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
 
+      - name: check_py310_typing_imports
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_py310_typing_imports.py
+
       - name: check_e2e_no_raw_requests
         run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
 
+      - name: check_migrations_no_data_rewrites
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 
@@ -139,3 +157,33 @@ jobs:
 
       - name: documentation_test_api_docs
         run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
+
+  python-310-import-smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.10"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        run: uv sync --frozen --extra proxy --python 3.10
+
+      - run: uv run --no-sync python --version
+
+      - name: Import litellm
+        run: uv run --no-sync python -c "import litellm"
+
+      - name: Check litellm CLI
+        run: uv run --no-sync litellm --version

.github/workflows/test-e2e-changed.yml

@@ -0,0 +1,240 @@
+name: e2e-changed-tests
+
+on:
+  pull_request:
+
+concurrency:
+  group: e2e-changed-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  detect:
+    name: Detect changed e2e tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: read
+    outputs:
+      tests: ${{ steps.changed.outputs.tests }}
+      any: ${{ steps.changed.outputs.any }}
+    steps:
+      - name: Checkout the selector and the canary suite
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          sparse-checkout: |
+            .github/e2e-stack
+            tests/e2e/access_control
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      - name: List the e2e test files this PR added or modified
+        id: changed
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" \
+            --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \
+            | grep -Fxq "${HEAD_SHA}"
+          files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \
+            --jq '.[] | select(.status != "removed") | .filename')"
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
+          tests="$(printf '%s\n' "${files}" \
+            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)"
+          echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
+          if [ -n "${tests}" ]; then
+            echo "any=true" >> "${GITHUB_OUTPUT}"
+            echo "selected e2e tests: ${tests}"
+          else
+            echo "any=false" >> "${GITHUB_OUTPUT}"
+            echo "no changed e2e test files supported by this stack; nothing to run"
+          fi
+
+  run:
+    name: Run changed e2e tests against the stage-mirror stack
+    needs: detect
+    if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    environment: e2e-changed
+    permissions:
+      contents: read
+      id-token: write
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: litellm
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U litellm"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+      jaeger:
+        image: jaegertracing/jaeger:2.10.0
+        ports:
+          - 4318:4318
+          - 16686:16686
+    steps:
+      - name: Validate configuration
+        env:
+          ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+        run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; }
+
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.13"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen \
+            --extra proxy --extra proxy-runtime --extra extra_proxy \
+            --extra semantic-router --extra bedrock-realtime \
+            --group ci --group proxy-dev --group e2e-dev
+          uv pip install "pipecat-ai[openai]==1.4.0"
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Install Playwright chromium
+        run: uv run --no-sync playwright install --with-deps chromium
+
+      - name: Configure AWS credentials
+        id: aws
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+        with:
+          role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+          aws-region: us-east-1
+          role-session-name: litellm-e2e-changed-${{ github.run_id }}
+          role-duration-seconds: 900
+          output-env-credentials: false
+          output-credentials: true
+
+      - name: Fetch provider credentials from AWS Secrets Manager
+        env:
+          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+          umask 077
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \
+            --query SecretString --output text \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \
+            --query SecretString --output text \
+            | jq -R -s '{"LITELLM_LICENSE": .}' \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+
+      - name: Boot the stage-mirror stack
+        id: boot
+        run: |
+          umask 077
+          if ! bash .github/e2e-stack/up.sh > "${RUNNER_TEMP}/e2e-boot.log" 2>&1; then
+            echo "::error::stage-mirror stack failed to boot; raw logs are not published"
+            exit 1
+          fi
+
+      - name: Export stack environment
+        run: |
+          master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)"
+          echo "::add-mask::${master_key}"
+          cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}"
+
+      - name: Run the selected tests three times
+        env:
+          TESTS: ${{ needs.detect.outputs.tests }}
+          E2E_FIXTURE_MODE: live
+        run: |
+          umask 077
+          read -r -a test_files <<< "${TESTS}"
+          for pass in 1 2 3; do
+            report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
+            log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
+            echo "::group::pass ${pass} of 3"
+            set +e
+            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
+              -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
+            status=$?
+            uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
+            verified=$?
+            set -e
+            grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
+            echo "::endgroup::"
+            if [ "${status}" = "5" ]; then
+              echo "::error::the selected files collected no runnable tests, so nothing was verified"
+              exit 1
+            fi
+            if [ "${status}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 failed with exit code ${status}"
+              exit "${status}"
+            fi
+            if [ "${verified}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 did not verify every selected file"
+              exit 1
+            fi
+            echo "pass ${pass} of 3 passed"
+          done
+
+      - name: Stop the stack
+        if: always() && steps.boot.outcome != 'skipped'
+        run: bash .github/e2e-stack/down.sh
+
+      - name: Remove credentials and raw output
+        if: always()
+        run: |
+          rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
+          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack"
+
+  gate:
+    name: e2e-changed-tests
+    needs: [detect, run]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Require three successful passes when tests changed
+        env:
+          DETECT_RESULT: ${{ needs.detect.result }}
+          ANY_TESTS: ${{ needs.detect.outputs.any }}
+          RUN_RESULT: ${{ needs.run.result }}
+        run: |
+          if [ "${DETECT_RESULT}" != "success" ]; then
+            echo "::error::changed-test detection did not succeed"
+            exit 1
+          fi
+          if [ "${ANY_TESTS}" = "false" ]; then
+            echo "no changed e2e test files supported by this stack; nothing to run"
+            exit 0
+          fi
+          if [ "${ANY_TESTS}" != "true" ] || [ "${RUN_RESULT}" != "success" ]; then
+            echo "::error::selected e2e tests require an approved, successful run; fork PRs must run from a reviewed same-repository branch"
+            exit 1
+          fi

.github/workflows/test-linting.yml

@@ -67,6 +67,17 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-lint-
+
       - name: Clean Python cache
         if: steps.changes.outputs.decision != 'skip'
         run: |
@@ -78,6 +89,10 @@ jobs:
         run: |
           uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1)
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-litellm-ui-build.yml

@@ -19,9 +19,6 @@ jobs:
   build-ui:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    defaults:
-      run:
-        working-directory: ui/litellm-dashboard
 
     steps:
       - name: Checkout repository
@@ -35,18 +32,11 @@ jobs:
         with:
           category: ui
 
-      - name: Setup Node.js
+      # Built through the image stage rather than the checkout, because the
+      # stage copies ui/litellm-dashboard/ alone: an import reaching above the
+      # dashboard root resolves in a checkout and fails in every image we ship.
+      # Dockerfile, docker/Dockerfile.non_root and ui/Dockerfile share this
+      # stage verbatim, so building one covers all three.
+      - name: Build the dashboard as the shipped images build it
         if: steps.changes.outputs.decision != 'skip'
-        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
-        with:
-          node-version-file: ui/litellm-dashboard/.nvmrc
-          cache: "npm"
-          cache-dependency-path: ui/litellm-dashboard/package-lock.json
-
-      - name: Install dependencies
-        if: steps.changes.outputs.decision != 'skip'
-        run: npm ci
-
-      - name: Build
-        if: steps.changes.outputs.decision != 'skip'
-        run: npm run build
+        run: docker build --target ui-builder -f Dockerfile .

.github/workflows/test-litellm-ui-unit.yml

@@ -12,6 +12,7 @@ on:
       - "litellm_**"
   push:
     branches:
+      - main
       - litellm_internal_staging
 
 concurrency:

.github/workflows/test-mcp.yml

@@ -47,6 +47,10 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |
@@ -56,4 +60,4 @@ jobs:
       - name: Run MCP tests
         if: steps.changes.outputs.decision != 'skip'
         run: |
-          uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5
+          uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml --durations=5

.github/workflows/test-model-map.yml

@@ -1,37 +0,0 @@
-name: Validate model_prices_and_context_window.json
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  validate-model-prices-json:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Validate model_prices_and_context_window.json
-        run: |
-          jq empty model_prices_and_context_window.json
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Check model_prices_and_context_window.schema.json is in sync
-        run: |
-          uv run --frozen python ci_cd/generate_model_prices_schema.py --check

.github/workflows/test-postgres.yml

@@ -0,0 +1,145 @@
+name: "Postgres Tests"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  postgres:
+    name: ${{ matrix.shard }}
+    runs-on: ubuntu-latest
+    timeout-minutes: ${{ matrix.job-timeout-minutes }}
+    permissions:
+      contents: read
+
+    services:
+      postgres:
+        image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: litellm_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - shard: proxy-behavior
+            test-path: "tests/proxy_behavior"
+            seed: db-push
+            workers: 0
+            timeout-minutes: 25
+            job-timeout-minutes: 50
+
+          - shard: proxy-security
+            test-path: "tests/proxy_security_tests"
+            seed: db-push
+            workers: 0
+            timeout-minutes: 15
+            job-timeout-minutes: 40
+
+          - shard: schema-migration
+            test-path: "tests/proxy_migration_tests"
+            seed: none
+            workers: 0
+            timeout-minutes: 20
+            job-timeout-minutes: 45
+
+    env:
+      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        timeout-minutes: 3
+        with:
+          persist-credentials: false
+
+      - name: Detect relevant changes
+        id: changes
+        timeout-minutes: 2
+        uses: ./.github/actions/detect-changes
+
+      - name: Set up Python
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-postgres-
+
+      - name: Install dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 12
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
+
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Seed database schema
+        if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
+        timeout-minutes: 10
+        run: |
+          uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
+
+      - name: Run tests
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: ${{ matrix.timeout-minutes }}
+        env:
+          TEST_PATH: ${{ matrix.test-path }}
+          WORKERS: ${{ matrix.workers }}
+        run: |
+          if [ "${WORKERS}" = "0" ]; then
+            uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
+          else
+            uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
+          fi

.github/workflows/test-redis-compat.yml

@@ -0,0 +1,77 @@
+name: "Unit Tests: Redis Client Version Compatibility"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+    paths:
+      - "litellm/_redis.py"
+      - "litellm/_redis_credential_provider.py"
+      - "tests/test_litellm/test_redis.py"
+      - "tests/test_litellm/caching/test_redis_connection_pool.py"
+      - ".github/workflows/test-redis-compat.yml"
+      - "pyproject.toml"
+      - "uv.lock"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  redis-compat:
+    name: "redis-py ${{ matrix.redis-version }}"
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # 5.3.1 is the version pinned in uv.lock (redisvl caps it below 6); the
+        # newer legs prove the inspect.signature introspection in litellm/_redis.py
+        # keeps extracting kwargs on the redis-py releases people actually run now.
+        # Only the exact release 6.0.0 is skipped: rq (pulled by the proxy extra)
+        # specifies `redis != 6`, which excludes 6.0.0 alone, so 6.4.0 stands in
+        # for the 6.x line.
+        redis-version: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"]
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+
+      - name: Pin redis-py to the matrix version
+        env:
+          REDIS_VERSION: ${{ matrix.redis-version }}
+        run: |
+          uv pip install "redis==${REDIS_VERSION:?}"
+          uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)"
+
+      - name: Run redis unit tests
+        run: |
+          uv run --no-sync pytest \
+            tests/test_litellm/test_redis.py \
+            tests/test_litellm/caching/test_redis_connection_pool.py \
+            --tb=short -vv \
+            --reruns 2 \
+            --reruns-delay 1 \
+            --durations=20

.github/workflows/test-rust.yml

@@ -4,6 +4,13 @@ on:
   push:
     paths:
       - "litellm-rust/**"
+      - ".cargo/**"
+      - "pyproject.toml"
+      - "rust-toolchain.toml"
+      - ".github/actions/setup-uv-with-retries/**"
+      - ".github/scripts/smoke_test_native_wheel.py"
+      - ".github/scripts/verify_linux_native_wheel.py"
+      - "tests/test_litellm/rust_bridge/native_route_wheel_test.py"
       - ".github/workflows/test-rust.yml"
   pull_request:
     branches:
@@ -13,6 +20,13 @@ on:
       - "litellm_**"
     paths:
       - "litellm-rust/**"
+      - ".cargo/**"
+      - "pyproject.toml"
+      - "rust-toolchain.toml"
+      - ".github/actions/setup-uv-with-retries/**"
+      - ".github/scripts/smoke_test_native_wheel.py"
+      - ".github/scripts/verify_linux_native_wheel.py"
+      - "tests/test_litellm/rust_bridge/native_route_wheel_test.py"
       - ".github/workflows/test-rust.yml"
 
 permissions:
@@ -22,50 +36,89 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  CARGO_TERM_COLOR: always
+
 jobs:
-  rust-checks:
-    name: rustfmt, clippy, test
+  rust-lint:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     defaults:
       run:
         working-directory: litellm-rust
-    env:
-      CARGO_TERM_COLOR: always
-
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           persist-credentials: false
 
-      - name: Set up Rust
-        run: |
-          rustup toolchain install stable --profile minimal --component clippy,rustfmt
-          rustup default stable
+      - run: rustup toolchain install --no-self-update
 
-      - name: Cache Cargo registry and target
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      - run: cargo fmt --check
+
+      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             litellm-rust/target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('litellm-rust/Cargo.lock') }}
+          key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }}
           restore-keys: |
-            ${{ runner.os }}-cargo-
+            ${{ runner.os }}-cargo-${{ github.job }}-
 
-      - name: Check Rust formatting
-        run: cargo fmt --check
+      - run: cargo clippy --workspace --all-targets --locked -- -D warnings
 
-      - name: Run Clippy
-        run: cargo clippy --workspace --all-targets --locked -- -D warnings
+      - run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
 
-      - name: Run Clippy with Bedrock auth
-        run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
+      - run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings
 
-      - name: Run Rust tests
-        run: cargo test --workspace --locked
+  rust-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
 
-      - name: Run core tests with Bedrock auth
-        run: cargo test -p litellm-core --features bedrock-auth --locked
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - run: rustup toolchain install --no-self-update
+
+      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            litellm-rust/target
+          key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-${{ github.job }}-
+
+      - run: cargo test --workspace --locked
+        working-directory: litellm-rust
+
+      - run: cargo test -p litellm-core --features bedrock-auth --locked
+        working-directory: litellm-rust
+
+      - run: cargo test -p litellm-ai-gateway --features server --locked
+        working-directory: litellm-rust
+
+      - run: uv build --wheel --out-dir dist
+
+      - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl
+        env:
+          RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - run: python tests/test_litellm/rust_bridge/native_route_wheel_test.py dist/*.whl
+
+      - run: >-
+          uv build --wheel --out-dir panic-dist
+          --config-setting "maturin.build-args=--features panic-test,extension-module"
+
+      - run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl

.github/workflows/test-terraform-modules.yml

@@ -4,6 +4,7 @@ on:
   push:
     paths:
       - "terraform/litellm/aws/**"
+      - "terraform/litellm/gcp/**"
       - ".github/workflows/test-terraform-modules.yml"
   pull_request:
     branches:
@@ -13,6 +14,7 @@ on:
       - "litellm_**"
     paths:
       - "terraform/litellm/aws/**"
+      - "terraform/litellm/gcp/**"
       - ".github/workflows/test-terraform-modules.yml"
 
 permissions:
@@ -52,3 +54,32 @@ jobs:
       # Plan-only, mock_provider-backed: no AWS credentials, no API calls.
       - name: test
         run: terraform test
+
+  gcp-module:
+    name: fmt, validate, test (gcp)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: terraform/litellm/gcp
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
+        with:
+          terraform_version: 1.13.3
+          terraform_wrapper: false
+
+      - name: fmt
+        run: terraform fmt -recursive -check -diff
+
+      - name: init
+        run: terraform init -backend=false -input=false
+
+      - name: validate
+        run: terraform validate
+
+      - name: test
+        run: terraform test

.github/workflows/test-terraform-provider.yml

@@ -88,6 +88,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
@@ -111,4 +114,4 @@ jobs:
 
       - name: Audit provider endpoints against the schema
         working-directory: terraform/provider
-        run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json"
+        run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json" -coverage-allowlist ./tools/endpointaudit/coverage_allowlist.txt

.github/workflows/test-unit-documentation.yml

@@ -67,6 +67,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-unit.yml

@@ -55,7 +55,7 @@ jobs:
             workers: 2
             reruns: 1
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: enterprise-routing
             artifact-name: enterprise-routing
@@ -67,7 +67,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: integrations
             artifact-name: integrations
@@ -75,7 +75,7 @@ jobs:
             workers: 2
             reruns: 3
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: Vertex AI
             artifact-name: llm-vertex-ai
@@ -83,7 +83,7 @@ jobs:
             workers: 1
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: All Other Providers
             artifact-name: llm-other-providers
@@ -91,7 +91,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: misc
             artifact-name: misc
@@ -103,6 +103,7 @@ jobs:
               tests/test_litellm/completion_extras
               tests/test_litellm/compression
               tests/test_litellm/containers
+              tests/test_litellm/endpoints
               tests/test_litellm/experimental_mcp_client
               tests/test_litellm/models
               tests/test_litellm/repositories
@@ -115,6 +116,7 @@ jobs:
               tests/test_litellm/rerank_api
               tests/test_litellm/rust_bridge
               tests/test_litellm/sandbox
+              tests/test_litellm/skills
               tests/test_litellm/test_router
               tests/test_litellm/vector_stores
               tests/test_litellm/videos
@@ -122,7 +124,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: proxy-auth
             artifact-name: proxy-auth
@@ -134,13 +136,14 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: proxy-endpoints
             artifact-name: proxy-endpoints
             test-path: >-
               tests/test_litellm/proxy/analytics_endpoints
               tests/test_litellm/proxy/management_endpoints
+              tests/test_litellm/proxy/list_api
               tests/test_litellm/proxy/memory
               tests/test_litellm/proxy/guardrails
               tests/test_litellm/proxy/management_helpers
@@ -148,6 +151,7 @@ jobs:
               tests/test_litellm/proxy/google_endpoints
               tests/test_litellm/proxy/openai_files_endpoint
               tests/test_litellm/proxy/batches_endpoints
+              tests/test_litellm/proxy/container_endpoints
               tests/test_litellm/proxy/fine_tuning_endpoints
               tests/test_litellm/proxy/vector_store_files_endpoints
               tests/test_litellm/proxy/video_endpoints
@@ -164,14 +168,15 @@ jobs:
               tests/test_litellm/proxy/public_endpoints
               tests/test_litellm/proxy/prompts
               tests/test_litellm/proxy/rag_endpoints
+              tests/test_litellm/proxy/rerank_endpoints
               tests/test_litellm/proxy/realtime_endpoints
               tests/test_litellm/proxy/ui_crud_endpoints
               tests/test_litellm/proxy/config_resolvers
               tests/test_litellm/proxy/utils
-            workers: 2
+            workers: 4
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: proxy-server
             artifact-name: proxy-server
@@ -179,7 +184,7 @@ jobs:
             workers: 4
             reruns: 2
             timeout-minutes: 60
-            job-timeout-minutes: 95
+            job-timeout-minutes: 100
 
           - shard: proxy-infra
             artifact-name: proxy-infra
@@ -195,10 +200,39 @@ jobs:
               tests/test_litellm/proxy/types_utils
               tests/test_litellm/proxy/logging_endpoints
               tests/test_litellm/proxy/test_*.py
+            workers: 4
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: caching-local
+            artifact-name: caching-local
+            test-path: >-
+              tests/local_testing/test_cache_preset_key.py
+              tests/local_testing/test_caching_handler.py
+              tests/local_testing/test_prompt_caching.py
+              tests/local_testing/test_responses_stream_cache_keys.py
+              tests/local_testing/test_unit_test_caching.py
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
+
+          - shard: proxy-extras
+            artifact-name: proxy-extras
+            test-path: "tests/litellm-proxy-extras"
+            workers: 2
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: enterprise-package
+            artifact-name: enterprise-package
+            test-path: "tests/enterprise"
+            workers: 4
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
 
           - shard: responses-caching-types
             artifact-name: responses-caching-types
@@ -209,7 +243,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
     uses: ./.github/workflows/_test-unit-base.yml
     with:
       test-path: ${{ matrix.test-path }}

.github/workflows/weekly_load_anomaly.yml

@@ -47,6 +47,9 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy

.gitignore

@@ -3,6 +3,8 @@
 tests/e2e/.fixtures/
 .venv-typecheck
 .venv_policy_test
+.venv-mutmut
+mutants/
 .env
 .claude
 CLAUDE.local.md

ARCHITECTURE.md

@@ -149,7 +149,7 @@ graph TD
 | `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user |
 | `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation |
 | `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation |
-| `litellm_skills` | `proxy/hooks/skills_injection.py` | Skills injection |
+| `litellm_skills` | `proxy/hooks/litellm_skills/main.py` | Skills injection |
 
 To add a new proxy hook, implement `CustomLogger` and register in `PROXY_HOOKS`.
 
@@ -220,20 +220,20 @@ graph LR
 | Job | Interval | Purpose | Key Files |
 |-----|----------|---------|-----------|
 | `update_spend` | 60s | Batch write spend logs to PostgreSQL | `proxy/db/db_spend_update_writer.py` |
-| `reset_budget` | 10-12min | Reset budgets for keys/users/teams | `proxy/management_helpers/budget_reset_job.py` |
+| `reset_budget` | 10-12min | Reset budgets for keys/users/teams | `proxy/common_utils/reset_budget_job.py` |
 | `add_deployment` | 10s | Sync new model deployments from DB | `proxy/proxy_server.py` (`ProxyConfig`) |
-| `cleanup_old_spend_logs` | cron/interval | Delete old spend logs | `proxy/management_helpers/spend_log_cleanup.py` |
-| `check_batch_cost` | 30min | Calculate costs for batch jobs | `proxy/management_helpers/check_batch_cost_job.py` |
-| `check_responses_cost` | 30min | Calculate costs for responses API | `proxy/management_helpers/check_responses_cost_job.py` |
-| `process_rotations` | 1hr | Auto-rotate API keys | `proxy/management_helpers/key_rotation_manager.py` |
+| `cleanup_old_spend_logs` | cron/interval | Delete old spend logs | `proxy/db/db_transaction_queue/spend_log_cleanup.py` |
+| `check_batch_cost` | 30min | Calculate costs for batch jobs | `enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py` |
+| `check_responses_cost` | 30min | Calculate costs for responses API | `enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py` |
+| `process_rotations` | 1hr | Auto-rotate API keys | `proxy/common_utils/key_rotation_manager.py` |
 | `_run_background_health_check` | continuous | Health check model deployments | `proxy/proxy_server.py` |
 | `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
 | `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
 
 **Cost Attribution Flow:**
 1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes
-2. `update_response_metadata()` (`llm_response_utils/response_metadata.py`) is called
-3. `logging_obj._response_cost_calculator()` (`litellm_logging.py`) calculates cost via `litellm.completion_cost()` (`cost_calculator.py`)
+2. `update_response_metadata()` (`litellm_core_utils/llm_response_utils/response_metadata.py`) is called
+3. `logging_obj._response_cost_calculator()` (`litellm_core_utils/litellm_logging.py`) calculates cost via `litellm.completion_cost()` (`cost_calculator.py`)
 4. Cost is stored in `response._hidden_params["response_cost"]`
 5. `proxy/common_request_processing.py` extracts cost from `hidden_params` and adds to response headers (`x-litellm-response-cost`)
 6. `logging_obj.async_success_handler()` triggers callbacks including `_ProxyDBLogger.async_log_success_event()`

CLAUDE.md

@@ -23,11 +23,13 @@ When adding new features, add meaningful tests. Don't add tests that don't check
 
 Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression)
 
+Never test structure of code only function of it
+
 `tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
 
 End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`
 
-When creating PRs, don't set base to `main`. `litellm_internal_staging` is the default base branch and serves that purpose for both internal and external / OSS contributions
+When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
 
 When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
 
@@ -35,21 +37,22 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
 
-Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
+Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
 
-If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
+If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
 - don't use emojis
 - don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
-- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
+- unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
+- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
 
 Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
 
 Python max line length is 120, not 88
 
-When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing
+Never edit or commit `ruff-strict-budget.json`, `type-discipline-budget.json`, `basedpyright-code-budget.json`, or `test-quality-budget.json` on a PR branch, and don't run `make lint-budget-update` there. A scheduled Devin automation lowers the limits on the default branch in its own PR by exactly what landed since the last ratchet, so concurrent PRs don't fight over the same `"limit"` lines. Keep the hosted automation's target in sync when the repository default changes. If your branch already carries a budget edit, drop it before opening the PR
 
 `make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
 
@@ -65,7 +68,9 @@ Commit and push your work when you're done without asking
 
 When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
 
-If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
+Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
+
+If you're an internal contributor, when creating a new PR, the typical flow is to branch off the repository's current default branch and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
 
 Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch
 
@@ -79,6 +84,8 @@ Do not put names of customers or customer company names in code, PR descriptions
 
 CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
 
+Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
+
 Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
 
 - Composition over inheritance

CONTRIBUTING.md

@@ -315,10 +315,12 @@ Ensure the UI builds successfully before submitting your PR:
 npm run build
 ```
 
+Local lint and budget checks follow origin's current default branch. They refresh it from the remote instead of trusting cached `origin/HEAD`. For an intentional comparison against another branch or commit, use `make check BASE_REF=<ref>` or the standalone gate's `--base <ref>` option. An explicit ref can also be used offline once it has been fetched locally. Without an override, unavailable remote metadata stops the check
+
 ## Submitting Your PR
 
 1. **Push your branch**: `git push origin your-feature-branch`
-2. **Create a PR**: Go to GitHub and open a pull request against [`litellm_internal_staging`](https://github.com/BerriAI/litellm/tree/litellm_internal_staging), which is the default base branch. Do not target `main`.
+2. **Create a PR**: Go to GitHub and open a pull request against the repository's current default branch. Run `python3 scripts/default_branch.py --branch` to check its name
 3. **Fill out the PR template**: Provide clear description of changes
 4. **Wait for review**: Maintainers will review and provide feedback
 5. **Address feedback**: Make requested changes and push updates

Dockerfile

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -40,8 +40,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 RUN apk add --no-cache \
     bash \
     gcc \
-    python3 \
-    python3-dev \
+    python-3.13 \
+    python-3.13-dev \
     rust \
     openssl \
     openssl-dev \
@@ -51,6 +51,7 @@ RUN apk add --no-cache \
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
@@ -65,7 +66,9 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --extra bedrock-realtime \
+    --extra mongodb \
+    --python python3.13
 
 # Copy full source tree
 COPY . .
@@ -86,7 +89,9 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --extra bedrock-realtime \
+    --extra mongodb \
+    --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -100,8 +105,14 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
 USER root
 
+# The base image only configures Chainguard's authenticated apk repo, which
+# requires an enterprise subscription. Add the public Wolfi repo so `apk add`
+# also works for anyone installing extra packages into a running container.
+# https://github.com/BerriAI/litellm/issues/33518
+RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
+
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
+RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
 
 WORKDIR /app
 ENV PATH="/app/.venv/bin:${PATH}" \

Makefile

@@ -34,7 +34,7 @@ help:
 	@echo "  make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed"
 	@echo "  make lint-format        - Check ruff format formatting (matches CI)"
 	@echo "  make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
-	@echo "  make lint-gate        - Strict ruff gate in CI-parity mode (fetches staging, simulates the merge)"
+	@echo "  make lint-gate        - Strict ruff gate in CI-parity mode (fetches the default branch, simulates the merge)"
 	@echo "  make lint-ruff-budget-update - Ratchet ruff-strict-budget.json limits down by what this branch fixed"
 	@echo "  make lint-test-quality  - Gate the test suite against test-quality-budget.json"
 	@echo "  make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + test quality + basedpyright)"
@@ -60,6 +60,9 @@ help:
 
 UV := uv
 UV_RUN := $(UV) run --no-sync
+BASE_REF ?=
+export BASE_REF
+RESOLVE_BASE = python3 scripts/default_branch.py --base "$(BASE_REF)"
 
 # Machine-wide slot queue for the heavy targets below; python3 + stdlib only, so
 # it runs before any venv exists. See scripts/gate_slot_lock.py.
@@ -67,7 +70,7 @@ GATE_SLOT_LOCK := python3 scripts/gate_slot_lock.py
 
 LINT_DEP_INSTALL ?= install-dev
 LINT_E2E_DEP_INSTALL ?= lint-install
-LINT_DEP_BASE ?= lint-fetch-base
+LINT_DEP_BASE ?=
 LINT_JOBS := $(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)
 LINT_OUTPUT_SYNC := $(if $(filter output-sync,$(.FEATURES)),--output-sync=target,)
 
@@ -130,10 +133,8 @@ format: install-dev
 format-check: install-dev
 	cd litellm && $(UV_RUN) ruff format --check --exclude '/enterprise/' . && cd ..
 
-# Single fetch of the PR base so the delta-based gates below share one network round
-# trip instead of each re-fetching when chained from `lint`.
 lint-fetch-base:
-	git fetch origin litellm_internal_staging
+	@$(RESOLVE_BASE)
 
 # Mirror test-linting.yml's lint job environment: the proxy-dev group plus a generated
 # Prisma client, so `basedpyright tests/e2e` resolves the same modules CI does. The
@@ -150,7 +151,9 @@ lint-install:
 # recursively, so 'litellm/*.py' covers nested modules and the top-level files that
 # CI's 'litellm/**/*.py' skips, which makes this target a superset of the CI step.
 lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
-	@files=$$(git diff --name-only --diff-filter=ACMR origin/litellm_internal_staging...HEAD -- 'litellm/*.py' | grep -v '^litellm/enterprise/' || true); \
+	@base_ref=$$($(RESOLVE_BASE)) && \
+	changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
+	files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \
 	if [ -z "$$files" ]; then \
 		echo "No changed litellm Python files to format-check."; \
 	else \
@@ -167,7 +170,9 @@ lint-ruff: $(LINT_DEP_INSTALL)
 # https://github.com/astral-sh/ruff/discussions/10977
 # https://github.com/astral-sh/ruff/discussions/4049
 lint-format-changed: install-dev
-	@git diff origin/main --unified=0 --no-color -- '*.py' | \
+	@base_ref=$$($(RESOLVE_BASE)) && \
+	diff=$$(git diff "$$base_ref" --unified=0 --no-color -- '*.py') && \
+	printf '%s\n' "$$diff" | \
 	perl -ne '\
 		if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
 		if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
@@ -182,20 +187,22 @@ lint-format-changed: install-dev
 		done
 
 lint-ruff-dev: install-dev
-	@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
+	@base_ref=$$($(RESOLVE_BASE)) || exit $$?; \
+	tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
 	cd litellm && \
 	($(UV_RUN) ruff check . --output-format=pylint || true) > "$$tmpfile" && \
-	$(UV_RUN) diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
+	$(UV_RUN) diff-quality --violations=pylint "$$tmpfile" --compare-branch="$$base_ref" && \
 	cd .. ; \
 	rm -f "$$tmpfile"
 
 lint-ruff-FULL-dev: install-dev
-	@files=$$(git diff --name-only origin/main -- '*.py'); \
+	@base_ref=$$($(RESOLVE_BASE)) && \
+	files=$$(git diff --name-only "$$base_ref" -- '*.py') || exit $$?; \
 	if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \
 	else echo "No changed .py files to check."; fi
 
 lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
-	$(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
+	$(UV_RUN) python scripts/type_check_gate.py --base "$(BASE_REF)"
 
 lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
 	$(UV_RUN) basedpyright tests/e2e
@@ -203,37 +210,37 @@ lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
 # Type-discipline budget (mutable collections / casts / type guards / kwargs /
 # unexplained suppressions), the test-linting.yml step `make lint` used to omit.
 lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
-	$(UV_RUN) python scripts/type_discipline_gate.py --base origin/litellm_internal_staging
+	$(UV_RUN) python scripts/type_discipline_gate.py --base "$(BASE_REF)"
 
 # Test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
 # litellm module-global mutation, credential-gated skips, conftest snapshot
 # inventory), counted across tests/ the same delta-vs-base way.
 lint-test-quality: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
-	$(UV_RUN) python scripts/test_quality_gate.py --base origin/litellm_internal_staging
+	$(UV_RUN) python scripts/test_quality_gate.py --base "$(BASE_REF)"
 
 # --update lowers each limit by what this branch fixed since its branch point, so
 # it needs the base ref fetched to resolve the merge-base.
-lint-basedpyright-budget-update: install-dev lint-fetch-base
-	$(UV_RUN) python scripts/type_check_gate.py --update
+lint-basedpyright-budget-update: install-dev
+	$(UV_RUN) python scripts/type_check_gate.py --update --base "$(BASE_REF)"
 
 lint-format: format-check
 
 lint-ruff-budget: install-dev
-	$(UV_RUN) python scripts/ruff_strict_gate.py
+	$(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)"
 
 # Strict gate, invoked the same way CI does in test-linting.yml so a local pass
 # means the CI check will pass too.
 lint-gate: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
-	$(UV_RUN) python scripts/ruff_strict_gate.py --base origin/litellm_internal_staging
+	$(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)"
 
-lint-ruff-budget-update: install-dev lint-fetch-base
-	$(UV_RUN) python scripts/ruff_strict_gate.py --update
+lint-ruff-budget-update: install-dev
+	$(UV_RUN) python scripts/ruff_strict_gate.py --update --base "$(BASE_REF)"
 
-lint-type-discipline-budget-update: install-dev lint-fetch-base
-	$(UV_RUN) python scripts/type_discipline_gate.py --update
+lint-type-discipline-budget-update: install-dev
+	$(UV_RUN) python scripts/type_discipline_gate.py --update --base "$(BASE_REF)"
 
-lint-test-quality-budget-update: install-dev lint-fetch-base
-	$(UV_RUN) python scripts/test_quality_gate.py --update
+lint-test-quality-budget-update: install-dev
+	$(UV_RUN) python scripts/test_quality_gate.py --update --base "$(BASE_REF)"
 
 # Ratchet all budgets in one shot (ruff strict + type-discipline + test quality + basedpyright)
 lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-test-quality-budget-update lint-basedpyright-budget-update
@@ -249,14 +256,15 @@ check-import-safety: $(LINT_DEP_INSTALL)
 # runs the diff-scoped ruff format check, whole-tree ruff check, the strict-rule /
 # type-discipline / basedpyright budgets as a delta vs the base, then the circular-import
 # and import-safety checks. Steps that compare against the base resolve it the same way CI
-# does (merge-base with origin/litellm_internal_staging). Setup (env sync, Prisma client,
+# does (merge-base with origin's current default branch). Setup (env sync, Prisma client,
 # base fetch) runs once up front; the checks themselves are independent, so a sub-make
 # fans them out with -j and the fast ones finish under basedpyright's shadow.
 lint:
 	@$(GATE_SLOT_LOCK) $(MAKE) lint-inner
 
-lint-inner: lint-install lint-fetch-base
-	$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks
+lint-inner: lint-install
+	@base_ref=$$($(RESOLVE_BASE)) && \
+	$(MAKE) BASE_REF="$$base_ref" -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks
 
 lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-test-quality lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
 

README.md

@@ -354,6 +354,8 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 | [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Predibase (`predibase`)](https://docs.litellm.ai/docs/providers/predibase) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Qwen AI Platform (`qwen_ai_platform`)](https://docs.litellm.ai/docs/providers/qwencloud) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  | ✅ |
+| [QwenCloud (`qwencloud`)](https://docs.litellm.ai/docs/providers/qwencloud) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  | ✅ |
 | [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) |  |  |  |  | ✅ |  |  |  |  |  |
 | [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

backend/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
@@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
 # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
 # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -46,7 +46,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
-        --python python3
+        --extra saml \
+        --python python3.13
 
 # Stage 2 — copy source and install the project + workspace members.
 COPY . .
@@ -57,7 +58,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
-        --python python3
+        --extra saml \
+        --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -71,7 +73,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
+      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done

basedpyright-code-budget.json

@@ -1,36 +1,36 @@
 {
   "reportAny": {
-    "limit": 19955
+    "limit": 13429
   },
   "reportArgumentType": {
-    "limit": 2566
+    "limit": 2198
   },
   "reportAssignmentType": {
-    "limit": 320
+    "limit": 319
   },
   "reportAttributeAccessIssue": {
-    "limit": 488
+    "limit": 480
   },
   "reportCallIssue": {
-    "limit": 114
+    "limit": 112
   },
   "reportConstantRedefinition": {
     "limit": 40
   },
   "reportDeprecated": {
-    "limit": 213
+    "limit": 209
   },
   "reportDuplicateImport": {
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 6049
+    "limit": 3369
   },
   "reportFunctionMemberAccess": {
     "limit": 7
   },
   "reportGeneralTypeIssues": {
-    "limit": 154
+    "limit": 101
   },
   "reportIncompatibleMethodOverride": {
     "limit": 56
@@ -42,22 +42,22 @@
     "limit": 12
   },
   "reportIndexIssue": {
-    "limit": 35
+    "limit": 24
   },
   "reportInvalidTypeForm": {
-    "limit": 35
+    "limit": 30
   },
   "reportInvalidTypeVarUse": {
-    "limit": 2
+    "limit": 1
   },
   "reportMatchNotExhaustive": {
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5663
+    "limit": 5570
   },
   "reportMissingTypeArgument": {
-    "limit": 15555
+    "limit": 15281
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1061
+    "limit": 0
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -84,46 +84,46 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1823
+    "limit": 1804
   },
   "reportRedeclaration": {
     "limit": 8
   },
   "reportReturnType": {
-    "limit": 213
+    "limit": 180
   },
   "reportTypedDictNotRequiredAccess": {
-    "limit": 26
+    "limit": 22
   },
   "reportUndefinedVariable": {
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44655
+    "limit": 44358
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 39011
+    "limit": 38271
   },
   "reportUnknownParameterType": {
-    "limit": 19885
+    "limit": 19584
   },
   "reportUnknownVariableType": {
-    "limit": 30569
+    "limit": 29814
   },
   "reportUnnecessaryCast": {
-    "limit": 117
+    "limit": 110
   },
   "reportUnnecessaryComparison": {
-    "limit": 699
+    "limit": 687
   },
   "reportUnnecessaryContains": {
-    "limit": 5
+    "limit": 4
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 836
+    "limit": 816
   },
   "reportUntypedBaseClass": {
     "limit": 0
@@ -135,12 +135,12 @@
     "limit": 21
   },
   "reportUnusedFunction": {
-    "limit": 139
+    "limit": 136
   },
   "reportUnusedImport": {
-    "limit": 545
+    "limit": 542
   },
   "reportUnusedVariable": {
-    "limit": 146
+    "limit": 137
   }
 }

ci_cd/cost_map_guard.py

@@ -0,0 +1,146 @@
+"""Guard the cost map on pull requests.
+
+Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file,
+and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named
+litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Final
+
+from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
+
+COST_MAP_PATH: Final = "model_prices_and_context_window.json"
+BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
+SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
+GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
+BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+
+CostMap = dict[str, object]
+
+
+@dataclass(frozen=True, slots=True)
+class Snapshot:
+    cost_map: str
+    backup: str
+    schema: str
+
+
+def _parse_object(text: str, path: str) -> CostMap | str:
+    try:
+        parsed: Final = json.loads(text)
+    except json.JSONDecodeError as error:
+        return f"{path} is not valid JSON: {error}"
+    return parsed if isinstance(parsed, dict) else f"{path} must be a JSON object at the root"
+
+
+def _rendered_schema(cost_map: CostMap) -> str:
+    try:
+        return render(build_schema(cost_map))
+    except SystemExit as error:
+        return str(error)
+
+
+def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]:
+    schema_text: Final = _rendered_schema(head_map)
+    if not schema_text.startswith("{"):
+        return (schema_text,)
+    backup_failure: Final = (
+        ()
+        if head.backup == head.cost_map
+        else (f"{BACKUP_PATH} differs from {COST_MAP_PATH}; copy the root file over it",)
+    )
+    schema_failure: Final = (
+        ()
+        if head.schema == schema_text
+        else (
+            f"{SCHEMA_PATH} is out of sync with {COST_MAP_PATH}; "
+            "run `python ci_cd/generate_model_prices_schema.py` and commit the result",
+        )
+    )
+    return (
+        *backup_failure,
+        *schema_failure,
+        *(
+            f"{COST_MAP_PATH} does not validate against its schema: {error}"
+            for error in validation_errors(head_map, json.loads(schema_text))[:20]
+        ),
+    )
+
+
+def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
+    return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
+
+
+def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
+    base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
+    if isinstance(base_map, str):
+        return (f"merge base: {base_map}",)
+    base_entries: Final = _entries(base_map)
+    head_entries: Final = _entries(head_map)
+    removed_fields: Final = tuple(
+        f"{key}.{field}"
+        for key, entry in base_entries.items()
+        if key in head_entries
+        for field in entry
+        if field not in head_entries[key]
+    )
+    return (
+        *(
+            f"bot PRs may only change the cost map files, not {path}"
+            for path in changed_files
+            if path not in GUARDED_PATHS
+        ),
+        *(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
+        *(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),
+        *(
+            f"bot PRs may not change {key}"
+            for key in sorted(SPECIAL_ROOT_KEYS)
+            if base_map.get(key) != head_map.get(key)
+        ),
+    )
+
+
+def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
+    head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
+    if isinstance(head_map, str):
+        return (head_map,)
+    return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
+
+
+def _git(*args: str) -> str:
+    result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True)
+    return result.stdout if result.returncode == 0 else ""
+
+
+def snapshot(revision: str) -> Snapshot:
+    return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS))
+
+
+def main(argv: Sequence[str]) -> int:
+    parser: Final = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--base", required=True, help="merge base of the pull request")
+    parser.add_argument("--head", required=True, help="head commit of the pull request")
+    parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
+    args: Final = parser.parse_args(argv)
+    bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
+    changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines())
+    failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot)
+    contract: Final = "bot contract enforced" if bot else "human PR, file checks only"
+    if failures:
+        print(f"cost map guard failed ({contract}):")
+        print("\n".join(f"- {failure}" for failure in failures))
+        return 1
+    print(f"cost map guard passed ({contract})")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

ci_cd/generate_model_prices_schema.py

@@ -27,6 +27,7 @@ EXTRA_BOOLEAN_KEYS = frozenset(
         "uses_embed_content",
         "use_openai_responses_path",
         "bedrock_converse_supports_strict_tools",
+        "thinking_always_on",
     }
 )
 
@@ -57,6 +58,11 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
 }
 
 ARRAY_KEYS: dict[str, JsonSchema] = {
+    "supported_audio_formats": {
+        "type": "array",
+        "description": "Audio container formats the model can return.",
+        "items": {"type": "string", "enum": ["mp3", "wav"]},
+    },
     "supported_endpoints": {
         "type": "array",
         "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
@@ -72,6 +78,11 @@ ARRAY_KEYS: dict[str, JsonSchema] = {
         "description": "Output modalities the model can produce.",
         "items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
     },
+    "reasoning_effort_levels": {
+        "type": "array",
+        "description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
+        "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
+    },
     "supported_regions": {
         "type": "array",
         "description": "Cloud regions the model is available in ('global' or region ids).",
@@ -156,6 +167,9 @@ COST_DESCRIPTIONS: dict[str, str] = {
     "input_cost_per_token": "USD per prompt token.",
     "output_cost_per_token": "USD per generated token.",
     "output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
+    "google_maps_grounding_cost_per_query": (
+        "USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
+    ),
     "cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
     "cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
     "input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
@@ -211,8 +225,21 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
             "description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
             "enum": ["low", "medium", "high", "max", "xhigh"],
         },
+        "default_reasoning_effort": {
+            "type": "string",
+            "description": (
+                "Reasoning effort the provider applies when the request omits reasoning_effort. "
+                "Gates whether a non-default temperature or the top_p/logprobs sampling params are "
+                "accepted, which hold only when the effort resolves to 'none'."
+            ),
+            "enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
+        },
         "comment": STRING,
         "audio_transcription_config": STRING,
+        "vertex_ai_audio_api": {
+            "type": "string",
+            "enum": ["lyria_predict", "lyria_interactions"],
+        },
     }
 
 

ci_cd/run_migration.py

@@ -6,12 +6,9 @@ import subprocess
 import sys
 from datetime import datetime
 from pathlib import Path
-
-import testing.postgresql
-
+from typing import Final
 
 DESTRUCTIVE_PATTERN = re.compile(r"\bDROP\s+(COLUMN|TABLE|INDEX)\b", re.IGNORECASE)
-DEFAULT_BASE_BRANCH = "litellm_internal_staging"
 
 
 def _find_destructive_statements(sql: str) -> list:
@@ -94,31 +91,57 @@ def _print_stale_branch_refusal(base_branch: str, behind: int) -> None:
     print(banner, file=out)
 
 
-def _check_branch_freshness(root_dir: Path, base_branch: str) -> None:
+def _default_base_branch(root_dir: Path) -> str:
+    try:
+        result: Final = subprocess.run(
+            [
+                sys.executable,
+                str(Path(__file__).resolve().parents[1] / "scripts" / "default_branch.py"),
+                "--repo-root",
+                str(root_dir),
+                "--branch",
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+            timeout=90,
+        )
+    except (OSError, subprocess.SubprocessError) as exc:
+        _print_freshness_failure(
+            "default branch",
+            "Could not discover origin's default branch. Pass --base-branch <name> to choose one.",
+            exc.stderr if isinstance(exc, subprocess.CalledProcessError) else str(exc),
+        )
+        sys.exit(3)
+    return result.stdout.strip()
+
+
+def _check_branch_freshness(root_dir: Path, base_branch: str | None = None) -> None:
     """Fetch origin/<base_branch> and exit 3 if HEAD is behind it."""
+    resolved_branch: Final = base_branch or _default_base_branch(root_dir)
     cwd = str(root_dir)
     try:
         subprocess.run(
-            ["git", "fetch", "origin", base_branch],
+            ["git", "fetch", "origin", f"+refs/heads/{resolved_branch}:refs/remotes/origin/{resolved_branch}"],
             check=True,
             capture_output=True,
             text=True,
             cwd=cwd,
         )
     except FileNotFoundError:
-        _print_freshness_failure(base_branch, "git executable not found on PATH")
+        _print_freshness_failure(resolved_branch, "git executable not found on PATH")
         sys.exit(3)
     except subprocess.CalledProcessError as e:
         _print_freshness_failure(
-            base_branch,
-            f"`git fetch origin {base_branch}` failed",
+            resolved_branch,
+            f"`git fetch origin {resolved_branch}` failed",
             e.stderr or "",
         )
         sys.exit(3)
 
     try:
         result = subprocess.run(
-            ["git", "rev-list", "--count", f"HEAD..origin/{base_branch}"],
+            ["git", "rev-list", "--count", f"HEAD..origin/{resolved_branch}"],
             check=True,
             capture_output=True,
             text=True,
@@ -127,23 +150,23 @@ def _check_branch_freshness(root_dir: Path, base_branch: str) -> None:
         behind = int(result.stdout.strip())
     except subprocess.CalledProcessError as e:
         _print_freshness_failure(
-            base_branch,
-            f"`git rev-list HEAD..origin/{base_branch}` failed",
+            resolved_branch,
+            f"`git rev-list HEAD..origin/{resolved_branch}` failed",
             e.stderr or "",
         )
         sys.exit(3)
     except ValueError:
         _print_freshness_failure(
-            base_branch,
+            resolved_branch,
             "could not parse commit count from `git rev-list`",
         )
         sys.exit(3)
 
     if behind > 0:
-        _print_stale_branch_refusal(base_branch, behind)
+        _print_stale_branch_refusal(resolved_branch, behind)
         sys.exit(3)
 
-    print(f"Branch freshness OK: up to date with origin/{base_branch}.")
+    print(f"Branch freshness OK: up to date with origin/{resolved_branch}.")
 
 
 def _print_destructive_refusal(destructive_lines: list) -> None:
@@ -198,7 +221,7 @@ def _print_destructive_refusal(destructive_lines: list) -> None:
 def create_migration(
     migration_name: str = None,
     allow_destructive: bool = False,
-    base_branch: str = DEFAULT_BASE_BRANCH,
+    base_branch: str | None = None,
     skip_freshness_check: bool = False,
 ):
     """
@@ -211,7 +234,7 @@ def create_migration(
             DROP COLUMN, DROP TABLE, or DROP INDEX statements. Without this
             flag, the script exits non-zero and prints guidance.
         base_branch (str): Branch to check freshness against
-            (default: "litellm_internal_staging").
+            (default: origin's current default branch).
         skip_freshness_check (bool): Skip the "branch is up to date" check.
             Only for intentional migrations against an older base.
     """
@@ -225,6 +248,8 @@ def create_migration(
     else:
         _check_branch_freshness(root_dir, base_branch)
 
+    import testing.postgresql
+
     try:
         migrations_dir = (
             root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
@@ -342,9 +367,8 @@ if __name__ == "__main__":
     )
     parser.add_argument(
         "--base-branch",
-        default=DEFAULT_BASE_BRANCH,
         help=(
-            f"Branch to check freshness against (default: {DEFAULT_BASE_BRANCH}). "
+            "Branch to check freshness against (default: origin's current default branch). "
             "The script fetches origin/<base-branch> and refuses to run if HEAD "
             "is behind it."
         ),

codecov.yaml

@@ -25,6 +25,8 @@ flag_management:
       carryforward: false
     - name: proxy-db-schema-migration
       carryforward: false
+    - name: circleci
+      carryforward: false
 
 component_management:
   individual_components:

db_scripts/partition_spend_logs.sql

@@ -10,6 +10,11 @@
 -- partitioned, so existing installs are unaffected until you run this.
 --
 -- IMPORTANT
+--   * After partitioning, `prisma db push` (including the proxy's
+--     --use_prisma_db_push startup mode) is NOT supported: it tries to rewrite
+--     the primary key back to ("request_id"), which Postgres rejects on a
+--     partitioned table. The proxy detects this and exits with guidance.
+--     Use the default startup path (`prisma migrate deploy`) instead.
 --   * Test on a staging copy first and take a backup.
 --   * Postgres cannot convert a populated table to partitioned in place, so this
 --     renames the old table aside and creates a fresh partitioned table.

docker/Dockerfile.database

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -39,8 +39,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 RUN apk add --no-cache \
     bash \
     gcc \
-    python3 \
-    python3-dev \
+    python-3.13 \
+    python-3.13-dev \
     openssl \
     openssl-dev \
     nodejs \
@@ -49,6 +49,7 @@ RUN apk add --no-cache \
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
@@ -63,7 +64,9 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --extra bedrock-realtime \
+    --extra mongodb \
+    --python python3.13
 
 # Copy full source tree
 COPY . .
@@ -84,7 +87,9 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --extra bedrock-realtime \
+    --extra mongodb \
+    --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -98,7 +103,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
+RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
 
 WORKDIR /app
 ENV PATH="/app/.venv/bin:${PATH}" \

docker/Dockerfile.non_root

@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1.7
 
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
@@ -37,8 +37,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 
 RUN for i in 1 2 3; do \
       apk add --no-cache \
-        python3 \
-        python3-dev \
+        python-3.13 \
+        python-3.13-dev \
         gcc \
         rust \
         bash \
@@ -52,6 +52,7 @@ RUN for i in 1 2 3; do \
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
     XDG_CACHE_HOME=/app/.cache
@@ -69,7 +70,9 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --extra bedrock-realtime \
+    --extra mongodb \
+    --python python3.13
 
 # Copy full source tree
 COPY . .
@@ -96,7 +99,9 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra extra_proxy \
         --extra semantic-router \
         --extra saml \
-        --python python3 \
+        --extra bedrock-realtime \
+        --extra mongodb \
+        --python python3.13 \
         --no-sources-package litellm-proxy-extras; \
     else \
       uv sync --frozen --no-default-groups --no-editable \
@@ -105,7 +110,9 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra extra_proxy \
         --extra semantic-router \
         --extra saml \
-        --python python3; \
+        --extra bedrock-realtime \
+        --extra mongodb \
+        --python python3.13; \
     fi
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
@@ -124,7 +131,7 @@ RUN for i in 1 2 3; do \
       apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
+      apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
     done
 
 # Copy only what runtime needs. The application is installed inside the venv;

docker/build_from_pip/Dockerfile.build_from_pip

@@ -55,7 +55,10 @@ RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/
 
 ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
+COPY docker/prod_entrypoint.sh /app/docker/prod_entrypoint.sh
+RUN sed -i 's/\r$//' /app/docker/prod_entrypoint.sh && chmod +x /app/docker/prod_entrypoint.sh
+
 EXPOSE 4000/tcp
 
-ENTRYPOINT ["litellm"]
+ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

docker/component_entrypoint.sh

@@ -1,8 +1,10 @@
 #!/bin/sh
 
-if [ "$USE_DDTRACE" = "true" ]; then
-    export DD_TRACE_OPENAI_ENABLED="False"
-    exec ddtrace-run "$@"
-fi
+case "$USE_DDTRACE" in
+    [Tt][Rr][Uu][Ee])
+        export DD_TRACE_OPENAI_ENABLED="False"
+        exec ddtrace-run "$@"
+        ;;
+esac
 
 exec "$@"

docker/prod_entrypoint.sh

@@ -1,8 +1,10 @@
 #!/bin/sh
 
-if [ "$USE_DDTRACE" = "true" ]; then
-    export DD_TRACE_OPENAI_ENABLED="False"
-    exec ddtrace-run litellm "$@"
-else
-    exec litellm "$@"
-fi
+case "$USE_DDTRACE" in
+    [Tt][Rr][Uu][Ee])
+        export DD_TRACE_OPENAI_ENABLED="False"
+        exec ddtrace-run litellm "$@"
+        ;;
+esac
+
+exec litellm "$@"

enterprise/enterprise_hooks/banned_keywords.py

@@ -21,6 +21,7 @@ from fastapi import HTTPException
 
 
 class _ENTERPRISE_BannedKeywords(CustomLogger):
+    enforces_request_content: bool = True
     # Class variables or attributes
     def __init__(self):
         banned_keywords_list = litellm.banned_keywords_list

enterprise/enterprise_hooks/blocked_user_list.py

@@ -18,6 +18,7 @@ from fastapi import HTTPException
 
 
 class _ENTERPRISE_BlockedUserList(CustomLogger):
+    enforces_request_content: bool = True
     # Class variables or attributes
     def __init__(self, prisma_client: Optional[PrismaClient]):
         self.prisma_client = prisma_client

enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py

@@ -11,17 +11,35 @@ import sys
 sys.path.insert(
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
+import functools
 import tempfile
-from typing import Optional
+from contextvars import ContextVar
+from typing import TYPE_CHECKING, ClassVar, Literal, Optional
 
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
-from litellm.integrations.custom_guardrail import CustomGuardrail
+from litellm.integrations.custom_guardrail import (
+    CustomGuardrail,
+    log_guardrail_information,
+)
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.guardrails._content_utils import walk_user_text
+from litellm.types.guardrails import GuardrailEventHooks
+from litellm.types.utils import GenericGuardrailAPIInputs
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 
 GUARDRAIL_NAME = "hide_secrets"
 
+GUARDRAIL_PROVIDER = "hide-secrets"
+
+# Per-invocation tally of redacted secrets by detect-secrets plugin type; None
+# means the guardrail did not run, so _process_response records nothing.
+_masked_entity_count: ContextVar[Optional[dict]] = ContextVar(
+    "hide_secrets_masked_entity_count", default=None
+)
+
 _custom_plugins_path = "file://" + os.path.join(
     os.path.dirname(os.path.abspath(__file__)), "secrets_plugins"
 )
@@ -415,13 +433,17 @@ _default_detect_secrets_config = {
             "name": "ZendeskSecretKeyDetector",
             "path": _custom_plugins_path + "/zendesk_secret_key.py",
         },
-        {"name": "Base64HighEntropyString", "limit": 3.0},
+        {"name": "Base64HighEntropyString", "limit": 4.5},
         {"name": "HexHighEntropyString", "limit": 3.0},
-    ]
+    ],
 }
 
 
 class _ENTERPRISE_SecretDetection(CustomGuardrail):
+    # Keeps proxied traffic on async_pre_call_hook (the unified apply_guardrail
+    # path skips should_run_check and never sees data["prompt"]).
+    use_native_lifecycle_hooks: ClassVar[bool] = True
+
     def __init__(self, detect_secrets_config: Optional[dict] = None, **kwargs):
         self.user_defined_detect_secrets_config = detect_secrets_config
         super().__init__(**kwargs)
@@ -444,16 +466,39 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
 
         os.remove(temp_file.name)
 
-        detected_secrets = []
-        for file in secrets.files:
-            for found_secret in secrets[file]:
-                if found_secret.secret_value is None:
-                    continue
-                detected_secrets.append(
-                    {"type": found_secret.type, "value": found_secret.secret_value}
-                )
+        return [
+            {"type": found_secret.type, "value": found_secret.secret_value}
+            for file in sorted(secrets.files)
+            for found_secret in sorted(
+                secrets[file],
+                key=lambda secret: (
+                    -len(secret.secret_value or ""),
+                    secret.type,
+                    secret.secret_value or "",
+                ),
+            )
+            if found_secret.secret_value is not None
+        ]
 
-        return detected_secrets
+    def redact_text(self, text: str, source: str = "message") -> str:
+        """Replace every detected secret in ``text`` with ``[REDACTED]`` and
+        tally the detected types into the per-invocation masked-entity count."""
+        detected_secrets = self.scan_message_for_secrets(text)
+        if not detected_secrets:
+            return text
+        counts = _masked_entity_count.get()
+        if counts is not None:
+            for secret in detected_secrets:
+                counts[secret["type"]] = counts.get(secret["type"], 0) + 1
+        secret_types = [secret["type"] for secret in detected_secrets]
+        verbose_proxy_logger.warning(
+            f"Detected and redacted secrets in {source}: {secret_types}"
+        )
+        return functools.reduce(
+            lambda redacted, secret: redacted.replace(secret["value"], "[REDACTED]"),
+            detected_secrets,
+            text,
+        )
 
     async def should_run_check(self, user_api_key_dict: UserAPIKeyAuth) -> bool:
         if user_api_key_dict.permissions is not None:
@@ -463,7 +508,45 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
 
         return True
 
+    @log_guardrail_information
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional["LiteLLMLoggingObj"] = None,
+    ) -> GenericGuardrailAPIInputs:
+        """Unified-interface entrypoint, used by /guardrails/apply_guardrail
+        (the UI test playground). Proxied traffic keeps using
+        ``async_pre_call_hook``, see ``use_native_lifecycle_hooks``."""
+        texts = inputs.get("texts")
+        if not texts or not any(texts):
+            return inputs
+        _masked_entity_count.set({})
+        return {**inputs, "texts": [self.redact_text(text) for text in texts]}
+
+    def _redact_prompt(self, data: dict) -> int:
+        """Redact ``data["prompt"]`` (the text-completion shape, which
+        ``walk_user_text`` does not cover) and return how many non-empty
+        strings were inspected."""
+        prompt = data.get("prompt")
+        if isinstance(prompt, str):
+            if not prompt:
+                return 0
+            data["prompt"] = self.redact_text(prompt, source="prompt")
+            return 1
+        if isinstance(prompt, list):
+            data["prompt"] = [  # mutable-ok: data["prompt"] is a list on the wire
+                self.redact_text(item, source="prompt")
+                if isinstance(item, str) and item
+                else item
+                for item in prompt
+            ]
+            return sum(1 for item in prompt if isinstance(item, str) and item)
+        return 0
+
     #### CALL HOOKS - proxy only ####
+    @log_guardrail_information
     async def async_pre_call_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -471,53 +554,84 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
         data: dict,
         call_type: str,  # "completion", "embeddings", "image_generation", "moderation"
     ):
+        _masked_entity_count.set(None)
         if await self.should_run_check(user_api_key_dict) is False:
             return
 
+        _masked_entity_count.set({})
+
         # Covers multimodal list content + Responses-API input.
-        def _redact_message_text(text: str) -> str:
-            detected_secrets = self.scan_message_for_secrets(text)
-            for secret in detected_secrets:
-                text = text.replace(secret["value"], "[REDACTED]")
-            if detected_secrets:
-                secret_types = [secret["type"] for secret in detected_secrets]
-                verbose_proxy_logger.warning(
-                    f"Detected and redacted secrets in message: {secret_types}"
-                )
-            return text
+        inspected = walk_user_text(data, self.redact_text) + self._redact_prompt(data)
 
-        walk_user_text(data, _redact_message_text)
+        if inspected == 0:
+            # Image-only, empty-text, and unsupported payloads inspected
+            # nothing, so recording "allow" would count a run that never
+            # looked at any content.
+            _masked_entity_count.set(None)
 
-        if "prompt" in data:
-            if isinstance(data["prompt"], str):
-                detected_secrets = self.scan_message_for_secrets(data["prompt"])
-                for secret in detected_secrets:
-                    data["prompt"] = data["prompt"].replace(
-                        secret["value"], "[REDACTED]"
-                    )
-                if len(detected_secrets) > 0:
-                    secret_types = [secret["type"] for secret in detected_secrets]
-                    verbose_proxy_logger.warning(
-                        f"Detected and redacted secrets in prompt: {secret_types}"
-                    )
-            elif isinstance(data["prompt"], list):
-                # Index back into the list — assigning to ``item`` would only
-                # rebind the loop variable and leave ``data["prompt"]``
-                # carrying the unredacted secret.
-                for idx, item in enumerate(data["prompt"]):
-                    if isinstance(item, str):
-                        detected_secrets = self.scan_message_for_secrets(item)
-                        for secret in detected_secrets:
-                            item = item.replace(secret["value"], "[REDACTED]")
-                        data["prompt"][idx] = item
-                        if len(detected_secrets) > 0:
-                            secret_types = [
-                                secret["type"] for secret in detected_secrets
-                            ]
-                            verbose_proxy_logger.warning(
-                                f"Detected and redacted secrets in prompt: {secret_types}"
-                            )
-
-        # ``data["input"]`` (Responses API and embeddings/moderation) is
-        # already covered by ``walk_user_text`` above.
         return
+
+    def _process_response(
+        self,
+        response: Optional[dict],
+        request_data: dict,
+        start_time: Optional[float] = None,
+        end_time: Optional[float] = None,
+        duration: Optional[float] = None,
+        event_type: Optional[GuardrailEventHooks] = None,
+        original_inputs: Optional[dict] = None,
+    ):
+        """Record allow/mask plus the masked-entity tally for a completed run.
+
+        Records nothing when the guardrail inspected nothing (opted-out key,
+        empty inputs) or when the instance has no guardrail_name (legacy
+        ``litellm_settings.callbacks`` deployments, which predate guardrail
+        telemetry and stay without it).
+        """
+        counts = _masked_entity_count.get()
+        _masked_entity_count.set(None)
+        if counts is None or self.guardrail_name is None:
+            return response
+        self.add_standard_logging_guardrail_information_to_request_data(
+            guardrail_json_response="mask" if counts else "allow",
+            request_data=request_data,
+            guardrail_status="success",
+            duration=duration,
+            start_time=start_time,
+            end_time=end_time,
+            event_type=event_type,
+            guardrail_provider=GUARDRAIL_PROVIDER,
+            masked_entity_count=counts,
+        )
+        return response
+
+    def _process_error(
+        self,
+        e: Exception,
+        request_data: dict,
+        start_time: Optional[float] = None,
+        end_time: Optional[float] = None,
+        duration: Optional[float] = None,
+        event_type: Optional[GuardrailEventHooks] = None,
+    ):
+        """Label the failed run with this guardrail's provider so error rows
+        group with the successful ones in the monitor. Nameless legacy
+        instances record nothing, matching ``_process_response``."""
+        _masked_entity_count.set(None)
+        if self.guardrail_name is None:
+            raise e
+        self.add_standard_logging_guardrail_information_to_request_data(
+            guardrail_json_response=e,
+            request_data=request_data,
+            guardrail_status=(
+                "guardrail_intervened"
+                if self._is_guardrail_intervention(e)
+                else "guardrail_failed_to_respond"
+            ),
+            duration=duration,
+            start_time=start_time,
+            end_time=end_time,
+            event_type=event_type,
+            guardrail_provider=GUARDRAIL_PROVIDER,
+        )
+        raise e

enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/openai_api_key.py

@@ -3,6 +3,7 @@ This plugin searches for OpenAI API Keys.
 """
 
 import re
+from collections.abc import Generator
 
 from detect_secrets.plugins.base import RegexBasedDetector
 
@@ -16,4 +17,16 @@ class OpenAIApiKeyDetector(RegexBasedDetector):
 
     @property
     def denylist(self) -> list[re.Pattern]:
-        return [re.compile(r"""(sk-[a-zA-Z0-9]{5,})""")]
+        return [
+            re.compile(
+                r"((?:(?<![a-zA-Z0-9])|(?<=%[0-9A-Fa-f]{2}))"
+                r"sk[-_]"
+                r"[a-zA-Z0-9_-]{5,}"
+                r"(?![a-zA-Z0-9_-]))"
+            )
+        ]
+
+    def analyze_string(self, string: str) -> Generator[str, None, None]:
+        # the digit check lives outside the regex: a lookahead re-scans the token
+        # from every `sk` inside it, which is quadratic on `-sk-sk-sk-...` input
+        yield from (match for match in super().analyze_string(string) if re.search(r"[0-9]", match))

enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py

@@ -7,7 +7,7 @@ GET - /audit/{id} - Get audit log by id
 GET - /audit - Get all audit logs
 """
 
-from typing import Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Final
 
 #### AUDIT LOGGING ####
 from fastapi import APIRouter, Depends, HTTPException, Query
@@ -18,11 +18,16 @@ from litellm_enterprise.types.proxy.audit_logging_endpoints import (
 
 from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.repositories.prisma_protocols import TableActions
+from litellm.repositories.table_repositories import AuditLogRepository
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
 
 router = APIRouter()
 
 
-def _build_json_field_or_condition(json_key: str, value: str) -> Dict[str, Any]:
+def _build_json_field_or_condition(json_key: str, value: str) -> dict[str, object]:
     """
     Build an OR condition that matches a value inside a JSON column at the
     given key, checking both before_value and updated_values.
@@ -43,6 +48,18 @@ def _build_json_field_or_condition(json_key: str, value: str) -> Dict[str, Any]:
     }
 
 
+def _build_search_condition(search: str) -> dict[str, object]:
+    """Match a row whose id, changed_by, object_id, or changed_by_api_key equals the search value."""
+    return {
+        "OR": (
+            {"id": search},
+            {"changed_by": search},
+            {"object_id": search},
+            {"changed_by_api_key": search},
+        )
+    }
+
+
 @router.get(
     "/audit",
     tags=["Audit Logging"],
@@ -53,33 +70,37 @@ async def get_audit_logs(
     page: int = Query(1, ge=1),
     page_size: int = Query(10, ge=1, le=100),
     # Filter parameters
-    changed_by: Optional[str] = Query(
+    changed_by: str | None = Query(
         None, description="Filter by user or system that performed the action"
     ),
-    changed_by_api_key: Optional[str] = Query(
+    changed_by_api_key: str | None = Query(
         None, description="Filter by API key hash that performed the action"
     ),
-    action: Optional[str] = Query(
+    action: str | None = Query(
         None, description="Filter by action type (create, update, delete)"
     ),
-    table_name: Optional[str] = Query(
+    table_name: str | None = Query(
         None, description="Filter by table name that was modified"
     ),
-    object_id: Optional[str] = Query(
+    object_id: str | None = Query(
         None, description="Filter by ID of the object that was modified"
     ),
-    start_date: Optional[str] = Query(None, description="Filter logs after this date"),
-    end_date: Optional[str] = Query(None, description="Filter logs before this date"),
-    object_team_id: Optional[str] = Query(
+    start_date: str | None = Query(None, description="Filter logs after this date"),
+    end_date: str | None = Query(None, description="Filter logs before this date"),
+    object_team_id: str | None = Query(
         None,
         description="Filter by team_id present in before_value or updated_values JSON (PostgreSQL only)",
     ),
-    object_key_hash: Optional[str] = Query(
+    object_key_hash: str | None = Query(
         None,
         description="Filter by token (key hash) present in before_value or updated_values JSON (PostgreSQL only)",
     ),
+    search: str | None = Query(
+        None,
+        description="Match a row whose id, object_id, changed_by, or changed_by_api_key equals this value",
+    ),
     # Sorting parameters
-    sort_by: Optional[str] = Query(
+    sort_by: str | None = Query(
         None,
         description="Column to sort by (e.g. 'updated_at', 'action', 'table_name')",
     ),
@@ -101,46 +122,42 @@ async def get_audit_logs(
             detail={"message": CommonProxyErrors.db_not_connected_error.value},
         )
 
-    # Build filter conditions
-    where_conditions: Dict[str, Any] = {}
-    if changed_by:
-        where_conditions["changed_by"] = changed_by
-    if changed_by_api_key:
-        where_conditions["changed_by_api_key"] = changed_by_api_key
-    if action:
-        where_conditions["action"] = action
-    if table_name:
-        where_conditions["table_name"] = table_name
-    if object_id:
-        where_conditions["object_id"] = object_id
-    if start_date or end_date:
-        date_filter: Dict[str, Any] = {}
-        if start_date:
-            date_filter["gte"] = start_date
-        if end_date:
-            date_filter["lte"] = end_date
-        where_conditions["updated_at"] = date_filter
+    date_filter: Final[dict[str, str]] = {
+        **({"gte": start_date} if start_date else {}),
+        **({"lte": end_date} if end_date else {}),
+    }
 
     # JSON field filters (PostgreSQL only) — each filter is AND'd with the
     # others, but checks both before_value and updated_values internally (OR).
-    if object_team_id:
-        where_conditions["AND"] = where_conditions.get("AND", []) + [
-            _build_json_field_or_condition("team_id", object_team_id)
-        ]
-    if object_key_hash:
-        where_conditions["AND"] = where_conditions.get("AND", []) + [
-            _build_json_field_or_condition("token", object_key_hash)
-        ]
+    json_field_conditions: Final[list[dict[str, object]]] = [
+        *([_build_json_field_or_condition("team_id", object_team_id)] if object_team_id else []),
+        *([_build_json_field_or_condition("token", object_key_hash)] if object_key_hash else []),
+    ]
 
-    # Build sort conditions
-    order_by: Dict[str, Any] = {}
-    if sort_by and isinstance(sort_by, str):
-        order_by[sort_by] = sort_order
-    else:
-        order_by["updated_at"] = sort_order  # Default sort by updated_at
+    and_conditions: Final[tuple[dict[str, object], ...]] = (
+        *json_field_conditions,
+        *((_build_search_condition(search),) if search else ()),
+    )
+
+    # Build filter conditions
+    where_conditions: Final[dict[str, object]] = {
+        **({"changed_by": changed_by} if changed_by else {}),
+        **({"changed_by_api_key": changed_by_api_key} if changed_by_api_key else {}),
+        **({"action": action} if action else {}),
+        **({"table_name": table_name} if table_name else {}),
+        **({"object_id": object_id} if object_id else {}),
+        **({"updated_at": date_filter} if start_date or end_date else {}),
+        **({"AND": and_conditions} if and_conditions else {}),
+    }
+
+    order_by: Final[dict[str, str]] = (
+        {sort_by: sort_order} if sort_by and isinstance(sort_by, str) else {"updated_at": sort_order}
+    )
+
+    audit_log_table: Final[TableActions[prisma_models.LiteLLM_AuditLog]] = AuditLogRepository(prisma_client).table
 
     # Get paginated results
-    audit_logs = await prisma_client.db.litellm_auditlog.find_many(
+    audit_logs: Final = await audit_log_table.find_many(
         where=where_conditions,
         order=order_by,
         skip=(page - 1) * page_size,
@@ -148,13 +165,14 @@ async def get_audit_logs(
     )
 
     # Get total count for pagination
-    total_count = await prisma_client.db.litellm_auditlog.count(where=where_conditions)
-    total_pages = -(-total_count // page_size)  # Ceiling division
+    total_count: Final = await audit_log_table.count(where=where_conditions)
+    total_pages: Final = -(-total_count // page_size)  # Ceiling division
 
     # Return paginated response
     return PaginatedAuditLogResponse(
         audit_logs=[
-            AuditLogResponse(**audit_log.model_dump()) for audit_log in audit_logs
+            AuditLogResponse.model_validate(audit_log.model_dump())
+            for audit_log in audit_logs
         ]
         if audit_logs
         else [],
@@ -198,8 +216,10 @@ async def get_audit_log_by_id(
             detail={"message": CommonProxyErrors.db_not_connected_error.value},
         )
 
+    audit_log_table: Final[TableActions[prisma_models.LiteLLM_AuditLog]] = AuditLogRepository(prisma_client).table
+
     # Get the audit log by ID
-    audit_log = await prisma_client.db.litellm_auditlog.find_unique(where={"id": id})
+    audit_log: Final = await audit_log_table.find_unique(where={"id": id})
 
     if audit_log is None:
         raise HTTPException(
@@ -207,4 +227,4 @@ async def get_audit_log_by_id(
         )
 
     # Convert to response model
-    return AuditLogResponse(**audit_log.model_dump())
+    return AuditLogResponse.model_validate(audit_log.model_dump())

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -2,9 +2,10 @@
 Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
 """
 
+from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Tuple
+from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -14,6 +15,8 @@ from litellm.constants import (
 )
 
 if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
     from litellm.integrations.prometheus import PrometheusLogger
     from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -84,7 +87,7 @@ class CheckBatchCost:
             return
         self.batch_processed_support_confirmed = True
 
-    async def _get_user_info(self, batch_id: str, user_id: Optional[str]) -> Dict[str, Any]:
+    async def _get_user_info(self, batch_id: str, user_id: Optional[str]) -> dict[str, str | None]:
         """
         Look up user email and key alias by user_id for enriching the S3 callback metadata.
         Returns a dict with user_api_key_user_email and user_api_key_alias (both may be None).
@@ -94,8 +97,10 @@ class CheckBatchCost:
         if not user_id:
             return {}
         try:
-            user_row = await self.prisma_client.db.litellm_usertable.find_unique(
-                where={"user_id": user_id}
+            user_row: prisma_models.LiteLLM_UserTable | None = (
+                await self.prisma_client.db.litellm_usertable.find_unique(
+                    where={"user_id": user_id}
+                )
             )
             if user_row is None:
                 return {}
@@ -112,8 +117,10 @@ class CheckBatchCost:
         if not api_key:
             return None
         try:
-            key_row = await self.prisma_client.db.litellm_verificationtoken.find_unique(
-                where={"token": api_key}
+            key_row: prisma_models.LiteLLM_VerificationToken | None = (
+                await self.prisma_client.db.litellm_verificationtoken.find_unique(
+                    where={"token": api_key}
+                )
             )
             return getattr(key_row, "key_alias", None) if key_row is not None else None
         except Exception as e:
@@ -125,8 +132,10 @@ class CheckBatchCost:
         if not team_id:
             return None
         try:
-            team_row = await self.prisma_client.db.litellm_teamtable.find_unique(
-                where={"team_id": team_id}
+            team_row: prisma_models.LiteLLM_TeamTable | None = (
+                await self.prisma_client.db.litellm_teamtable.find_unique(
+                    where={"team_id": team_id}
+                )
             )
             return getattr(team_row, "team_alias", None) if team_row is not None else None
         except Exception as e:
@@ -135,7 +144,7 @@ class CheckBatchCost:
 
     async def _build_creator_attribution_metadata(
         self, job: "LiteLLM_ManagedObjectTable", batch_id: str
-    ) -> Dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Rebuild the spend-tracking metadata for the key, team, and tags that created the
         batch so the batch-cost spend log is attributed the same way a non-batch request
@@ -149,9 +158,10 @@ class CheckBatchCost:
         team_id = getattr(job, "team_id", None)
         request_tags = getattr(job, "request_tags", None)
 
-        metadata: Dict[str, Any] = {
+        metadata: dict[str, object] = {
             "user_api_key_user_id": job.created_by,
             "user_api_key": api_key,
+            "user_api_key_hash": api_key,
             "user_api_key_team_id": team_id,
             **(await self._get_user_info(batch_id, job.created_by)),
         }
@@ -351,7 +361,7 @@ class CheckBatchCost:
         return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
 
     async def _finalize_unbilled_terminal_job(
-        self, job: "LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
+        self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
     ) -> None:
         """Persist a terminal batch that has nothing billable, converting any raw
         provider file ids to managed ids, and take it out of the poll page."""
@@ -624,6 +634,7 @@ class CheckBatchCost:
         later poll.
         """
         from litellm.batches.batch_utils import (
+            count_error_file_failed_requests,
             _get_file_content_as_dictionary,
             calculate_batch_cost_and_usage,
         )
@@ -759,16 +770,33 @@ class CheckBatchCost:
             model_id=model_id,
             deployment_model=litellm_model_name,
         )
-        batch_cost, batch_usage, batch_models = (
-            await calculate_batch_cost_and_usage(
-                file_content_dictionary=file_content_as_dict,
-                custom_llm_provider=llm_provider,  # type: ignore
-                model_name=model_name,
-                model_info=deployment_model_info,
+        batch_file_provider: Final = cast(
+            Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"], llm_provider
+        )
+        output_file_result: Final = await calculate_batch_cost_and_usage(
+            file_content_dictionary=file_content_as_dict,
+            custom_llm_provider=batch_file_provider,
+            model_name=model_name,
+            model_info=deployment_model_info,
+        )
+        error_file_failed_requests: Final = await count_error_file_failed_requests(
+            response,
+            custom_llm_provider=batch_file_provider,
+            litellm_params={
+                **credentials,
+                "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
+            },
+        )
+        batch_result: Final = (
+            output_file_result
+            if not error_file_failed_requests
+            else dataclasses_replace(
+                output_file_result,
+                failed_requests=output_file_result.failed_requests + error_file_failed_requests,
             )
         )
         logging_obj = LiteLLMLogging(
-            model=batch_models[0],
+            model=batch_result.models[0],
             messages=[{"role": "user", "content": "<retrieve_batch>"}],
             stream=False,
             call_type="aretrieve_batch",
@@ -800,9 +828,11 @@ class CheckBatchCost:
         try:
             await logging_obj.async_success_handler(
                 result=response,
-                batch_cost=batch_cost,
-                batch_usage=batch_usage,
-                batch_models=batch_models,
+                batch_cost=batch_result.cost,
+                batch_usage=batch_result.usage,
+                batch_models=batch_result.models,
+                batch_successful_requests=batch_result.successful_requests,
+                batch_failed_requests=batch_result.failed_requests,
             )
         except Exception:
             await self._release_job_claim(job)
@@ -966,6 +996,16 @@ class CheckBatchCost:
                     )
 
             elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
+                from litellm.proxy.openai_files_endpoints.common_utils import (
+                    _completed_batch_safe_to_retire,
+                )
+
+                if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
+                    verbose_proxy_logger.info(
+                        f"CheckBatchCost: batch {batch_id} is completed but its output file id "
+                        f"has not appeared yet; leaving job {job.id} for the next poll cycle"
+                    )
+                    continue
                 await self._finalize_unbilled_terminal_job(job, response)
 
         # Record polling run metrics (always, even if nothing was processed)

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -1,6 +1,8 @@
 """
 Polls LiteLLM_ManagedObjectTable to check if the response is complete.
-Cost tracking is handled automatically by the get-responses call.
+Cost tracking is handled by the get-responses call, which prices normally only because the
+poll stamps itself with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN; user-facing reads of the
+same route are non-inference and free.
 """
 
 from datetime import datetime, timedelta, timezone
@@ -9,12 +11,14 @@ from typing import TYPE_CHECKING, Dict, Optional, cast
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import (
+    INTERNAL_CALL_ORIGIN_METADATA_KEY,
     MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
     MAX_OBJECTS_PER_POLL_CYCLE,
     STALE_OBJECT_CLEANUP_BATCH_SIZE,
 )
 from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.types.llms.openai import ResponsesAPIResponse
+from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -113,7 +117,8 @@ class CheckResponsesCost:
         Check if background responses are complete and track their cost.
         - Get all status="queued" or "in_progress" and file_purpose="response" jobs
         - Query the provider to check if response is complete
-        - Cost is automatically tracked by the get-responses call
+        - Cost is tracked by the get-responses call, billed because the poll is stamped
+          with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
         - Mark responses in a terminal state as complete in the database
         """
         try:
@@ -153,6 +158,7 @@ class CheckResponsesCost:
                 # Prepare metadata with model information for cost tracking
                 litellm_metadata = {
                     "user_api_key_user_id": job.created_by or "default-user-id",
+                    INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
                 }
                 
                 # Add model information if available

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -36,6 +36,7 @@ from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
     build_owner_filter,
     can_access_resource,
+    resolve_resource_owner_id,
 )
 from litellm.proxy._types import (
     CallTypes,
@@ -45,15 +46,22 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.openai_files_endpoints.common_utils import (
+    BATCH_CREATE_HIDDEN_PARAM,
+    FILE_LIST_CONTINUATION_CHUNK_SIZE,
+    MAX_FILE_LIST_LIMIT,
     _is_base64_encoded_unified_file_id,
     apply_unified_file_ids,
+    decode_model_from_file_id,
     ensure_batch_response_managed_file_ids,
     get_batch_id_from_unified_batch_id,
     get_content_type_from_file_object,
     get_model_id_from_unified_batch_id,
+    get_original_file_id,
     map_raw_file_ids_to_unified,
     normalize_mime_type_for_provider,
     resolve_managed_output_file_model_name,
+    validate_file_list_limit,
+    validate_file_list_purpose,
 )
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
     request_tags_from_metadata,
@@ -63,9 +71,9 @@ from litellm.types.llms.openai import (  # pyright: ignore[reportAttributeAccess
     AsyncCursorPage,
     ChatCompletionFileObject,
     CreateFileRequest,
+    FileListPage,
     FileObject,
     OpenAIFileObject,
-    OpenAIFilesPurpose,
     ResponsesAPIResponse,
 )
 from litellm.types.utils import (
@@ -144,7 +152,14 @@ class _ManagedFileRow(Protocol):
 class _ManagedFileTableActions(Protocol):
     async def find_first(self, where: Mapping[str, object]) -> Optional[_ManagedFileRow]: ...
 
-    async def find_many(self, where: Mapping[str, object]) -> Sequence[_ManagedFileRow]: ...
+    async def find_many(
+        self,
+        where: Mapping[str, object],
+        take: int = ...,
+        order: Union[Mapping[str, str], Sequence[Mapping[str, str]]] = ...,
+        cursor: Mapping[str, str] = ...,
+        skip: int = ...,
+    ) -> Sequence[_ManagedFileRow]: ...
 
     async def upsert(self, where: Mapping[str, str], data: Mapping[str, Mapping[str, object]]) -> _ManagedFileRow: ...
 
@@ -170,6 +185,10 @@ class _ManagedObjectTableActions(Protocol):
     async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
 
 
+class _SchedulerWithJobLookup(Protocol):
+    def get_job(self, job_id: str) -> object: ...
+
+
 class _CursorPageArgs(TypedDict, total=False):
     cursor: Mapping[str, str]
     skip: int
@@ -211,7 +230,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 file_object=file_object,
                 model_mappings=model_mappings,
                 flat_model_file_ids=list(model_mappings.values()),
-                created_by=user_api_key_dict.user_id,
+                created_by=resolve_resource_owner_id(user_api_key_dict),
                 team_id=user_api_key_dict.team_id,
                 updated_by=user_api_key_dict.user_id,
             )
@@ -227,7 +246,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             "unified_file_id": file_id,
             "model_mappings": json.dumps(model_mappings),
             "flat_model_file_ids": list(model_mappings.values()),
-            "created_by": user_api_key_dict.user_id,
+            "created_by": resolve_resource_owner_id(user_api_key_dict),
             "team_id": user_api_key_dict.team_id,
             "updated_by": user_api_key_dict.user_id,
         }
@@ -331,7 +350,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "file_object": file_object.model_dump_json(),
                     "model_object_id": model_object_id,
                     "file_purpose": file_purpose,
-                    "created_by": user_api_key_dict.user_id,
+                    "created_by": resolve_resource_owner_id(user_api_key_dict),
                     "team_id": user_api_key_dict.team_id,
                     "updated_by": user_api_key_dict.user_id,
                     "status": file_object.status,
@@ -411,6 +430,103 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             detail=f"Object not found: {unified_object_id}",
         )
 
+    async def enforce_batch_object_access(
+        self, object_id: str, user_api_key_dict: UserAPIKeyAuth
+    ) -> None:
+        """Deny access to a provider-format batch id owned by another caller.
+
+        Ids with no ownership row (batches created before ownership tracking,
+        or directly on the provider account) stay accessible so pass-through
+        reads keep working.
+        """
+        if self.prisma_client is None:
+            return
+        managed_object = (
+            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]}
+            )
+        )
+        if managed_object is None:
+            return
+        if not can_access_resource(
+            user_api_key_dict=user_api_key_dict,
+            created_by=managed_object.created_by,
+            resource_team_id=managed_object.team_id,
+        ):
+            raise HTTPException(
+                status_code=403,
+                detail=f"User {user_api_key_dict.user_id} does not have access to the object {object_id}",
+            )
+
+    async def enforce_provider_file_access(
+        self, file_id: str, user_api_key_dict: UserAPIKeyAuth
+    ) -> None:
+        """Deny access to a provider-format file id owned by another caller.
+
+        Ownership rows for provider-format ids are written when a managed
+        batch's output/error files are first synced; ids with no row stay
+        accessible so pass-through reads keep working.
+        """
+        if self.prisma_client is None:
+            return
+        managed_file = (
+            await self.prisma_client.db.litellm_managedfiletable.find_first(
+                where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]}
+            )
+        )
+        if managed_file is None:
+            return
+        if not can_access_resource(
+            user_api_key_dict=user_api_key_dict,
+            created_by=managed_file.created_by,
+            resource_team_id=managed_file.team_id,
+        ):
+            raise HTTPException(
+                status_code=403,
+                detail=f"User {user_api_key_dict.user_id} does not have access to the file {file_id}",
+            )
+
+    async def store_batch_output_file_ownership(
+        self, response: LiteLLMBatch, litellm_parent_otel_span: Optional[Span]
+    ) -> None:
+        """Record ownership rows for a batch's provider-format output/error
+        file ids, inherited from the owning batch row (never the caller), so
+        file reads can be isolation-checked."""
+        provider_file_ids = tuple(
+            file_id
+            for file_id in (
+                getattr(response, "output_file_id", None),
+                getattr(response, "error_file_id", None),
+            )
+            if file_id and not _is_base64_encoded_unified_file_id(file_id)
+        )
+        if not provider_file_ids:
+            return
+        if self.prisma_client is None:
+            return
+        batch_row = (
+            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                where={"unified_object_id": response.id}
+            )
+        )
+        if batch_row is None or (
+            batch_row.created_by is None and batch_row.team_id is None
+        ):
+            return
+        owner_identity = UserAPIKeyAuth(
+            user_id=batch_row.created_by, team_id=batch_row.team_id
+        )
+        for file_id in provider_file_ids:
+            model_name = decode_model_from_file_id(file_id)
+            raw_file_id = get_original_file_id(file_id)
+            await self.store_unified_file_id(
+                file_id=file_id,
+                file_object=None,
+                litellm_parent_otel_span=litellm_parent_otel_span,
+                model_mappings={model_name: raw_file_id} if model_name else {},
+                user_api_key_dict=owner_identity,
+            )
+
     async def list_user_batches(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -462,19 +578,56 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 )
 
         page_size: Final = min(limit or 20, 100)
-        cursor_args: _CursorPageArgs = {"cursor": {"unified_object_id": after}, "skip": 1} if after else {}
-
-        batches = await _managed_object_table(self.prisma_client).find_many(
-            where=where_clause,
-            take=page_size + 1,
-            order=[{"created_at": "desc"}, {"unified_object_id": "desc"}],
-            **cursor_args,
+        matches: Final = await self._collect_listed_batches(
+            where_clause=where_clause,
+            after=after,
+            wanted=page_size + 1,
+            user_api_key_dict=user_api_key_dict,
         )
+        return build_list_page(list(matches[:page_size]), has_more=len(matches) > page_size)
 
-        has_more = len(batches) > page_size
+    async def _collect_listed_batches(
+        self,
+        where_clause: Mapping[str, object],
+        after: Optional[str],
+        wanted: int,
+        user_api_key_dict: UserAPIKeyAuth,
+    ) -> tuple[LiteLLMBatch, ...]:
+        """Read chunks newest-first until ``wanted`` batches survive parsing and
+        file-id resolution or the caller's rows run out, so a run of rows that will
+        not parse refills the page instead of emptying it. The first chunk is
+        ``wanted`` rows, so a healthy page still costs one query; a scan that has to
+        continue widens to ``FILE_LIST_CONTINUATION_CHUNK_SIZE`` like ``afile_list``,
+        and every chunk advances the keyset cursor, so the walk ends once the
+        caller's rows are exhausted."""
+        matches: tuple[LiteLLMBatch, ...] = ()  # rebind-ok: accumulates survivors across chunks
+        cursor_id: Optional[str] = after  # rebind-ok: keyset cursor advances to each chunk's last row
+        chunk_size: int = wanted  # rebind-ok: widens once a scan has to continue past the first chunk
+        while len(matches) < wanted:
+            cursor_args: _CursorPageArgs = {"cursor": {"unified_object_id": cursor_id}, "skip": 1} if cursor_id else {}
+            chunk = await _managed_object_table(self.prisma_client).find_many(
+                where=where_clause,
+                take=chunk_size,
+                order=[{"created_at": "desc"}, {"unified_object_id": "desc"}],
+                **cursor_args,
+            )
+            matches = matches + await self._resolve_listed_rows(
+                rows=chunk, wanted=wanted - len(matches), user_api_key_dict=user_api_key_dict
+            )
+            if len(chunk) < chunk_size:
+                break
+            cursor_id = chunk[-1].unified_object_id
+            chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
+        return matches
 
+    async def _resolve_listed_rows(
+        self,
+        rows: "Sequence[PrismaManagedObjectRow]",
+        wanted: int,
+        user_api_key_dict: UserAPIKeyAuth,
+    ) -> tuple[LiteLLMBatch, ...]:
         parsed_rows: Final = tuple(
-            (row, batch_obj) for row in batches[:page_size] if (batch_obj := _parse_managed_batch_row(row)) is not None
+            (row, batch_obj) for row in rows if (batch_obj := _parse_managed_batch_row(row)) is not None
         )
         unified_id_by_raw_id: Final = await map_raw_file_ids_to_unified(
             raw_file_ids=frozenset(
@@ -485,19 +638,19 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             ),
             prisma_client=self.prisma_client,
         )
-        resolved_batches: Final = [
-            await self._resolve_listed_batch(
+        resolved: Final[list[LiteLLMBatch]] = []  # mutable-ok: resolution stops as soon as the page is full
+        for row, batch_obj in parsed_rows:
+            if len(resolved) == wanted:
+                break
+            resolved_batch = await self._resolve_listed_batch(
                 row=row,
                 batch_obj=batch_obj,
                 unified_id_by_raw_id=unified_id_by_raw_id,
                 user_api_key_dict=user_api_key_dict,
             )
-            for row, batch_obj in parsed_rows
-        ]
-        return build_list_page(
-            [batch_obj for batch_obj in resolved_batches if batch_obj is not None],
-            has_more=has_more,
-        )
+            if resolved_batch is not None:
+                resolved.append(resolved_batch)
+        return tuple(resolved)
 
     async def _resolve_listed_batch(
         self,
@@ -560,6 +713,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     status_code=403,
                     detail=f"User {user_api_key_dict.user_id} does not have access to the file {retrieve_file_id}",
                 )
+        if retrieve_file_id:
+            await self.enforce_provider_file_access(
+                retrieve_file_id, user_api_key_dict
+            )
         return False
 
     async def check_file_ids_access(self, file_ids: List[str], user_api_key_dict: UserAPIKeyAuth) -> None:
@@ -712,6 +869,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     )
                 data["model"] = potential_model_id
                 data[accessor_key] = get_batch_id_from_unified_batch_id(potential_llm_object_id)
+            elif retrieve_object_id and accessor_key == "batch_id":
+                await self.enforce_batch_object_access(retrieve_object_id, user_api_key_dict)
         elif call_type == CallTypes.acreate_fine_tuning_job.value:
             input_file_id = cast(Optional[str], data.get("training_file"))
             if input_file_id:
@@ -804,7 +963,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                                 file_ids.append(file_id)
         return file_ids
 
-    def get_file_ids_from_responses_input(self, input: Union[str, List[Dict[str, Any]]]) -> List[str]:
+    def get_file_ids_from_responses_input(self, input: Union[str, List[Dict[str, object]]]) -> List[str]:
         """
         Gets file ids from responses API input.
 
@@ -829,7 +988,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             # Check for direct input_file type
             if item.get("type") == "input_file":
                 file_id = item.get("file_id")
-                if file_id:
+                if isinstance(file_id, str) and file_id:
                     file_ids.append(file_id)
 
             # Check for input_file in content array
@@ -838,7 +997,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 for content_item in content:
                     if isinstance(content_item, dict) and content_item.get("type") == "input_file":
                         file_id = content_item.get("file_id")
-                        if file_id:
+                        if isinstance(file_id, str) and file_id:
                             file_ids.append(file_id)
 
         return file_ids
@@ -1163,7 +1322,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             ## Check if unified_file_id is in the response
             unified_file_id = response._hidden_params.get("unified_file_id")  # managed file id
             unified_batch_id = response._hidden_params.get("unified_batch_id")  # managed batch id
-            is_batch_create: Final = unified_file_id is not None
+            is_batch_create: Final = response._hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True
             model_id = cast(Optional[str], response._hidden_params.get("model_id"))
             model_name = cast(Optional[str], response._hidden_params.get("model_name"))
 
@@ -1178,7 +1337,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
 
                 # Handle both output_file_id and error_file_id
                 for file_attr in ["output_file_id", "error_file_id"]:
-                    file_id_value = getattr(response, file_attr, None)
+                    file_id_value: str | None = getattr(response, file_attr, None)
                     if file_id_value and model_id:
                         decoded_output_file_id = _is_base64_encoded_unified_file_id(file_id_value)
                         if decoded_output_file_id and "llm_output_file_id," in decoded_output_file_id:
@@ -1244,12 +1403,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 user_api_key_dict=user_api_key_dict,
                 request_tags=request_tags_from_metadata(request_metadata if isinstance(request_metadata, dict) else {}),
                 persist_attribution=is_batch_create,
+                create_if_missing=is_batch_create,
             )
+            if not is_batch_create:
+                await self.store_batch_output_file_ownership(
+                    response=response,
+                    litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
+                )
 
-            # Only record batch creation metric on actual create (not retrieve/cancel).
-            # unified_file_id in _hidden_params is only set by the create_batch endpoint.
-            original_unified_file_id = response._hidden_params.get("unified_file_id")
-            if original_unified_file_id:
+            if is_batch_create:
                 prom_logger = self._get_prometheus_logger()
                 if prom_logger:
                     batch_provider = ""
@@ -1365,12 +1527,76 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
 
     async def afile_list(
         self,
-        purpose: Optional[OpenAIFilesPurpose],
+        purpose: Optional[str],
         litellm_parent_otel_span: Optional[Span],
+        user_api_key_dict: UserAPIKeyAuth,
+        limit: Optional[int] = None,
+        after: Optional[str] = None,
         **data: Dict,
-    ) -> List[OpenAIFileObject]:
-        """Handled in files_endpoints.py"""
-        return []
+    ) -> FileListPage:
+        """List the managed files the caller owns, newest first.
+
+        Pagination is keyset based on ``unified_file_id`` so a key that owns
+        every file on the proxy still reads one bounded page at a time.
+        ``purpose`` is applied after parsing, because the managed file table
+        keeps it inside the ``file_object`` blob instead of a column, and rows
+        whose blob will not parse drop out there too, so a chunk of rows can
+        yield fewer matches than the page holds. Successive chunks are read
+        until the page is full or the caller's rows run out, which keeps
+        ``data`` non-empty while matches remain and its last id usable as the
+        next cursor. A first chunk that fills the page costs one query; once a
+        scan has to continue past it, the chunk widens to
+        ``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
+        that many rows instead of one per page. That bound is per query, not
+        per request: the work is still linear in the rows the caller owns, and
+        a filter matching nothing reads every one of them, with no index
+        covering either the owner filter or the sort.
+        """
+        validate_file_list_limit(limit)
+        validate_file_list_purpose(purpose)
+
+        owner_filter: Final = build_owner_filter(user_api_key_dict)
+        if owner_filter is None:
+            return FileListPage(**build_list_page([]))
+
+        if after:
+            cursor_row = await _managed_file_table(self.prisma_client).find_first(
+                where={**owner_filter, "unified_file_id": after}
+            )
+            if cursor_row is None:
+                raise ProxyException(
+                    message=f"Invalid 'after' cursor: no file found with id '{after}'.",
+                    type="invalid_request_error",
+                    param="after",
+                    code=400,
+                    openai_code="invalid_value",
+                )
+
+        page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
+        matches: Final[List[OpenAIFileObject]] = []
+        cursor_id = after
+        chunk_size = page_size + 1
+
+        while len(matches) <= page_size:
+            cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
+            chunk = await _managed_file_table(self.prisma_client).find_many(
+                where=owner_filter,
+                take=chunk_size,
+                order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
+                **cursor_args,
+            )
+            matches.extend(
+                parsed_file_object.model_copy(update={"id": row.unified_file_id})
+                for row in chunk
+                if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
+                and (purpose is None or parsed_file_object.purpose == purpose)
+            )
+            if len(chunk) < chunk_size:
+                break
+            cursor_id = chunk[-1].unified_file_id
+            chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
+
+        return FileListPage(**build_list_page(matches[:page_size], has_more=len(matches) > page_size))
 
     def _is_batch_polling_enabled(self) -> bool:
         """
@@ -1383,7 +1609,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             import litellm.proxy.proxy_server as proxy_server_module
 
             # Check if the scheduler has the batch cost checking job registered
-            scheduler = getattr(proxy_server_module, "scheduler", None)
+            scheduler: Final[_SchedulerWithJobLookup | None] = getattr(proxy_server_module, "scheduler", None)
             if scheduler is None:
                 return False
 
@@ -1429,7 +1655,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             )
         MAX_MATCHES_TO_RETURN = 10
 
-        batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
+        batches = await _managed_object_table(self.prisma_client).find_many(
             where={
                 "file_purpose": "batch",
                 "batch_processed": False,
@@ -1439,11 +1665,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             order={"created_at": "desc"},
         )
 
-        referencing_batches = []
+        referencing_batches: Final[list[dict[str, object]]] = []
         for batch in batches:
             try:
                 # Parse the batch file_object to check for file references
-                batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
+                decoded_file_object = _decode_json_blob(batch.file_object)
+                batch_data: Mapping[str, object] = (
+                    decoded_file_object if isinstance(decoded_file_object, Mapping) else {}
+                )
 
                 # Extract file IDs from batch
                 # Batches typically reference the unified file ID in input_file_id

enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py

@@ -16,6 +16,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
     is_base64_encoded_unified_id,
 )
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.types.utils import LLMResponseTypes
 from litellm.types.vector_stores import (
     VectorStoreCreateOptionalRequestParams,
     VectorStoreCreateResponse,
@@ -24,6 +25,7 @@ from litellm.types.vector_stores import (
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
+    from litellm.caching.caching import DualCache
     from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
     from litellm.proxy.utils import PrismaClient as _PrismaClient
 
@@ -156,7 +158,7 @@ class _PROXY_LiteLLMManagedVectorStores(
 
         # Create vector store for each model
         # Convert TypedDict to Dict[str, Any] for base class compatibility
-        request_data_dict: Dict[str, Any] = dict(create_request)
+        request_data_dict: Dict[str, object] = dict(create_request)
         responses = await self.create_resource_for_each_model(
             llm_router=llm_router,
             request_data=request_data_dict,
@@ -209,7 +211,7 @@ class _PROXY_LiteLLMManagedVectorStores(
         limit: Optional[int] = None,
         after: Optional[str] = None,
         order: Optional[str] = None,
-    ) -> Dict[str, Any]:
+    ) -> Dict[str, object]:
         """
         List vector stores created by a user.
         
@@ -301,7 +303,7 @@ class _PROXY_LiteLLMManagedVectorStores(
     async def async_pre_call_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
-        cache: Any,
+        cache: "DualCache",
         data: Dict,
         call_type: str,
     ) -> Union[Exception, str, Dict, None]:
@@ -403,8 +405,8 @@ class _PROXY_LiteLLMManagedVectorStores(
         self,
         data: Dict,
         user_api_key_dict: UserAPIKeyAuth,
-        response: Any,
-    ) -> Any:
+        response: LLMResponseTypes,
+    ) -> LLMResponseTypes:
         """
         Post-call hook to transform responses.
         

enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py

@@ -39,9 +39,9 @@ async def available_enterprise_users(
 
     if not premium_user:
         # check if SSO is enabled - show 5 user limit
-        from litellm.proxy.auth.auth_utils import _has_user_setup_sso
+        from litellm.proxy.auth.auth_utils import has_user_setup_sso
 
-        if _has_user_setup_sso():
+        if has_user_setup_sso():
             premium_user_data = EnterpriseLicenseData(
                 max_users=5,
             )

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -12,9 +12,10 @@ Endpoints for /project operations
 
 import json
 from collections.abc import Sequence
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Final
 
 from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import TypeAdapter
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -26,37 +27,50 @@ from litellm.proxy.management_helpers.utils import (
     management_endpoint_wrapper,
 )
 from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
+from litellm.repositories.budget_repository import BudgetRepository
+from litellm.repositories.object_permission_repository import ObjectPermissionRepository
+from litellm.repositories.prisma_protocols import TableActions
+from litellm.repositories.project_repository import ProjectRepository
+from litellm.repositories.team_repository import TeamRepository
+from litellm.repositories.user_repository import UserRepository
+from litellm.repositories.verification_token_repository import VerificationTokenRepository
 
 if TYPE_CHECKING:
     from prisma import models as prisma_models
-    from prisma.actions import (
-        LiteLLM_ProjectTableActions,
-        LiteLLM_TeamTableActions,
-        LiteLLM_VerificationTokenActions,
-    )
+
+    from litellm import Router
 
 router = APIRouter()
 
-
-def _team_table(prisma_client: PrismaClient) -> "LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable]":
-    team_table: LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable] = prisma_client.db.litellm_teamtable
-    return team_table
+_OBJECT_PERMISSION_PAYLOAD: Final = TypeAdapter(dict[str, object])
 
 
-def _project_table(prisma_client: PrismaClient) -> "LiteLLM_ProjectTableActions[prisma_models.LiteLLM_ProjectTable]":
-    project_table: LiteLLM_ProjectTableActions[prisma_models.LiteLLM_ProjectTable] = (
-        prisma_client.db.litellm_projecttable
-    )
-    return project_table
+def _team_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_TeamTable"]:
+    return TeamRepository(prisma_client).table
+
+
+def _project_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_ProjectTable"]:
+    return ProjectRepository(prisma_client).table
 
 
 def _verification_token_table(
     prisma_client: PrismaClient,
-) -> "LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken]":
-    verification_token_table: LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken] = (
-        prisma_client.db.litellm_verificationtoken
-    )
-    return verification_token_table
+) -> TableActions["prisma_models.LiteLLM_VerificationToken"]:
+    return VerificationTokenRepository(prisma_client).table
+
+
+def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]:
+    return BudgetRepository(prisma_client).table
+
+
+def _object_permission_table(
+    prisma_client: PrismaClient,
+) -> TableActions["prisma_models.LiteLLM_ObjectPermissionTable"]:
+    return ObjectPermissionRepository(prisma_client).table
+
+
+def _user_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_UserTable"]:
+    return UserRepository(prisma_client).table
 
 
 def _jsonified(prisma_client: PrismaClient, payload: dict[str, object]) -> dict[str, object]:
@@ -205,6 +219,114 @@ def _check_team_project_limits(
         )
 
 
+def _project_models_missing_positive_quota(
+    models: list[str] | None,
+    rpm_limits: Mapping[str, object] | None,
+    tpm_limits: Mapping[str, object] | None,
+) -> list[str]:
+    """Return the models that lack a positive `rpm` AND `tpm` quota.
+
+    A valid quota is a positive integer; null, zero, and negative are rejected
+    because downstream rate limiters treat a non-positive limit as immediately
+    exhausted (every request blocked).
+    """
+
+    def _is_positive(value: object) -> bool:
+        return isinstance(value, int) and not isinstance(value, bool) and value > 0
+
+    rpm = rpm_limits or {}
+    tpm = tpm_limits or {}
+    return [model for model in (models or []) if not _is_positive(rpm.get(model)) or not _is_positive(tpm.get(model))]
+
+
+def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
+    return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
+
+
+def _project_models_expanding_at_request_time(
+    models: Sequence[str] | None, access_group_names: frozenset[str]
+) -> tuple[str, ...]:
+    """Entries project auth expands to many concrete models (`all-proxy-models`, `*` patterns,
+    access groups). The rate limiter looks quotas up by the exact requested model name, so a
+    quota keyed on one of these entries is never applied."""
+    return tuple(
+        model
+        for model in (models or ())
+        if model == SpecialModelNames.all_proxy_models.value or "*" in model or model in access_group_names
+    )
+
+
+def _raise_on_project_models_expanding_at_request_time(
+    models: Sequence[str] | None, access_group_names: frozenset[str]
+) -> None:
+    expanding: Final = _project_models_expanding_at_request_time(models, access_group_names)
+    if not expanding:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while 'enforce_project_model_quota' is enabled. List concrete model names instead."
+        },
+    )
+
+
+def _raise_on_missing_project_model_quota(
+    data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset()
+) -> None:
+    """Require a positive `rpm`/`tpm` quota for every model on project CREATE.
+
+    `model_rpm_limit`/`model_tpm_limit` are relocated into `metadata` by the request
+    model's `set_model_info` validator, so they are read from there.
+
+    Only invoked when `general_settings.enforce_project_model_quota` is enabled
+    (default off), so it is opt-in and does not change behavior for existing users.
+    """
+    _raise_on_project_models_expanding_at_request_time(data.models, access_group_names)
+    metadata = data.metadata or {}
+    missing = _project_models_missing_positive_quota(
+        data.models, metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit")
+    )
+    if not missing:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {missing} added to project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
+        },
+    )
+
+
+def _raise_on_missing_project_model_quota_on_update(
+    data: UpdateProjectRequest, existing_project: object, access_group_names: frozenset[str] = frozenset()
+) -> None:
+    """Require a positive `rpm`/`tpm` quota over the RESULTING state on project UPDATE.
+
+    `/project/update` replaces `models` and `metadata` when they are provided, so the
+    check runs on what the project WILL look like: a partial update that doesn't touch
+    models/quota keeps the existing values, while one that adds a model or clears a
+    model's quota must leave every resulting model with a positive limit.
+
+    Only invoked when `general_settings.enforce_project_model_quota` is enabled
+    (default off), so it is opt-in and does not change behavior for existing users.
+    """
+    resulting_models = data.models if data.models is not None else (getattr(existing_project, "models", None) or [])
+    resulting_metadata = (
+        data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {})
+    )
+    _raise_on_project_models_expanding_at_request_time(resulting_models, access_group_names)
+    missing = _project_models_missing_positive_quota(
+        resulting_models, resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit")
+    )
+    if not missing:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {missing} would be left on the project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
+        },
+    )
+
+
 async def _create_budget_for_project(
     data: NewProjectRequest,
     user_id: str | None,
@@ -219,7 +341,7 @@ async def _create_budget_for_project(
 
     new_budget = _jsonified(prisma_client, budget_row.model_dump(exclude_none=True))
 
-    _budget: prisma_models.LiteLLM_BudgetTable = await prisma_client.db.litellm_budgettable.create(
+    _budget: Final = await _budget_table(prisma_client).create(
         data={
             **new_budget,
             "created_by": user_id or litellm_proxy_admin_name,
@@ -242,10 +364,8 @@ async def _set_project_object_permission(
         return None
 
     if data.object_permission is not None:
-        created_object_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
-            await prisma_client.db.litellm_objectpermissiontable.create(
-                data=data.object_permission.model_dump(exclude_none=True),
-            )
+        created_object_permission: Final = await _object_permission_table(prisma_client).create(
+            data=data.object_permission.model_dump(exclude_none=True),
         )
         del data.object_permission
         return created_object_permission.object_permission_id
@@ -352,7 +472,9 @@ async def new_project(
     ```
     """
     from litellm.proxy.proxy_server import (
+        general_settings,
         litellm_proxy_admin_name,
+        llm_router,
         premium_user,
         prisma_client,
     )
@@ -399,6 +521,10 @@ async def new_project(
             data=data,
         )
 
+        # Opt-in (default off): require rpm/tpm for every model added to the project.
+        if general_settings.get("enforce_project_model_quota", False):
+            _raise_on_missing_project_model_quota(data, _router_access_group_names(llm_router))
+
         # Check if user has permission to create projects for this team
         # only team admins can create projects for their team
         has_permission = await _check_user_permission_for_project(
@@ -470,10 +596,8 @@ async def new_project(
         new_project_row = _remove_budget_fields_from_project_data(new_project_row)
 
         verbose_proxy_logger.info(f"new_project_row: {json.dumps(new_project_row, indent=2)}")
-        response: prisma_models.LiteLLM_ProjectTable = await prisma_client.db.litellm_projecttable.create(
-            data={
-                **new_project_row,  # type: ignore
-            },
+        response: Final = await _project_table(prisma_client).create(
+            data={**new_project_row},
             include={"litellm_budget_table": True},
         )
 
@@ -538,7 +662,9 @@ async def update_project(
     ```
     """
     from litellm.proxy.proxy_server import (
+        general_settings,
         litellm_proxy_admin_name,
+        llm_router,
         premium_user,
         prisma_client,
         user_api_key_cache,
@@ -642,6 +768,12 @@ async def update_project(
                 data=data,
             )
 
+        # Opt-in (default off): require rpm/tpm for every model the update would leave on the project.
+        if general_settings.get("enforce_project_model_quota", False):
+            _raise_on_missing_project_model_quota_on_update(
+                data, existing_project, _router_access_group_names(llm_router)
+            )
+
         # Prepare update data
         update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
         update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
@@ -652,7 +784,7 @@ async def update_project(
 
         if budget_updates and existing_project.budget_id:
             # Update existing budget
-            await prisma_client.db.litellm_budgettable.update(
+            await _budget_table(prisma_client).update(
                 where={"budget_id": existing_project.budget_id},
                 data={
                     **budget_updates,
@@ -667,18 +799,17 @@ async def update_project(
         if "object_permission" in update_data:
             object_permission_data = update_data.pop("object_permission")
             if object_permission_data:
+                object_permission_payload: Final = _OBJECT_PERMISSION_PAYLOAD.validate_python(object_permission_data)
                 if existing_project.object_permission_id:
                     # Update existing permission
-                    await prisma_client.db.litellm_objectpermissiontable.update(
+                    await _object_permission_table(prisma_client).update(
                         where={"object_permission_id": existing_project.object_permission_id},
-                        data=object_permission_data,
+                        data=object_permission_payload,
                     )
                 else:
                     # Create new permission
-                    created_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
-                        await prisma_client.db.litellm_objectpermissiontable.create(
-                            data=object_permission_data,
-                        )
+                    created_permission: Final = await _object_permission_table(prisma_client).create(
+                        data=object_permission_payload,
                     )
                     update_data["object_permission_id"] = created_permission.object_permission_id
 
@@ -694,7 +825,7 @@ async def update_project(
         update_data = _remove_budget_fields_from_project_data(update_data)
 
         # Update project
-        updated_project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.update(
+        updated_project: Final = await _project_table(prisma_client).update(
             where={"project_id": data.project_id},
             data=update_data,
             include={"litellm_budget_table": True, "object_permission": True},
@@ -934,7 +1065,7 @@ async def list_projects(
             # Look up the user's team memberships via the reverse-index on
             # LiteLLM_UserTable.teams (maintained by team_member_add alongside
             # members_with_roles). This avoids a full scan of all team rows.
-            user_record: prisma_models.LiteLLM_UserTable | None = await prisma_client.db.litellm_usertable.find_unique(
+            user_record: Final = await _user_table(prisma_client).find_unique(
                 where={"user_id": user_api_key_dict.user_id},
             )
             user_team_ids: list[str] = user_record.teams if user_record is not None and user_record.teams else []

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.58"
+version = "0.1.65"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.58"
+version = "0.1.65"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
@@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
 # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
 # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -47,7 +47,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
-        --python python3
+        --extra mongodb \
+        --python python3.13
 
 # Stage 2 — copy source and install the project + workspace members.
 COPY . .
@@ -59,7 +60,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
-        --python python3
+        --extra mongodb \
+        --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -73,7 +75,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
+      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done

gateway/routes/allowlist.py

@@ -86,6 +86,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/comprehendmedical",
     "/cohere/",
     "/gemini/",
+    "/gigachat/",
     "/google/",
     "/vertex_ai/",
     "/vertex-ai/",

helm/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.1.2
+version: 1.1.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/litellm-helm/README.md

@@ -26,7 +26,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `replicaCount`              | The number of LiteLLM Proxy pods to be deployed                                                                                                                                                                                               | `1`                       |
 | `masterkeySecretName`       | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name.                                                                                                              | N/A                       |
 | `masterkeySecretKey`        | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key.                                                                                                              | N/A                       |
-| `masterkey`                 | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated.                                                                                                                                           | N/A                       |
+| `masterkey`                 | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated on first install and reused on upgrades. | N/A                       |
 | `environmentSecrets`        | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                                   | `[]`                      |
 | `environmentConfigMaps`     | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                             | `[]`                      |
 | `image.repository`          | LiteLLM Proxy image repository                                                                                                                                                                                                                | `ghcr.io/berriai/litellm` |
@@ -212,6 +212,8 @@ service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`
 The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
 was not provided to the helm command line, the `masterkey` is a randomly
 generated string in the `sk-...` format stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
+The key is generated once on the first install; later `helm upgrade` runs reuse the
+value already in that Secret, so upgrading never rotates the master key.
 
 ```bash
 kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.masterkey}"

helm/litellm-helm/templates/secret-masterkey.yaml

@@ -1,9 +1,11 @@
 {{- if not .Values.masterkeySecretName }}
-{{ $masterkey := (.Values.masterkey | default (printf "sk-%s" (randAlphaNum 18))) }}
+{{- $secretName := printf "%s-masterkey" (include "litellm.fullname" .) }}
+{{- $existing := lookup "v1" "Secret" .Release.Namespace $secretName }}
+{{- $masterkey := .Values.masterkey | default (dig "data" "masterkey" "" $existing | b64dec) | default (printf "sk-%s" (randAlphaNum 18)) }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ include "litellm.fullname" . }}-masterkey
+  name: {{ $secretName }}
 data:
   masterkey: {{ $masterkey | b64enc }}
 type: Opaque

helm/litellm-helm/tests/hpa_tests.yaml

@@ -1,4 +1,4 @@
-suite: "hpa with behavior"
+suite: "hpa"
 templates:
   - hpa.yaml
 tests:
@@ -23,14 +23,44 @@ tests:
       - equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 }
       - equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 }
 
----
-suite: "hpa without behavior"
-templates:
-  - hpa.yaml
-tests:
   - it: "does not render behavior when not set"
     set:
       autoscaling.enabled: true
     asserts:
       - isKind: { of: HorizontalPodAutoscaler }
       - isNull: { path: spec.behavior }
+
+  - it: "scales on cpu at the documented 60 percent by default"
+    set:
+      autoscaling.enabled: true
+    asserts:
+      - isKind: { of: HorizontalPodAutoscaler }
+      - equal: { path: "spec.metrics[0].resource.name", value: cpu }
+      - equal: { path: "spec.metrics[0].resource.target.type", value: Utilization }
+      - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 }
+
+  - it: "does not scale on memory by default"
+    set:
+      autoscaling.enabled: true
+    asserts:
+      - lengthEqual: { path: spec.metrics, count: 1 }
+
+  - it: "honours an explicit cpu target override"
+    set:
+      autoscaling.enabled: true
+      autoscaling.targetCPUUtilizationPercentage: 75
+    asserts:
+      - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 }
+
+  - it: "renders a memory metric only when a memory target is set"
+    set:
+      autoscaling.enabled: true
+      autoscaling.targetMemoryUtilizationPercentage: 80
+    asserts:
+      - lengthEqual: { path: spec.metrics, count: 2 }
+      - equal: { path: "spec.metrics[1].resource.name", value: memory }
+      - equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 }
+
+  - it: "renders no hpa when autoscaling is disabled"
+    asserts:
+      - hasDocuments: { count: 0 }

helm/litellm-helm/tests/masterkey-secret_tests.yaml

@@ -15,6 +15,53 @@ tests:
           # Note: The masterkey is generated as "sk-<18-random-chars>" in plain text,
           # but stored as base64 encoded in Kubernetes secret (requirement).
           # "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern.
+  - it: should reuse the master key already stored in the cluster instead of generating a new one on upgrade
+    template: secret-masterkey.yaml
+    set:
+      masterkeySecretName: ""
+    kubernetesProvider:
+      scheme:
+        "v1/Secret":
+          gvr:
+            version: "v1"
+            resource: "secrets"
+          namespaced: true
+      objects:
+        - kind: Secret
+          apiVersion: v1
+          metadata:
+            name: RELEASE-NAME-litellm-masterkey
+            namespace: NAMESPACE
+          data:
+            masterkey: c2stZXhpc3Rpbmcta2V5
+    asserts:
+      - equal:
+          path: data.masterkey
+          value: c2stZXhpc3Rpbmcta2V5
+  - it: should let an explicit masterkey value override the one already stored in the cluster
+    template: secret-masterkey.yaml
+    set:
+      masterkeySecretName: ""
+      masterkey: sk-explicit
+    kubernetesProvider:
+      scheme:
+        "v1/Secret":
+          gvr:
+            version: "v1"
+            resource: "secrets"
+          namespaced: true
+      objects:
+        - kind: Secret
+          apiVersion: v1
+          metadata:
+            name: RELEASE-NAME-litellm-masterkey
+            namespace: NAMESPACE
+          data:
+            masterkey: c2stZXhpc3Rpbmcta2V5
+    asserts:
+      - equal:
+          path: data.masterkey
+          value: c2stZXhwbGljaXQ=
   - it: should not create a secret if masterkeySecretName is set
     template: secret-masterkey.yaml
     set:

helm/litellm-helm/values.yaml

@@ -200,7 +200,16 @@ autoscaling:
   enabled: false
   minReplicas: 1
   maxReplicas: 100
-  targetCPUUtilizationPercentage: 80
+  # 60 is the documented recommendation. See "Recommended Machine Specifications"
+  # in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe
+  # above only after up to failureThreshold x periodSeconds = 300 seconds, so a target
+  # high enough to trip near saturation adds capacity minutes after it was needed.
+  targetCPUUtilizationPercentage: 60
+  # Deliberately left unset rather than given a value. The prisma query engine's
+  # resident memory is a high-water mark that ratchets to the pod's worst-ever write
+  # and is never returned, so a memory target reads the largest write a pod ever did
+  # rather than what it is doing now, and replicas ratchet up without scaling back in.
+  # Memory is a floor to provision under 'resources', not a signal to scale on.
   # targetMemoryUtilizationPercentage: 80
   # behavior: {}
 

helm/litellm/templates/_helpers.tpl

@@ -428,3 +428,16 @@ envFrom:
 {{- end }}
 {{- end }}
 {{- end -}}
+
+{{/*
+ingress-nginx's admission webhook rejects a dot in an Exact or Prefix path
+(strict-validate-path-type) and serves ImplementationSpecific as a plain
+prefix location, so a dotted path takes that type there.
+*/}}
+{{- define "litellm.ingress.pathType" -}}
+{{- if and (eq .controller "nginx") (contains "." .path) -}}
+ImplementationSpecific
+{{- else -}}
+{{- .pathType -}}
+{{- end -}}
+{{- end -}}

helm/litellm/templates/backend/deployment.yaml

@@ -7,6 +7,10 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: backend
 spec:
+  {{- with .Values.backend.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "litellm.backend.selectorLabels" . | nindent 6 }}

helm/litellm/templates/gateway/deployment.yaml

@@ -7,6 +7,10 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: gateway
 spec:
+  {{- with .Values.gateway.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "litellm.gateway.selectorLabels" . | nindent 6 }}

helm/litellm/templates/ingress.yaml

@@ -5,6 +5,48 @@
 {{- $gatewayPort := .Values.gateway.service.port -}}
 {{- $backendPort := .Values.backend.service.port -}}
 {{- $uiPort := .Values.ui.service.port -}}
+{{- $controller := .Values.ingress.controller | default "alb" -}}
+{{- if not (has $controller (list "alb" "nginx")) }}
+{{- fail (printf "ingress.controller: unknown controller %q, expected one of alb, nginx" $controller) }}
+{{- end }}
+{{/*
+  Backends addressable from ingress.extraPaths, keyed by the `service` field.
+*/}}
+{{- $extraPathBackends := dict
+    "gateway" (dict "name" $gatewayName "port" $gatewayPort)
+    "backend" (dict "name" $backendName "port" $backendPort)
+    "ui" (dict "name" $uiName "port" $uiPort)
+-}}
+{{/*
+  UI paths (Next.js static export).
+
+  /ui/* is where the SPA serves its login + dashboard routes (e.g. /ui/login).
+  Without it, /ui/* falls into the catch-all → backend → 404.
+
+  The App Router (output: "export", basePath: "") emits the RSC/flight payload
+  for every route as a ROOT-level <route>.txt (/index.txt, /teams.txt,
+  /__next._tree.txt, ...). The client router fetches these on every soft
+  navigation / prefetch as <route>.txt?_rsc=<hash> (the query string is
+  irrelevant to path matching). They are not under /ui, /_next, or
+  /litellm-asset-prefix, so without /*.txt they fall to the backend catch-all
+  → 404 → client-side navigation never settles and the login flow spins in an
+  infinite redirect loop (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt
+  from the export; the rule only routes the request to it. It needs an
+  ingress controller whose ImplementationSpecific path is a wildcard pattern
+  (AWS ALB: `*` = 0+ chars), so it is rendered for ingress.controller=alb
+  only: ingress-nginx serves ImplementationSpecific as a literal prefix
+  location, where /*.txt can never match.
+*/}}
+{{- $uiPaths := list
+    (dict "path" "/" "pathType" "Exact")
+    (dict "path" "/favicon.ico" "pathType" "Exact")
+    (dict "path" "/litellm-asset-prefix" "pathType" "Prefix")
+    (dict "path" "/_next" "pathType" "Prefix")
+    (dict "path" "/ui" "pathType" "Prefix")
+-}}
+{{- if eq $controller "alb" }}
+{{- $uiPaths = append $uiPaths (dict "path" "/*.txt" "pathType" "ImplementationSpecific") }}
+{{- end }}
 {{/*
   Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
   Versioned paths are listed explicitly to avoid routing management routes
@@ -39,6 +81,15 @@
   routes at startup -> 404. So /test is rendered as a standalone Exact path
   and /test/* falls through to the backend catch-all.
 */}}
+{{/*
+  Every "<path>|<pathType>" this template renders on its own. An
+  ingress.extraPaths entry that repeats one of these is rejected: duplicates
+  in a single rule are resolved by position or by controller-specific tie
+  breaking, so the operator entry could take over a built-in route (an entry
+  at "/" Prefix would swallow the whole backend management API) instead of
+  adding to it.
+*/}}
+{{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -64,65 +115,17 @@ spec:
       http:
         paths:
           # --- UI (Next.js static export) ---
-          - path: /
-            pathType: Exact
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          - path: /favicon.ico
-            pathType: Exact
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          - path: /litellm-asset-prefix
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          - path: /_next
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          # /ui/* is where the Next.js SPA serves its login + dashboard
-          # routes (e.g. /ui/login). Without this, /ui/* falls into the
-          # catch-all → backend → 404.
-          - path: /ui
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          # Next.js App Router (output: "export", basePath: "") emits the
-          # RSC/flight payload for every route as a ROOT-level <route>.txt
-          # (/index.txt, /teams.txt, /__next._tree.txt, ...). The client
-          # router fetches these on every soft navigation / prefetch as
-          # <route>.txt?_rsc=<hash> (the query string is irrelevant to path
-          # matching). They are not under /ui, /_next, or
-          # /litellm-asset-prefix, so without this rule they fall to the
-          # backend catch-all → 404 → client-side navigation never settles
-          # and the login flow spins in an infinite redirect loop
-          # (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt from the
-          # export; this rule only routes the request to it. Needs an
-          # ingress controller whose ImplementationSpecific path is a
-          # wildcard pattern (AWS ALB: `*` = 0+ chars); this chart targets
-          # the AWS Load Balancer Controller.
-          - path: /*.txt
-            pathType: ImplementationSpecific
+          {{- range $uiPaths }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" .path "pathType" .pathType) }}
+          {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path $pathType) }}
+          - path: {{ .path }}
+            pathType: {{ $pathType }}
             backend:
               service:
                 name: {{ $uiName }}
                 port:
                   number: {{ $uiPort }}
+          {{- end }}
           # --- Gateway data plane ---
           # Exact /test only (see the $gatewayPrefixes comment above);
           # /test/* MCP management endpoints fall to the backend catch-all.
@@ -134,14 +137,58 @@ spec:
                 port:
                   number: {{ $gatewayPort }}
           {{- range $gatewayPrefixes }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }}
+          {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }}
           - path: {{ . }}
-            pathType: Prefix
+            pathType: {{ $pathType }}
             backend:
               service:
                 name: {{ $gatewayName }}
                 port:
                   number: {{ $gatewayPort }}
           {{- end }}
+          {{- /*
+            --- Operator-supplied extra paths (ingress.extraPaths) ---
+            Rendered after every built-in path so an entry can never take
+            precedence over a default, and before the backend catch-all.
+            Position only decides the match on controllers that honour manifest
+            order: the AWS Load Balancer Controller (ingress.controller=alb)
+            sorts Exact paths first and Prefix paths longest-first, but keeps
+            ImplementationSpecific paths in manifest order, which is what the
+            /*.txt rule above already depends on. ingress-nginx ignores order
+            and serves the longest matching location.
+          */}}
+          {{- range $idx, $extra := .Values.ingress.extraPaths }}
+          {{- if not (kindIs "map" $extra) }}
+          {{- fail (printf "ingress.extraPaths[%d]: each entry must be a mapping with a 'path' key" $idx) }}
+          {{- end }}
+          {{- if not $extra.path }}
+          {{- fail (printf "ingress.extraPaths[%d]: 'path' is required" $idx) }}
+          {{- end }}
+          {{- $service := $extra.service | default "gateway" }}
+          {{- $target := get $extraPathBackends $service }}
+          {{- if not $target }}
+          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown service %q, expected one of backend, gateway, ui" $idx $extra.path $service) }}
+          {{- end }}
+          {{- $requestedPathType := $extra.pathType | default "Prefix" }}
+          {{- if not (has $requestedPathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
+          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $requestedPathType) }}
+          {{- end }}
+          {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" $extra.path "pathType" $requestedPathType) }}
+          {{- if eq $extra.path "/" }}
+          {{- fail (printf "ingress.extraPaths[%d]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture" $idx) }}
+          {{- end }}
+          {{- if has (printf "%s|%s" $extra.path $pathType) $builtinPathKeys }}
+          {{- fail (printf "ingress.extraPaths[%d]: path %s with pathType %s is already routed by this chart, and a duplicate would take it over rather than add to it" $idx $extra.path $pathType) }}
+          {{- end }}
+          - path: {{ $extra.path | quote }}
+            pathType: {{ $pathType }}
+            backend:
+              service:
+                name: {{ $target.name }}
+                port:
+                  number: {{ $target.port }}
+          {{- end }}
           # --- Catch-all → backend (management API: /key/*, /user/*, /team/*, ...) ---
           - path: /
             pathType: Prefix

helm/litellm/templates/migrations-job.yaml

@@ -7,6 +7,8 @@
 #
 # Running this pre-upgrade closes the window where new application pods would
 # otherwise serve traffic against the previous release's unmigrated schema.
+# Argo CD users can swap the Helm hook for a PreSync hook through
+# `migrationJob.hooks`, which re-runs the Job on every sync.
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -14,10 +16,18 @@ metadata:
   labels:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: migrations
+  {{- if or .Values.migrationJob.hooks.helm.enabled .Values.migrationJob.hooks.argocd.enabled }}
   annotations:
+    {{- if .Values.migrationJob.hooks.helm.enabled }}
     helm.sh/hook: pre-install,pre-upgrade
     helm.sh/hook-delete-policy: before-hook-creation
-    helm.sh/hook-weight: "0"
+    helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "0" | quote }}
+    {{- end }}
+    {{- if .Values.migrationJob.hooks.argocd.enabled }}
+    argocd.argoproj.io/hook: PreSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+    {{- end }}
+  {{- end }}
 spec:
   backoffLimit: {{ .Values.migrationJob.backoffLimit }}
   ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
@@ -67,4 +77,16 @@ spec:
       volumes:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.migrationJob.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.migrationJob.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.migrationJob.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

helm/litellm/templates/ui/deployment.yaml

@@ -7,6 +7,10 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: ui
 spec:
+  {{- with .Values.ui.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "litellm.ui.selectorLabels" . | nindent 6 }}

helm/litellm/tests/ingress_controller_tests.yaml

@@ -0,0 +1,205 @@
+suite: test ingress.controller
+templates:
+  - ingress.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: keeps the AWS Load Balancer Controller path types by default
+    set:
+      ingress.enabled: true
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /favicon.ico
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /*.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: renders no dotted Exact or Prefix path for ingress-nginx, whose admission webhook rejects them
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+    asserts:
+      - notMatchRegexRaw:
+          pattern: 'path: /\S*\.\S*\n\s+pathType: (Exact|Prefix)\n'
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /favicon.ico
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+
+  - it: drops the /*.txt wildcard for ingress-nginx and keeps every other route as is
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+    asserts:
+      - notContains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /*.txt
+          any: true
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /ui
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /test
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - equal:
+          path: spec.rules[0].http.paths[-1]
+          value:
+            path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+
+  - it: rejects an extraPaths entry that repeats a built-in path at the pathType ingress-nginx renders it with
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /favicon.ico
+          service: ui
+          pathType: ImplementationSpecific
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects a controller it has no path types for
+    set:
+      ingress.enabled: true
+      ingress.controller: traefik
+    asserts:
+      - failedTemplate:
+          errorMessage: 'ingress.controller: unknown controller "traefik", expected one of alb, nginx'
+
+  - it: rejects an extraPaths entry that repeats a built-in path once ingress-nginx normalizes its pathType
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /favicon.ico
+          service: ui
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /favicon.ico with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: renders a dotted extraPaths entry as ImplementationSpecific for ingress-nginx
+    set:
+      ingress.enabled: true
+      ingress.controller: nginx
+      ingress.extraPaths:
+        - path: /eu.assemblyai.custom
+          service: gateway
+        - path: /robots.txt
+          service: ui
+          pathType: Exact
+    asserts:
+      - notMatchRegexRaw:
+          pattern: 'path: "?/\S*\.\S*"?\n\s+pathType: (Exact|Prefix)\n'
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai.custom
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /robots.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: keeps the requested pathType of a dotted extraPaths entry for the AWS Load Balancer Controller
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /eu.assemblyai.custom
+          service: gateway
+        - path: /robots.txt
+          service: ui
+          pathType: Exact
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /eu.assemblyai.custom
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /robots.txt
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000

helm/litellm/tests/ingress_extra_paths_tests.yaml

@@ -0,0 +1,317 @@
+suite: test ingress.extraPaths
+templates:
+  - ingress.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: renders nothing extra between the built-in gateway prefixes and the backend catch-all when unset
+    set:
+      ingress.enabled: true
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-1]
+          value:
+            path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /metrics
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+
+  - it: routes an extra path to the gateway by default, immediately before the backend catch-all
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /watsonx
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - equal:
+          path: spec.rules[0].http.paths[-1]
+          value:
+            path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+
+  - it: keeps every built-in path when extra paths are supplied
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /ui
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /test
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /v1/chat
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /vertex_ai
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+
+  - it: renders every entry in order and honours the service and pathType selectors
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+          service: gateway
+        - path: /my-passthrough
+          pathType: Exact
+          service: backend
+        - path: /brand.txt
+          pathType: ImplementationSpecific
+          service: ui
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-4]
+          value:
+            path: /watsonx
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - equal:
+          path: spec.rules[0].http.paths[-3]
+          value:
+            path: /my-passthrough
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /brand.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: addresses the component services by their configured ports
+    set:
+      ingress.enabled: true
+      gateway.service.port: 8000
+      backend.service.port: 8001
+      ui.service.port: 8080
+      ingress.extraPaths:
+        - path: /watsonx
+        - path: /my-passthrough
+          service: backend
+        - path: /brand.txt
+          service: ui
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-4].backend.service.port.number
+          value: 8000
+      - equal:
+          path: spec.rules[0].http.paths[-3].backend.service.port.number
+          value: 8001
+      - equal:
+          path: spec.rules[0].http.paths[-2].backend.service.port.number
+          value: 8080
+
+  - it: rejects an entry naming a service the chart does not deploy
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+          service: proxy
+    asserts:
+      - failedTemplate:
+          errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown service "proxy", expected one of backend, gateway, ui'
+
+  - it: rejects an entry whose pathType is not a kubernetes pathType
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+          pathType: prefix
+    asserts:
+      - failedTemplate:
+          errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix'
+
+  - it: rejects an entry with no path
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: 'path' is required"
+
+
+  - it: rejects a root entry that would take over the backend catch-all
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
+
+  - it: rejects a root entry that would take over the UI root
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /
+          pathType: Exact
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
+
+  # A root ImplementationSpecific entry duplicates no built-in pair, so the
+  # duplicate check alone would admit it. It is still dead: the built-in
+  # Exact / sorts ahead of it on the AWS Load Balancer Controller and claims
+  # the only request its pattern matches, so it renders and never routes.
+  - it: rejects a root entry that would render but never match
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /
+          pathType: ImplementationSpecific
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
+
+  - it: rejects an entry that would take over a UI prefix
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /ui
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /ui with pathType Prefix is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects an entry that would take over the UI RSC payload rule
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /*.txt
+          pathType: ImplementationSpecific
+          service: backend
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /*.txt with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects an entry that would take over a gateway data-plane prefix
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /v1/chat
+          service: backend
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /v1/chat with pathType Prefix is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects an entry that would take over the exact /test route
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /test
+          pathType: Exact
+          service: backend
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /test with pathType Exact is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: allows a built-in path under a different pathType, which is a distinct rule
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /ui
+          pathType: Exact
+          service: ui
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /ui
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: rejects a bare string entry instead of failing on template internals
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - /watsonx
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: each entry must be a mapping with a 'path' key"

helm/litellm/tests/migration_job_hooks_tests.yaml

@@ -0,0 +1,63 @@
+suite: test migrations Job hook annotations
+templates:
+  - migrations-job.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: runs as a Helm pre-install / pre-upgrade hook by default
+    asserts:
+      - equal:
+          path: metadata.annotations["helm.sh/hook"]
+          value: pre-install,pre-upgrade
+      - equal:
+          path: metadata.annotations["helm.sh/hook-delete-policy"]
+          value: before-hook-creation
+      - equal:
+          path: metadata.annotations["helm.sh/hook-weight"]
+          value: "0"
+      - notExists:
+          path: metadata.annotations["argocd.argoproj.io/hook"]
+
+  - it: adds the Argo CD PreSync hook when asked
+    set:
+      migrationJob.hooks.argocd.enabled: true
+    asserts:
+      - equal:
+          path: metadata.annotations["argocd.argoproj.io/hook"]
+          value: PreSync
+      - equal:
+          path: metadata.annotations["argocd.argoproj.io/hook-delete-policy"]
+          value: BeforeHookCreation
+
+  - it: drops the Helm hook so Argo CD owns the Job
+    set:
+      migrationJob.hooks.argocd.enabled: true
+      migrationJob.hooks.helm.enabled: false
+    asserts:
+      - equal:
+          path: metadata.annotations["argocd.argoproj.io/hook"]
+          value: PreSync
+      - notExists:
+          path: metadata.annotations["helm.sh/hook"]
+      - notExists:
+          path: metadata.annotations["helm.sh/hook-delete-policy"]
+      - notExists:
+          path: metadata.annotations["helm.sh/hook-weight"]
+
+  - it: renders an ordinary Job when both hooks are disabled
+    set:
+      migrationJob.hooks.helm.enabled: false
+    asserts:
+      - notExists:
+          path: metadata.annotations
+      - equal:
+          path: kind
+          value: Job
+
+  - it: honours a custom Helm hook weight
+    set:
+      migrationJob.hooks.helm.weight: "-5"
+    asserts:
+      - equal:
+          path: metadata.annotations["helm.sh/hook-weight"]
+          value: "-5"

helm/litellm/tests/migration_job_tests.yaml

@@ -1,4 +1,4 @@
-suite: test migrations Job ServiceAccount resolution and pod hardening
+suite: test migrations Job ServiceAccount resolution, pod hardening, and scheduling
 templates:
   - migrations-job.yaml
 values:
@@ -188,3 +188,69 @@ tests:
     asserts:
       - notExists:
           path: spec.activeDeadlineSeconds
+
+  - it: renders no scheduling fields by default
+    asserts:
+      - isNull:
+          path: spec.template.spec.nodeSelector
+      - isNull:
+          path: spec.template.spec.tolerations
+      - isNull:
+          path: spec.template.spec.affinity
+
+  - it: renders nodeSelector, tolerations, and affinity from the migrationJob values
+    set:
+      migrationJob.nodeSelector:
+        intent: no-csi-nodes
+      migrationJob.tolerations:
+        - key: intent
+          operator: Equal
+          value: no-csi-nodes
+          effect: NoSchedule
+      migrationJob.affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: intent
+                    operator: In
+                    values:
+                      - no-csi-nodes
+    asserts:
+      - equal:
+          path: spec.template.spec.nodeSelector
+          value:
+            intent: no-csi-nodes
+      - equal:
+          path: spec.template.spec.tolerations
+          value:
+            - key: intent
+              operator: Equal
+              value: no-csi-nodes
+              effect: NoSchedule
+      - equal:
+          path: spec.template.spec.affinity
+          value:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                  - matchExpressions:
+                      - key: intent
+                        operator: In
+                        values:
+                          - no-csi-nodes
+
+  - it: does not inherit the gateway's scheduling values
+    set:
+      gateway.nodeSelector:
+        intent: no-csi-nodes
+      gateway.tolerations:
+        - key: intent
+          operator: Equal
+          value: no-csi-nodes
+          effect: NoSchedule
+    asserts:
+      - isNull:
+          path: spec.template.spec.nodeSelector
+      - isNull:
+          path: spec.template.spec.tolerations

helm/litellm/tests/rollout_strategy_tests.yaml

@@ -0,0 +1,66 @@
+suite: test rolling update strategy on the component deployments
+templates:
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - ui/deployment.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: leaves the strategy to Kubernetes defaults when unset
+    asserts:
+      - notExists:
+          path: spec.strategy
+
+  - it: renders the configured strategy on each deployment
+    set:
+      gateway.strategy:
+        type: RollingUpdate
+        rollingUpdate:
+          maxUnavailable: 0
+          maxSurge: 1
+      backend.strategy:
+        type: RollingUpdate
+        rollingUpdate:
+          maxUnavailable: "25%"
+          maxSurge: 2
+      ui.strategy:
+        type: Recreate
+    asserts:
+      - equal:
+          path: spec.strategy
+          value:
+            type: RollingUpdate
+            rollingUpdate:
+              maxUnavailable: 0
+              maxSurge: 1
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.strategy
+          value:
+            type: RollingUpdate
+            rollingUpdate:
+              maxUnavailable: 25%
+              maxSurge: 2
+        template: backend/deployment.yaml
+      - equal:
+          path: spec.strategy
+          value:
+            type: Recreate
+        template: ui/deployment.yaml
+
+  - it: keeps a component on the cluster default when only another one sets a strategy
+    set:
+      gateway.strategy:
+        type: Recreate
+    asserts:
+      - equal:
+          path: spec.strategy.type
+          value: Recreate
+        template: gateway/deployment.yaml
+      - notExists:
+          path: spec.strategy
+        template: backend/deployment.yaml
+      - notExists:
+          path: spec.strategy
+        template: ui/deployment.yaml

helm/litellm/values.yaml

@@ -10,9 +10,43 @@ imagePullSecrets: []
 ingress:
   enabled: false
   className: ""
+  # Which ingress controller serves this Ingress. Controllers disagree on the
+  # pathTypes they accept, so this picks the pathType of the dotted paths, the
+  # built-in ones and any dotted extraPaths entry alike:
+  #   alb    AWS Load Balancer Controller (default): Exact and Prefix paths plus
+  #          the /*.txt wildcard that routes the UI's RSC payloads.
+  #   nginx  ingress-nginx: its admission webhook rejects a dot in an Exact or
+  #          Prefix path (strict-validate-path-type, on by default from v1.12.0
+  #          until v1.12.6 / v1.13.2 allowed dots again), so /favicon.ico and
+  #          /eu.assemblyai render as ImplementationSpecific, which nginx serves
+  #          as a plain prefix location. /*.txt is dropped: nginx has no
+  #          wildcard pathType, so that rule could never match there.
+  controller: alb
   annotations: {}
   host: ""        # optional; if set, becomes the rule's host
   tls: []
+  # Extra HTTP paths appended to the ingress rule. Additive: every built-in
+  # UI / gateway / backend path is still rendered, these entries are placed
+  # after them and before the backend catch-all, and an entry that repeats a
+  # path the chart already routes is rejected at render time rather than
+  # silently taking it over.
+  #
+  # The chart's built-in gateway prefix list is a snapshot of the data-plane
+  # surface at release time. Use extraPaths for passthrough routes it does not
+  # cover: a provider prefix added upstream after this chart version, or a
+  # custom general_settings.pass_through_endpoints route.
+  #
+  #   path      required; the HTTP path to route
+  #   service   which component serves it: gateway (default), backend, or ui
+  #   pathType  Prefix (default), Exact, or ImplementationSpecific; a dotted
+  #             path renders as ImplementationSpecific when controller is nginx
+  #
+  # The target component only answers paths its own route allowlist keeps, so
+  # a path here still has to be one that component serves.
+  extraPaths: []
+  # - path: /watsonx
+  #   pathType: Prefix
+  #   service: gateway
 
 # Per-component ServiceAccounts for gateway, backend, and ui.
 #
@@ -54,6 +88,22 @@ serviceAccounts:
 # generate` — the migration engine doesn't need the generated client.
 migrationJob:
   enabled: true
+  # Which controller is responsible for running the Job.
+  #
+  # `helm.enabled` renders the Helm pre-install / pre-upgrade hook, so the Job
+  # runs whenever `helm upgrade` sees a change to apply. `argocd.enabled`
+  # renders an Argo CD PreSync hook instead, which runs the Job on every sync
+  # even when the rendered manifests are unchanged: the way to re-run
+  # migrations on demand from a GitOps pipeline. Turning the Helm hook off
+  # while the Argo CD hook is on leaves the Job out of Helm's own upgrade
+  # path, which is what Argo CD users want since Argo, not Helm, applies the
+  # manifests.
+  hooks:
+    helm:
+      enabled: true
+      weight: "0"
+    argocd:
+      enabled: false
   backoffLimit: 4
   ttlSecondsAfterFinished: 120
   # Wall-clock budget for the whole Job, shared across every `backoffLimit`
@@ -102,6 +152,13 @@ migrationJob:
   # the writable scratch space a read-only root filesystem needs.
   volumes: []
   volumeMounts: []
+  # Scheduling for the Job pod, same shape as gateway.nodeSelector /
+  # gateway.tolerations / gateway.affinity. The Job does not inherit the other
+  # components' scheduling values: a migration usually needs a larger node
+  # than the gateway, so pin it here explicitly.
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
   image:
     repository: ghcr.io/berriai/litellm-migrations
     tag: ""              # defaults to .Chart.AppVersion
@@ -236,6 +293,15 @@ gateway:
     initialDelaySeconds: 5
     periodSeconds: 10
     timeoutSeconds: 10
+  # Rolling update tuning for the gateway Deployment. Empty by default, so
+  # Kubernetes applies its own RollingUpdate defaults (25% maxSurge /
+  # 25% maxUnavailable). Example, for a surge-only rollout behind a load
+  # balancer that must never lose capacity:
+  #   type: RollingUpdate
+  #   rollingUpdate:
+  #     maxUnavailable: 0
+  #     maxSurge: 1
+  strategy: {}
   # Optional startupProbe. Empty by default, so existing installs are unchanged
   # and liveness/readiness apply from container start. Set it to gate
   # liveness/readiness until a slow cold start finishes — a high failureThreshold
@@ -348,6 +414,8 @@ backend:
     initialDelaySeconds: 5
     periodSeconds: 10
     timeoutSeconds: 10
+  # Same shape as gateway.strategy.
+  strategy: {}
   # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
   startupProbe: {}
   hpa:
@@ -412,6 +480,8 @@ ui:
     httpGet: { path: /, port: http }
     initialDelaySeconds: 2
     periodSeconds: 10
+  # Same shape as gateway.strategy.
+  strategy: {}
   # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
   startupProbe: {}
   hpa:
@@ -428,9 +498,11 @@ ui:
     maxUnavailable: ""
   podAnnotations: {}
   # Same shape as the gateway blocks of the same name. The nginx runtime
-  # writes its pid, cache, and proxy temp files under the image's root
-  # filesystem, so `securityContext.readOnlyRootFilesystem: true` here needs
-  # emptyDir volumes mounted over those paths.
+  # writes its pid, cache, and proxy temp files under /tmp, so it boots as
+  # any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem:
+  # true` here needs an emptyDir volume mounted over /tmp. Images before
+  # the /tmp move instead need emptyDirs over /var/cache/nginx and /run to
+  # run as a non-root uid at all.
   podLabels: {}
   podSecurityContext: {}
   securityContext: {}

litellm-proxy-extras/litellm_proxy_extras/migrations/20260804162853_add_budget_window_spend_table/migration.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_BudgetWindowSpend" (
+    "entity_type" TEXT NOT NULL,
+    "entity_id" TEXT NOT NULL,
+    "window_duration" TEXT NOT NULL,
+    "window_start" TIMESTAMP(3) NOT NULL,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_BudgetWindowSpend_pkey" PRIMARY KEY ("entity_type","entity_id","window_duration")
+);
+

litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_backfill_spend_log_timestamps/migration.sql

@@ -1,4 +0,0 @@
-UPDATE "LiteLLM_SpendLogs"
-SET "created_at" = "endTime",
-    "updated_at" = "endTime"
-WHERE "created_at" > "endTime" + interval '1 hour';

litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql

@@ -0,0 +1,18 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+

litellm-proxy-extras/litellm_proxy_extras/migrations/20260828000000_shadow_eval_cost_comparison/migration.sql

@@ -0,0 +1,22 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_cost" DOUBLE PRECISION;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "shadow_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_cache_hit" BOOLEAN NOT NULL DEFAULT false;
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalFunnel" (
+    "job_id" TEXT NOT NULL,
+    "not_sampled" INTEGER NOT NULL DEFAULT 0,
+    "unjudgeable" INTEGER NOT NULL DEFAULT 0,
+    "shed" INTEGER NOT NULL DEFAULT 0,
+    "withheld" INTEGER NOT NULL DEFAULT 0,
+
+    CONSTRAINT "LiteLLM_ShadowEvalFunnel_pkey" PRIMARY KEY ("job_id")
+);