Merge branch 'BerriAI:litellm_internal_staging' into docs-enhancement-docker-compose-hardened

.circleci/config.yml

@@ -112,10 +112,10 @@ commands:
             node --version
             npm --version
   install_rust:
-    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.97.1) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
+    description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself."
     steps:
       - run:
-          name: Install Rust (rustup 1.28.2, toolchain 1.97.1)
+          name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
           command: |
             case "$(uname -m)" in
               x86_64)
@@ -135,7 +135,7 @@ commands:
               "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init"
             echo "${RUSTUP_SHA256}  /tmp/rustup-init" | sha256sum -c -
             chmod +x /tmp/rustup-init
-            /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.97.1
+            /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
             rm -f /tmp/rustup-init
             echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
             export PATH="$HOME/.cargo/bin:$PATH"
@@ -300,7 +300,7 @@ jobs:
             if ($rustupActual -ne $rustupExpected) {
               throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual"
             }
-            & $rustupInit -y --profile minimal --default-toolchain stable
+            & $rustupInit -y --profile minimal --default-toolchain 1.98.0
             if ($LASTEXITCODE -ne 0) {
               exit $LASTEXITCODE
             }
@@ -430,7 +430,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise \
                 --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
@@ -504,7 +504,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise \
                 --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
@@ -631,7 +631,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -v -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -651,126 +651,6 @@ jobs:
             - auth_ui_unit_tests_coverage.xml
             - auth_ui_unit_tests_coverage
 
-  proxy_behavior_tests:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Seed DB schema via prisma db push
-          command: |
-            uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Run proxy management behavior tests
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_behavior \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
-  proxy_security_tests:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Seed DB schema via prisma db push
-          command: |
-            uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Run proxy security tests
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_security_tests \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
-  schema_migration_check:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      # An empty database; the test applies every committed migration itself.
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Check schema.prisma is in sync with committed migrations
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_migration_tests \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
   litellm_router_testing: # Runs all tests with the "router" keyword
     docker:
       - *python312_image
@@ -858,7 +738,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -v -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -985,7 +865,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
                 -n 4 \
@@ -1030,7 +910,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -1074,7 +954,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2 \
@@ -1120,7 +1000,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 --retries 3 --retry-delay 5"
@@ -1211,7 +1091,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1255,7 +1135,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1274,40 +1154,6 @@ jobs:
           paths:
             - search_coverage.xml
             - search_coverage
-  litellm_mapped_enterprise_tests:
-    docker:
-      - *python312_image
-    working_directory: ~/project
-    resource_class: large
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - setup_litellm_enterprise_pip
-      - run:
-          name: Run enterprise tests
-          command: |
-            uv run --no-sync python -m prisma generate
-            mkdir -p test-results
-            TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
-            echo "$TEST_FILES" | circleci tests run \
-              --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
-                --junitxml=test-results/junit-enterprise.xml \
-                --durations=10 \
-                -n 4"
-          no_output_timeout: 15m
-      # Store test results
-      - store_test_results:
-          path: test-results
   batches_testing:
     docker:
       - *python312_image
@@ -1333,7 +1179,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -1377,7 +1223,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -1422,7 +1268,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1501,7 +1347,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 -n 4 \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
@@ -1546,7 +1392,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -1599,7 +1445,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 -n 2 \
                 --reruns 2 --reruns-delay 1"
@@ -2575,45 +2421,6 @@ jobs:
       - wait_for_service:
           url: http://localhost:4000
           timeout: "300"
-      # Add Ruby installation and testing before the existing Node.js and Python tests
-      - run:
-          name: Install Ruby and Bundler
-          command: |
-            # Clone RVM at pinned tag and verify the commit SHA matches the
-            # published tag before running its install script.
-            RVM_VERSION="1.29.12"
-            RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81"
-            git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm
-            RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)"
-            if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then
-              echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2
-              exit 1
-            fi
-
-            # Import RVM signing keys (used by `rvm install` to verify Ruby tarballs)
-            gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-
-            # Install RVM from the verified checkout. The install script
-            # sources `scripts/functions/installer` using paths relative to
-            # its own working directory, so it must be run from /tmp/rvm.
-            (cd /tmp/rvm && ./install --path "$HOME/.rvm")
-            source "$HOME/.rvm/scripts/rvm"
-
-            # Install Ruby 3.2.2 (RVM verifies the tarball PGP signature)
-            rvm install 3.2.2
-            rvm use 3.2.2 --default
-
-            # Install latest Bundler
-            gem install bundler
-
-      - run:
-          name: Run Ruby tests
-          command: |
-            source $HOME/.rvm/scripts/rvm
-            cd tests/pass_through_tests/ruby_passthrough_tests
-            bundle install
-            bundle exec rspec
-          no_output_timeout: 30m
       # Install Node.js directly from nodejs.org with SHA256 verification,
       # instead of piping NodeSource's setup_24.x apt-repo installer into
       # sudo bash (which runs a mutable upstream script unattended).
@@ -3105,12 +2912,6 @@ workflows:
           filters: *main_branches
       - auth_ui_unit_tests:
           filters: *main_branches
-      - proxy_behavior_tests:
-          filters: *main_branches
-      - proxy_security_tests:
-          filters: *main_branches
-      - schema_migration_check:
-          filters: *main_branches
       - build_docker_database_image:
           filters: *main_branches
       - e2e_ui_testing:
@@ -3167,8 +2968,6 @@ workflows:
           filters: *main_branches
       - search_testing:
           filters: *main_branches
-      - litellm_mapped_enterprise_tests:
-          filters: *main_branches
       - batches_testing:
           filters: *main_branches
       - litellm_utils_testing:
@@ -3191,7 +2990,6 @@ workflows:
             - guardrails_testing
             - ocr_testing
             - search_testing
-            - litellm_mapped_enterprise_tests
             - batches_testing
             - litellm_utils_testing
             - pass_through_unit_testing

.github/CODEOWNERS

@@ -1,6 +1,9 @@
 /ui/ @yuneng-berri @ryan-crabbe-berri
 /litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
+/ui/Dockerfile
+/ui/nginx.conf
 /ui/litellm-dashboard/src/lib/http/schema.d.ts
+/ui/litellm-dashboard/tsconfig.tsbuildinfo
 /model_prices_and_context_window.json @mateo-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri
 /litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri

.github/actions/cache-cargo-build/action.yml

@@ -4,17 +4,16 @@ description: >-
   so only the first job on a given Cargo.lock compiles the bridge from scratch.
 
   litellm builds through maturin, which compiles litellm-rust/crates/python-bridge
-  in release mode before it can produce a wheel. `uv sync` therefore pays a full
-  build in every job that installs the workspace: measured at 2m40s per unit shard
-  on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught
-  it, because the uv cache holds wheels uv downloads rather than wheels it builds,
-  and a path dependency whose source moves every commit could never hit that cache
-  anyway. Cargo rebuilds only what changed when its target directory survives, so a
-  warm job pays for the bridge crate alone.
+  in the dev profile for editable installs. `uv sync` therefore pays a full build
+  in every job that installs the workspace. Nothing caught it, because the uv cache
+  holds wheels uv downloads rather than wheels it builds, and a path dependency
+  whose source moves every commit could never hit that cache anyway. Cargo rebuilds
+  only what changed when its target directory survives, so a warm job pays for the
+  bridge crate alone.
 
-  The key namespace is separate from test-rust.yml's. Both cache the same directory,
-  but that workflow fills it with debug and clippy artifacts, which a release build
-  cannot reuse, and a shared key would let whichever ran first deny the other a save.
+  The key namespace is separate from test-rust.yml's check and release caches. They
+  cache the same directory for different workloads, and a shared key would let
+  whichever ran first deny the others a save.
 
 runs:
   using: composite
@@ -26,6 +25,6 @@ runs:
           ~/.cargo/registry
           ~/.cargo/git
           litellm-rust/target
-        key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }}
+        key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }}
         restore-keys: |
-          ${{ runner.os }}-cargo-release-
+          ${{ runner.os }}-maturin-dev-

.github/ci-coverage-allowlist.yml

@@ -5,24 +5,21 @@ description: >-
 
 test_paths:
   - reason: >-
-      The caching suite in tests/local_testing, which runs nowhere. Every job that globs that
-      directory either deselects it (local_testing_part1 and part2 carry `-k "... and not caching
-      and not cache"`) or keeps only another keyword (langfuse, router, assistants), and no job
-      names these files the way redis_caching_unit_tests names test_dual_cache.py. Measured
-      2026-08-20 by collecting the directory under each job's own selector: 118 tests across
-      these eight files are selected by none of them. Listed so the gap is a decision rather
-      than an accident, and so the --slices guard has a baseline to ratchet down from. Revisit
-      when tests/local_testing is ported off CircleCI, where the keyless part of this suite
-      belongs in a real job
+      What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
+      globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
+      not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
+      and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
+      The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
+      run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
+      credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
+      test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
+      test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
+      split that porting tests/local_testing off CircleCI will force, not a job that is red by
+      construction
     paths:
-      - tests/local_testing/test_cache_preset_key.py
       - tests/local_testing/test_caching.py
-      - tests/local_testing/test_caching_handler.py
       - tests/local_testing/test_disk_cache_unit_tests.py
       - tests/local_testing/test_gcs_cache_unit_tests.py
-      - tests/local_testing/test_prompt_caching.py
-      - tests/local_testing/test_responses_stream_cache_keys.py
-      - tests/local_testing/test_unit_test_caching.py
   - reason: >-
       The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
       from a pull request; it needs a live gateway and provider credentials no PR job holds
@@ -92,17 +89,6 @@ test_paths:
       - tests/integration/sandbox/test_e2b_sandbox.py
       - tests/integration/test_oci_integration.py
       - tests/integration/test_oci_proxy_integration.py
-  - reason: >-
-      A unit test for the proxy-extras package that no job invokes, while the package's other
-      tests live under tests/proxy_migration_tests. Measured 2026-08-20: 24 of its 28 tests pass
-      and the 4 in TestMigrationSQLIdempotency fail, because 13 migrations from 2026-03 onward use
-      bare CREATE TABLE, ADD COLUMN, CREATE INDEX and ADD CONSTRAINT rather than the guarded forms
-      this file requires. It also matches those keywords inside SQL comments, so two further
-      migrations are reported that are in fact fine. Wiring it up means deciding what to do about
-      the 13 first, and they cannot simply be edited: Prisma checksums an applied migration, so a
-      changed one breaks migrate deploy for existing installs
-    paths:
-      - tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
 
 dockerfiles:
   - reason: >-

.github/mutmut-coverage.rc

@@ -0,0 +1,5 @@
+# mutmut's gather_coverage() looks covered lines up by absolute path, so the
+# repo's `relative_files = true` makes every lookup miss and mutmut generates
+# zero mutants. Point COVERAGE_RCFILE here for mutation runs only.
+[run]
+relative_files = false

.github/pull_request_template.md

@@ -1,7 +1,10 @@
+<!-- The whole description's target audience is humans, not AI agents: write it in plain, simple,
+     everyday engineering language, extremely parsable and readable at a glance. This goes double for
+     the TLDR, User Flow, and Caveats sections -->
+
 ## TLDR
 
-<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
-     This section must be extremely human parsable, comprehensible, and readable: its target audience is humans, not AI agents -->
+<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max -->
 
 Problem this solves:
 
@@ -110,8 +113,20 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 
 ## Caveats (if any)
 
-<!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
+<!-- Group caveats under severity subheadings (### Severe, ### High, ### Medium, ### Low), with
+     short bullet points inside each, just like the TLDR: one line per bullet, roughly 10 words max
      Call out known limitations, follow-up work, or anything a reviewer should watch out for
+     Include only the tiers that have caveats; drop the empty ones
+     - Severe: inherent to what the PR deliberately ships, there even when the code works as intended:
+       it can degrade or take down a running deployment (e.g. a slow or table-locking boot migration),
+       rewrite data by design, break an existing workflow on purpose, or change auth behavior. An
+       operator must plan around it before rollout
+     - High: an unintended hole: a correctness, security, data-loss, or backward-compatibility bug,
+       unsafe to ship as is
+     - Medium: a real gap someone can hit, but with a workaround or a narrow blast radius
+     - Low: anything else worth noting: naming, cleanup, an edge case nobody hits
+     Nest bullets as deep as helps: hierarchy beats one long line when it makes things clearer to a
+     human reader
      Leave this section empty if there are none -->
 
 ## QA runbook
@@ -134,6 +149,6 @@ Example checklists:
   - [ ] Sanity check: this test makes sense to add and is not hand-wavey (e.g., assert actual expected spend instead of just spend > 0) or potentially flaky
 -->
 
-### Final Attestation
+## Final Attestation
 
 - [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR

.github/scripts/assert_ci_coverage.py

@@ -312,8 +312,23 @@ def _matchable_names(relative_path: str) -> frozenset[str]:
     )
 
 
+def _workflow_named_tokens() -> frozenset[str]:
+    """Test tokens a GitHub Actions job names directly.
+
+    A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a
+    workflow names it, so the slice check has to credit those the same way the census does.
+    """
+    return _invoked_test_tokens(
+        scalar
+        for path in _config_files()
+        if path != CIRCLECI_CONFIG
+        for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
+    )
+
+
 def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
     slices: Final = _slices()
+    named_by_workflow: Final = _workflow_named_tokens()
     globbed: Final = tuple(
         path
         for path in _test_files()
@@ -326,6 +341,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
         )
         for path in globbed
         if not allowlist.covers_test(path)
+        and not any(_token_covers(token, path) for token in named_by_workflow)
         and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices)
     )
 

.github/scripts/close_duplicate_issues.py

@@ -1,230 +0,0 @@
-#!/usr/bin/env python3
-"""
-Detect and close duplicate GitHub issues using title similarity.
-
-Modes:
-  --scan           Compare all open issues against each other (batch)
-  --issue-number N Check a single issue against older open issues
-
-Requires the `gh` CLI to be authenticated.
-"""
-
-import argparse
-import difflib
-import json
-import re
-import subprocess
-import sys
-
-
-def normalize_title(title: str) -> str:
-    """Strip common prefixes, lowercase, and collapse whitespace."""
-    title = re.sub(
-        r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
-        "",
-        title,
-        flags=re.IGNORECASE,
-    )
-    return " ".join(title.lower().split())
-
-
-def gh(*args: str) -> str:
-    """Run a gh CLI command and return stdout."""
-    result = subprocess.run(
-        ["gh", *args],
-        capture_output=True,
-        text=True,
-        check=True,
-    )
-    return result.stdout
-
-
-def fetch_open_issues(repo: str | None) -> list[dict]:
-    """Fetch all open issues (excluding PRs) via gh api --paginate."""
-    if repo:
-        endpoint = (
-            f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
-        )
-    else:
-        endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
-    cmd = ["api", "--paginate", endpoint]
-
-    raw = gh(*cmd)
-    # gh --paginate concatenates JSON arrays, so we may get multiple arrays
-    issues = []
-    for line in raw.strip().splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        parsed = json.loads(line)
-        if isinstance(parsed, list):
-            issues.extend(parsed)
-        else:
-            issues.append(parsed)
-
-    # Filter out pull requests (they also appear in the issues endpoint)
-    return [i for i in issues if "pull_request" not in i]
-
-
-def close_as_duplicate(
-    issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
-) -> None:
-    """Close an issue as duplicate of another, adding a comment and label."""
-    repo_args = ["--repo", repo] if repo else []
-
-    if dry_run:
-        print(
-            f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}"
-        )
-        return
-
-    # Add comment
-    comment_body = (
-        f"Closing as duplicate of #{duplicate_of}.\n\n"
-        "If you believe this is not a duplicate, please reopen and add context "
-        "explaining how this differs."
-    )
-    gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
-
-    # Add label
-    gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
-
-    # Close with not_planned reason
-    gh(
-        "api",
-        f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
-        "-X",
-        "PATCH",
-        "-f",
-        "state=closed",
-        "-f",
-        "state_reason=not_planned",
-    )
-
-    print(f"  Closed #{issue_number} as duplicate of #{duplicate_of}")
-
-
-def find_duplicate(
-    issue: dict, candidates: list[dict], threshold: float
-) -> dict | None:
-    """Return the first candidate whose normalized title is above threshold."""
-    norm = normalize_title(issue["title"])
-    for candidate in candidates:
-        if candidate["number"] == issue["number"]:
-            continue
-        cand_norm = normalize_title(candidate["title"])
-        ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
-        if ratio >= threshold:
-            return candidate
-    return None
-
-
-def scan_all(
-    issues: list[dict], threshold: float, repo: str | None, dry_run: bool
-) -> int:
-    """Compare every issue against all older issues. Returns count of duplicates found."""
-    # Sort oldest first
-    issues.sort(key=lambda i: i["number"])
-    closed_count = 0
-
-    for idx, issue in enumerate(issues):
-        older = issues[:idx]
-        if not older:
-            continue
-        dup = find_duplicate(issue, older, threshold)
-        if dup:
-            ratio = difflib.SequenceMatcher(
-                None,
-                normalize_title(issue["title"]),
-                normalize_title(dup["title"]),
-            ).ratio()
-            print(
-                f"#{issue['number']}: \"{issue['title']}\"\n"
-                f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
-                f"({ratio:.0%} similar)"
-            )
-            close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
-            closed_count += 1
-
-    return closed_count
-
-
-def check_single(
-    issue_number: int,
-    issues: list[dict],
-    threshold: float,
-    repo: str | None,
-    dry_run: bool,
-) -> bool:
-    """Check a single issue against all older open issues. Returns True if duplicate found."""
-    target = None
-    for i in issues:
-        if i["number"] == issue_number:
-            target = i
-            break
-
-    if target is None:
-        print(f"Issue #{issue_number} not found among open issues.")
-        return False
-
-    older = [i for i in issues if i["number"] < issue_number]
-    dup = find_duplicate(target, older, threshold)
-    if dup:
-        ratio = difflib.SequenceMatcher(
-            None,
-            normalize_title(target["title"]),
-            normalize_title(dup["title"]),
-        ).ratio()
-        print(
-            f"#{target['number']}: \"{target['title']}\"\n"
-            f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
-            f"({ratio:.0%} similar)"
-        )
-        close_as_duplicate(issue_number, dup["number"], repo, dry_run)
-        return True
-
-    print(f"#{issue_number}: no duplicate found above threshold {threshold}")
-    return False
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Detect and close duplicate GitHub issues"
-    )
-    mode = parser.add_mutually_exclusive_group(required=True)
-    mode.add_argument("--scan", action="store_true", help="Scan all open issues")
-    mode.add_argument("--issue-number", type=int, help="Check a single issue number")
-    parser.add_argument(
-        "--threshold", type=float, default=0.85, help="Similarity threshold (0-1)"
-    )
-    parser.add_argument(
-        "--close",
-        action="store_true",
-        help="Actually close duplicates (default is dry-run)",
-    )
-    parser.add_argument(
-        "--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted."
-    )
-    args = parser.parse_args()
-
-    dry_run = not args.close
-
-    if dry_run:
-        print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
-
-    print("Fetching open issues...")
-    issues = fetch_open_issues(args.repo)
-    print(f"Found {len(issues)} open issues.\n")
-
-    if args.scan:
-        count = scan_all(issues, args.threshold, args.repo, dry_run)
-        print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
-    else:
-        found = check_single(
-            args.issue_number, issues, args.threshold, args.repo, dry_run
-        )
-        sys.exit(0 if found else 0)  # Always exit 0; finding no dup is not an error
-
-
-if __name__ == "__main__":
-    main()

.github/scripts/e2e_egress_sentinel.py

@@ -0,0 +1,198 @@
+"""Prove an e2e replay run makes zero outbound provider calls, by counting them.
+
+`serve` pins each provider host (`--host`) to a local sink address in the hosts
+file and binds a counting listener on that address, so any connection the proxy
+or the record/replay edge opens to a real provider is redirected to the sink,
+recorded as one line in `--hits-file`, and never leaves the box. The record and
+replay edge only ever dials `127.0.0.1:<edge-port>` (a different host than the
+pinned provider names), so in a clean replay the sink sees nothing; a single hit
+means a provider call escaped the bundle. `assert-empty` turns that hit file into
+the pass/fail check.
+
+Stdlib only, so CI runs it under the system interpreter as root (binding :443 and
+editing the hosts file both need root); `--sink-address`, `--port`, and
+`--hosts-file` are injectable so it runs unprivileged against a temp hosts file on
+a high port under test.
+"""
+
+# ruff: noqa: T201  # CLI script: its stdout/stderr progress and results are the interface
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import signal
+import socket
+import sys
+import threading
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from types import FrameType
+from typing import Final
+
+_BLOCK_BEGIN: Final = "# BEGIN e2e-egress-sentinel"
+_BLOCK_END: Final = "# END e2e-egress-sentinel"
+
+
+@dataclass(frozen=True, slots=True)
+class ServeConfig:
+    hosts: tuple[str, ...]
+    sink_address: str
+    ports: tuple[int, ...]
+    hits_file: Path
+    hosts_file: Path
+    ready_file: Path | None
+    pid_file: Path | None
+
+
+def _pin_block(sink_address: str, hosts: tuple[str, ...]) -> str:
+    lines = "\n".join(f"{sink_address}\t{host}" for host in hosts)
+    return f"\n{_BLOCK_BEGIN}\n{lines}\n{_BLOCK_END}\n"
+
+
+def _install_pins(hosts_file: Path, sink_address: str, hosts: tuple[str, ...]) -> bytes:
+    original = hosts_file.read_bytes() if hosts_file.exists() else b""
+    hosts_file.write_bytes(original + _pin_block(sink_address, hosts).encode())
+    return original
+
+
+def _restore_pins(hosts_file: Path, original: bytes) -> None:
+    hosts_file.write_bytes(original)
+
+
+def _bind(sink_address: str, port: int) -> socket.socket:
+    listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    listener.bind((sink_address, port))
+    listener.listen(128)
+    return listener
+
+
+@dataclass(frozen=True, slots=True)
+class _HitLog:
+    path: Path
+    _lock: threading.Lock
+
+    def record(self, *, port: int, peer: tuple[str, int]) -> None:
+        entry = json.dumps({"ts": time.time(), "port": port, "peer": list(peer)})
+        with self._lock:
+            with self.path.open("a", encoding="utf-8") as handle:
+                handle.write(entry + "\n")
+
+
+def _serve_socket(listener: socket.socket, port: int, hits: _HitLog, stop: threading.Event) -> None:
+    while not stop.is_set():
+        try:
+            conn, peer = listener.accept()
+        except OSError:
+            return
+        hits.record(port=port, peer=(peer[0], peer[1]))
+        try:
+            conn.close()
+        except OSError:
+            pass
+
+
+def serve(config: ServeConfig) -> int:
+    config.hits_file.write_text("", encoding="utf-8")
+    original_hosts = _install_pins(config.hosts_file, config.sink_address, config.hosts)
+    try:
+        listeners = tuple(_bind(config.sink_address, port) for port in config.ports)
+    except OSError as exc:
+        _restore_pins(config.hosts_file, original_hosts)
+        print(f"egress sentinel could not bind a sink: {exc}", file=sys.stderr)
+        return 1
+
+    stop = threading.Event()
+    hits = _HitLog(path=config.hits_file, _lock=threading.Lock())
+    threads = tuple(
+        threading.Thread(target=_serve_socket, args=(listener, port, hits, stop), daemon=True)
+        for listener, port in zip(listeners, config.ports)
+    )
+    for thread in threads:
+        thread.start()
+
+    def _handle(_signum: int, _frame: FrameType | None) -> None:
+        stop.set()
+        for listener in listeners:
+            try:
+                listener.close()
+            except OSError:
+                pass
+
+    signal.signal(signal.SIGTERM, _handle)
+    signal.signal(signal.SIGINT, _handle)
+
+    if config.pid_file is not None:
+        config.pid_file.write_text(str(os.getpid()), encoding="utf-8")
+    if config.ready_file is not None:
+        config.ready_file.write_text("ready", encoding="utf-8")
+    print(
+        f"egress sentinel up: pinned {', '.join(config.hosts)} to {config.sink_address} "
+        f"on port(s) {', '.join(str(p) for p in config.ports)}",
+        flush=True,
+    )
+
+    stop.wait()
+    _restore_pins(config.hosts_file, original_hosts)
+    if config.ready_file is not None and config.ready_file.exists():
+        config.ready_file.unlink()
+    if config.pid_file is not None and config.pid_file.exists():
+        config.pid_file.unlink()
+    return 0
+
+
+def assert_empty(hits_file: Path) -> int:
+    if not hits_file.exists():
+        print(f"egress sentinel recorded no provider calls ({hits_file} absent): zero egress")
+        return 0
+    hits = [line for line in hits_file.read_text(encoding="utf-8").splitlines() if line.strip()]
+    if not hits:
+        print("egress sentinel recorded no provider calls: zero egress")
+        return 0
+    print(f"egress sentinel recorded {len(hits)} provider call(s); replay was not hermetic:", file=sys.stderr)
+    for line in hits:
+        print(f"  {line}", file=sys.stderr)
+    return 1
+
+
+def _serve_from_args(args: argparse.Namespace) -> int:
+    config = ServeConfig(
+        hosts=tuple(args.host),
+        sink_address=args.sink_address,
+        ports=tuple(args.port),
+        hits_file=Path(args.hits_file),
+        hosts_file=Path(args.hosts_file),
+        ready_file=Path(args.ready_file) if args.ready_file else None,
+        pid_file=Path(args.pid_file) if args.pid_file else None,
+    )
+    return serve(config)
+
+
+def main(argv: tuple[str, ...]) -> int:
+    parser = argparse.ArgumentParser(description="count outbound provider calls during an e2e replay")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    serve_parser = sub.add_parser("serve", help="pin provider hosts and count connection attempts")
+    serve_parser.add_argument("--host", action="append", required=True, help="provider host to pin and watch")
+    serve_parser.add_argument("--sink-address", default="127.0.0.1")
+    serve_parser.add_argument("--port", action="append", type=int, default=None)
+    serve_parser.add_argument("--hits-file", required=True)
+    serve_parser.add_argument("--hosts-file", default="/etc/hosts")
+    serve_parser.add_argument("--ready-file", default=None)
+    serve_parser.add_argument("--pid-file", default=None)
+
+    assert_parser = sub.add_parser("assert-empty", help="exit non-zero if any provider call was recorded")
+    assert_parser.add_argument("--hits-file", required=True)
+
+    args = parser.parse_args(argv)
+    if args.command == "serve":
+        if args.port is None:
+            args.port = [443]
+        return _serve_from_args(args)
+    return assert_empty(Path(args.hits_file))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(tuple(sys.argv[1:])))

.github/scripts/e2e_fetch_fixture_bundle.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="${1:-${GITHUB_REPOSITORY:?REPO required}}"
+ARTIFACT_NAME="${2:-e2e-fixtures-bundle}"
+BASE_BRANCH="${3:?base branch required}"
+DEST_DIR="${4:?destination bundle dir required}"
+
+: "${GH_TOKEN:?GH_TOKEN required to query and download artifacts}"
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "${WORKDIR}"' EXIT
+
+echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}"
+
+SELECTED="$(
+  gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \
+    --jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \
+    | jq -s 'sort_by(.created_at) | reverse | .[0] // empty'
+)"
+
+if [[ -z "${SELECTED}" ]]; then
+  echo "no usable '${ARTIFACT_NAME}' artifact on ${BASE_BRANCH}: the last record run produced none (a red Saturday), so there is nothing fresh to replay; failing loudly instead of replaying a stale bundle" >&2
+  exit 1
+fi
+
+RUN_ID="$(echo "${SELECTED}" | jq -r '.run_id')"
+RUN_NUMBER="$(echo "${SELECTED}" | jq -r '.run_number')"
+ARTIFACT_ID="$(echo "${SELECTED}" | jq -r '.id')"
+GH_DIGEST="$(echo "${SELECTED}" | jq -r '.digest // "unknown"')"
+CREATED_AT="$(echo "${SELECTED}" | jq -r '.created_at')"
+
+echo "pinned bundle: run #${RUN_NUMBER} (run_id=${RUN_ID}, artifact_id=${ARTIFACT_ID}), recorded ${CREATED_AT}, github digest ${GH_DIGEST}"
+
+gh run download "${RUN_ID}" --repo "${REPO}" -n "${ARTIFACT_NAME}" -D "${WORKDIR}"
+
+TARBALL="$(find "${WORKDIR}" -name '*.tar.gz' -type f | head -n 1)"
+if [[ -z "${TARBALL}" ]]; then
+  echo "downloaded artifact contained no tarball" >&2
+  exit 1
+fi
+SIDECAR="${TARBALL}.sha256"
+if [[ ! -f "${SIDECAR}" ]]; then
+  echo "downloaded artifact has no ${SIDECAR}: cannot verify the bundle digest" >&2
+  exit 1
+fi
+
+echo "verifying bundle against its recorded sha256 digest"
+( cd "$(dirname "${TARBALL}")" && sha256sum -c "$(basename "${SIDECAR}")" )
+
+mkdir -p "${DEST_DIR}"
+tar xzf "${TARBALL}" -C "${DEST_DIR}"
+
+echo "extracted bundle into ${DEST_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  recorded_at', m['recorded_at'], 'harness', m['harness_version'], 'format_version', m['format_version'])" "${DEST_DIR}/manifest.json"

.github/scripts/e2e_pack_fixture_bundle.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 2 ]]; then
+  echo "usage: $0 <bundle-dir> <out-tarball>" >&2
+  exit 2
+fi
+
+BUNDLE_DIR="$1"
+OUT_TARBALL="$2"
+
+MANIFEST="${BUNDLE_DIR}/manifest.json"
+if [[ ! -f "${MANIFEST}" ]]; then
+  echo "no ${MANIFEST}: refusing to publish a bundle with no manifest (record produced nothing)" >&2
+  exit 1
+fi
+
+echo "packing fixture bundle from ${BUNDLE_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  format_version', m['format_version'], 'recorded_at', m['recorded_at'], 'harness', m['harness_version'])" "${MANIFEST}"
+
+TEST_DIRS=$(find "${BUNDLE_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')
+if [[ "${TEST_DIRS}" -eq 0 ]]; then
+  echo "bundle at ${BUNDLE_DIR} has a manifest but no recorded interactions; refusing to publish an empty bundle" >&2
+  exit 1
+fi
+echo "  ${TEST_DIRS} recorded test director(ies)"
+
+mkdir -p "$(dirname "${OUT_TARBALL}")"
+tar czf "${OUT_TARBALL}" -C "${BUNDLE_DIR}" .
+
+OUT_DIR="$(cd "$(dirname "${OUT_TARBALL}")" && pwd)"
+OUT_BASE="$(basename "${OUT_TARBALL}")"
+( cd "${OUT_DIR}" && sha256sum "${OUT_BASE}" > "${OUT_BASE}.sha256" )
+
+echo "wrote ${OUT_TARBALL} ($(du -h "${OUT_TARBALL}" | cut -f1)) and ${OUT_BASE}.sha256"
+cat "${OUT_DIR}/${OUT_BASE}.sha256"

.github/scripts/smoke_test_native_wheel.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import Final
+
+CHILD_SCRIPT: Final = """
+from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
+import sys
+
+native_path = Path(sys.argv[1])
+spec = spec_from_file_location("litellm.rust_bridge._native", native_path)
+if spec is None or spec.loader is None:
+    raise RuntimeError("cannot create native extension import specification")
+module = module_from_spec(spec)
+spec.loader.exec_module(module)
+
+before = module.gil_stats()
+if not isinstance(before.get("releases"), int):
+    raise AssertionError(f"unexpected gil_stats result: {before!r}")
+
+try:
+    module._panic_for_test()
+except BaseException as error:
+    if type(error).__name__ != "PanicException":
+        raise AssertionError(f"expected PanicException, got {type(error).__name__}") from error
+else:
+    raise AssertionError("Rust panic returned without raising")
+
+after = module.gil_stats()
+if not isinstance(after.get("releases"), int):
+    raise AssertionError(f"native module unusable after panic: {after!r}")
+"""
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        sys.stderr.write(f"usage: {Path(sys.argv[0]).name} WHEEL\n")
+        return 2
+
+    wheel: Final = Path(sys.argv[1])
+    with tempfile.TemporaryDirectory() as temporary_directory, zipfile.ZipFile(wheel) as archive:
+        native_members: Final = tuple(
+            member
+            for member in archive.infolist()
+            if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so")
+        )
+        if len(native_members) != 1:
+            sys.stderr.write(f"expected one native extension, found {len(native_members)}\n")
+            return 1
+
+        native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name
+        native_path.write_bytes(archive.read(native_members[0]))
+        result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False)
+
+    if result.returncode != 0:
+        sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n")
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/verify_linux_native_wheel.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import importlib.util
+import os
+import re
+import subprocess
+import sys
+import zipfile
+from collections.abc import Callable, Mapping, Sequence
+from itertools import product
+from pathlib import Path, PurePosixPath
+from types import MappingProxyType, ModuleType
+from typing import Final, Protocol
+
+EXPECTED_PYTHON_TAG: Final = "cp310"
+EXPECTED_ABI_TAG: Final = "abi3"
+EXPECTED_PLATFORM_TAG: Final = "linux_x86_64"
+
+
+class CommandRunner(Protocol):
+    def __call__(
+        self,
+        command: tuple[str, ...],
+        *,
+        check: bool,
+        capture_output: bool,
+        text: bool,
+    ) -> subprocess.CompletedProcess[str]: ...
+
+
+def _run_command(
+    command: tuple[str, ...],
+    *,
+    check: bool,
+    capture_output: bool,
+    text: bool,
+) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(command, check=check, capture_output=capture_output, text=text)
+
+
+def _dist_info_directory(member: zipfile.ZipInfo) -> str | None:
+    parts: Final = PurePosixPath(member.filename).parts
+    if not parts or not parts[0].endswith(".dist-info"):
+        return None
+    return parts[0]
+
+
+def _wheel_metadata_tags(archive: zipfile.ZipFile, members: tuple[zipfile.ZipInfo, ...]) -> tuple[str, ...]:
+    if len(members) != 1:
+        return ()
+    lines: Final = archive.read(members[0]).splitlines()
+    return tuple(line.removeprefix(b"Tag:").strip().decode("ascii") for line in lines if line.startswith(b"Tag:"))
+
+
+def _load_native_module(native_path: Path) -> ModuleType | None:
+    module_spec: Final = importlib.util.spec_from_file_location("litellm.rust_bridge._native", native_path)
+    if module_spec is None or module_spec.loader is None:
+        return None
+    try:
+        native_module: Final = importlib.util.module_from_spec(module_spec)
+        module_spec.loader.exec_module(native_module)
+    except Exception as error:  # noqa: BLE001  # native module initialization can raise arbitrary exceptions
+        sys.stderr.write(f"native module load failed: {error}\n")
+        return None
+    return native_module
+
+
+def main(
+    argv: Sequence[str] | None = None,
+    environment: Mapping[str, str] | None = None,
+    load_native_module: Callable[[Path], ModuleType | None] = _load_native_module,
+    run_command: CommandRunner = _run_command,
+) -> int:
+    arguments: Final = tuple(sys.argv if argv is None else argv)
+    resolved_environment: Final = os.environ if environment is None else environment
+    if len(arguments) != 2:
+        sys.stderr.write(f"usage: {Path(arguments[0]).name} WHEEL\n")
+        return 2
+
+    wheel: Final = Path(arguments[1])
+    wheel_tags: Final = wheel.stem.rsplit("-", maxsplit=3)
+    if len(wheel_tags) != 4:
+        sys.stderr.write(f"cannot parse wheel tags from {wheel.name}\n")
+        return 1
+
+    wheel_identity: Final = wheel_tags[0].split("-")
+    if len(wheel_identity) != 2 or wheel_identity[0] != "litellm" or not wheel_identity[1]:
+        sys.stderr.write(f"unexpected wheel identity: {wheel_tags[0]}\n")
+        return 1
+
+    expected_dist_info_directory: Final = f"{wheel_tags[0]}.dist-info"
+    expected_dist_info_directories: Final = frozenset((expected_dist_info_directory,))
+    python_tag: Final = wheel_tags[1]
+    abi_tag: Final = wheel_tags[2]
+    platform_tag: Final = wheel_tags[3]
+    expanded_filename_tags: Final = frozenset(
+        "-".join(tag) for tag in product(python_tag.split("."), abi_tag.split("."), platform_tag.split("."))
+    )
+
+    with zipfile.ZipFile(wheel) as archive:
+        wheel_members: Final = archive.infolist()
+        dist_info_directories: Final = frozenset(
+            directory for member in wheel_members if (directory := _dist_info_directory(member)) is not None
+        )
+        required_dist_info_files: Final = ("METADATA", "RECORD", "WHEEL")
+        dist_info_file_counts: Final = MappingProxyType(
+            {
+                filename: sum(
+                    member.filename == f"{expected_dist_info_directory}/{filename}" for member in wheel_members
+                )
+                for filename in required_dist_info_files
+            }
+        )
+        wheel_metadata_members: Final = tuple(
+            member for member in wheel_members if member.filename == f"{expected_dist_info_directory}/WHEEL"
+        )
+        wheel_metadata_tags: Final = _wheel_metadata_tags(archive, wheel_metadata_members)
+        native_members: Final = tuple(
+            member
+            for member in wheel_members
+            if member.filename.startswith("litellm/rust_bridge/_native.") and member.filename.endswith(".so")
+        )
+        if len(native_members) != 1:
+            sys.stderr.write(f"expected one native extension, found {len(native_members)}\n")
+            return 1
+
+        unexpected_members: Final = tuple(
+            member.filename
+            for member in wheel_members
+            if member.filename.endswith((".pdb", ".dwp", ".rlib", ".rmeta", "Cargo.toml", "Cargo.lock"))
+            or any(part.endswith(".dSYM") for part in PurePosixPath(member.filename).parts)
+        )
+        native_member: Final = native_members[0]
+        uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members)
+        native_path: Final = wheel.parent / "native" / Path(native_member.filename).name
+        native_path.parent.mkdir(parents=True, exist_ok=True)
+        native_path.write_bytes(archive.read(native_member))
+
+    wheel_metadata_tags_match: Final = (
+        len(wheel_metadata_tags) == len(expanded_filename_tags)
+        and frozenset(wheel_metadata_tags) == expanded_filename_tags
+    )
+    commit_sha: Final = resolved_environment.get(
+        "RELEASE_WHEEL_COMMIT_SHA", resolved_environment.get("GITHUB_SHA", "unknown")
+    )
+    rustc_version: Final = run_command(
+        ("rustc", "--version"),
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    pyproject: Final = (Path(__file__).parents[2] / "pyproject.toml").read_text()
+    maturin_match: Final = re.search(r'"maturin==([^";]+)', pyproject)
+    if maturin_match is None:
+        sys.stderr.write("build-system does not pin an exact Maturin version\n")
+        return 1
+
+    maturin_version: Final = maturin_match.group(1)
+    native_percentage: Final = native_member.file_size / uncompressed_wheel_size * 100
+    size_report: Final = "\n".join(
+        (
+            "## Native wheel build report",
+            "",
+            "| Build | Value |",
+            "| --- | --- |",
+            f"| Commit | `{commit_sha}` |",
+            f"| Platform | `{platform_tag}` |",
+            f"| Python ABI | `{python_tag}-{abi_tag}` |",
+            f"| Rust compiler | `{rustc_version}` |",
+            f"| Maturin | `{maturin_version}` |",
+            "| Cargo profile | `release` |",
+            "",
+            "| Artifact | Size |",
+            "| --- | ---: |",
+            f"| Compressed wheel | {wheel.stat().st_size / 1_000_000:.2f} MB |",
+            f"| Uncompressed wheel | {uncompressed_wheel_size / 1_000_000:.2f} MB |",
+            f"| Native extension | {native_member.file_size / 1_000_000:.2f} MB |",
+            f"| Native share | {native_percentage:.2f}% |",
+            "",
+        )
+    )
+    summary_path: Final = resolved_environment.get("GITHUB_STEP_SUMMARY")
+    if summary_path is None:
+        sys.stdout.write(size_report)
+    else:
+        Path(summary_path).write_text(size_report)
+
+    sections: Final = run_command(
+        ("readelf", "--sections", "--wide", str(native_path)),
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout
+    debug_sections: Final = tuple(section for section in (".debug_", ".zdebug_") if section in sections)
+    debug_sections_absent: Final = not debug_sections
+    static_symbol_table_absent: Final = ".symtab" not in sections
+
+    dynamic_symbols: Final = run_command(
+        ("readelf", "--dyn-syms", "--wide", str(native_path)),
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout
+    extension_entry_point_present: Final = "PyInit__native" in dynamic_symbols
+    native_module: Final = load_native_module(native_path)
+    native_module_loads: Final = native_module is not None
+    panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
+    native_size_limit: Final = 20_000_000
+    native_size_within_limit: Final = native_member.file_size <= native_size_limit
+    validations: Final = (
+        (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
+        (f"ABI tag is {EXPECTED_ABI_TAG}", abi_tag == EXPECTED_ABI_TAG),
+        (f"Platform tag is {EXPECTED_PLATFORM_TAG}", platform_tag == EXPECTED_PLATFORM_TAG),
+        ("Wheel dist-info directory matches the filename", dist_info_directories == expected_dist_info_directories),
+        (
+            "Required dist-info files are present exactly once",
+            all(count == 1 for count in dist_info_file_counts.values()),
+        ),
+        ("Wheel metadata tags match the filename", wheel_metadata_tags_match),
+        ("Debug sections are absent", debug_sections_absent),
+        ("Static symbol table is absent", static_symbol_table_absent),
+        ("Python extension entry point is present", extension_entry_point_present),
+        ("Native module loads", native_module_loads),
+        ("Production module omits the panic test hook", panic_test_hook_absent),
+        ("Native extension does not exceed 20 MB", native_size_within_limit),
+        ("Wheel contents are valid", not unexpected_members),
+    )
+
+    verified_report: Final = size_report + "\n".join(
+        ("", "| Validation | Expected | Result |", "| --- | --- | :---: |")
+        + tuple(f"| {label} | Yes | {'O' if passed else 'X'} |" for label, passed in validations)
+        + ("",)
+    )
+    if summary_path is not None:
+        Path(summary_path).write_text(verified_report)
+
+    invalid_dist_info_files: Final = any(count != 1 for count in dist_info_file_counts.values())
+    validation_errors: Final = tuple(
+        message
+        for failed, message in (
+            (bool(debug_sections), f"{native_member.filename} contains debug sections: {', '.join(debug_sections)}"),
+            (not static_symbol_table_absent, f"{native_member.filename} contains a static symbol table"),
+            (not extension_entry_point_present, "native extension does not export PyInit__native"),
+            (
+                python_tag != EXPECTED_PYTHON_TAG,
+                f"unexpected Python tag: expected {EXPECTED_PYTHON_TAG}, found {python_tag}",
+            ),
+            (abi_tag != EXPECTED_ABI_TAG, f"unexpected ABI tag: expected {EXPECTED_ABI_TAG}, found {abi_tag}"),
+            (
+                platform_tag != EXPECTED_PLATFORM_TAG,
+                f"unexpected platform tag: expected {EXPECTED_PLATFORM_TAG}, found {platform_tag}",
+            ),
+            (
+                dist_info_directories != expected_dist_info_directories,
+                f"unexpected dist-info directories: expected {expected_dist_info_directory}, "
+                f"found {', '.join(sorted(dist_info_directories))}",
+            ),
+            (invalid_dist_info_files, f"required dist-info file counts are invalid: {dist_info_file_counts}"),
+            (
+                not invalid_dist_info_files and not wheel_metadata_tags_match,
+                f"WHEEL tags do not match filename: expected {', '.join(sorted(expanded_filename_tags))}, "
+                f"found {', '.join(sorted(wheel_metadata_tags))}",
+            ),
+            (
+                native_module is not None and not panic_test_hook_absent,
+                "production native module exposes _panic_for_test",
+            ),
+            (
+                not native_size_within_limit,
+                f"native extension exceeds 20 MB: {native_member.file_size / 1_000_000:.2f} MB",
+            ),
+            (bool(unexpected_members), f"wheel contains unexpected build artifacts: {', '.join(unexpected_members)}"),
+        )
+        if failed
+    )
+    sys.stderr.write("".join(f"{message}\n" for message in validation_errors))
+
+    return 0 if all(passed for _, passed in validations) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/_test-unit-base.yml

@@ -149,7 +149,7 @@ jobs:
               --reruns "${RERUNS}" \
               --reruns-delay 1 \
               --durations=20 \
-              --cov=./litellm \
+              --cov=./litellm --cov=./enterprise/litellm_enterprise \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           else
@@ -161,7 +161,7 @@ jobs:
               --reruns-delay 1 \
               --dist="${DIST}" \
               --durations=20 \
-              --cov=./litellm \
+              --cov=./litellm --cov=./enterprise/litellm_enterprise \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           fi

.github/workflows/auto-close-duplicates.yml

@@ -0,0 +1,69 @@
+name: Auto-close duplicate issues
+
+on:
+  schedule:
+    - cron: "0 9 * * *"
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: Log which issues would close without closing anything
+        type: boolean
+        default: true
+      grace_period_days:
+        description: Days a duplicate notice must go unanswered before the close
+        type: number
+        default: 3
+  pull_request:
+    paths:
+      - .github/workflows/auto-close-duplicates.yml
+      - scripts/auto-close-duplicates.ts
+      - scripts/auto-close-duplicates.test.ts
+
+permissions: {}
+
+jobs:
+  test:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          bun-version: "1.4.0"
+
+      - name: Test the sweep
+        run: bun test scripts/auto-close-duplicates.test.ts
+
+  sweep:
+    if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          # Exact version, never latest: the next step holds an issues: write token
+          bun-version: "1.4.0"
+
+      - name: Close unanswered duplicates, reopen ones the reporter answered
+        run: bun run scripts/auto-close-duplicates.ts
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DRY_RUN: ${{ inputs.dry_run == true }}
+          GRACE_PERIOD_DAYS: ${{ inputs.grace_period_days }}

.github/workflows/check-ui-api-types.yml

@@ -83,6 +83,24 @@ jobs:
         if: steps.changes.outputs.relevant == 'true'
         run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
+      - name: Regenerate the lazy OpenAPI snapshot
+        if: steps.changes.outputs.relevant == 'true'
+        run: uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
+
+      - name: Fail if the lazy OpenAPI snapshot is stale
+        if: steps.changes.outputs.relevant == 'true'
+        run: |
+          if ! git diff --exit-code -- litellm/proxy/_lazy_openapi_snapshot.json; then
+            echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::The lazy OpenAPI snapshot is out of sync with the lazily loaded routes."
+            echo ""
+            echo "A lazily loaded route or model changed without regenerating the snapshot that /openapi.json serves for unloaded features."
+            echo "To fix, run from the repo root:"
+            echo "  uv run python -m litellm.proxy._lazy_openapi_snapshot"
+            echo "then run npm run gen:api from ui/litellm-dashboard and commit both files."
+            exit 1
+          fi
+          echo "_lazy_openapi_snapshot.json is in sync with the lazily loaded routes."
+
       - name: Set up Node.js
         if: steps.changes.outputs.relevant == 'true'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0

.github/workflows/check_duplicate_issues.yml

@@ -1,12 +1,19 @@
 name: Check Duplicate Issues
 
+# Flagging only. "Auto-close duplicate issues" closes a flagged issue 3 days later,
+# and only when its title is identical to an older open issue and nobody replied.
+# The HTML marker below is the handshake between the two, so keep it in the template.
+
 on:
   issues:
     types: [opened, edited]
 
+permissions: {}
+
 jobs:
   check-duplicate:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     permissions:
       issues: write
       contents: read
@@ -19,35 +26,12 @@ jobs:
           threshold: 0.6
           reaction: eyes
           comment: |
-            **⚠️ Potential duplicate detected**
+            <!-- litellm:potential-duplicate candidates={{#issues}}{{number}},{{/issues}} -->
+            **Potential duplicate detected**
 
-            This issue appears similar to existing issue(s):
+            This looks similar to:
             {{#issues}}
-            - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
+            - #{{number}} - {{title}}
             {{/issues}}
 
-            Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
-
-      - name: Checkout close script
-        if: github.event.action == 'opened'
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          sparse-checkout: .github/scripts
-          persist-credentials: false
-
-      - name: Set up Python
-        if: github.event.action == 'opened'
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Auto-close if high-confidence duplicate
-        if: github.event.action == 'opened'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python3 .github/scripts/close_duplicate_issues.py \
-            --issue-number ${{ github.event.issue.number }} \
-            --repo ${{ github.repository }} \
-            --threshold 0.85 \
-            --close
+            If this is a duplicate, add a thumbs-up reaction to the existing issue and follow along there. When the title is identical to an older open issue, this issue closes automatically in 3 days unless someone responds. If it is not a duplicate, comment here or add a thumbs-down reaction to this comment and it stays open.

.github/workflows/codspeed.yml

@@ -12,6 +12,7 @@ on:
       - "uv.lock"
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
+      - ".github/actions/cache-cargo-build/**"
   pull_request:
     branches:
       - main
@@ -23,6 +24,7 @@ on:
       - "uv.lock"
       - ".github/workflows/codspeed.yml"
       - ".github/actions/setup-uv-with-retries/**"
+      - ".github/actions/cache-cargo-build/**"
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:
@@ -55,6 +57,26 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      # Build the wheel and resolve every dependency outside the CodSpeed
+      # runner: the same maturin build took 42 minutes inside `codspeed run`
+      # versus under 3 minutes as a plain step (LIT-6183)
+      - name: Build environment
+        run: >
+          env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
+          uv run --frozen --no-default-groups
+          --with pytest==8.3.5
+          --with pytest-codspeed==4.3.0
+          --with "mcp>=1.26.0,<2.0"
+          --with "a2a-sdk>=1.1.0,<2.0"
+          pytest
+          -p pytest_codspeed.plugin
+          tests/benchmarks/
+          --codspeed
+          --collect-only -q
+
       - name: Run benchmarks
         uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
         with:

.github/workflows/e2e_record_replay.yml

@@ -0,0 +1,237 @@
+name: "E2E Record and Replay"
+
+on:
+  schedule:
+    - cron: "0 8 * * 6"
+    - cron: "0 8 * * 1-5"
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "record (hits real providers and publishes a fresh bundle) or replay (bundle only, zero provider egress)"
+        type: choice
+        options:
+          - record
+          - replay
+        default: record
+
+permissions:
+  contents: read
+
+jobs:
+  record:
+    name: "Record the e2e suite against real providers"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 6' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'record'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Record the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: record
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Pack the fixture bundle
+        run: |
+          .github/scripts/e2e_pack_fixture_bundle.sh tests/e2e/.fixtures "${RUNNER_TEMP}/bundle/e2e-fixtures.tar.gz"
+
+      - name: Publish the fixture bundle
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: e2e-fixtures-bundle
+          path: |
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz.sha256
+          if-no-files-found: error
+          retention-days: 30
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log
+
+  replay:
+    name: "Replay the e2e suite from the pinned bundle with zero egress"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 1-5' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'replay'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      contents: read
+      actions: read
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      GH_TOKEN: ${{ github.token }}
+      OPENAI_API_KEY: sk-replay-must-never-reach-a-provider
+      ANTHROPIC_API_KEY: sk-ant-replay-must-never-reach-a-provider
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Fetch the pinned fixture bundle by digest
+        env:
+          BASE_BRANCH: ${{ github.ref_name }}
+        run: |
+          .github/scripts/e2e_fetch_fixture_bundle.sh \
+            "${GITHUB_REPOSITORY}" \
+            e2e-fixtures-bundle \
+            "${BASE_BRANCH}" \
+            tests/e2e/.fixtures
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Start the egress sentinel
+        run: |
+          # shellcheck disable=SC2024  # the log redirect is deliberately the runner user's, so a later non-sudo cat can read it
+          sudo python3 .github/scripts/e2e_egress_sentinel.py serve \
+            --host api.openai.com \
+            --host api.anthropic.com \
+            --hits-file "${RUNNER_TEMP}/egress-hits.jsonl" \
+            --ready-file "${RUNNER_TEMP}/egress-ready" \
+            --pid-file "${RUNNER_TEMP}/egress.pid" \
+            > "${RUNNER_TEMP}/egress-sentinel.log" 2>&1 &
+          for _ in $(seq 1 30); do
+            if [[ -f "${RUNNER_TEMP}/egress-ready" ]]; then
+              cat "${RUNNER_TEMP}/egress-sentinel.log"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "egress sentinel never became ready"
+          cat "${RUNNER_TEMP}/egress-sentinel.log"
+          exit 1
+
+      - name: Replay the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: replay
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Stop the egress sentinel and assert zero provider egress
+        if: always()
+        run: |
+          if [[ -f "${RUNNER_TEMP}/egress.pid" ]]; then
+            sudo kill -TERM "$(cat "${RUNNER_TEMP}/egress.pid")" 2>/dev/null || true
+            sleep 2
+          fi
+          python3 .github/scripts/e2e_egress_sentinel.py assert-empty --hits-file "${RUNNER_TEMP}/egress-hits.jsonl"
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log

.github/workflows/image-scan.yml

@@ -17,10 +17,14 @@ on:
       - backend/Dockerfile
       - backend/main.py
       - docker/component_entrypoint.sh
+      - docker/entrypoint.sh
+      - litellm/proxy/prisma_migration.py
       - litellm-proxy-extras/**
       - tests/proxy_migration_tests/**
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
+      - ui/Dockerfile
+      - ui/nginx.conf
       - .github/workflows/image-scan.yml
   schedule:
     - cron: "41 6 * * *"
@@ -76,7 +80,7 @@ jobs:
           LITELLM_IMAGE: litellm-image-scan:${{ github.sha }}
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
 
       # Scans the whole shipped artifact: OS/apk plus every language package
       # baked into the image, including ones no lockfile declares (e.g. prisma's
@@ -120,7 +124,7 @@ jobs:
           LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
 
   migrations-image:
     name: migrations-image
@@ -181,7 +185,36 @@ jobs:
           LITELLM_COMPONENT_PORT: "4000"
         run: |
           python -m pip install "pytest==9.0.3"
-          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
+
+  ui-image:
+    name: ui-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build UI image
+        run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
+        env:
+          LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
 
   backend-image:
     name: backend-image

.github/workflows/mutation-test.yml

@@ -87,11 +87,20 @@ jobs:
         run: |
           uv pip uninstall pytest-retry || true
 
+      # Ends before the job's own deadline so a run that outlasts the budget is
+      # still followed by the report and upload steps. mutmut saves after every
+      # mutant result, to mutants/<source path>.meta, so an interrupted run
+      # still scores the mutants it finished and export-cicd-stats can read
+      # them; a cancelled job skips those steps and publishes nothing at all.
       - name: Run mutmut
+        timeout-minutes: 300
         env:
           # Make the mutants/ sandbox win over site-packages on sys.path so the
           # trampolined files are imported instead of the installed copy.
           PYTHONPATH: ${{ github.workspace }}/mutants
+          # Without this mutmut finds no covered lines and generates 0 mutants.
+          # See the file itself for why.
+          COVERAGE_RCFILE: ${{ github.workspace }}/.github/mutmut-coverage.rc
         run: |
           set -o pipefail
           mkdir -p mutants
@@ -130,6 +139,7 @@ jobs:
             mutmut-run.log
             mutants/mutmut-stats.json
             mutants/mutmut-cicd-stats.json
+            mutants/**/*.meta
             mutants/litellm/proxy/management_endpoints/**/*.py
           if-no-files-found: warn
           retention-days: 14

.github/workflows/report-rust-release-wheel.yml

@@ -0,0 +1,130 @@
+name: Report LiteLLM Rust release wheel
+
+on: # zizmor: ignore[dangerous-triggers] reporter executes no PR code and consumes no PR artifacts or outputs
+  workflow_run:
+    workflows:
+      - LiteLLM Rust
+    types:
+      - completed
+
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || github.event.workflow_run.id }}
+  cancel-in-progress: false
+
+jobs:
+  report-release-wheel:
+    name: report release wheel
+    if: >-
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.path == '.github/workflows/test-rust.yml' &&
+      github.event.workflow_run.head_repository.full_name == github.repository &&
+      github.event.workflow_run.pull_requests[0].number != null
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      issues: write # PR comments use the issues API
+      pull-requests: read # Current-head validation rejects stale workflow runs
+
+    steps:
+      - name: Link release wheel report on PR
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        env:
+          COMMENT_MARKER: "<!-- litellm-release-wheel-size -->"
+        with:
+          script: |
+            const marker = process.env.COMMENT_MARKER;
+            const workflowRun = context.payload.workflow_run;
+            const allowedConclusions = new Set([
+              "action_required",
+              "cancelled",
+              "failure",
+              "neutral",
+              "skipped",
+              "stale",
+              "startup_failure",
+              "success",
+              "timed_out",
+            ]);
+            if (
+              !allowedConclusions.has(workflowRun.conclusion) ||
+              workflowRun.event !== "pull_request" ||
+              workflowRun.path !== ".github/workflows/test-rust.yml" ||
+              workflowRun.head_repository?.full_name !==
+                `${context.repo.owner}/${context.repo.repo}` ||
+              workflowRun.pull_requests?.length !== 1
+            ) {
+              throw new Error("unexpected source workflow");
+            }
+            const pullRequest = workflowRun.pull_requests[0];
+            const pullRequestNumber = pullRequest.number;
+            const headSha = workflowRun.head_sha;
+            const runId = workflowRun.id;
+            if (
+              !Number.isSafeInteger(pullRequestNumber) ||
+              pullRequestNumber <= 0 ||
+              !Number.isSafeInteger(runId) ||
+              runId <= 0 ||
+              !/^[0-9a-f]{40}$/.test(headSha) ||
+              pullRequest.head?.sha !== headSha
+            ) {
+              throw new Error("invalid source workflow metadata");
+            }
+            const runUrl =
+              `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` +
+              `/actions/runs/${runId}`;
+            const result =
+              workflowRun.conclusion === "success"
+                ? "successfully"
+                : `with \`${workflowRun.conclusion}\``;
+            const body = [
+              marker,
+              "## LiteLLM Rust workflow",
+              "",
+              `Workflow completed ${result} for \`${headSha}\``,
+              "",
+              `[View workflow run](${runUrl})`,
+            ].join("\n");
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: pullRequestNumber,
+              per_page: 100,
+            });
+            const existing = comments.find(
+              (comment) =>
+                comment.user?.login === "github-actions[bot]" &&
+                comment.body?.startsWith(marker),
+            );
+            const currentPullRequest = (
+              await github.rest.pulls.get({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: pullRequestNumber,
+              })
+            ).data;
+            if (
+              currentPullRequest.state !== "open" ||
+              currentPullRequest.head.repo?.full_name !==
+                `${context.repo.owner}/${context.repo.repo}` ||
+              currentPullRequest.head.sha !== headSha
+            ) {
+              core.info("source workflow no longer matches the current pull request head");
+              return;
+            }
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: pullRequestNumber,
+                body,
+              });
+            }

.github/workflows/sync-together-ai-models.yml

@@ -0,0 +1,68 @@
+name: Sync Together AI model registry
+
+on:
+  schedule:
+    - cron: "30 6 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  sync_together_ai_models:
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          ref: litellm_internal_staging
+          persist-credentials: false
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+      - name: Look for an already-open sync PR
+        id: existing
+        run: |
+          open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \
+            --jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')"
+          echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT"
+          if [ -n "$open_pr" ]; then
+            echo "An open sync PR already exists on branch $open_pr; skipping this run."
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
+      - name: Run the sync
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
+        env:
+          TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+      - name: Regenerate the JSON schema
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          uv run --frozen python ci_cd/generate_model_prices_schema.py
+      - name: Create a pull request when the registry changed
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          if git diff --quiet; then
+            echo "Registry already in sync; no PR needed."
+            exit 0
+          fi
+          branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$branch"
+          git add model_prices_and_context_window.json \
+            litellm/model_prices_and_context_window_backup.json \
+            model_prices_and_context_window.schema.json
+          git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
+          gh auth setup-git
+          git push origin "$branch"
+          gh pr create --title "feat(models): sync together_ai model registry" \
+            --body-file "$RUNNER_TEMP/pr_body.md" \
+            --head "$branch" \
+            --base litellm_internal_staging
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}

.github/workflows/test-code-quality.yml

@@ -131,6 +131,9 @@ jobs:
       - name: check_e2e_no_raw_requests
         run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
 
+      - name: check_migrations_no_data_rewrites
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

.github/workflows/test-linting.yml

@@ -67,6 +67,17 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-lint-
+
       - name: Clean Python cache
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-mcp.yml

@@ -60,4 +60,4 @@ jobs:
       - name: Run MCP tests
         if: steps.changes.outputs.decision != 'skip'
         run: |
-          uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5
+          uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml --durations=5

.github/workflows/test-postgres.yml

@@ -0,0 +1,145 @@
+name: "Postgres Tests"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  postgres:
+    name: ${{ matrix.shard }}
+    runs-on: ubuntu-latest
+    timeout-minutes: ${{ matrix.job-timeout-minutes }}
+    permissions:
+      contents: read
+
+    services:
+      postgres:
+        image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: litellm_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - shard: proxy-behavior
+            test-path: "tests/proxy_behavior"
+            seed: db-push
+            workers: 0
+            timeout-minutes: 25
+            job-timeout-minutes: 50
+
+          - shard: proxy-security
+            test-path: "tests/proxy_security_tests"
+            seed: db-push
+            workers: 0
+            timeout-minutes: 15
+            job-timeout-minutes: 40
+
+          - shard: schema-migration
+            test-path: "tests/proxy_migration_tests"
+            seed: none
+            workers: 0
+            timeout-minutes: 20
+            job-timeout-minutes: 45
+
+    env:
+      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        timeout-minutes: 3
+        with:
+          persist-credentials: false
+
+      - name: Detect relevant changes
+        id: changes
+        timeout-minutes: 2
+        uses: ./.github/actions/detect-changes
+
+      - name: Set up Python
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-postgres-
+
+      - name: Install dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 12
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
+
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Seed database schema
+        if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
+        timeout-minutes: 10
+        run: |
+          uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
+
+      - name: Run tests
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: ${{ matrix.timeout-minutes }}
+        env:
+          TEST_PATH: ${{ matrix.test-path }}
+          WORKERS: ${{ matrix.workers }}
+        run: |
+          if [ "${WORKERS}" = "0" ]; then
+            uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
+          else
+            uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
+          fi

.github/workflows/test-redis-compat.yml

@@ -0,0 +1,77 @@
+name: "Unit Tests: Redis Client Version Compatibility"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+    paths:
+      - "litellm/_redis.py"
+      - "litellm/_redis_credential_provider.py"
+      - "tests/test_litellm/test_redis.py"
+      - "tests/test_litellm/caching/test_redis_connection_pool.py"
+      - ".github/workflows/test-redis-compat.yml"
+      - "pyproject.toml"
+      - "uv.lock"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  redis-compat:
+    name: "redis-py ${{ matrix.redis-version }}"
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # 5.3.1 is the version pinned in uv.lock (redisvl caps it below 6); the
+        # newer legs prove the inspect.signature introspection in litellm/_redis.py
+        # keeps extracting kwargs on the redis-py releases people actually run now.
+        # Only the exact release 6.0.0 is skipped: rq (pulled by the proxy extra)
+        # specifies `redis != 6`, which excludes 6.0.0 alone, so 6.4.0 stands in
+        # for the 6.x line.
+        redis-version: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"]
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+
+      - name: Pin redis-py to the matrix version
+        env:
+          REDIS_VERSION: ${{ matrix.redis-version }}
+        run: |
+          uv pip install "redis==${REDIS_VERSION:?}"
+          uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)"
+
+      - name: Run redis unit tests
+        run: |
+          uv run --no-sync pytest \
+            tests/test_litellm/test_redis.py \
+            tests/test_litellm/caching/test_redis_connection_pool.py \
+            --tb=short -vv \
+            --reruns 2 \
+            --reruns-delay 1 \
+            --durations=20

.github/workflows/test-rust.yml

@@ -4,6 +4,11 @@ on:
   push:
     paths:
       - "litellm-rust/**"
+      - ".cargo/**"
+      - "pyproject.toml"
+      - "rust-toolchain.toml"
+      - ".github/scripts/smoke_test_native_wheel.py"
+      - ".github/scripts/verify_linux_native_wheel.py"
       - ".github/workflows/test-rust.yml"
   pull_request:
     branches:
@@ -13,6 +18,11 @@ on:
       - "litellm_**"
     paths:
       - "litellm-rust/**"
+      - ".cargo/**"
+      - "pyproject.toml"
+      - "rust-toolchain.toml"
+      - ".github/scripts/smoke_test_native_wheel.py"
+      - ".github/scripts/verify_linux_native_wheel.py"
       - ".github/workflows/test-rust.yml"
 
 permissions:
@@ -40,9 +50,7 @@ jobs:
           persist-credentials: false
 
       - name: Set up Rust
-        run: |
-          rustup toolchain install stable --profile minimal --component clippy,rustfmt
-          rustup default stable
+        run: rustup toolchain install
 
       - name: Cache Cargo registry and target
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
@@ -51,7 +59,7 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             litellm-rust/target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('litellm-rust/Cargo.lock') }}
+          key: ${{ runner.os }}-cargo-${{ hashFiles('rust-toolchain.toml', 'litellm-rust/Cargo.lock') }}
           restore-keys: |
             ${{ runner.os }}-cargo-
 
@@ -69,3 +77,47 @@ jobs:
 
       - name: Run core tests with Bedrock auth
         run: cargo test -p litellm-core --features bedrock-auth --locked
+
+  release-wheel:
+    name: release wheel
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    env:
+      CARGO_TERM_COLOR: always
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Set up Rust
+        run: rustup toolchain install
+
+      - name: Build release wheel
+        run: uv build --wheel --out-dir dist
+
+      - name: Build panic contract wheel
+        run: >-
+          uv build --wheel --out-dir panic-dist
+          --config-setting "maturin.build-args=--features panic-test,extension-module"
+
+      - name: Smoke-test native panic unwinding
+        run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl
+
+      - name: Verify stripped native extension
+        env:
+          RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl

.github/workflows/test-terraform-provider.yml

@@ -114,4 +114,4 @@ jobs:
 
       - name: Audit provider endpoints against the schema
         working-directory: terraform/provider
-        run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json"
+        run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json" -coverage-allowlist ./tools/endpointaudit/coverage_allowlist.txt

.github/workflows/test-unit.yml

@@ -103,6 +103,7 @@ jobs:
               tests/test_litellm/completion_extras
               tests/test_litellm/compression
               tests/test_litellm/containers
+              tests/test_litellm/endpoints
               tests/test_litellm/experimental_mcp_client
               tests/test_litellm/models
               tests/test_litellm/repositories
@@ -141,6 +142,7 @@ jobs:
             test-path: >-
               tests/test_litellm/proxy/analytics_endpoints
               tests/test_litellm/proxy/management_endpoints
+              tests/test_litellm/proxy/list_api
               tests/test_litellm/proxy/memory
               tests/test_litellm/proxy/guardrails
               tests/test_litellm/proxy/management_helpers
@@ -164,11 +166,12 @@ jobs:
               tests/test_litellm/proxy/public_endpoints
               tests/test_litellm/proxy/prompts
               tests/test_litellm/proxy/rag_endpoints
+              tests/test_litellm/proxy/rerank_endpoints
               tests/test_litellm/proxy/realtime_endpoints
               tests/test_litellm/proxy/ui_crud_endpoints
               tests/test_litellm/proxy/config_resolvers
               tests/test_litellm/proxy/utils
-            workers: 2
+            workers: 4
             reruns: 2
             timeout-minutes: 20
             job-timeout-minutes: 60
@@ -195,11 +198,40 @@ jobs:
               tests/test_litellm/proxy/types_utils
               tests/test_litellm/proxy/logging_endpoints
               tests/test_litellm/proxy/test_*.py
+            workers: 4
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: caching-local
+            artifact-name: caching-local
+            test-path: >-
+              tests/local_testing/test_cache_preset_key.py
+              tests/local_testing/test_caching_handler.py
+              tests/local_testing/test_prompt_caching.py
+              tests/local_testing/test_responses_stream_cache_keys.py
+              tests/local_testing/test_unit_test_caching.py
             workers: 2
             reruns: 2
             timeout-minutes: 20
             job-timeout-minutes: 60
 
+          - shard: proxy-extras
+            artifact-name: proxy-extras
+            test-path: "tests/litellm-proxy-extras"
+            workers: 2
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: enterprise-package
+            artifact-name: enterprise-package
+            test-path: "tests/enterprise"
+            workers: 4
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
           - shard: responses-caching-types
             artifact-name: responses-caching-types
             test-path: >-

.gitignore

@@ -3,6 +3,8 @@
 tests/e2e/.fixtures/
 .venv-typecheck
 .venv_policy_test
+.venv-mutmut
+mutants/
 .env
 .claude
 CLAUDE.local.md

CLAUDE.md

@@ -23,6 +23,8 @@ When adding new features, add meaningful tests. Don't add tests that don't check
 
 Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression)
 
+Never test structure of code only function of it
+
 `tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
 
 End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`
@@ -37,13 +39,14 @@ If you're resolving a linear ticket, in the "## Linear ticket" section of the PR
 
 Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
 
-If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
+If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
 - don't use emojis
 - don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
-- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
+- unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
+- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
 
 Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
 
@@ -65,6 +68,8 @@ Commit and push your work when you're done without asking
 
 When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
 
+Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
+
 If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
 
 Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch
@@ -79,6 +84,8 @@ Do not put names of customers or customer company names in code, PR descriptions
 
 CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
 
+Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
+
 Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
 
 - Composition over inheritance

Dockerfile

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -40,8 +40,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 RUN apk add --no-cache \
     bash \
     gcc \
-    python3 \
-    python3-dev \
+    python-3.13 \
+    python-3.13-dev \
     rust \
     openssl \
     openssl-dev \
@@ -51,6 +51,7 @@ RUN apk add --no-cache \
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
@@ -65,7 +66,8 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --extra bedrock-realtime \
+    --python python3.13
 
 # Copy full source tree
 COPY . .
@@ -86,7 +88,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --extra bedrock-realtime \
+    --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -100,8 +103,14 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
 USER root
 
+# The base image only configures Chainguard's authenticated apk repo, which
+# requires an enterprise subscription. Add the public Wolfi repo so `apk add`
+# also works for anyone installing extra packages into a running container.
+# https://github.com/BerriAI/litellm/issues/33518
+RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
+
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
+RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
 
 WORKDIR /app
 ENV PATH="/app/.venv/bin:${PATH}" \

README.md

@@ -354,6 +354,8 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 | [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Predibase (`predibase`)](https://docs.litellm.ai/docs/providers/predibase) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Qwen AI Platform (`qwen_ai_platform`)](https://docs.litellm.ai/docs/providers/qwencloud) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  | ✅ |
+| [QwenCloud (`qwencloud`)](https://docs.litellm.ai/docs/providers/qwencloud) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  | ✅ |
 | [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) |  |  |  |  | ✅ |  |  |  |  |  |
 | [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

backend/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
@@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
 # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
 # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -46,7 +46,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
-        --python python3
+        --extra saml \
+        --python python3.13
 
 # Stage 2 — copy source and install the project + workspace members.
 COPY . .
@@ -57,7 +58,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
-        --python python3
+        --extra saml \
+        --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -71,7 +73,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
+      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done

basedpyright-code-budget.json

@@ -1,36 +1,36 @@
 {
   "reportAny": {
-    "limit": 19955
+    "limit": 14076
   },
   "reportArgumentType": {
-    "limit": 2566
+    "limit": 2216
   },
   "reportAssignmentType": {
-    "limit": 320
+    "limit": 319
   },
   "reportAttributeAccessIssue": {
-    "limit": 488
+    "limit": 480
   },
   "reportCallIssue": {
-    "limit": 114
+    "limit": 112
   },
   "reportConstantRedefinition": {
     "limit": 40
   },
   "reportDeprecated": {
-    "limit": 213
+    "limit": 211
   },
   "reportDuplicateImport": {
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 6049
+    "limit": 4128
   },
   "reportFunctionMemberAccess": {
     "limit": 7
   },
   "reportGeneralTypeIssues": {
-    "limit": 154
+    "limit": 101
   },
   "reportIncompatibleMethodOverride": {
     "limit": 56
@@ -42,10 +42,10 @@
     "limit": 12
   },
   "reportIndexIssue": {
-    "limit": 35
+    "limit": 25
   },
   "reportInvalidTypeForm": {
-    "limit": 35
+    "limit": 34
   },
   "reportInvalidTypeVarUse": {
     "limit": 2
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5663
+    "limit": 5601
   },
   "reportMissingTypeArgument": {
-    "limit": 15555
+    "limit": 15306
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1061
+    "limit": 0
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -84,46 +84,46 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1822
+    "limit": 1808
   },
   "reportRedeclaration": {
     "limit": 8
   },
   "reportReturnType": {
-    "limit": 213
+    "limit": 181
   },
   "reportTypedDictNotRequiredAccess": {
-    "limit": 26
+    "limit": 24
   },
   "reportUndefinedVariable": {
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44655
+    "limit": 44364
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 39011
+    "limit": 38350
   },
   "reportUnknownParameterType": {
-    "limit": 19885
+    "limit": 19626
   },
   "reportUnknownVariableType": {
-    "limit": 30569
+    "limit": 29890
   },
   "reportUnnecessaryCast": {
-    "limit": 117
+    "limit": 111
   },
   "reportUnnecessaryComparison": {
-    "limit": 699
+    "limit": 692
   },
   "reportUnnecessaryContains": {
     "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 836
+    "limit": 826
   },
   "reportUntypedBaseClass": {
     "limit": 0
@@ -135,12 +135,12 @@
     "limit": 21
   },
   "reportUnusedFunction": {
-    "limit": 139
+    "limit": 138
   },
   "reportUnusedImport": {
-    "limit": 545
+    "limit": 543
   },
   "reportUnusedVariable": {
-    "limit": 146
+    "limit": 137
   }
 }

ci_cd/generate_model_prices_schema.py

@@ -73,6 +73,11 @@ ARRAY_KEYS: dict[str, JsonSchema] = {
         "description": "Output modalities the model can produce.",
         "items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
     },
+    "reasoning_effort_levels": {
+        "type": "array",
+        "description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
+        "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
+    },
     "supported_regions": {
         "type": "array",
         "description": "Cloud regions the model is available in ('global' or region ids).",
@@ -157,6 +162,9 @@ COST_DESCRIPTIONS: dict[str, str] = {
     "input_cost_per_token": "USD per prompt token.",
     "output_cost_per_token": "USD per generated token.",
     "output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
+    "google_maps_grounding_cost_per_query": (
+        "USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
+    ),
     "cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
     "cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
     "input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
@@ -212,6 +220,15 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
             "description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
             "enum": ["low", "medium", "high", "max", "xhigh"],
         },
+        "default_reasoning_effort": {
+            "type": "string",
+            "description": (
+                "Reasoning effort the provider applies when the request omits reasoning_effort. "
+                "Gates whether a non-default temperature or the top_p/logprobs sampling params are "
+                "accepted, which hold only when the effort resolves to 'none'."
+            ),
+            "enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
+        },
         "comment": STRING,
         "audio_transcription_config": STRING,
     }

codecov.yaml

@@ -25,6 +25,8 @@ flag_management:
       carryforward: false
     - name: proxy-db-schema-migration
       carryforward: false
+    - name: circleci
+      carryforward: false
 
 component_management:
   individual_components:

db_scripts/partition_spend_logs.sql

@@ -10,6 +10,11 @@
 -- partitioned, so existing installs are unaffected until you run this.
 --
 -- IMPORTANT
+--   * After partitioning, `prisma db push` (including the proxy's
+--     --use_prisma_db_push startup mode) is NOT supported: it tries to rewrite
+--     the primary key back to ("request_id"), which Postgres rejects on a
+--     partitioned table. The proxy detects this and exits with guidance.
+--     Use the default startup path (`prisma migrate deploy`) instead.
 --   * Test on a staging copy first and take a backup.
 --   * Postgres cannot convert a populated table to partitioned in place, so this
 --     renames the old table aside and creates a fresh partitioned table.

docker/Dockerfile.database

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -39,8 +39,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 RUN apk add --no-cache \
     bash \
     gcc \
-    python3 \
-    python3-dev \
+    python-3.13 \
+    python-3.13-dev \
     openssl \
     openssl-dev \
     nodejs \
@@ -49,6 +49,7 @@ RUN apk add --no-cache \
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
@@ -63,7 +64,8 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --extra bedrock-realtime \
+    --python python3.13
 
 # Copy full source tree
 COPY . .
@@ -84,7 +86,8 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --extra bedrock-realtime \
+    --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -98,7 +101,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
+RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
 
 WORKDIR /app
 ENV PATH="/app/.venv/bin:${PATH}" \

docker/Dockerfile.non_root

@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1.7
 
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
@@ -37,8 +37,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 
 RUN for i in 1 2 3; do \
       apk add --no-cache \
-        python3 \
-        python3-dev \
+        python-3.13 \
+        python-3.13-dev \
         gcc \
         rust \
         bash \
@@ -52,6 +52,7 @@ RUN for i in 1 2 3; do \
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
     XDG_CACHE_HOME=/app/.cache
@@ -69,7 +70,8 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --extra bedrock-realtime \
+    --python python3.13
 
 # Copy full source tree
 COPY . .
@@ -96,7 +98,8 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra extra_proxy \
         --extra semantic-router \
         --extra saml \
-        --python python3 \
+        --extra bedrock-realtime \
+        --python python3.13 \
         --no-sources-package litellm-proxy-extras; \
     else \
       uv sync --frozen --no-default-groups --no-editable \
@@ -105,7 +108,8 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra extra_proxy \
         --extra semantic-router \
         --extra saml \
-        --python python3; \
+        --extra bedrock-realtime \
+        --python python3.13; \
     fi
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
@@ -124,7 +128,7 @@ RUN for i in 1 2 3; do \
       apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
+      apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
     done
 
 # Copy only what runtime needs. The application is installed inside the venv;

enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py

@@ -7,7 +7,7 @@ GET - /audit/{id} - Get audit log by id
 GET - /audit - Get all audit logs
 """
 
-from typing import Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Final
 
 #### AUDIT LOGGING ####
 from fastapi import APIRouter, Depends, HTTPException, Query
@@ -18,11 +18,16 @@ from litellm_enterprise.types.proxy.audit_logging_endpoints import (
 
 from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.repositories.prisma_protocols import TableActions
+from litellm.repositories.table_repositories import AuditLogRepository
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
 
 router = APIRouter()
 
 
-def _build_json_field_or_condition(json_key: str, value: str) -> Dict[str, Any]:
+def _build_json_field_or_condition(json_key: str, value: str) -> dict[str, object]:
     """
     Build an OR condition that matches a value inside a JSON column at the
     given key, checking both before_value and updated_values.
@@ -53,33 +58,33 @@ async def get_audit_logs(
     page: int = Query(1, ge=1),
     page_size: int = Query(10, ge=1, le=100),
     # Filter parameters
-    changed_by: Optional[str] = Query(
+    changed_by: str | None = Query(
         None, description="Filter by user or system that performed the action"
     ),
-    changed_by_api_key: Optional[str] = Query(
+    changed_by_api_key: str | None = Query(
         None, description="Filter by API key hash that performed the action"
     ),
-    action: Optional[str] = Query(
+    action: str | None = Query(
         None, description="Filter by action type (create, update, delete)"
     ),
-    table_name: Optional[str] = Query(
+    table_name: str | None = Query(
         None, description="Filter by table name that was modified"
     ),
-    object_id: Optional[str] = Query(
+    object_id: str | None = Query(
         None, description="Filter by ID of the object that was modified"
     ),
-    start_date: Optional[str] = Query(None, description="Filter logs after this date"),
-    end_date: Optional[str] = Query(None, description="Filter logs before this date"),
-    object_team_id: Optional[str] = Query(
+    start_date: str | None = Query(None, description="Filter logs after this date"),
+    end_date: str | None = Query(None, description="Filter logs before this date"),
+    object_team_id: str | None = Query(
         None,
         description="Filter by team_id present in before_value or updated_values JSON (PostgreSQL only)",
     ),
-    object_key_hash: Optional[str] = Query(
+    object_key_hash: str | None = Query(
         None,
         description="Filter by token (key hash) present in before_value or updated_values JSON (PostgreSQL only)",
     ),
     # Sorting parameters
-    sort_by: Optional[str] = Query(
+    sort_by: str | None = Query(
         None,
         description="Column to sort by (e.g. 'updated_at', 'action', 'table_name')",
     ),
@@ -101,46 +106,37 @@ async def get_audit_logs(
             detail={"message": CommonProxyErrors.db_not_connected_error.value},
         )
 
-    # Build filter conditions
-    where_conditions: Dict[str, Any] = {}
-    if changed_by:
-        where_conditions["changed_by"] = changed_by
-    if changed_by_api_key:
-        where_conditions["changed_by_api_key"] = changed_by_api_key
-    if action:
-        where_conditions["action"] = action
-    if table_name:
-        where_conditions["table_name"] = table_name
-    if object_id:
-        where_conditions["object_id"] = object_id
-    if start_date or end_date:
-        date_filter: Dict[str, Any] = {}
-        if start_date:
-            date_filter["gte"] = start_date
-        if end_date:
-            date_filter["lte"] = end_date
-        where_conditions["updated_at"] = date_filter
+    date_filter: Final[dict[str, str]] = {
+        **({"gte": start_date} if start_date else {}),
+        **({"lte": end_date} if end_date else {}),
+    }
 
     # JSON field filters (PostgreSQL only) — each filter is AND'd with the
     # others, but checks both before_value and updated_values internally (OR).
-    if object_team_id:
-        where_conditions["AND"] = where_conditions.get("AND", []) + [
-            _build_json_field_or_condition("team_id", object_team_id)
-        ]
-    if object_key_hash:
-        where_conditions["AND"] = where_conditions.get("AND", []) + [
-            _build_json_field_or_condition("token", object_key_hash)
-        ]
+    json_field_conditions: Final[list[dict[str, object]]] = [
+        *([_build_json_field_or_condition("team_id", object_team_id)] if object_team_id else []),
+        *([_build_json_field_or_condition("token", object_key_hash)] if object_key_hash else []),
+    ]
 
-    # Build sort conditions
-    order_by: Dict[str, Any] = {}
-    if sort_by and isinstance(sort_by, str):
-        order_by[sort_by] = sort_order
-    else:
-        order_by["updated_at"] = sort_order  # Default sort by updated_at
+    # Build filter conditions
+    where_conditions: Final[dict[str, object]] = {
+        **({"changed_by": changed_by} if changed_by else {}),
+        **({"changed_by_api_key": changed_by_api_key} if changed_by_api_key else {}),
+        **({"action": action} if action else {}),
+        **({"table_name": table_name} if table_name else {}),
+        **({"object_id": object_id} if object_id else {}),
+        **({"updated_at": date_filter} if start_date or end_date else {}),
+        **({"AND": json_field_conditions} if json_field_conditions else {}),
+    }
+
+    order_by: Final[dict[str, str]] = (
+        {sort_by: sort_order} if sort_by and isinstance(sort_by, str) else {"updated_at": sort_order}
+    )
+
+    audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table
 
     # Get paginated results
-    audit_logs = await prisma_client.db.litellm_auditlog.find_many(
+    audit_logs: Final = await audit_log_table.find_many(
         where=where_conditions,
         order=order_by,
         skip=(page - 1) * page_size,
@@ -148,13 +144,14 @@ async def get_audit_logs(
     )
 
     # Get total count for pagination
-    total_count = await prisma_client.db.litellm_auditlog.count(where=where_conditions)
-    total_pages = -(-total_count // page_size)  # Ceiling division
+    total_count: Final = await audit_log_table.count(where=where_conditions)
+    total_pages: Final = -(-total_count // page_size)  # Ceiling division
 
     # Return paginated response
     return PaginatedAuditLogResponse(
         audit_logs=[
-            AuditLogResponse(**audit_log.model_dump()) for audit_log in audit_logs
+            AuditLogResponse.model_validate(audit_log.model_dump())
+            for audit_log in audit_logs
         ]
         if audit_logs
         else [],
@@ -198,8 +195,10 @@ async def get_audit_log_by_id(
             detail={"message": CommonProxyErrors.db_not_connected_error.value},
         )
 
+    audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table
+
     # Get the audit log by ID
-    audit_log = await prisma_client.db.litellm_auditlog.find_unique(where={"id": id})
+    audit_log: Final = await audit_log_table.find_unique(where={"id": id})
 
     if audit_log is None:
         raise HTTPException(
@@ -207,4 +206,4 @@ async def get_audit_log_by_id(
         )
 
     # Convert to response model
-    return AuditLogResponse(**audit_log.model_dump())
+    return AuditLogResponse.model_validate(audit_log.model_dump())

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -2,9 +2,10 @@
 Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
 """
 
+from dataclasses import replace as dataclasses_replace
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Tuple
+from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -14,6 +15,8 @@ from litellm.constants import (
 )
 
 if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
     from litellm.integrations.prometheus import PrometheusLogger
     from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -84,7 +87,7 @@ class CheckBatchCost:
             return
         self.batch_processed_support_confirmed = True
 
-    async def _get_user_info(self, batch_id: str, user_id: Optional[str]) -> Dict[str, Any]:
+    async def _get_user_info(self, batch_id: str, user_id: Optional[str]) -> dict[str, str | None]:
         """
         Look up user email and key alias by user_id for enriching the S3 callback metadata.
         Returns a dict with user_api_key_user_email and user_api_key_alias (both may be None).
@@ -94,8 +97,10 @@ class CheckBatchCost:
         if not user_id:
             return {}
         try:
-            user_row = await self.prisma_client.db.litellm_usertable.find_unique(
-                where={"user_id": user_id}
+            user_row: prisma_models.LiteLLM_UserTable | None = (
+                await self.prisma_client.db.litellm_usertable.find_unique(
+                    where={"user_id": user_id}
+                )
             )
             if user_row is None:
                 return {}
@@ -112,8 +117,10 @@ class CheckBatchCost:
         if not api_key:
             return None
         try:
-            key_row = await self.prisma_client.db.litellm_verificationtoken.find_unique(
-                where={"token": api_key}
+            key_row: prisma_models.LiteLLM_VerificationToken | None = (
+                await self.prisma_client.db.litellm_verificationtoken.find_unique(
+                    where={"token": api_key}
+                )
             )
             return getattr(key_row, "key_alias", None) if key_row is not None else None
         except Exception as e:
@@ -125,8 +132,10 @@ class CheckBatchCost:
         if not team_id:
             return None
         try:
-            team_row = await self.prisma_client.db.litellm_teamtable.find_unique(
-                where={"team_id": team_id}
+            team_row: prisma_models.LiteLLM_TeamTable | None = (
+                await self.prisma_client.db.litellm_teamtable.find_unique(
+                    where={"team_id": team_id}
+                )
             )
             return getattr(team_row, "team_alias", None) if team_row is not None else None
         except Exception as e:
@@ -135,7 +144,7 @@ class CheckBatchCost:
 
     async def _build_creator_attribution_metadata(
         self, job: "LiteLLM_ManagedObjectTable", batch_id: str
-    ) -> Dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Rebuild the spend-tracking metadata for the key, team, and tags that created the
         batch so the batch-cost spend log is attributed the same way a non-batch request
@@ -149,7 +158,7 @@ class CheckBatchCost:
         team_id = getattr(job, "team_id", None)
         request_tags = getattr(job, "request_tags", None)
 
-        metadata: Dict[str, Any] = {
+        metadata: dict[str, object] = {
             "user_api_key_user_id": job.created_by,
             "user_api_key": api_key,
             "user_api_key_team_id": team_id,
@@ -351,7 +360,7 @@ class CheckBatchCost:
         return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
 
     async def _finalize_unbilled_terminal_job(
-        self, job: "LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
+        self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
     ) -> None:
         """Persist a terminal batch that has nothing billable, converting any raw
         provider file ids to managed ids, and take it out of the poll page."""
@@ -624,6 +633,7 @@ class CheckBatchCost:
         later poll.
         """
         from litellm.batches.batch_utils import (
+            count_error_file_failed_requests,
             _get_file_content_as_dictionary,
             calculate_batch_cost_and_usage,
         )
@@ -759,16 +769,33 @@ class CheckBatchCost:
             model_id=model_id,
             deployment_model=litellm_model_name,
         )
-        batch_cost, batch_usage, batch_models = (
-            await calculate_batch_cost_and_usage(
-                file_content_dictionary=file_content_as_dict,
-                custom_llm_provider=llm_provider,  # type: ignore
-                model_name=model_name,
-                model_info=deployment_model_info,
+        batch_file_provider: Final = cast(
+            Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"], llm_provider
+        )
+        output_file_result: Final = await calculate_batch_cost_and_usage(
+            file_content_dictionary=file_content_as_dict,
+            custom_llm_provider=batch_file_provider,
+            model_name=model_name,
+            model_info=deployment_model_info,
+        )
+        error_file_failed_requests: Final = await count_error_file_failed_requests(
+            response,
+            custom_llm_provider=batch_file_provider,
+            litellm_params={
+                **credentials,
+                "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
+            },
+        )
+        batch_result: Final = (
+            output_file_result
+            if not error_file_failed_requests
+            else dataclasses_replace(
+                output_file_result,
+                failed_requests=output_file_result.failed_requests + error_file_failed_requests,
             )
         )
         logging_obj = LiteLLMLogging(
-            model=batch_models[0],
+            model=batch_result.models[0],
             messages=[{"role": "user", "content": "<retrieve_batch>"}],
             stream=False,
             call_type="aretrieve_batch",
@@ -800,9 +827,11 @@ class CheckBatchCost:
         try:
             await logging_obj.async_success_handler(
                 result=response,
-                batch_cost=batch_cost,
-                batch_usage=batch_usage,
-                batch_models=batch_models,
+                batch_cost=batch_result.cost,
+                batch_usage=batch_result.usage,
+                batch_models=batch_result.models,
+                batch_successful_requests=batch_result.successful_requests,
+                batch_failed_requests=batch_result.failed_requests,
             )
         except Exception:
             await self._release_job_claim(job)
@@ -966,6 +995,16 @@ class CheckBatchCost:
                     )
 
             elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
+                from litellm.proxy.openai_files_endpoints.common_utils import (
+                    _completed_batch_safe_to_retire,
+                )
+
+                if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
+                    verbose_proxy_logger.info(
+                        f"CheckBatchCost: batch {batch_id} is completed but its output file id "
+                        f"has not appeared yet; leaving job {job.id} for the next poll cycle"
+                    )
+                    continue
                 await self._finalize_unbilled_terminal_job(job, response)
 
         # Record polling run metrics (always, even if nothing was processed)

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -1,6 +1,8 @@
 """
 Polls LiteLLM_ManagedObjectTable to check if the response is complete.
-Cost tracking is handled automatically by the get-responses call.
+Cost tracking is handled by the get-responses call, which prices normally only because the
+poll stamps itself with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN; user-facing reads of the
+same route are non-inference and free.
 """
 
 from datetime import datetime, timedelta, timezone
@@ -9,12 +11,14 @@ from typing import TYPE_CHECKING, Dict, Optional, cast
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import (
+    INTERNAL_CALL_ORIGIN_METADATA_KEY,
     MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
     MAX_OBJECTS_PER_POLL_CYCLE,
     STALE_OBJECT_CLEANUP_BATCH_SIZE,
 )
 from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.types.llms.openai import ResponsesAPIResponse
+from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -113,7 +117,8 @@ class CheckResponsesCost:
         Check if background responses are complete and track their cost.
         - Get all status="queued" or "in_progress" and file_purpose="response" jobs
         - Query the provider to check if response is complete
-        - Cost is automatically tracked by the get-responses call
+        - Cost is tracked by the get-responses call, billed because the poll is stamped
+          with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
         - Mark responses in a terminal state as complete in the database
         """
         try:
@@ -153,6 +158,7 @@ class CheckResponsesCost:
                 # Prepare metadata with model information for cost tracking
                 litellm_metadata = {
                     "user_api_key_user_id": job.created_by or "default-user-id",
+                    INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
                 }
                 
                 # Add model information if available

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -36,6 +36,7 @@ from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
     build_owner_filter,
     can_access_resource,
+    resolve_resource_owner_id,
 )
 from litellm.proxy._types import (
     CallTypes,
@@ -45,6 +46,8 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.openai_files_endpoints.common_utils import (
+    FILE_LIST_CONTINUATION_CHUNK_SIZE,
+    MAX_FILE_LIST_LIMIT,
     _is_base64_encoded_unified_file_id,
     apply_unified_file_ids,
     ensure_batch_response_managed_file_ids,
@@ -54,6 +57,8 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
     map_raw_file_ids_to_unified,
     normalize_mime_type_for_provider,
     resolve_managed_output_file_model_name,
+    validate_file_list_limit,
+    validate_file_list_purpose,
 )
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
     request_tags_from_metadata,
@@ -63,9 +68,9 @@ from litellm.types.llms.openai import (  # pyright: ignore[reportAttributeAccess
     AsyncCursorPage,
     ChatCompletionFileObject,
     CreateFileRequest,
+    FileListPage,
     FileObject,
     OpenAIFileObject,
-    OpenAIFilesPurpose,
     ResponsesAPIResponse,
 )
 from litellm.types.utils import (
@@ -144,7 +149,14 @@ class _ManagedFileRow(Protocol):
 class _ManagedFileTableActions(Protocol):
     async def find_first(self, where: Mapping[str, object]) -> Optional[_ManagedFileRow]: ...
 
-    async def find_many(self, where: Mapping[str, object]) -> Sequence[_ManagedFileRow]: ...
+    async def find_many(
+        self,
+        where: Mapping[str, object],
+        take: int = ...,
+        order: Union[Mapping[str, str], Sequence[Mapping[str, str]]] = ...,
+        cursor: Mapping[str, str] = ...,
+        skip: int = ...,
+    ) -> Sequence[_ManagedFileRow]: ...
 
     async def upsert(self, where: Mapping[str, str], data: Mapping[str, Mapping[str, object]]) -> _ManagedFileRow: ...
 
@@ -170,6 +182,10 @@ class _ManagedObjectTableActions(Protocol):
     async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
 
 
+class _SchedulerWithJobLookup(Protocol):
+    def get_job(self, job_id: str) -> object: ...
+
+
 class _CursorPageArgs(TypedDict, total=False):
     cursor: Mapping[str, str]
     skip: int
@@ -211,7 +227,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 file_object=file_object,
                 model_mappings=model_mappings,
                 flat_model_file_ids=list(model_mappings.values()),
-                created_by=user_api_key_dict.user_id,
+                created_by=resolve_resource_owner_id(user_api_key_dict),
                 team_id=user_api_key_dict.team_id,
                 updated_by=user_api_key_dict.user_id,
             )
@@ -227,7 +243,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             "unified_file_id": file_id,
             "model_mappings": json.dumps(model_mappings),
             "flat_model_file_ids": list(model_mappings.values()),
-            "created_by": user_api_key_dict.user_id,
+            "created_by": resolve_resource_owner_id(user_api_key_dict),
             "team_id": user_api_key_dict.team_id,
             "updated_by": user_api_key_dict.user_id,
         }
@@ -331,7 +347,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "file_object": file_object.model_dump_json(),
                     "model_object_id": model_object_id,
                     "file_purpose": file_purpose,
-                    "created_by": user_api_key_dict.user_id,
+                    "created_by": resolve_resource_owner_id(user_api_key_dict),
                     "team_id": user_api_key_dict.team_id,
                     "updated_by": user_api_key_dict.user_id,
                     "status": file_object.status,
@@ -462,19 +478,56 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 )
 
         page_size: Final = min(limit or 20, 100)
-        cursor_args: _CursorPageArgs = {"cursor": {"unified_object_id": after}, "skip": 1} if after else {}
-
-        batches = await _managed_object_table(self.prisma_client).find_many(
-            where=where_clause,
-            take=page_size + 1,
-            order=[{"created_at": "desc"}, {"unified_object_id": "desc"}],
-            **cursor_args,
+        matches: Final = await self._collect_listed_batches(
+            where_clause=where_clause,
+            after=after,
+            wanted=page_size + 1,
+            user_api_key_dict=user_api_key_dict,
         )
+        return build_list_page(list(matches[:page_size]), has_more=len(matches) > page_size)
 
-        has_more = len(batches) > page_size
+    async def _collect_listed_batches(
+        self,
+        where_clause: Mapping[str, object],
+        after: Optional[str],
+        wanted: int,
+        user_api_key_dict: UserAPIKeyAuth,
+    ) -> tuple[LiteLLMBatch, ...]:
+        """Read chunks newest-first until ``wanted`` batches survive parsing and
+        file-id resolution or the caller's rows run out, so a run of rows that will
+        not parse refills the page instead of emptying it. The first chunk is
+        ``wanted`` rows, so a healthy page still costs one query; a scan that has to
+        continue widens to ``FILE_LIST_CONTINUATION_CHUNK_SIZE`` like ``afile_list``,
+        and every chunk advances the keyset cursor, so the walk ends once the
+        caller's rows are exhausted."""
+        matches: tuple[LiteLLMBatch, ...] = ()  # rebind-ok: accumulates survivors across chunks
+        cursor_id: Optional[str] = after  # rebind-ok: keyset cursor advances to each chunk's last row
+        chunk_size: int = wanted  # rebind-ok: widens once a scan has to continue past the first chunk
+        while len(matches) < wanted:
+            cursor_args: _CursorPageArgs = {"cursor": {"unified_object_id": cursor_id}, "skip": 1} if cursor_id else {}
+            chunk = await _managed_object_table(self.prisma_client).find_many(
+                where=where_clause,
+                take=chunk_size,
+                order=[{"created_at": "desc"}, {"unified_object_id": "desc"}],
+                **cursor_args,
+            )
+            matches = matches + await self._resolve_listed_rows(
+                rows=chunk, wanted=wanted - len(matches), user_api_key_dict=user_api_key_dict
+            )
+            if len(chunk) < chunk_size:
+                break
+            cursor_id = chunk[-1].unified_object_id
+            chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
+        return matches
 
+    async def _resolve_listed_rows(
+        self,
+        rows: "Sequence[PrismaManagedObjectRow]",
+        wanted: int,
+        user_api_key_dict: UserAPIKeyAuth,
+    ) -> tuple[LiteLLMBatch, ...]:
         parsed_rows: Final = tuple(
-            (row, batch_obj) for row in batches[:page_size] if (batch_obj := _parse_managed_batch_row(row)) is not None
+            (row, batch_obj) for row in rows if (batch_obj := _parse_managed_batch_row(row)) is not None
         )
         unified_id_by_raw_id: Final = await map_raw_file_ids_to_unified(
             raw_file_ids=frozenset(
@@ -485,19 +538,19 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             ),
             prisma_client=self.prisma_client,
         )
-        resolved_batches: Final = [
-            await self._resolve_listed_batch(
+        resolved: Final[list[LiteLLMBatch]] = []  # mutable-ok: resolution stops as soon as the page is full
+        for row, batch_obj in parsed_rows:
+            if len(resolved) == wanted:
+                break
+            resolved_batch = await self._resolve_listed_batch(
                 row=row,
                 batch_obj=batch_obj,
                 unified_id_by_raw_id=unified_id_by_raw_id,
                 user_api_key_dict=user_api_key_dict,
             )
-            for row, batch_obj in parsed_rows
-        ]
-        return build_list_page(
-            [batch_obj for batch_obj in resolved_batches if batch_obj is not None],
-            has_more=has_more,
-        )
+            if resolved_batch is not None:
+                resolved.append(resolved_batch)
+        return tuple(resolved)
 
     async def _resolve_listed_batch(
         self,
@@ -804,7 +857,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                                 file_ids.append(file_id)
         return file_ids
 
-    def get_file_ids_from_responses_input(self, input: Union[str, List[Dict[str, Any]]]) -> List[str]:
+    def get_file_ids_from_responses_input(self, input: Union[str, List[Dict[str, object]]]) -> List[str]:
         """
         Gets file ids from responses API input.
 
@@ -829,7 +882,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             # Check for direct input_file type
             if item.get("type") == "input_file":
                 file_id = item.get("file_id")
-                if file_id:
+                if isinstance(file_id, str) and file_id:
                     file_ids.append(file_id)
 
             # Check for input_file in content array
@@ -838,7 +891,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 for content_item in content:
                     if isinstance(content_item, dict) and content_item.get("type") == "input_file":
                         file_id = content_item.get("file_id")
-                        if file_id:
+                        if isinstance(file_id, str) and file_id:
                             file_ids.append(file_id)
 
         return file_ids
@@ -1178,7 +1231,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
 
                 # Handle both output_file_id and error_file_id
                 for file_attr in ["output_file_id", "error_file_id"]:
-                    file_id_value = getattr(response, file_attr, None)
+                    file_id_value: str | None = getattr(response, file_attr, None)
                     if file_id_value and model_id:
                         decoded_output_file_id = _is_base64_encoded_unified_file_id(file_id_value)
                         if decoded_output_file_id and "llm_output_file_id," in decoded_output_file_id:
@@ -1365,12 +1418,76 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
 
     async def afile_list(
         self,
-        purpose: Optional[OpenAIFilesPurpose],
+        purpose: Optional[str],
         litellm_parent_otel_span: Optional[Span],
+        user_api_key_dict: UserAPIKeyAuth,
+        limit: Optional[int] = None,
+        after: Optional[str] = None,
         **data: Dict,
-    ) -> List[OpenAIFileObject]:
-        """Handled in files_endpoints.py"""
-        return []
+    ) -> FileListPage:
+        """List the managed files the caller owns, newest first.
+
+        Pagination is keyset based on ``unified_file_id`` so a key that owns
+        every file on the proxy still reads one bounded page at a time.
+        ``purpose`` is applied after parsing, because the managed file table
+        keeps it inside the ``file_object`` blob instead of a column, and rows
+        whose blob will not parse drop out there too, so a chunk of rows can
+        yield fewer matches than the page holds. Successive chunks are read
+        until the page is full or the caller's rows run out, which keeps
+        ``data`` non-empty while matches remain and its last id usable as the
+        next cursor. A first chunk that fills the page costs one query; once a
+        scan has to continue past it, the chunk widens to
+        ``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
+        that many rows instead of one per page. That bound is per query, not
+        per request: the work is still linear in the rows the caller owns, and
+        a filter matching nothing reads every one of them, with no index
+        covering either the owner filter or the sort.
+        """
+        validate_file_list_limit(limit)
+        validate_file_list_purpose(purpose)
+
+        owner_filter: Final = build_owner_filter(user_api_key_dict)
+        if owner_filter is None:
+            return FileListPage(**build_list_page([]))
+
+        if after:
+            cursor_row = await _managed_file_table(self.prisma_client).find_first(
+                where={**owner_filter, "unified_file_id": after}
+            )
+            if cursor_row is None:
+                raise ProxyException(
+                    message=f"Invalid 'after' cursor: no file found with id '{after}'.",
+                    type="invalid_request_error",
+                    param="after",
+                    code=400,
+                    openai_code="invalid_value",
+                )
+
+        page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
+        matches: Final[List[OpenAIFileObject]] = []
+        cursor_id = after
+        chunk_size = page_size + 1
+
+        while len(matches) <= page_size:
+            cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
+            chunk = await _managed_file_table(self.prisma_client).find_many(
+                where=owner_filter,
+                take=chunk_size,
+                order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
+                **cursor_args,
+            )
+            matches.extend(
+                parsed_file_object.model_copy(update={"id": row.unified_file_id})
+                for row in chunk
+                if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
+                and (purpose is None or parsed_file_object.purpose == purpose)
+            )
+            if len(chunk) < chunk_size:
+                break
+            cursor_id = chunk[-1].unified_file_id
+            chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
+
+        return FileListPage(**build_list_page(matches[:page_size], has_more=len(matches) > page_size))
 
     def _is_batch_polling_enabled(self) -> bool:
         """
@@ -1383,7 +1500,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             import litellm.proxy.proxy_server as proxy_server_module
 
             # Check if the scheduler has the batch cost checking job registered
-            scheduler = getattr(proxy_server_module, "scheduler", None)
+            scheduler: Final[_SchedulerWithJobLookup | None] = getattr(proxy_server_module, "scheduler", None)
             if scheduler is None:
                 return False
 
@@ -1429,7 +1546,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             )
         MAX_MATCHES_TO_RETURN = 10
 
-        batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
+        batches = await _managed_object_table(self.prisma_client).find_many(
             where={
                 "file_purpose": "batch",
                 "batch_processed": False,
@@ -1439,11 +1556,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             order={"created_at": "desc"},
         )
 
-        referencing_batches = []
+        referencing_batches: Final[list[dict[str, object]]] = []
         for batch in batches:
             try:
                 # Parse the batch file_object to check for file references
-                batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
+                decoded_file_object = _decode_json_blob(batch.file_object)
+                batch_data: Mapping[str, object] = (
+                    decoded_file_object if isinstance(decoded_file_object, Mapping) else {}
+                )
 
                 # Extract file IDs from batch
                 # Batches typically reference the unified file ID in input_file_id

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -12,9 +12,10 @@ Endpoints for /project operations
 
 import json
 from collections.abc import Sequence
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Final
 
 from fastapi import APIRouter, Depends, HTTPException, Request
+from pydantic import TypeAdapter
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -26,37 +27,50 @@ from litellm.proxy.management_helpers.utils import (
     management_endpoint_wrapper,
 )
 from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
+from litellm.repositories.budget_repository import BudgetRepository
+from litellm.repositories.object_permission_repository import ObjectPermissionRepository
+from litellm.repositories.prisma_protocols import TableActions
+from litellm.repositories.project_repository import ProjectRepository
+from litellm.repositories.team_repository import TeamRepository
+from litellm.repositories.user_repository import UserRepository
+from litellm.repositories.verification_token_repository import VerificationTokenRepository
 
 if TYPE_CHECKING:
     from prisma import models as prisma_models
-    from prisma.actions import (
-        LiteLLM_ProjectTableActions,
-        LiteLLM_TeamTableActions,
-        LiteLLM_VerificationTokenActions,
-    )
+
+    from litellm import Router
 
 router = APIRouter()
 
-
-def _team_table(prisma_client: PrismaClient) -> "LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable]":
-    team_table: LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable] = prisma_client.db.litellm_teamtable
-    return team_table
+_OBJECT_PERMISSION_PAYLOAD: Final = TypeAdapter(dict[str, object])
 
 
-def _project_table(prisma_client: PrismaClient) -> "LiteLLM_ProjectTableActions[prisma_models.LiteLLM_ProjectTable]":
-    project_table: LiteLLM_ProjectTableActions[prisma_models.LiteLLM_ProjectTable] = (
-        prisma_client.db.litellm_projecttable
-    )
-    return project_table
+def _team_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_TeamTable"]:
+    return TeamRepository(prisma_client).table
+
+
+def _project_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_ProjectTable"]:
+    return ProjectRepository(prisma_client).table
 
 
 def _verification_token_table(
     prisma_client: PrismaClient,
-) -> "LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken]":
-    verification_token_table: LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken] = (
-        prisma_client.db.litellm_verificationtoken
-    )
-    return verification_token_table
+) -> TableActions["prisma_models.LiteLLM_VerificationToken"]:
+    return VerificationTokenRepository(prisma_client).table
+
+
+def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]:
+    return BudgetRepository(prisma_client).table
+
+
+def _object_permission_table(
+    prisma_client: PrismaClient,
+) -> TableActions["prisma_models.LiteLLM_ObjectPermissionTable"]:
+    return ObjectPermissionRepository(prisma_client).table
+
+
+def _user_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_UserTable"]:
+    return UserRepository(prisma_client).table
 
 
 def _jsonified(prisma_client: PrismaClient, payload: dict[str, object]) -> dict[str, object]:
@@ -205,6 +219,114 @@ def _check_team_project_limits(
         )
 
 
+def _project_models_missing_positive_quota(
+    models: list[str] | None,
+    rpm_limits: Mapping[str, object] | None,
+    tpm_limits: Mapping[str, object] | None,
+) -> list[str]:
+    """Return the models that lack a positive `rpm` AND `tpm` quota.
+
+    A valid quota is a positive integer; null, zero, and negative are rejected
+    because downstream rate limiters treat a non-positive limit as immediately
+    exhausted (every request blocked).
+    """
+
+    def _is_positive(value: object) -> bool:
+        return isinstance(value, int) and not isinstance(value, bool) and value > 0
+
+    rpm = rpm_limits or {}
+    tpm = tpm_limits or {}
+    return [model for model in (models or []) if not _is_positive(rpm.get(model)) or not _is_positive(tpm.get(model))]
+
+
+def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
+    return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
+
+
+def _project_models_expanding_at_request_time(
+    models: Sequence[str] | None, access_group_names: frozenset[str]
+) -> tuple[str, ...]:
+    """Entries project auth expands to many concrete models (`all-proxy-models`, `*` patterns,
+    access groups). The rate limiter looks quotas up by the exact requested model name, so a
+    quota keyed on one of these entries is never applied."""
+    return tuple(
+        model
+        for model in (models or ())
+        if model == SpecialModelNames.all_proxy_models.value or "*" in model or model in access_group_names
+    )
+
+
+def _raise_on_project_models_expanding_at_request_time(
+    models: Sequence[str] | None, access_group_names: frozenset[str]
+) -> None:
+    expanding: Final = _project_models_expanding_at_request_time(models, access_group_names)
+    if not expanding:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while 'enforce_project_model_quota' is enabled. List concrete model names instead."
+        },
+    )
+
+
+def _raise_on_missing_project_model_quota(
+    data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset()
+) -> None:
+    """Require a positive `rpm`/`tpm` quota for every model on project CREATE.
+
+    `model_rpm_limit`/`model_tpm_limit` are relocated into `metadata` by the request
+    model's `set_model_info` validator, so they are read from there.
+
+    Only invoked when `general_settings.enforce_project_model_quota` is enabled
+    (default off), so it is opt-in and does not change behavior for existing users.
+    """
+    _raise_on_project_models_expanding_at_request_time(data.models, access_group_names)
+    metadata = data.metadata or {}
+    missing = _project_models_missing_positive_quota(
+        data.models, metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit")
+    )
+    if not missing:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {missing} added to project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
+        },
+    )
+
+
+def _raise_on_missing_project_model_quota_on_update(
+    data: UpdateProjectRequest, existing_project: object, access_group_names: frozenset[str] = frozenset()
+) -> None:
+    """Require a positive `rpm`/`tpm` quota over the RESULTING state on project UPDATE.
+
+    `/project/update` replaces `models` and `metadata` when they are provided, so the
+    check runs on what the project WILL look like: a partial update that doesn't touch
+    models/quota keeps the existing values, while one that adds a model or clears a
+    model's quota must leave every resulting model with a positive limit.
+
+    Only invoked when `general_settings.enforce_project_model_quota` is enabled
+    (default off), so it is opt-in and does not change behavior for existing users.
+    """
+    resulting_models = data.models if data.models is not None else (getattr(existing_project, "models", None) or [])
+    resulting_metadata = (
+        data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {})
+    )
+    _raise_on_project_models_expanding_at_request_time(resulting_models, access_group_names)
+    missing = _project_models_missing_positive_quota(
+        resulting_models, resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit")
+    )
+    if not missing:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {missing} would be left on the project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
+        },
+    )
+
+
 async def _create_budget_for_project(
     data: NewProjectRequest,
     user_id: str | None,
@@ -219,7 +341,7 @@ async def _create_budget_for_project(
 
     new_budget = _jsonified(prisma_client, budget_row.model_dump(exclude_none=True))
 
-    _budget: prisma_models.LiteLLM_BudgetTable = await prisma_client.db.litellm_budgettable.create(
+    _budget: Final = await _budget_table(prisma_client).create(
         data={
             **new_budget,
             "created_by": user_id or litellm_proxy_admin_name,
@@ -242,10 +364,8 @@ async def _set_project_object_permission(
         return None
 
     if data.object_permission is not None:
-        created_object_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
-            await prisma_client.db.litellm_objectpermissiontable.create(
-                data=data.object_permission.model_dump(exclude_none=True),
-            )
+        created_object_permission: Final = await _object_permission_table(prisma_client).create(
+            data=data.object_permission.model_dump(exclude_none=True),
         )
         del data.object_permission
         return created_object_permission.object_permission_id
@@ -352,7 +472,9 @@ async def new_project(
     ```
     """
     from litellm.proxy.proxy_server import (
+        general_settings,
         litellm_proxy_admin_name,
+        llm_router,
         premium_user,
         prisma_client,
     )
@@ -399,6 +521,10 @@ async def new_project(
             data=data,
         )
 
+        # Opt-in (default off): require rpm/tpm for every model added to the project.
+        if general_settings.get("enforce_project_model_quota", False):
+            _raise_on_missing_project_model_quota(data, _router_access_group_names(llm_router))
+
         # Check if user has permission to create projects for this team
         # only team admins can create projects for their team
         has_permission = await _check_user_permission_for_project(
@@ -470,10 +596,8 @@ async def new_project(
         new_project_row = _remove_budget_fields_from_project_data(new_project_row)
 
         verbose_proxy_logger.info(f"new_project_row: {json.dumps(new_project_row, indent=2)}")
-        response: prisma_models.LiteLLM_ProjectTable = await prisma_client.db.litellm_projecttable.create(
-            data={
-                **new_project_row,  # type: ignore
-            },
+        response: Final = await _project_table(prisma_client).create(
+            data={**new_project_row},
             include={"litellm_budget_table": True},
         )
 
@@ -538,7 +662,9 @@ async def update_project(
     ```
     """
     from litellm.proxy.proxy_server import (
+        general_settings,
         litellm_proxy_admin_name,
+        llm_router,
         premium_user,
         prisma_client,
         user_api_key_cache,
@@ -642,6 +768,12 @@ async def update_project(
                 data=data,
             )
 
+        # Opt-in (default off): require rpm/tpm for every model the update would leave on the project.
+        if general_settings.get("enforce_project_model_quota", False):
+            _raise_on_missing_project_model_quota_on_update(
+                data, existing_project, _router_access_group_names(llm_router)
+            )
+
         # Prepare update data
         update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
         update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
@@ -652,7 +784,7 @@ async def update_project(
 
         if budget_updates and existing_project.budget_id:
             # Update existing budget
-            await prisma_client.db.litellm_budgettable.update(
+            await _budget_table(prisma_client).update(
                 where={"budget_id": existing_project.budget_id},
                 data={
                     **budget_updates,
@@ -667,18 +799,17 @@ async def update_project(
         if "object_permission" in update_data:
             object_permission_data = update_data.pop("object_permission")
             if object_permission_data:
+                object_permission_payload: Final = _OBJECT_PERMISSION_PAYLOAD.validate_python(object_permission_data)
                 if existing_project.object_permission_id:
                     # Update existing permission
-                    await prisma_client.db.litellm_objectpermissiontable.update(
+                    await _object_permission_table(prisma_client).update(
                         where={"object_permission_id": existing_project.object_permission_id},
-                        data=object_permission_data,
+                        data=object_permission_payload,
                     )
                 else:
                     # Create new permission
-                    created_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
-                        await prisma_client.db.litellm_objectpermissiontable.create(
-                            data=object_permission_data,
-                        )
+                    created_permission: Final = await _object_permission_table(prisma_client).create(
+                        data=object_permission_payload,
                     )
                     update_data["object_permission_id"] = created_permission.object_permission_id
 
@@ -694,7 +825,7 @@ async def update_project(
         update_data = _remove_budget_fields_from_project_data(update_data)
 
         # Update project
-        updated_project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.update(
+        updated_project: Final = await _project_table(prisma_client).update(
             where={"project_id": data.project_id},
             data=update_data,
             include={"litellm_budget_table": True, "object_permission": True},
@@ -934,7 +1065,7 @@ async def list_projects(
             # Look up the user's team memberships via the reverse-index on
             # LiteLLM_UserTable.teams (maintained by team_member_add alongside
             # members_with_roles). This avoids a full scan of all team rows.
-            user_record: prisma_models.LiteLLM_UserTable | None = await prisma_client.db.litellm_usertable.find_unique(
+            user_record: Final = await _user_table(prisma_client).find_unique(
                 where={"user_id": user_api_key_dict.user_id},
             )
             user_team_ids: list[str] = user_record.teams if user_record is not None and user_record.teams else []

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.59"
+version = "0.1.63"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.59"
+version = "0.1.63"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
@@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
 # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
 # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -47,7 +47,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
-        --python python3
+        --python python3.13
 
 # Stage 2 — copy source and install the project + workspace members.
 COPY . .
@@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
-        --python python3
+        --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -73,7 +73,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
+      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done

gateway/routes/allowlist.py

@@ -86,6 +86,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/comprehendmedical",
     "/cohere/",
     "/gemini/",
+    "/gigachat/",
     "/google/",
     "/vertex_ai/",
     "/vertex-ai/",

helm/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.1.2
+version: 1.1.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/litellm-helm/README.md

@@ -26,7 +26,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `replicaCount`              | The number of LiteLLM Proxy pods to be deployed                                                                                                                                                                                               | `1`                       |
 | `masterkeySecretName`       | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name.                                                                                                              | N/A                       |
 | `masterkeySecretKey`        | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key.                                                                                                              | N/A                       |
-| `masterkey`                 | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated.                                                                                                                                           | N/A                       |
+| `masterkey`                 | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated on first install and reused on upgrades. | N/A                       |
 | `environmentSecrets`        | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                                   | `[]`                      |
 | `environmentConfigMaps`     | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                             | `[]`                      |
 | `image.repository`          | LiteLLM Proxy image repository                                                                                                                                                                                                                | `ghcr.io/berriai/litellm` |
@@ -212,6 +212,8 @@ service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`
 The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
 was not provided to the helm command line, the `masterkey` is a randomly
 generated string in the `sk-...` format stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
+The key is generated once on the first install; later `helm upgrade` runs reuse the
+value already in that Secret, so upgrading never rotates the master key.
 
 ```bash
 kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.masterkey}"

helm/litellm-helm/templates/secret-masterkey.yaml

@@ -1,9 +1,11 @@
 {{- if not .Values.masterkeySecretName }}
-{{ $masterkey := (.Values.masterkey | default (printf "sk-%s" (randAlphaNum 18))) }}
+{{- $secretName := printf "%s-masterkey" (include "litellm.fullname" .) }}
+{{- $existing := lookup "v1" "Secret" .Release.Namespace $secretName }}
+{{- $masterkey := .Values.masterkey | default (dig "data" "masterkey" "" $existing | b64dec) | default (printf "sk-%s" (randAlphaNum 18)) }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ include "litellm.fullname" . }}-masterkey
+  name: {{ $secretName }}
 data:
   masterkey: {{ $masterkey | b64enc }}
 type: Opaque

helm/litellm-helm/tests/hpa_tests.yaml

@@ -1,4 +1,4 @@
-suite: "hpa with behavior"
+suite: "hpa"
 templates:
   - hpa.yaml
 tests:
@@ -23,14 +23,44 @@ tests:
       - equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 }
       - equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 }
 
----
-suite: "hpa without behavior"
-templates:
-  - hpa.yaml
-tests:
   - it: "does not render behavior when not set"
     set:
       autoscaling.enabled: true
     asserts:
       - isKind: { of: HorizontalPodAutoscaler }
       - isNull: { path: spec.behavior }
+
+  - it: "scales on cpu at the documented 60 percent by default"
+    set:
+      autoscaling.enabled: true
+    asserts:
+      - isKind: { of: HorizontalPodAutoscaler }
+      - equal: { path: "spec.metrics[0].resource.name", value: cpu }
+      - equal: { path: "spec.metrics[0].resource.target.type", value: Utilization }
+      - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 }
+
+  - it: "does not scale on memory by default"
+    set:
+      autoscaling.enabled: true
+    asserts:
+      - lengthEqual: { path: spec.metrics, count: 1 }
+
+  - it: "honours an explicit cpu target override"
+    set:
+      autoscaling.enabled: true
+      autoscaling.targetCPUUtilizationPercentage: 75
+    asserts:
+      - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 }
+
+  - it: "renders a memory metric only when a memory target is set"
+    set:
+      autoscaling.enabled: true
+      autoscaling.targetMemoryUtilizationPercentage: 80
+    asserts:
+      - lengthEqual: { path: spec.metrics, count: 2 }
+      - equal: { path: "spec.metrics[1].resource.name", value: memory }
+      - equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 }
+
+  - it: "renders no hpa when autoscaling is disabled"
+    asserts:
+      - hasDocuments: { count: 0 }

helm/litellm-helm/tests/masterkey-secret_tests.yaml

@@ -15,6 +15,53 @@ tests:
           # Note: The masterkey is generated as "sk-<18-random-chars>" in plain text,
           # but stored as base64 encoded in Kubernetes secret (requirement).
           # "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern.
+  - it: should reuse the master key already stored in the cluster instead of generating a new one on upgrade
+    template: secret-masterkey.yaml
+    set:
+      masterkeySecretName: ""
+    kubernetesProvider:
+      scheme:
+        "v1/Secret":
+          gvr:
+            version: "v1"
+            resource: "secrets"
+          namespaced: true
+      objects:
+        - kind: Secret
+          apiVersion: v1
+          metadata:
+            name: RELEASE-NAME-litellm-masterkey
+            namespace: NAMESPACE
+          data:
+            masterkey: c2stZXhpc3Rpbmcta2V5
+    asserts:
+      - equal:
+          path: data.masterkey
+          value: c2stZXhpc3Rpbmcta2V5
+  - it: should let an explicit masterkey value override the one already stored in the cluster
+    template: secret-masterkey.yaml
+    set:
+      masterkeySecretName: ""
+      masterkey: sk-explicit
+    kubernetesProvider:
+      scheme:
+        "v1/Secret":
+          gvr:
+            version: "v1"
+            resource: "secrets"
+          namespaced: true
+      objects:
+        - kind: Secret
+          apiVersion: v1
+          metadata:
+            name: RELEASE-NAME-litellm-masterkey
+            namespace: NAMESPACE
+          data:
+            masterkey: c2stZXhpc3Rpbmcta2V5
+    asserts:
+      - equal:
+          path: data.masterkey
+          value: c2stZXhwbGljaXQ=
   - it: should not create a secret if masterkeySecretName is set
     template: secret-masterkey.yaml
     set:

helm/litellm-helm/values.yaml

@@ -200,7 +200,16 @@ autoscaling:
   enabled: false
   minReplicas: 1
   maxReplicas: 100
-  targetCPUUtilizationPercentage: 80
+  # 60 is the documented recommendation. See "Recommended Machine Specifications"
+  # in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe
+  # above only after up to failureThreshold x periodSeconds = 300 seconds, so a target
+  # high enough to trip near saturation adds capacity minutes after it was needed.
+  targetCPUUtilizationPercentage: 60
+  # Deliberately left unset rather than given a value. The prisma query engine's
+  # resident memory is a high-water mark that ratchets to the pod's worst-ever write
+  # and is never returned, so a memory target reads the largest write a pod ever did
+  # rather than what it is doing now, and replicas ratchet up without scaling back in.
+  # Memory is a floor to provision under 'resources', not a signal to scale on.
   # targetMemoryUtilizationPercentage: 80
   # behavior: {}
 

helm/litellm/templates/backend/deployment.yaml

@@ -7,6 +7,10 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: backend
 spec:
+  {{- with .Values.backend.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "litellm.backend.selectorLabels" . | nindent 6 }}

helm/litellm/templates/gateway/deployment.yaml

@@ -7,6 +7,10 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: gateway
 spec:
+  {{- with .Values.gateway.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "litellm.gateway.selectorLabels" . | nindent 6 }}

helm/litellm/templates/ingress.yaml

@@ -5,6 +5,41 @@
 {{- $gatewayPort := .Values.gateway.service.port -}}
 {{- $backendPort := .Values.backend.service.port -}}
 {{- $uiPort := .Values.ui.service.port -}}
+{{/*
+  Backends addressable from ingress.extraPaths, keyed by the `service` field.
+*/}}
+{{- $extraPathBackends := dict
+    "gateway" (dict "name" $gatewayName "port" $gatewayPort)
+    "backend" (dict "name" $backendName "port" $backendPort)
+    "ui" (dict "name" $uiName "port" $uiPort)
+-}}
+{{/*
+  UI paths (Next.js static export).
+
+  /ui/* is where the SPA serves its login + dashboard routes (e.g. /ui/login).
+  Without it, /ui/* falls into the catch-all → backend → 404.
+
+  The App Router (output: "export", basePath: "") emits the RSC/flight payload
+  for every route as a ROOT-level <route>.txt (/index.txt, /teams.txt,
+  /__next._tree.txt, ...). The client router fetches these on every soft
+  navigation / prefetch as <route>.txt?_rsc=<hash> (the query string is
+  irrelevant to path matching). They are not under /ui, /_next, or
+  /litellm-asset-prefix, so without /*.txt they fall to the backend catch-all
+  → 404 → client-side navigation never settles and the login flow spins in an
+  infinite redirect loop (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt
+  from the export; the rule only routes the request to it. Needs an ingress
+  controller whose ImplementationSpecific path is a wildcard pattern
+  (AWS ALB: `*` = 0+ chars); this chart targets the AWS Load Balancer
+  Controller.
+*/}}
+{{- $uiPaths := list
+    (dict "path" "/" "pathType" "Exact")
+    (dict "path" "/favicon.ico" "pathType" "Exact")
+    (dict "path" "/litellm-asset-prefix" "pathType" "Prefix")
+    (dict "path" "/_next" "pathType" "Prefix")
+    (dict "path" "/ui" "pathType" "Prefix")
+    (dict "path" "/*.txt" "pathType" "ImplementationSpecific")
+-}}
 {{/*
   Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
   Versioned paths are listed explicitly to avoid routing management routes
@@ -39,6 +74,21 @@
   routes at startup -> 404. So /test is rendered as a standalone Exact path
   and /test/* falls through to the backend catch-all.
 */}}
+{{/*
+  Every "<path>|<pathType>" this template renders on its own. An
+  ingress.extraPaths entry that repeats one of these is rejected: duplicates
+  in a single rule are resolved by position or by controller-specific tie
+  breaking, so the operator entry could take over a built-in route (an entry
+  at "/" Prefix would swallow the whole backend management API) instead of
+  adding to it.
+*/}}
+{{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}}
+{{- range $uiPaths }}
+{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path .pathType) }}
+{{- end }}
+{{- range $gatewayPrefixes }}
+{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }}
+{{- end }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -64,65 +114,15 @@ spec:
       http:
         paths:
           # --- UI (Next.js static export) ---
-          - path: /
-            pathType: Exact
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          - path: /favicon.ico
-            pathType: Exact
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          - path: /litellm-asset-prefix
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          - path: /_next
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          # /ui/* is where the Next.js SPA serves its login + dashboard
-          # routes (e.g. /ui/login). Without this, /ui/* falls into the
-          # catch-all → backend → 404.
-          - path: /ui
-            pathType: Prefix
-            backend:
-              service:
-                name: {{ $uiName }}
-                port:
-                  number: {{ $uiPort }}
-          # Next.js App Router (output: "export", basePath: "") emits the
-          # RSC/flight payload for every route as a ROOT-level <route>.txt
-          # (/index.txt, /teams.txt, /__next._tree.txt, ...). The client
-          # router fetches these on every soft navigation / prefetch as
-          # <route>.txt?_rsc=<hash> (the query string is irrelevant to path
-          # matching). They are not under /ui, /_next, or
-          # /litellm-asset-prefix, so without this rule they fall to the
-          # backend catch-all → 404 → client-side navigation never settles
-          # and the login flow spins in an infinite redirect loop
-          # (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt from the
-          # export; this rule only routes the request to it. Needs an
-          # ingress controller whose ImplementationSpecific path is a
-          # wildcard pattern (AWS ALB: `*` = 0+ chars); this chart targets
-          # the AWS Load Balancer Controller.
-          - path: /*.txt
-            pathType: ImplementationSpecific
+          {{- range $uiPaths }}
+          - path: {{ .path }}
+            pathType: {{ .pathType }}
             backend:
               service:
                 name: {{ $uiName }}
                 port:
                   number: {{ $uiPort }}
+          {{- end }}
           # --- Gateway data plane ---
           # Exact /test only (see the $gatewayPrefixes comment above);
           # /test/* MCP management endpoints fall to the backend catch-all.
@@ -142,6 +142,46 @@ spec:
                 port:
                   number: {{ $gatewayPort }}
           {{- end }}
+          {{- /*
+            --- Operator-supplied extra paths (ingress.extraPaths) ---
+            Rendered after every built-in path so an entry can never take
+            precedence over a default, and before the backend catch-all.
+            Position only decides the match on controllers that honour manifest
+            order: the AWS Load Balancer Controller this chart targets sorts
+            Exact paths first and Prefix paths longest-first, but keeps
+            ImplementationSpecific paths in manifest order, which is what the
+            /*.txt rule above already depends on.
+          */}}
+          {{- range $idx, $extra := .Values.ingress.extraPaths }}
+          {{- if not (kindIs "map" $extra) }}
+          {{- fail (printf "ingress.extraPaths[%d]: each entry must be a mapping with a 'path' key" $idx) }}
+          {{- end }}
+          {{- if not $extra.path }}
+          {{- fail (printf "ingress.extraPaths[%d]: 'path' is required" $idx) }}
+          {{- end }}
+          {{- $service := $extra.service | default "gateway" }}
+          {{- $target := get $extraPathBackends $service }}
+          {{- if not $target }}
+          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown service %q, expected one of backend, gateway, ui" $idx $extra.path $service) }}
+          {{- end }}
+          {{- $pathType := $extra.pathType | default "Prefix" }}
+          {{- if not (has $pathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
+          {{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $pathType) }}
+          {{- end }}
+          {{- if eq $extra.path "/" }}
+          {{- fail (printf "ingress.extraPaths[%d]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture" $idx) }}
+          {{- end }}
+          {{- if has (printf "%s|%s" $extra.path $pathType) $builtinPathKeys }}
+          {{- fail (printf "ingress.extraPaths[%d]: path %s with pathType %s is already routed by this chart, and a duplicate would take it over rather than add to it" $idx $extra.path $pathType) }}
+          {{- end }}
+          - path: {{ $extra.path | quote }}
+            pathType: {{ $pathType }}
+            backend:
+              service:
+                name: {{ $target.name }}
+                port:
+                  number: {{ $target.port }}
+          {{- end }}
           # --- Catch-all → backend (management API: /key/*, /user/*, /team/*, ...) ---
           - path: /
             pathType: Prefix

helm/litellm/templates/migrations-job.yaml

@@ -7,6 +7,8 @@
 #
 # Running this pre-upgrade closes the window where new application pods would
 # otherwise serve traffic against the previous release's unmigrated schema.
+# Argo CD users can swap the Helm hook for a PreSync hook through
+# `migrationJob.hooks`, which re-runs the Job on every sync.
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -14,10 +16,18 @@ metadata:
   labels:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: migrations
+  {{- if or .Values.migrationJob.hooks.helm.enabled .Values.migrationJob.hooks.argocd.enabled }}
   annotations:
+    {{- if .Values.migrationJob.hooks.helm.enabled }}
     helm.sh/hook: pre-install,pre-upgrade
     helm.sh/hook-delete-policy: before-hook-creation
-    helm.sh/hook-weight: "0"
+    helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "0" | quote }}
+    {{- end }}
+    {{- if .Values.migrationJob.hooks.argocd.enabled }}
+    argocd.argoproj.io/hook: PreSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+    {{- end }}
+  {{- end }}
 spec:
   backoffLimit: {{ .Values.migrationJob.backoffLimit }}
   ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}

helm/litellm/templates/ui/deployment.yaml

@@ -7,6 +7,10 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: ui
 spec:
+  {{- with .Values.ui.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "litellm.ui.selectorLabels" . | nindent 6 }}

helm/litellm/tests/ingress_extra_paths_tests.yaml

@@ -0,0 +1,317 @@
+suite: test ingress.extraPaths
+templates:
+  - ingress.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: renders nothing extra between the built-in gateway prefixes and the backend catch-all when unset
+    set:
+      ingress.enabled: true
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-1]
+          value:
+            path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /metrics
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+
+  - it: routes an extra path to the gateway by default, immediately before the backend catch-all
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /watsonx
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - equal:
+          path: spec.rules[0].http.paths[-1]
+          value:
+            path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+
+  - it: keeps every built-in path when extra paths are supplied
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+    asserts:
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /ui
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /test
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /v1/chat
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /vertex_ai
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+
+  - it: renders every entry in order and honours the service and pathType selectors
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+          service: gateway
+        - path: /my-passthrough
+          pathType: Exact
+          service: backend
+        - path: /brand.txt
+          pathType: ImplementationSpecific
+          service: ui
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-4]
+          value:
+            path: /watsonx
+            pathType: Prefix
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
+      - equal:
+          path: spec.rules[0].http.paths[-3]
+          value:
+            path: /my-passthrough
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-backend
+                port:
+                  number: 4001
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /brand.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: addresses the component services by their configured ports
+    set:
+      ingress.enabled: true
+      gateway.service.port: 8000
+      backend.service.port: 8001
+      ui.service.port: 8080
+      ingress.extraPaths:
+        - path: /watsonx
+        - path: /my-passthrough
+          service: backend
+        - path: /brand.txt
+          service: ui
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-4].backend.service.port.number
+          value: 8000
+      - equal:
+          path: spec.rules[0].http.paths[-3].backend.service.port.number
+          value: 8001
+      - equal:
+          path: spec.rules[0].http.paths[-2].backend.service.port.number
+          value: 8080
+
+  - it: rejects an entry naming a service the chart does not deploy
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+          service: proxy
+    asserts:
+      - failedTemplate:
+          errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown service "proxy", expected one of backend, gateway, ui'
+
+  - it: rejects an entry whose pathType is not a kubernetes pathType
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /watsonx
+          pathType: prefix
+    asserts:
+      - failedTemplate:
+          errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix'
+
+  - it: rejects an entry with no path
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: 'path' is required"
+
+
+  - it: rejects a root entry that would take over the backend catch-all
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
+
+  - it: rejects a root entry that would take over the UI root
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /
+          pathType: Exact
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
+
+  # A root ImplementationSpecific entry duplicates no built-in pair, so the
+  # duplicate check alone would admit it. It is still dead: the built-in
+  # Exact / sorts ahead of it on the AWS Load Balancer Controller and claims
+  # the only request its pattern matches, so it renders and never routes.
+  - it: rejects a root entry that would render but never match
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /
+          pathType: ImplementationSpecific
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
+
+  - it: rejects an entry that would take over a UI prefix
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /ui
+          service: gateway
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /ui with pathType Prefix is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects an entry that would take over the UI RSC payload rule
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /*.txt
+          pathType: ImplementationSpecific
+          service: backend
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /*.txt with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects an entry that would take over a gateway data-plane prefix
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /v1/chat
+          service: backend
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /v1/chat with pathType Prefix is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: rejects an entry that would take over the exact /test route
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /test
+          pathType: Exact
+          service: backend
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /test with pathType Exact is already routed by this chart, and a duplicate would take it over rather than add to it"
+
+  - it: allows a built-in path under a different pathType, which is a distinct rule
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /ui
+          pathType: Exact
+          service: ui
+    asserts:
+      - equal:
+          path: spec.rules[0].http.paths[-2]
+          value:
+            path: /ui
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-ui
+                port:
+                  number: 3000
+
+  - it: rejects a bare string entry instead of failing on template internals
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - /watsonx
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: each entry must be a mapping with a 'path' key"

helm/litellm/tests/migration_job_hooks_tests.yaml

@@ -0,0 +1,63 @@
+suite: test migrations Job hook annotations
+templates:
+  - migrations-job.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: runs as a Helm pre-install / pre-upgrade hook by default
+    asserts:
+      - equal:
+          path: metadata.annotations["helm.sh/hook"]
+          value: pre-install,pre-upgrade
+      - equal:
+          path: metadata.annotations["helm.sh/hook-delete-policy"]
+          value: before-hook-creation
+      - equal:
+          path: metadata.annotations["helm.sh/hook-weight"]
+          value: "0"
+      - notExists:
+          path: metadata.annotations["argocd.argoproj.io/hook"]
+
+  - it: adds the Argo CD PreSync hook when asked
+    set:
+      migrationJob.hooks.argocd.enabled: true
+    asserts:
+      - equal:
+          path: metadata.annotations["argocd.argoproj.io/hook"]
+          value: PreSync
+      - equal:
+          path: metadata.annotations["argocd.argoproj.io/hook-delete-policy"]
+          value: BeforeHookCreation
+
+  - it: drops the Helm hook so Argo CD owns the Job
+    set:
+      migrationJob.hooks.argocd.enabled: true
+      migrationJob.hooks.helm.enabled: false
+    asserts:
+      - equal:
+          path: metadata.annotations["argocd.argoproj.io/hook"]
+          value: PreSync
+      - notExists:
+          path: metadata.annotations["helm.sh/hook"]
+      - notExists:
+          path: metadata.annotations["helm.sh/hook-delete-policy"]
+      - notExists:
+          path: metadata.annotations["helm.sh/hook-weight"]
+
+  - it: renders an ordinary Job when both hooks are disabled
+    set:
+      migrationJob.hooks.helm.enabled: false
+    asserts:
+      - notExists:
+          path: metadata.annotations
+      - equal:
+          path: kind
+          value: Job
+
+  - it: honours a custom Helm hook weight
+    set:
+      migrationJob.hooks.helm.weight: "-5"
+    asserts:
+      - equal:
+          path: metadata.annotations["helm.sh/hook-weight"]
+          value: "-5"

helm/litellm/tests/rollout_strategy_tests.yaml

@@ -0,0 +1,66 @@
+suite: test rolling update strategy on the component deployments
+templates:
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - ui/deployment.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: leaves the strategy to Kubernetes defaults when unset
+    asserts:
+      - notExists:
+          path: spec.strategy
+
+  - it: renders the configured strategy on each deployment
+    set:
+      gateway.strategy:
+        type: RollingUpdate
+        rollingUpdate:
+          maxUnavailable: 0
+          maxSurge: 1
+      backend.strategy:
+        type: RollingUpdate
+        rollingUpdate:
+          maxUnavailable: "25%"
+          maxSurge: 2
+      ui.strategy:
+        type: Recreate
+    asserts:
+      - equal:
+          path: spec.strategy
+          value:
+            type: RollingUpdate
+            rollingUpdate:
+              maxUnavailable: 0
+              maxSurge: 1
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.strategy
+          value:
+            type: RollingUpdate
+            rollingUpdate:
+              maxUnavailable: 25%
+              maxSurge: 2
+        template: backend/deployment.yaml
+      - equal:
+          path: spec.strategy
+          value:
+            type: Recreate
+        template: ui/deployment.yaml
+
+  - it: keeps a component on the cluster default when only another one sets a strategy
+    set:
+      gateway.strategy:
+        type: Recreate
+    asserts:
+      - equal:
+          path: spec.strategy.type
+          value: Recreate
+        template: gateway/deployment.yaml
+      - notExists:
+          path: spec.strategy
+        template: backend/deployment.yaml
+      - notExists:
+          path: spec.strategy
+        template: ui/deployment.yaml

helm/litellm/values.yaml

@@ -13,6 +13,27 @@ ingress:
   annotations: {}
   host: ""        # optional; if set, becomes the rule's host
   tls: []
+  # Extra HTTP paths appended to the ingress rule. Additive: every built-in
+  # UI / gateway / backend path is still rendered, these entries are placed
+  # after them and before the backend catch-all, and an entry that repeats a
+  # path the chart already routes is rejected at render time rather than
+  # silently taking it over.
+  #
+  # The chart's built-in gateway prefix list is a snapshot of the data-plane
+  # surface at release time. Use extraPaths for passthrough routes it does not
+  # cover: a provider prefix added upstream after this chart version, or a
+  # custom general_settings.pass_through_endpoints route.
+  #
+  #   path      required; the HTTP path to route
+  #   service   which component serves it: gateway (default), backend, or ui
+  #   pathType  Prefix (default), Exact, or ImplementationSpecific
+  #
+  # The target component only answers paths its own route allowlist keeps, so
+  # a path here still has to be one that component serves.
+  extraPaths: []
+  # - path: /watsonx
+  #   pathType: Prefix
+  #   service: gateway
 
 # Per-component ServiceAccounts for gateway, backend, and ui.
 #
@@ -54,6 +75,22 @@ serviceAccounts:
 # generate` — the migration engine doesn't need the generated client.
 migrationJob:
   enabled: true
+  # Which controller is responsible for running the Job.
+  #
+  # `helm.enabled` renders the Helm pre-install / pre-upgrade hook, so the Job
+  # runs whenever `helm upgrade` sees a change to apply. `argocd.enabled`
+  # renders an Argo CD PreSync hook instead, which runs the Job on every sync
+  # even when the rendered manifests are unchanged: the way to re-run
+  # migrations on demand from a GitOps pipeline. Turning the Helm hook off
+  # while the Argo CD hook is on leaves the Job out of Helm's own upgrade
+  # path, which is what Argo CD users want since Argo, not Helm, applies the
+  # manifests.
+  hooks:
+    helm:
+      enabled: true
+      weight: "0"
+    argocd:
+      enabled: false
   backoffLimit: 4
   ttlSecondsAfterFinished: 120
   # Wall-clock budget for the whole Job, shared across every `backoffLimit`
@@ -236,6 +273,15 @@ gateway:
     initialDelaySeconds: 5
     periodSeconds: 10
     timeoutSeconds: 10
+  # Rolling update tuning for the gateway Deployment. Empty by default, so
+  # Kubernetes applies its own RollingUpdate defaults (25% maxSurge /
+  # 25% maxUnavailable). Example, for a surge-only rollout behind a load
+  # balancer that must never lose capacity:
+  #   type: RollingUpdate
+  #   rollingUpdate:
+  #     maxUnavailable: 0
+  #     maxSurge: 1
+  strategy: {}
   # Optional startupProbe. Empty by default, so existing installs are unchanged
   # and liveness/readiness apply from container start. Set it to gate
   # liveness/readiness until a slow cold start finishes — a high failureThreshold
@@ -348,6 +394,8 @@ backend:
     initialDelaySeconds: 5
     periodSeconds: 10
     timeoutSeconds: 10
+  # Same shape as gateway.strategy.
+  strategy: {}
   # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
   startupProbe: {}
   hpa:
@@ -412,6 +460,8 @@ ui:
     httpGet: { path: /, port: http }
     initialDelaySeconds: 2
     periodSeconds: 10
+  # Same shape as gateway.strategy.
+  strategy: {}
   # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
   startupProbe: {}
   hpa:
@@ -428,9 +478,11 @@ ui:
     maxUnavailable: ""
   podAnnotations: {}
   # Same shape as the gateway blocks of the same name. The nginx runtime
-  # writes its pid, cache, and proxy temp files under the image's root
-  # filesystem, so `securityContext.readOnlyRootFilesystem: true` here needs
-  # emptyDir volumes mounted over those paths.
+  # writes its pid, cache, and proxy temp files under /tmp, so it boots as
+  # any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem:
+  # true` here needs an emptyDir volume mounted over /tmp. Images before
+  # the /tmp move instead need emptyDirs over /var/cache/nginx and /run to
+  # run as a non-root uid at all.
   podLabels: {}
   podSecurityContext: {}
   securityContext: {}

litellm-proxy-extras/litellm_proxy_extras/migrations/20260804162853_add_budget_window_spend_table/migration.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_BudgetWindowSpend" (
+    "entity_type" TEXT NOT NULL,
+    "entity_id" TEXT NOT NULL,
+    "window_duration" TEXT NOT NULL,
+    "window_start" TIMESTAMP(3) NOT NULL,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_BudgetWindowSpend_pkey" PRIMARY KEY ("entity_type","entity_id","window_duration")
+);
+

litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql

@@ -0,0 +1,18 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+

litellm-proxy-extras/litellm_proxy_extras/migrations/20260828000000_shadow_eval_cost_comparison/migration.sql

@@ -0,0 +1,22 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_cost" DOUBLE PRECISION;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "shadow_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_cache_hit" BOOLEAN NOT NULL DEFAULT false;
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalFunnel" (
+    "job_id" TEXT NOT NULL,
+    "not_sampled" INTEGER NOT NULL DEFAULT 0,
+    "unjudgeable" INTEGER NOT NULL DEFAULT 0,
+    "shed" INTEGER NOT NULL DEFAULT 0,
+    "withheld" INTEGER NOT NULL DEFAULT 0,
+
+    CONSTRAINT "LiteLLM_ShadowEvalFunnel_pkey" PRIMARY KEY ("job_id")
+);

litellm-proxy-extras/litellm_proxy_extras/migrations/20260829000000_add_model_access_group_budget_table/migration.sql

@@ -0,0 +1,20 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_ModelAccessGroupBudgetTable" (
+    "access_group_name" TEXT NOT NULL,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "budget_id" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_by" TEXT,
+
+    CONSTRAINT "LiteLLM_ModelAccessGroupBudgetTable_pkey" PRIMARY KEY ("access_group_name")
+);
+
+-- AddForeignKey
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_ModelAccessGroupBudgetTable_budget_id_fkey') THEN
+        ALTER TABLE "LiteLLM_ModelAccessGroupBudgetTable" ADD CONSTRAINT "LiteLLM_ModelAccessGroupBudgetTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260831000000_shadow_eval_typed_targets/migration.sql

@@ -0,0 +1,21 @@
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1 FROM information_schema.columns
+        WHERE table_name = 'LiteLLM_ShadowEvalJob' AND column_name = 'api_key_id'
+    ) THEN
+        ALTER TABLE "LiteLLM_ShadowEvalJob" RENAME COLUMN "api_key_id" TO "target_id";
+    END IF;
+END $$;
+
+ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "target_type" TEXT NOT NULL DEFAULT 'key';
+
+DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key_direction";
+
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_one_active_per_target_direction"
+    ON "LiteLLM_ShadowEvalJob"("target_type", "target_id", "direction") WHERE "stopped_at" IS NULL;
+
+DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_api_key_id_idx";
+
+CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_target_type_target_id_idx"
+    ON "LiteLLM_ShadowEvalJob"("target_type", "target_id");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000000_shadow_eval_multi_router/migration.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "router_names" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[];
+
+ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "router_name" TEXT;

litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py

@@ -14,9 +14,22 @@ then fails on a Node binary that was never written. Deleting a cache directory
 that exists without a Node binary is what turns a killed bootstrap back into a
 recoverable one.
 
-Both budgets are overridable so an operator can widen them without a release:
-``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install and
-``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every individual Prisma command.
+``prisma migrate deploy`` is the other command whose runtime is not a
+constant: it grows with the number of pending migrations, so a fresh database
+that has to replay every migration this package ships overruns a per-command
+budget sized for the short bookkeeping commands, on a laptop as much as on a
+slow CI runner. The Python ``prisma`` wrapper spawns Node and the schema engine
+as separate children, so killing the wrapper on timeout leaves them running:
+the retry then contends with that orphan for Prisma's advisory lock and cannot
+finish any sooner. Migrate deploy therefore runs under its own budget.
+
+All three budgets are overridable so an operator can widen them without a
+release: ``LITELLM_PRISMA_BOOTSTRAP_TIMEOUT`` for the toolchain install,
+``LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT`` for ``prisma migrate deploy`` and
+``LITELLM_PRISMA_COMMAND_TIMEOUT`` for every other Prisma command. The
+per-command budget used to bound migrate deploy as well, so a deployment that
+raised it above the deploy default keeps that larger budget for deploy unless
+the deploy override says otherwise.
 """
 
 import math
@@ -36,10 +49,12 @@ except ImportError:
 
 PRISMA_COMMAND_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_COMMAND_TIMEOUT"
 PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_BOOTSTRAP_TIMEOUT"
+PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT"
 NODEENV_CACHE_DIR_ENV_VAR = "PRISMA_NODEENV_CACHE_DIR"
 
 DEFAULT_PRISMA_COMMAND_TIMEOUT = 60.0
 DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT = 600.0
+DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT = 600.0
 
 BOOTSTRAP_ARG = "--version"
 
@@ -88,6 +103,15 @@ def prisma_bootstrap_timeout() -> float:
     )
 
 
+def prisma_migrate_deploy_timeout() -> float:
+    """Seconds one ``prisma migrate deploy`` may run for, however many migrations are pending."""
+    if os.getenv(PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR) is not None:
+        return _timeout_from_env(
+            PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT
+        )
+    return max(DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT, prisma_command_timeout())
+
+
 def nodeenv_cache_dir() -> Optional[Path]:
     """Where Prisma installs its private Node runtime, or None if unknowable."""
     override = os.getenv(NODEENV_CACHE_DIR_ENV_VAR)

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -29,6 +29,7 @@ model LiteLLM_BudgetTable {
   keys LiteLLM_VerificationToken[] // multiple keys can have the same budget
   end_users LiteLLM_EndUserTable[] // multiple end-users can have the same budget
   tags LiteLLM_TagTable[] // multiple tags can have the same budget
+  model_access_groups LiteLLM_ModelAccessGroupBudgetTable[] // multiple model access groups can have the same budget
   team_membership LiteLLM_TeamMembership[] // budgets of Users within a Team 
   organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization
 }
@@ -585,6 +586,20 @@ model LiteLLM_EndUserTable {
   blocked Boolean @default(false)
 }
 
+// Budget and shared spend for a model access group. The groups themselves are not rows anywhere:
+// they are free-text strings in LiteLLM_ProxyModelTable.model_info.access_groups, so a row here
+// exists only once someone gives that group a budget.
+model LiteLLM_ModelAccessGroupBudgetTable {
+  access_group_name String @id
+  spend Float @default(0.0)
+  budget_id String?
+  litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
+  created_at DateTime @default(now()) @map("created_at")
+  created_by String?
+  updated_at DateTime @default(now()) @updatedAt
+  updated_by String?
+}
+
 // Track tags with budgets and spend
 model LiteLLM_TagTable {
   tag_name String @id
@@ -649,6 +664,18 @@ model LiteLLM_SpendLogs {
   @@index([session_id])
 }
 
+model LiteLLM_BudgetWindowSpend {
+  entity_type     String
+  entity_id       String
+  window_duration String
+  window_start    DateTime
+  spend           Float    @default(0.0)
+  created_at      DateTime @default(now())
+  updated_at      DateTime @updatedAt
+
+  @@id([entity_type, entity_id, window_duration])
+}
+
 // View spend, model, api_key per request
 model LiteLLM_ErrorLogs {
   request_id          String   @id @default(uuid())
@@ -754,6 +781,7 @@ model LiteLLM_DailyUserSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -789,6 +817,7 @@ model LiteLLM_DailyOrganizationSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -824,6 +853,7 @@ model LiteLLM_DailyEndUserSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -858,6 +888,7 @@ model LiteLLM_DailyAgentSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -892,6 +923,7 @@ model LiteLLM_DailyTeamSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -929,6 +961,7 @@ model LiteLLM_DailyTagSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -1496,14 +1529,16 @@ model LiteLLM_AutoRouterSession {
 model LiteLLM_ShadowEvalJob {
   id                String   @id @default(cuid())
   group_id          String   // legs of one job share this; the API's job id
-  api_key_id        String   // hashed virtual key whose traffic this leg shadows
-  router_name       String   // the auto-router under evaluation, in either direction
+  target_type       String   @default("key") // key | team | user
+  target_id         String   // hashed virtual key, team_id, or user_id whose traffic this leg shadows
+  router_name       String   // first (often only) auto-router under evaluation; router_names is the full set
+  router_names      String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name)
   direction         String   @default("forward") // forward | reverse
   baseline_model    String?  // reverse only: the fixed model the router is judged against
   judge_model       String
   shadow_percentage Float
   max_turns         Int      // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
-  max_budget        Float?   // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
+  max_budget        Float?   // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
   created_at        DateTime @default(now())
   created_by        String?
   ends_at           DateTime
@@ -1511,7 +1546,7 @@ model LiteLLM_ShadowEvalJob {
   stopped_by        String?  // operator who stopped it early; null when it ended on its own
 
   @@index([group_id])
-  @@index([api_key_id])
+  @@index([target_type, target_id])
   @@index([created_at])
 }
 
@@ -1521,18 +1556,34 @@ model LiteLLM_ShadowEvalAttempt {
   job_id       String
   request_id   String   // the judged real request
   outcome      String   // real | shadow | tie | error
+  router_name  String?  // the arm this verdict scores; NULL on legacy rows, meaning the job's own router
   tier         String?  // router's tier for the prompt, when classified
   real_model   String?
   shadow_model String?
   confidence   Float?
   judge_cost   Float    @default(0)
   shadow_cost  Float    @default(0)
+  real_cost    Float?   // NULL = row predates cost measurement; comparisons read only measured rows
+  real_classifier_cost   Float @default(0)
+  shadow_classifier_cost Float @default(0)
+  real_cache_hit Boolean @default(false)
   error        String?
   created_at   DateTime @default(now())
 
   @@index([job_id])
 }
 
+// Per-leg sampling funnel counters the attempt rows cannot derive: requests an
+// admitting job saw but did not judge. attempted = the leg's attempt rows; the
+// leg's eligible traffic = not_sampled + unjudgeable + shed + withheld + attempted.
+model LiteLLM_ShadowEvalFunnel {
+  job_id      String @id
+  not_sampled Int    @default(0)
+  unjudgeable Int    @default(0)
+  shed        Int    @default(0)
+  withheld    Int    @default(0)
+}
+
 // ---------------------------------------------------------------------------
 // Workflow Run Tracking
 //

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -15,8 +15,11 @@ from litellm_proxy_extras.replica_identity import (
     apply_replica_identity_full,
 )
 from litellm_proxy_extras.prisma_toolchain import (
+    PRISMA_COMMAND_TIMEOUT_ENV_VAR,
+    PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR,
     ensure_prisma_toolchain,
     prisma_command_timeout,
+    prisma_migrate_deploy_timeout,
 )
 
 
@@ -40,6 +43,65 @@ def _get_prisma_env() -> dict:
 
 _MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
 
+_SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
+_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(
+    r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE
+)
+_SPEND_LOGS_PK_CLAUSE_RE = re.compile(
+    r'^(?:DROP\s+CONSTRAINT\s+"[^"]*_pkey"'
+    r'|ADD\s+(?:CONSTRAINT\s+"[^"]*"\s+)?PRIMARY\s+KEY\s*\([^)]*\))$',
+    re.IGNORECASE,
+)
+
+PARTITIONED_SPEND_LOGS_PUSH_ERROR = (
+    "LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), "
+    "so its primary key must include the partition key (\"startTime\"). `prisma db push` "
+    "reconciles the database against schema.prisma, which declares the unpartitioned "
+    "primary key (\"request_id\"), and Postgres rejects that rewrite with: unique "
+    "constraint on partitioned table must include all partitioning columns. Start the "
+    "proxy without --use_prisma_db_push so it uses `prisma migrate deploy`, which only "
+    "applies shipped migrations and leaves the partitioned primary key alone."
+)
+
+
+def _without_sql_comments(statement: str) -> str:
+    return "\n".join(
+        line
+        for line in statement.splitlines()
+        if line.strip() and not line.strip().startswith("--")
+    ).strip()
+
+
+def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
+    prefix_match = _SPEND_LOGS_ALTER_RE.match(statement)
+    if not prefix_match:
+        return statement
+    kept = tuple(
+        clause.strip()
+        for clause in statement[prefix_match.end():].split(",\n")
+        if not _SPEND_LOGS_PK_CLAUSE_RE.match(clause.strip())
+    )
+    if not kept:
+        return None
+    return statement[: prefix_match.end()] + ",\n".join(kept)
+
+
+def filter_partitioned_spend_logs_diff(diff_sql: str) -> str:
+    """Drop statements from a `prisma migrate diff` script that fight the
+    SpendLogs partitioning runbook (db_scripts/partition_spend_logs.sql): the
+    primary-key rewrite on "LiteLLM_SpendLogs", which Postgres rejects on a
+    partitioned table, and drops of runbook artifacts such as
+    "LiteLLM_SpendLogs_legacy"."""
+    kept = tuple(
+        filtered
+        for statement in diff_sql.split(";")
+        for bare in (_without_sql_comments(statement),)
+        if bare and not _SPEND_LOGS_ARTIFACT_DROP_RE.match(bare)
+        for filtered in (_without_spend_logs_pk_clauses(bare),)
+        if filtered is not None
+    )
+    return "".join(f"{statement};\n\n" for statement in kept)
+
 
 def _migration_timestamp(name: str) -> int:
     """Extract the leading `YYYYMMDDHHMMSS` timestamp from a migration name.
@@ -355,7 +417,24 @@ class ProxyExtrasDBManager:
             return
         logger.info(f"Migration diff created at {diff_sql_path}")
 
+        if ProxyExtrasDBManager.spend_logs_is_partitioned():
+            filtered_sql = filter_partitioned_spend_logs_diff(
+                diff_sql_path.read_text()
+            )
+            diff_sql_path.write_text(filtered_sql)
+            logger.info(
+                "LiteLLM_SpendLogs is partitioned; removed its primary-key "
+                "rewrite and partitioning artifacts from the drift script"
+            )
+            if not filtered_sql.strip():
+                logger.info("Drift script is empty after filtering; nothing to apply")
+                if not mark_all_applied:
+                    return
+                ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
+                return
+
         # 2. Run prisma db execute to apply the migration
+        applied_ok = False
         try:
             logger.info("Running prisma db execute to apply the migration diff...")
             result = subprocess.run(
@@ -376,6 +455,7 @@ class ProxyExtrasDBManager:
             )
             logger.info(f"prisma db execute stdout: {result.stdout}")
             logger.info("✅ Migration diff applied successfully")
+            applied_ok = True
         except subprocess.CalledProcessError as e:
             logger.warning(f"Failed to apply migration diff: {e.stderr}")
         except subprocess.TimeoutExpired:
@@ -384,6 +464,16 @@ class ProxyExtrasDBManager:
         # 3. Mark all migrations as applied
         if not mark_all_applied:
             return
+        if not applied_ok:
+            logger.warning(
+                "Drift script failed to apply; NOT marking migrations as "
+                "applied so a later migration run can retry them"
+            )
+            return
+        ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
+
+    @staticmethod
+    def _mark_migrations_applied(migrations_dir: str) -> None:
         migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir)
         logger.info(f"Resolving {len(migration_names)} migrations")
         for migration_name in migration_names:
@@ -410,6 +500,62 @@ class ProxyExtrasDBManager:
                         f"Failed to resolve migration {migration_name}: {e.stderr}"
                     )
 
+    @staticmethod
+    def spend_logs_is_partitioned() -> bool:
+        """True when the connected database's LiteLLM_SpendLogs is a
+        partitioned table in Prisma's target schema (the `schema` URL param,
+        falling back to Prisma's default target, public), i.e. the operator
+        ran db_scripts/partition_spend_logs.sql. Returns False when psycopg is
+        unavailable or the database cannot be reached, preserving the
+        pre-existing behavior in those cases."""
+        database_url = os.getenv("DATABASE_URL")
+        if not database_url:
+            return False
+
+        try:
+            import psycopg
+        except ImportError:
+            logger.warning(
+                "psycopg is not installed; skipping the LiteLLM_SpendLogs "
+                "partition check. If this table is partitioned (see "
+                "db_scripts/partition_spend_logs.sql), schema reconciliation "
+                "will try to rewrite its primary key and fail. Install the "
+                "litellm[extra_proxy] extra, which now includes psycopg."
+            )
+            return False
+
+        cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
+        try:
+            with psycopg.connect(
+                cleaned_url, connect_timeout=10, autocommit=True
+            ) as conn:
+                row = conn.execute(
+                    "SELECT 1 "
+                    "FROM pg_partitioned_table pt "
+                    "JOIN pg_class c ON c.oid = pt.partrelid "
+                    "JOIN pg_namespace n ON n.oid = c.relnamespace "
+                    "WHERE c.relname = 'LiteLLM_SpendLogs' "
+                    "  AND n.nspname = %s",
+                    (
+                        ProxyExtrasDBManager._prisma_schema_param(database_url)
+                        or "public",
+                    ),
+                ).fetchone()
+        except (psycopg.OperationalError, psycopg.DatabaseError):
+            return False
+        return row is not None
+
+    @staticmethod
+    def _prisma_schema_param(url: str) -> Optional[str]:
+        """The `schema` query param Prisma uses to pick its target schema,
+        or None when the URL does not set one."""
+        from urllib.parse import urlparse, parse_qsl
+
+        return next(
+            (v for k, v in parse_qsl(urlparse(url).query) if k == "schema"),
+            None,
+        )
+
     @staticmethod
     def _strip_prisma_query_params(url: str) -> str:
         """Remove Prisma-specific query params (connection_limit, pool_timeout,
@@ -528,7 +674,8 @@ class ProxyExtrasDBManager:
         migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
 
         if not use_migrate:
-            # Preserve `prisma db push` path unchanged.
+            if ProxyExtrasDBManager.spend_logs_is_partitioned():
+                raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
             original_dir = os.getcwd()
             os.chdir(migrations_dir)
             try:
@@ -554,12 +701,13 @@ class ProxyExtrasDBManager:
 
         original_dir = os.getcwd()
         os.chdir(migrations_dir)
+        deploy_timeout = prisma_migrate_deploy_timeout()
         try:
             for attempt in range(4):
                 try:
                     result = subprocess.run(
                         [_get_prisma_command(), "migrate", "deploy"],
-                        timeout=prisma_command_timeout(),
+                        timeout=deploy_timeout,
                         check=True,
                         capture_output=True,
                         text=True,
@@ -569,8 +717,12 @@ class ProxyExtrasDBManager:
                     return True
 
                 except subprocess.TimeoutExpired:
-                    logger.info(
-                        f"prisma migrate deploy attempt {attempt + 1} timed out, retrying"
+                    logger.warning(
+                        "prisma migrate deploy attempt %s timed out after %ss, retrying. "
+                        "Raise %s if this database needs longer to apply its pending migrations.",
+                        attempt + 1,
+                        deploy_timeout,
+                        PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR,
                     )
                     time.sleep(random.randrange(5, 15))
                     continue
@@ -679,7 +831,8 @@ class ProxyExtrasDBManager:
                 "Database migration failed after 4 attempts (retry loop "
                 "exhausted by timeouts or repeated idempotent-recovery "
                 "continues). Check database connectivity, load, and "
-                "_prisma_migrations ledger state."
+                "_prisma_migrations ledger state, and raise "
+                f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out."
             )
         finally:
             os.chdir(original_dir)
@@ -764,7 +917,7 @@ class ProxyExtrasDBManager:
                         # Set migrations directory for Prisma
                         result = subprocess.run(
                             [_get_prisma_command(), "migrate", "deploy"],
-                            timeout=prisma_command_timeout(),
+                            timeout=prisma_migrate_deploy_timeout(),
                             check=True,
                             capture_output=True,
                             text=True,
@@ -972,6 +1125,8 @@ class ProxyExtrasDBManager:
                                 )
                                 raise
                 else:
+                    if ProxyExtrasDBManager.spend_logs_is_partitioned():
+                        raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
                     # Use prisma db push with increased timeout
                     subprocess.run(
                         [_get_prisma_command(), "db", "push", "--accept-data-loss"],
@@ -980,7 +1135,11 @@ class ProxyExtrasDBManager:
                     )
                     return True
             except subprocess.TimeoutExpired:
-                logger.info(f"Attempt {attempt + 1} timed out")
+                logger.warning(
+                    "Attempt %s timed out. Raise %s if this database needs longer to apply its schema.",
+                    attempt + 1,
+                    PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR if use_migrate else PRISMA_COMMAND_TIMEOUT_ENV_VAR,
+                )
                 time.sleep(random.randrange(5, 15))
             except subprocess.CalledProcessError as e:
                 attempts_left = 3 - attempt

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.89"
+version = "0.4.92"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.89"
+version = "0.4.92"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/AGENTS.md

@@ -1,6 +1,6 @@
 # AGENTS.md
 
-litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are MODULES inside the layers.
+litellm-rust has four crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers.
 
 ## Crates
 
@@ -8,9 +8,10 @@ litellm-rust has exactly THREE crates. A crate is a LAYER, not a route. Routes (
 |-------|------|
 | litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. |
 | litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. |
-| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. |
+| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
+| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. |
 
-Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge.
+Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
 
 ## Where a route lives
 
@@ -28,7 +29,7 @@ core/src/messages/
 
 Handlers never live in `ai-gateway`. `ocr`, `audio_transcription`, and `realtime` are still hosted there from before this rule; they move to `core` as they are touched.
 
-Adding a crate: default to a MODULE. New crate ONLY on a real trigger — separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these.
+Adding a crate: default to a module. A new crate requires a real trigger: separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these.
 
 Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional.
 

litellm-rust/CLAUDE.md

@@ -21,12 +21,13 @@ variants of it. The test for a good abstraction is that adding the next provider
 is a few declarative lines, not a new file of duplicated flow. Only diverge from
 the base when behavior is genuinely different, and say so explicitly in the PR.
 
-## Crates (exactly three — see AGENTS.md)
+## Crates (see AGENTS.md)
 
 `litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call.
 `litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and
-`litellm-python-bridge` exposes it to the Python SDK. A crate is a **layer**, not
-a route — add modules, not crates.
+`litellm-python-bridge` exposes it to the Python SDK. `litellm-python-interop`
+holds domain-neutral PyO3 primitives shared by Python-facing Rust code. A crate
+is a layer or shared foundation, not a route; add modules, not crates.
 
 ## Core Boundary
 
@@ -175,7 +176,7 @@ cd litellm-rust
 cargo fmt --check
 # the ai-gateway binary + server code is behind the `server` feature
 cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
-cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings
+cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings
 cargo test --workspace
 ```
 

litellm-rust/Cargo.lock

@@ -2,6 +2,36 @@
 # It is not intended for manual editing.
 version = 4
 
+[[package]]
+name = "aho-corasick"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "alloca"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
+[[package]]
+name = "anstyle"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
+
 [[package]]
 name = "arc-swap"
 version = "1.9.2"
@@ -506,6 +536,12 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
 version = "1.3.0"
@@ -541,6 +577,58 @@ dependencies = [
  "rand_core 0.10.1",
 ]
 
+[[package]]
+name = "ciborium"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
+[[package]]
+name = "clap"
+version = "4.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca"
+dependencies = [
+ "clap_builder",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.6.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889"
+dependencies = [
+ "anstyle",
+ "clap_lex",
+]
+
+[[package]]
+name = "clap_lex"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
+
 [[package]]
 name = "cmake"
 version = "0.1.58"
@@ -596,6 +684,72 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3"
+dependencies = [
+ "alloca",
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot",
+ "itertools",
+ "num-traits",
+ "oorandom",
+ "page_size",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
+
+[[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
 [[package]]
 name = "crypto-common"
 version = "0.1.7"
@@ -765,6 +919,12 @@ version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
 
+[[package]]
+name = "futures-timer"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
+
 [[package]]
 name = "futures-util"
 version = "0.3.33"
@@ -818,6 +978,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "glob"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
+
 [[package]]
 name = "h2"
 version = "0.3.27"
@@ -856,6 +1022,17 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "half"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+ "zerocopy",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.17.1"
@@ -1179,6 +1356,15 @@ version = "2.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
 
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.18"
@@ -1255,14 +1441,27 @@ dependencies = [
 name = "litellm-python-bridge"
 version = "0.1.0"
 dependencies = [
+ "criterion",
  "litellm-ai-gateway",
  "litellm-core",
+ "litellm-python-interop",
  "pyo3",
  "pyo3-async-runtimes",
  "serde_json",
  "tokio",
 ]
 
+[[package]]
+name = "litellm-python-interop"
+version = "0.1.0"
+dependencies = [
+ "pyo3",
+ "pythonize",
+ "rstest",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "litemap"
 version = "0.8.2"
@@ -1340,6 +1539,12 @@ version = "1.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
+[[package]]
+name = "oorandom"
+version = "11.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
+
 [[package]]
 name = "openssl-probe"
 version = "0.2.1"
@@ -1352,6 +1557,16 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
 
+[[package]]
+name = "page_size"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "2.3.2"
@@ -1376,6 +1591,34 @@ version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
 
+[[package]]
+name = "plotters"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "portable-atomic"
 version = "1.14.0"
@@ -1406,6 +1649,15 @@ dependencies = [
  "zerocopy",
 ]
 
+[[package]]
+name = "proc-macro-crate"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
+dependencies = [
+ "toml_edit",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.107"
@@ -1486,6 +1738,16 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "pythonize"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ec376e1216e0c929a74964ce2020012a1a39f32d80e78aa688721219ea7fb89"
+dependencies = [
+ "pyo3",
+ "serde",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.11"
@@ -1613,12 +1875,67 @@ dependencies = [
  "rand_core 0.10.1",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "regex"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
 [[package]]
 name = "regex-lite"
 version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
 
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
+[[package]]
+name = "relative-path"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2"
+
 [[package]]
 name = "reqwest"
 version = "0.12.28"
@@ -1676,6 +1993,35 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rstest"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49"
+dependencies = [
+ "futures-timer",
+ "futures-util",
+ "rstest_macros",
+]
+
+[[package]]
+name = "rstest_macros"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0"
+dependencies = [
+ "cfg-if",
+ "glob",
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "relative-path",
+ "rustc_version",
+ "syn 2.0.119",
+ "unicode-ident",
+]
+
 [[package]]
 name = "rustc-hash"
 version = "2.1.3"
@@ -1774,6 +2120,15 @@ version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "schannel"
 version = "0.1.29"
@@ -2099,6 +2454,16 @@ dependencies = [
  "zerovec",
 ]
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.12.0"
@@ -2189,6 +2554,36 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "toml_datetime"
+version = "1.1.1+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.25.13+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b"
+dependencies = [
+ "indexmap",
+ "toml_datetime",
+ "toml_parser",
+ "winnow",
+]
+
+[[package]]
+name = "toml_parser"
+version = "1.1.3+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
+dependencies = [
+ "winnow",
+]
+
 [[package]]
 name = "tower"
 version = "0.5.3"
@@ -2363,6 +2758,16 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
 
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
 [[package]]
 name = "want"
 version = "0.3.1"
@@ -2475,6 +2880,37 @@ dependencies = [
  "rustls-pki-types",
 ]
 
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
 [[package]]
 name = "windows-link"
 version = "0.2.1"
@@ -2563,6 +2999,15 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
+[[package]]
+name = "winnow"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "writeable"
 version = "0.6.3"

litellm-rust/Cargo.toml

@@ -2,6 +2,7 @@
 members = [
     "crates/core",
     "crates/ai-gateway",
+    "crates/python-interop",
     "crates/python-bridge",
 ]
 resolver = "2"
@@ -15,11 +16,14 @@ repository = "https://github.com/BerriAI/litellm"
 [workspace.dependencies]
 litellm-core = { path = "crates/core" }
 litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false }
+litellm-python-interop = { path = "crates/python-interop" }
 axum = "0.7"
 pyo3 = "0.29.0"
 pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
+pythonize = "0.29.0"
 rand = "0.8"
 reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] }
+rstest = "0.26.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 sha2 = "0.10"
@@ -29,3 +33,12 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"]
 tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
 futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
 base64 = "0.22"
+
+[profile.release]
+opt-level = 3
+lto = "thin"
+codegen-units = 1
+panic = "unwind"
+debug = false
+incremental = false
+strip = "symbols"

litellm-rust/README.md

@@ -26,9 +26,10 @@ coverage and production evidence.
 |-------|------|
 | litellm-core | The SDK. Per-route entrypoints (`messages::messages()`), types, provider transforms (modules under `providers/`), provider resolution, auth, the provider HTTP call, and the router. |
 | litellm-ai-gateway | The axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
-| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. |
+| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
+| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. |
 
-Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge.
+Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
 
 ## Layout
 
@@ -38,7 +39,8 @@ crates/
     src/messages/   mod.rs (entrypoint), types, transformation, prepare, handler, client
     src/providers/anthropic/messages/transformation.rs
   ai-gateway/     Axum server + WebSocket hosts; calls core entrypoints.
-  python-bridge/  PyO3 bridge for Python LiteLLM.
+  python-interop/ Domain-neutral PyO3 conversion and GIL primitives.
+  python-bridge/  PyO3 API adapter for Python LiteLLM.
 ```
 
 The folder shape follows the Python provider tree:

litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md

@@ -54,6 +54,6 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages`
     cd litellm-rust
     cargo fmt --check
     cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
-    cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings
+    cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings
     cargo test --workspace
     ```

litellm-rust/crates/ai-gateway/README.md

@@ -6,15 +6,16 @@ dials OpenAI upstream, and splices the two sockets frame-by-frame.
 
 ## Crates
 
-`litellm-rust` is exactly three crates (a crate is a **layer**, not a route):
+`litellm-rust` has four crates. A crate is a layer or shared foundation, not a route:
 
 | Crate | Role |
 |-------|------|
 | litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. |
 | litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. |
-| litellm-python-bridge | PyO3 cdylib exposing Rust to the litellm Python SDK — marshals Python objects and calls core entrypoints. |
+| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
+| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. |
 
-Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-python-bridge.
+Dependency direction is acyclic: `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`; the interop foundation depends on no LiteLLM domain crate.
 
 - **Client endpoint:** `wss://<host>/v1/realtime?model=<model>` (WebSocket)
 - **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset)

litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs

@@ -1,10 +1,8 @@
-use std::collections::BTreeMap;
-
-use litellm_core::CoreResult;
 use litellm_core::audio_transcription::transformation::AudioTranscriptionProviderConfig;
-use litellm_core::error::CoreError;
+use litellm_core::error::Error;
 use litellm_core::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG;
 use serde_json::{Map, Value};
+use std::collections::BTreeMap;
 
 pub(super) fn audio_transcription_provider_config(
     provider: &str,
@@ -17,7 +15,7 @@ pub(super) fn audio_transcription_provider_config(
 
 pub(super) fn string_headers(
     headers: Option<Map<String, Value>>,
-) -> CoreResult<BTreeMap<String, String>> {
+) -> Result<BTreeMap<String, String>, Error> {
     headers
         .unwrap_or_default()
         .into_iter()
@@ -26,7 +24,7 @@ pub(super) fn string_headers(
                 .as_str()
                 .map(|value| (key.clone(), value.to_string()))
                 .ok_or_else(|| {
-                    CoreError::InvalidRequest(format!(
+                    Error::InvalidRequest(format!(
                         "audio transcription extra_headers.{key} must be a string"
                     ))
                 })

litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs

@@ -1,11 +1,9 @@
-use std::time::SystemTime;
-
-use litellm_core::CoreResult;
 use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth;
-use litellm_core::error::CoreError;
+use litellm_core::error::Error;
 use litellm_core::providers::bedrock::audio_transcription::aws_auth_config;
 use litellm_core::providers::bedrock::aws_base::{resolve_credentials, sign_bedrock_post};
 use serde_json::Value;
+use std::time::SystemTime;
 
 use super::common_utils::truncate_error_body;
 use super::types::ProviderAudioTranscriptionRequest;
@@ -13,10 +11,9 @@ use crate::client::http_client;
 
 pub(crate) async fn execute_audio_transcription_provider_call(
     request: ProviderAudioTranscriptionRequest,
-) -> CoreResult<Value> {
-    let body = serde_json::to_vec(&request.body).map_err(|error| {
-        CoreError::InvalidRequest(format!("invalid audio request body: {error}"))
-    })?;
+) -> Result<Value, Error> {
+    let body = serde_json::to_vec(&request.body)
+        .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?;
     let mut request_builder = http_client().post(&request.url).body(body.clone());
     for (key, value) in &request.upstream_headers {
         request_builder = request_builder.header(key, value);
@@ -27,21 +24,20 @@ pub(crate) async fn execute_audio_transcription_provider_call(
     let response = request_builder
         .send()
         .await
-        .map_err(|error| CoreError::Network(error.to_string()))?;
+        .map_err(|error| Error::Network(error.to_string()))?;
     let status = response.status();
     let text = response
         .text()
         .await
-        .map_err(|error| CoreError::Network(error.to_string()))?;
+        .map_err(|error| Error::Network(error.to_string()))?;
     if !status.is_success() {
-        return Err(CoreError::Http {
+        return Err(Error::Http {
             status: status.as_u16(),
             body: truncate_error_body(&text),
         });
     }
-    let response_json: Value = serde_json::from_str(&text).map_err(|error| {
-        CoreError::InvalidResponse(format!("invalid audio response JSON: {error}"))
-    })?;
+    let response_json: Value = serde_json::from_str(&text)
+        .map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?;
     Ok(request
         .config
         .transform_transcription_response(&request.model, response_json)?
@@ -51,14 +47,13 @@ pub(crate) async fn execute_audio_transcription_provider_call(
 pub(crate) async fn sign_request(
     request: &ProviderAudioTranscriptionRequest,
     optional_params: &serde_json::Map<String, Value>,
-) -> CoreResult<ProviderAudioTranscriptionRequest> {
+) -> Result<ProviderAudioTranscriptionRequest, Error> {
     let env_lookup = environment_lookup;
     let auth = request
         .config
         .auth_strategy(&request.model, optional_params, &env_lookup)?;
-    let body = serde_json::to_vec(&request.body).map_err(|error| {
-        CoreError::InvalidRequest(format!("invalid audio request body: {error}"))
-    })?;
+    let body = serde_json::to_vec(&request.body)
+        .map_err(|error| Error::InvalidRequest(format!("invalid audio request body: {error}")))?;
     let mut headers = super::common_utils::string_headers(None)?;
     headers.insert("Content-Type".to_string(), "application/json".to_string());
     headers.extend(request.upstream_headers.iter().cloned());

litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs

@@ -1,11 +1,9 @@
-use std::future::Future;
-use std::pin::Pin;
-
-use litellm_core::CoreResult;
 use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth;
 use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming};
-use litellm_core::error::CoreError;
+use litellm_core::error::Error;
 use serde_json::{Map, Value, json};
+use std::future::Future;
+use std::pin::Pin;
 
 use super::common_utils::{audio_transcription_provider_config, has_header, string_headers};
 use super::handler::sign_request;
@@ -26,7 +24,7 @@ pub(crate) struct AudioTranscriptionLifecycleHooks {
     request_metadata: RequestMetadata,
 }
 
-type AudioFuture<'a, T> = Pin<Box<dyn Future<Output = CoreResult<T>> + Send + 'a>>;
+type AudioFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
 type AudioLogFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
 
 impl AudioTranscriptionLifecycleHooks {
@@ -45,7 +43,7 @@ impl AudioTranscriptionLifecycleHooks {
     async fn run_pre_call_guardrails(
         &self,
         request: PreparedAudioTranscriptionRequest,
-    ) -> CoreResult<PreparedAudioTranscriptionRequest> {
+    ) -> Result<PreparedAudioTranscriptionRequest, Error> {
         if self.guardrail_runner.is_empty() {
             return Ok(request);
         }
@@ -63,17 +61,17 @@ impl AudioTranscriptionLifecycleHooks {
             .await
             .map_err(guardrail_error_to_core_error)?;
         let Value::Object(mut data) = guardrail_request.data else {
-            return Err(CoreError::InvalidRequest(
+            return Err(Error::InvalidRequest(
                 "audio transcription pre_call guardrail must return an object".to_string(),
             ));
         };
         let audio = data.remove("audio").ok_or_else(|| {
-            CoreError::InvalidRequest("audio transcription guardrail removed audio".to_string())
+            Error::InvalidRequest("audio transcription guardrail removed audio".to_string())
         })?;
         let optional_params = match data.remove("optional_params") {
             Some(Value::Object(value)) => value,
             Some(_) => {
-                return Err(CoreError::InvalidRequest(
+                return Err(Error::InvalidRequest(
                     "audio transcription optional_params must be an object".to_string(),
                 ));
             }
@@ -89,9 +87,9 @@ impl AudioTranscriptionLifecycleHooks {
     async fn prepare_provider_request(
         &self,
         request: PreparedAudioTranscriptionRequest,
-    ) -> CoreResult<ProviderAudioTranscriptionRequest> {
+    ) -> Result<ProviderAudioTranscriptionRequest, Error> {
         let config = audio_transcription_provider_config(&request.custom_llm_provider)
-            .ok_or_else(|| CoreError::InvalidProvider(request.custom_llm_provider.clone()))?;
+            .ok_or_else(|| Error::InvalidProvider(request.custom_llm_provider.clone()))?;
         let env_lookup = super::handler::environment_lookup;
         let headers = string_headers(request.extra_headers)?;
         let url = config.complete_url(
@@ -135,7 +133,7 @@ impl AudioTranscriptionLifecycleHooks {
     async fn run_during_call_guardrails(
         &self,
         request: ProviderAudioTranscriptionRequest,
-    ) -> CoreResult<ProviderAudioTranscriptionRequest> {
+    ) -> Result<ProviderAudioTranscriptionRequest, Error> {
         if self.guardrail_runner.is_empty() {
             return Ok(request);
         }
@@ -153,12 +151,12 @@ impl AudioTranscriptionLifecycleHooks {
             .await
             .map_err(guardrail_error_to_core_error)?;
         let Value::Object(mut data) = guardrail_request.data else {
-            return Err(CoreError::InvalidRequest(
+            return Err(Error::InvalidRequest(
                 "audio transcription during_call guardrail must return an object".to_string(),
             ));
         };
         let body = data.remove("body").ok_or_else(|| {
-            CoreError::InvalidRequest("audio transcription guardrail removed body".to_string())
+            Error::InvalidRequest("audio transcription guardrail removed body".to_string())
         })?;
         Ok(ProviderAudioTranscriptionRequest { body, ..request })
     }
@@ -241,7 +239,7 @@ impl CallLifecycleHooks<PreparedAudioTranscriptionRequest, ProviderAudioTranscri
     fn async_log_failure_event<'a>(
         &'a self,
         context: &'a CallLifecycleContext,
-        error: &'a CoreError,
+        error: &'a Error,
         timing: &'a CallLifecycleTiming,
     ) -> Self::FailureFuture<'a> {
         Box::pin(async move {
@@ -281,22 +279,22 @@ fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext {
     }
 }
 
-fn guardrail_error_to_core_error(error: GuardrailError) -> CoreError {
-    CoreError::InvalidRequest(format!("{}: {}", error.kind, error.message))
+fn guardrail_error_to_core_error(error: GuardrailError) -> Error {
+    Error::InvalidRequest(format!("{}: {}", error.kind, error.message))
 }
 
-fn core_error_kind(error: &CoreError) -> &'static str {
+fn core_error_kind(error: &Error) -> &'static str {
     match error {
-        CoreError::Auth(_) => "AuthError",
-        CoreError::InvalidProvider(_) => "InvalidProvider",
-        CoreError::InvalidRequest(_) => "InvalidRequest",
-        CoreError::InvalidType { .. } => "InvalidType",
-        CoreError::MissingField(_) => "MissingField",
-        CoreError::Http { .. } => "HttpError",
-        CoreError::InvalidResponse(_) => "InvalidResponse",
-        CoreError::Network(_) => "NetworkError",
-        CoreError::Connect(_) => "ConnectError",
-        CoreError::Routing(_) => "RoutingError",
-        CoreError::Unsupported(_) => "UnsupportedRequest",
+        Error::Auth(_) => "AuthError",
+        Error::InvalidProvider(_) => "InvalidProvider",
+        Error::InvalidRequest(_) => "InvalidRequest",
+        Error::InvalidType { .. } => "InvalidType",
+        Error::MissingField(_) => "MissingField",
+        Error::Http { .. } => "HttpError",
+        Error::InvalidResponse(_) => "InvalidResponse",
+        Error::Network(_) => "NetworkError",
+        Error::Connect(_) => "ConnectError",
+        Error::Routing(_) => "RoutingError",
+        Error::Unsupported(_) => "UnsupportedRequest",
     }
 }

litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs

@@ -1,4 +1,4 @@
-use litellm_core::CoreResult;
+use litellm_core::Error;
 use litellm_core::call_lifecycle::CallLifecycle;
 use serde_json::Value;
 
@@ -13,7 +13,7 @@ pub use types::AudioTranscriptionRequest;
 use handler::execute_audio_transcription_provider_call;
 use prepare::{PreparedAudioTranscriptionCall, prepare_audio_transcription_call};
 
-pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> CoreResult<Value> {
+pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
     let PreparedAudioTranscriptionCall { request, hooks } =
         prepare_audio_transcription_call(request);
     CallLifecycle::default()

litellm-rust/crates/ai-gateway/src/io/realtime.rs

@@ -15,8 +15,7 @@ use std::time::Duration;
 
 use futures_util::stream::{SplitSink, SplitStream};
 use futures_util::{Sink, SinkExt, Stream, StreamExt};
-use litellm_core::CoreResult;
-use litellm_core::error::CoreError;
+use litellm_core::error::Error;
 use litellm_core::realtime::transformation::RealtimeProviderConfig;
 use litellm_core::realtime::types::RealtimeEvent;
 use tokio::net::TcpStream;
@@ -48,7 +47,7 @@ pub(crate) type UpstreamRx = SplitStream<UpstreamWs>;
 /// Resolve the OpenAI API key from the explicit param or the environment.
 ///
 /// Blank/whitespace values are treated as absent (guard at resolution time).
-pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult<String> {
+pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result<String, Error> {
     api_key
         .map(str::trim)
         .filter(|key| !key.is_empty())
@@ -58,7 +57,7 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult<String> {
                 .ok()
                 .filter(|key| !key.trim().is_empty())
         })
-        .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string()))
+        .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string()))
 }
 
 /// Open the upstream WebSocket to OpenAI for `(model, api_key, api_base)`.
@@ -70,24 +69,24 @@ pub(crate) async fn dial_upstream(
     model: &str,
     api_key: &str,
     api_base: Option<&str>,
-) -> CoreResult<UpstreamWs> {
+) -> Result<UpstreamWs, Error> {
     let url = OPENAI_REALTIME_CONFIG.complete_url(api_base, model);
 
     let mut request = url
         .as_str()
         .into_client_request()
-        .map_err(|err| CoreError::Network(err.to_string()))?;
+        .map_err(|err| Error::Network(err.to_string()))?;
     // GA realtime: only Authorization. The legacy OpenAI-Beta header triggers
     // beta_api_shape_disabled, so we do not send it.
     request.headers_mut().insert(
         AUTHORIZATION,
         HeaderValue::from_str(&format!("Bearer {api_key}"))
-            .map_err(|err| CoreError::Auth(err.to_string()))?,
+            .map_err(|err| Error::Auth(err.to_string()))?,
     );
 
     let (upstream, _response) = connect_async(request)
         .await
-        .map_err(|err| CoreError::Network(err.to_string()))?;
+        .map_err(|err| Error::Network(err.to_string()))?;
     Ok(upstream)
 }
 
@@ -96,22 +95,22 @@ pub(crate) async fn dial_upstream(
 /// Used by the pool to pre-read OpenAI's unprompted `session.created`. Returns an
 /// error on a non-text frame, a closed socket, or undecodable JSON so the pool can
 /// discard a misbehaving socket rather than warm it.
-pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> CoreResult<RealtimeEvent> {
+pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> Result<RealtimeEvent, Error> {
     loop {
         let message = upstream_rx
             .next()
             .await
-            .ok_or_else(|| CoreError::Network("upstream closed before first event".to_string()))?
-            .map_err(|err| CoreError::Network(err.to_string()))?;
+            .ok_or_else(|| Error::Network("upstream closed before first event".to_string()))?
+            .map_err(|err| Error::Network(err.to_string()))?;
         match message {
             Message::Text(text) => {
                 return serde_json::from_str(&text)
-                    .map_err(|err| CoreError::InvalidResponse(err.to_string()));
+                    .map_err(|err| Error::InvalidResponse(err.to_string()));
             }
             // Ignore protocol frames (ping/pong) while waiting for the first event.
             Message::Ping(_) | Message::Pong(_) => continue,
             Message::Close(_) => {
-                return Err(CoreError::Network(
+                return Err(Error::Network(
                     "upstream closed before first event".to_string(),
                 ));
             }
@@ -139,7 +138,7 @@ pub(crate) async fn splice<In, Out>(
     mut observe: impl FnMut(&RealtimeEvent) + Send,
     mut client_in: In,
     mut client_out: Out,
-) -> CoreResult<()>
+) -> Result<(), Error>
 where
     In: Stream<Item = RealtimeEvent> + Unpin + Send,
     Out: Sink<RealtimeEvent> + Unpin + Send,
@@ -154,7 +153,7 @@ where
             client_out
                 .send(outbound)
                 .await
-                .map_err(|err| CoreError::Network(err.to_string()))?;
+                .map_err(|err| Error::Network(err.to_string()))?;
         }
     }
 
@@ -175,26 +174,26 @@ where
                 // inflate its own spend log. Logging observes upstream events only.
                 for outbound in config.transform_realtime_request(&event, model)?.events {
                     let payload = serde_json::to_string(&outbound)
-                        .map_err(|err| CoreError::InvalidResponse(err.to_string()))?;
+                        .map_err(|err| Error::InvalidResponse(err.to_string()))?;
                     upstream_tx
                         .send(Message::Text(payload))
                         .await
-                        .map_err(|err| CoreError::Network(err.to_string()))?;
+                        .map_err(|err| Error::Network(err.to_string()))?;
                 }
             }
             // upstream -> client
             upstream_message = upstream_rx.next() => {
                 let Some(message) = upstream_message else { break }; // upstream closed
-                match message.map_err(|err| CoreError::Network(err.to_string()))? {
+                match message.map_err(|err| Error::Network(err.to_string()))? {
                     Message::Text(text) => {
                         let event: RealtimeEvent = serde_json::from_str(&text)
-                            .map_err(|err| CoreError::InvalidResponse(err.to_string()))?;
+                            .map_err(|err| Error::InvalidResponse(err.to_string()))?;
                         observe(&event);
                         for outbound in config.transform_realtime_response(&event, model)?.events {
                             client_out
                                 .send(outbound)
                                 .await
-                                .map_err(|err| CoreError::Network(err.to_string()))?;
+                                .map_err(|err| Error::Network(err.to_string()))?;
                         }
                     }
                     Message::Close(_) => break,
@@ -225,7 +224,7 @@ pub async fn realtime<In, Out>(
     observe: impl FnMut(&RealtimeEvent) + Send,
     client_in: In,
     client_out: Out,
-) -> CoreResult<()>
+) -> Result<(), Error>
 where
     In: Stream<Item = RealtimeEvent> + Unpin + Send,
     Out: Sink<RealtimeEvent> + Unpin + Send,
@@ -258,7 +257,7 @@ pub async fn realtime_warm<In, Out>(
     observe: impl FnMut(&RealtimeEvent) + Send,
     client_in: In,
     client_out: Out,
-) -> CoreResult<()>
+) -> Result<(), Error>
 where
     In: Stream<Item = RealtimeEvent> + Unpin + Send,
     Out: Sink<RealtimeEvent> + Unpin + Send,

litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs

@@ -28,7 +28,7 @@ use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
 
 use futures_util::StreamExt;
-use litellm_core::CoreResult;
+use litellm_core::Error;
 use litellm_core::realtime::types::RealtimeEvent;
 
 use crate::io::realtime::{
@@ -438,7 +438,7 @@ impl RealtimePool {
 ///
 /// `key.api_key` is already resolved (non-blank). The first frame OpenAI sends
 /// unprompted is `session.created`; we buffer exactly that and read nothing more.
-async fn warm_one(key: &UpstreamKey) -> CoreResult<WarmConnection> {
+async fn warm_one(key: &UpstreamKey) -> Result<WarmConnection, Error> {
     let upstream: UpstreamWs =
         dial_upstream(&key.model, &key.api_key, key.api_base.as_deref()).await?;
     let (tx, mut rx) = upstream.split();

litellm-rust/crates/ai-gateway/src/io/responses_ws.rs

@@ -4,10 +4,10 @@ use std::time::Duration;
 
 use futures_util::stream::{SplitSink, SplitStream};
 use futures_util::{Sink, SinkExt, Stream, StreamExt};
+use litellm_core::Error;
 use litellm_core::providers::openai::responses::transformation::OPENAI_RESPONSES_WS_CONFIG;
 use litellm_core::responses::types::ResponsesWsEvent;
 use litellm_core::responses::websocket::ResponsesWebSocketProviderConfig;
-use litellm_core::{CoreError, CoreResult};
 use tokio::net::TcpStream;
 use tokio::sync::Mutex;
 use tokio_tungstenite::tungstenite::Message;
@@ -37,51 +37,49 @@ impl ResponsesWebSocketConnection {
         url: &str,
         headers: &HashMap<String, String>,
         timeout: Option<Duration>,
-    ) -> CoreResult<Self> {
+    ) -> Result<Self, Error> {
         let mut request = url
             .into_client_request()
-            .map_err(|error| CoreError::Network(error.to_string()))?;
+            .map_err(|error| Error::Network(error.to_string()))?;
         for (name, value) in headers {
             let header_name = name
                 .parse::<HeaderName>()
-                .map_err(|error| CoreError::InvalidRequest(error.to_string()))?;
+                .map_err(|error| Error::InvalidRequest(error.to_string()))?;
             let header_value = HeaderValue::from_str(value)
-                .map_err(|error| CoreError::InvalidRequest(error.to_string()))?;
+                .map_err(|error| Error::InvalidRequest(error.to_string()))?;
             request.headers_mut().insert(header_name, header_value);
         }
         let connect = connect_async(request);
         let result = match timeout {
             Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| {
-                CoreError::Network("Responses WebSocket connection timed out".to_string())
+                Error::Network("Responses WebSocket connection timed out".to_string())
             })?,
             None => connect.await,
         };
         let (socket, _) = result.map_err(|error| match error {
-            tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http {
+            tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http {
                 status: response.status().as_u16(),
                 body: String::new(),
             },
-            other => CoreError::Network(other.to_string()),
+            other => Error::Network(other.to_string()),
         })?;
         Ok(Self {
             socket: Arc::new(Mutex::new(Some(socket))),
         })
     }
 
-    pub async fn send_text(&self, text: String) -> CoreResult<()> {
+    pub async fn send_text(&self, text: String) -> Result<(), Error> {
         let mut socket = self.socket.lock().await;
         let Some(socket) = socket.as_mut() else {
-            return Err(CoreError::Network(
-                "Responses WebSocket is closed".to_string(),
-            ));
+            return Err(Error::Network("Responses WebSocket is closed".to_string()));
         };
         socket
             .send(Message::Text(text))
             .await
-            .map_err(|error| CoreError::Network(error.to_string()))
+            .map_err(|error| Error::Network(error.to_string()))
     }
 
-    pub async fn recv_text(&self) -> CoreResult<Option<String>> {
+    pub async fn recv_text(&self) -> Result<Option<String>, Error> {
         let mut socket_guard = self.socket.lock().await;
         let Some(socket) = socket_guard.as_mut() else {
             return Ok(None);
@@ -90,27 +88,27 @@ impl ResponsesWebSocketConnection {
             Some(Ok(Message::Text(text))) => Ok(Some(text)),
             Some(Ok(Message::Binary(bytes))) => String::from_utf8(bytes.to_vec())
                 .map(Some)
-                .map_err(|error| CoreError::InvalidResponse(error.to_string())),
+                .map_err(|error| Error::InvalidResponse(error.to_string())),
             Some(Ok(Message::Close(_))) | None => Ok(None),
             Some(Ok(_)) => Ok(None),
-            Some(Err(error)) => Err(CoreError::Network(error.to_string())),
+            Some(Err(error)) => Err(Error::Network(error.to_string())),
         }
     }
 
-    pub async fn close(&self) -> CoreResult<()> {
+    pub async fn close(&self) -> Result<(), Error> {
         let mut socket = self.socket.lock().await;
         if let Some(socket) = socket.as_mut() {
             socket
                 .close(None)
                 .await
-                .map_err(|error| CoreError::Network(error.to_string()))?;
+                .map_err(|error| Error::Network(error.to_string()))?;
         }
         *socket = None;
         Ok(())
     }
 }
 
-pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult<String> {
+pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result<String, Error> {
     api_key
         .map(str::trim)
         .filter(|value| !value.is_empty())
@@ -120,38 +118,38 @@ pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult<String> {
                 .ok()
                 .filter(|value| !value.trim().is_empty())
         })
-        .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string()))
+        .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string()))
 }
 
 async fn dial_upstream(
     model: &str,
     api_key: &str,
     api_base: Option<&str>,
-) -> CoreResult<ResponsesUpstreamWs> {
+) -> Result<ResponsesUpstreamWs, Error> {
     let url = OPENAI_RESPONSES_WS_CONFIG.complete_websocket_url(api_base, model);
     let mut request = url
         .as_str()
         .into_client_request()
-        .map_err(|error| CoreError::Network(error.to_string()))?;
+        .map_err(|error| Error::Network(error.to_string()))?;
     request.headers_mut().insert(
         AUTHORIZATION,
         HeaderValue::from_str(&format!("Bearer {api_key}"))
-            .map_err(|error| CoreError::Auth(error.to_string()))?,
+            .map_err(|error| Error::Auth(error.to_string()))?,
     );
     let result = tokio::time::timeout(
         Duration::from_secs(DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS),
         connect_async(request),
     )
     .await
-    .map_err(|_| CoreError::Network("Responses WebSocket connection timed out".to_string()))?;
+    .map_err(|_| Error::Network("Responses WebSocket connection timed out".to_string()))?;
     result
         .map(|(socket, _)| socket)
         .map_err(|error| match error {
-            tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http {
+            tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http {
                 status: response.status().as_u16(),
                 body: String::new(),
             },
-            other => CoreError::Network(other.to_string()),
+            other => Error::Network(other.to_string()),
         })
 }
 
@@ -166,7 +164,7 @@ impl ResponsesWebSocketStreaming {
         observe: impl FnMut(&ResponsesWsEvent) + Send,
         client_in: In,
         client_out: Out,
-    ) -> CoreResult<()>
+    ) -> Result<(), Error>
     where
         In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
         Out: Sink<ResponsesWsEvent> + Unpin + Send,
@@ -193,7 +191,7 @@ pub(crate) async fn splice<In, Out>(
     mut observe: impl FnMut(&ResponsesWsEvent) + Send,
     mut client_in: In,
     mut client_out: Out,
-) -> CoreResult<()>
+) -> Result<(), Error>
 where
     In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
     Out: Sink<ResponsesWsEvent> + Unpin + Send,
@@ -210,18 +208,18 @@ where
                     .events
                 {
                     let payload = serde_json::to_string(&outbound)
-                        .map_err(|error| CoreError::InvalidResponse(error.to_string()))?;
+                        .map_err(|error| Error::InvalidResponse(error.to_string()))?;
                     upstream_tx.send(Message::Text(payload))
                         .await
-                        .map_err(|error| CoreError::Network(error.to_string()))?;
+                        .map_err(|error| Error::Network(error.to_string()))?;
                 }
             }
             message = upstream_rx.next() => {
                 let Some(message) = message else { break };
-                match message.map_err(|error| CoreError::Network(error.to_string()))? {
+                match message.map_err(|error| Error::Network(error.to_string()))? {
                     Message::Text(text) => {
                         let event = serde_json::from_str::<ResponsesWsEvent>(&text)
-                            .map_err(|error| CoreError::InvalidResponse(error.to_string()))?;
+                            .map_err(|error| Error::InvalidResponse(error.to_string()))?;
                         observe(&event);
                         for outbound in OPENAI_RESPONSES_WS_CONFIG
                             .transform_ws_response(&event, model)?
@@ -229,7 +227,7 @@ where
                         {
                             client_out.send(outbound)
                                 .await
-                                .map_err(|error| CoreError::Network(error.to_string()))?;
+                                .map_err(|error| Error::Network(error.to_string()))?;
                         }
                     }
                     Message::Close(_) => break,
@@ -252,7 +250,7 @@ pub async fn async_responses_websocket<In, Out>(
     mut observe: impl FnMut(&ResponsesWsEvent) + Send,
     client_in: In,
     client_out: Out,
-) -> CoreResult<()>
+) -> Result<(), Error>
 where
     In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
     Out: Sink<ResponsesWsEvent> + Unpin + Send,
@@ -267,11 +265,11 @@ where
             .events
         {
             let payload = serde_json::to_string(&outbound)
-                .map_err(|error| CoreError::InvalidResponse(error.to_string()))?;
+                .map_err(|error| Error::InvalidResponse(error.to_string()))?;
             upstream_tx
                 .send(Message::Text(payload))
                 .await
-                .map_err(|error| CoreError::Network(error.to_string()))?;
+                .map_err(|error| Error::Network(error.to_string()))?;
         }
     }
     ResponsesWebSocketStreaming::bidirectional_forward(
@@ -296,7 +294,7 @@ pub async fn responses_ws<In, Out>(
     observe: impl FnMut(&ResponsesWsEvent) + Send,
     client_in: In,
     client_out: Out,
-) -> CoreResult<()>
+) -> Result<(), Error>
 where
     In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
     Out: Sink<ResponsesWsEvent> + Unpin + Send,
@@ -514,7 +512,7 @@ mod tests {
         )
         .await
         .expect_err("status error");
-        assert!(matches!(error, CoreError::Http { status: 401, .. }));
+        assert!(matches!(error, Error::Http { status: 401, .. }));
         server.await.expect("server task");
     }
 
@@ -543,7 +541,7 @@ mod tests {
         )
         .await
         .expect_err("status error");
-        assert!(matches!(error, CoreError::Http { status: 500, .. }));
+        assert!(matches!(error, Error::Http { status: 500, .. }));
         server.await.expect("server task");
     }
 }

litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs

@@ -3,8 +3,7 @@ use std::time::{Duration, Instant};
 
 use base64::Engine;
 use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
-use litellm_core::CoreResult;
-use litellm_core::error::CoreError;
+use litellm_core::error::Error;
 use litellm_core::ocr::transformation::OcrProviderConfig;
 use reqwest::Url;
 use serde_json::{Map, Value};
@@ -56,7 +55,7 @@ fn is_azure_document_intelligence_model(model: &str) -> bool {
 
 pub(super) fn string_headers(
     extra_headers: Option<Map<String, Value>>,
-) -> CoreResult<Vec<(String, String)>> {
+) -> Result<Vec<(String, String)>, Error> {
     extra_headers
         .unwrap_or_default()
         .into_iter()
@@ -65,7 +64,7 @@ pub(super) fn string_headers(
                 .as_str()
                 .map(|value| (key.clone(), value.to_string()))
                 .ok_or_else(|| {
-                    CoreError::InvalidRequest(format!(
+                    Error::InvalidRequest(format!(
                         "OCR extra_headers.{key} must be a string, got {}",
                         litellm_core::error::json_type_name(&value)
                     ))
@@ -80,7 +79,7 @@ pub(super) fn has_header(headers: &[(String, String)], name: &str) -> bool {
         .any(|(key, _)| key.eq_ignore_ascii_case(name))
 }
 
-fn document_url_field(document: &Value) -> CoreResult<Option<(&str, &str)>> {
+fn document_url_field(document: &Value) -> Result<Option<(&str, &str)>, Error> {
     let Some(object) = document.as_object() else {
         return Ok(None);
     };
@@ -138,13 +137,13 @@ fn is_blocked_ip(ip: IpAddr) -> bool {
     }
 }
 
-fn blocked_url_error(url: &Url) -> CoreError {
-    CoreError::InvalidRequest(format!(
+fn blocked_url_error(url: &Url) -> Error {
+    Error::InvalidRequest(format!(
         "OCR document URL rejected by SSRF protection: {url}"
     ))
 }
 
-async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> {
+async fn validate_safe_fetch_url(url: &Url) -> Result<(), Error> {
     if !matches!(url.scheme(), "http" | "https") {
         return Err(blocked_url_error(url));
     }
@@ -162,7 +161,7 @@ async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> {
         .ok_or_else(|| blocked_url_error(url))?;
     let addresses = tokio::net::lookup_host((host, port))
         .await
-        .map_err(|err| CoreError::Network(err.to_string()))?;
+        .map_err(|err| Error::Network(err.to_string()))?;
     let mut saw_address = false;
     for address in addresses {
         saw_address = true;
@@ -176,25 +175,25 @@ async fn validate_safe_fetch_url(url: &Url) -> CoreResult<()> {
     Ok(())
 }
 
-fn redirect_location(response: &reqwest::Response, url: &Url) -> CoreResult<Url> {
+fn redirect_location(response: &reqwest::Response, url: &Url) -> Result<Url, Error> {
     let location = response
         .headers()
         .get(reqwest::header::LOCATION)
         .and_then(|value| value.to_str().ok())
         .ok_or_else(|| {
-            CoreError::InvalidResponse("OCR document redirect missing Location header".to_string())
+            Error::InvalidResponse("OCR document redirect missing Location header".to_string())
         })?;
     url.join(location)
-        .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR document redirect: {err}")))
+        .map_err(|err| Error::InvalidResponse(format!("invalid OCR document redirect: {err}")))
 }
 
-async fn safe_get_document_url(url: &str) -> CoreResult<(Url, reqwest::Response)> {
+async fn safe_get_document_url(url: &str) -> Result<(Url, reqwest::Response), Error> {
     let client = reqwest::Client::builder()
         .redirect(reqwest::redirect::Policy::none())
         .build()
-        .map_err(|err| CoreError::Network(err.to_string()))?;
+        .map_err(|err| Error::Network(err.to_string()))?;
     let mut current_url = Url::parse(url)
-        .map_err(|err| CoreError::InvalidRequest(format!("invalid OCR document URL: {err}")))?;
+        .map_err(|err| Error::InvalidRequest(format!("invalid OCR document URL: {err}")))?;
 
     for _ in 0..MAX_SAFE_FETCH_REDIRECTS {
         validate_safe_fetch_url(&current_url).await?;
@@ -202,28 +201,28 @@ async fn safe_get_document_url(url: &str) -> CoreResult<(Url, reqwest::Response)
             .get(current_url.clone())
             .send()
             .await
-            .map_err(|err| CoreError::Network(err.to_string()))?;
+            .map_err(|err| Error::Network(err.to_string()))?;
         if !response.status().is_redirection() {
             return Ok((current_url, response));
         }
         current_url = redirect_location(&response, &current_url)?;
     }
 
-    Err(CoreError::InvalidRequest(
+    Err(Error::InvalidRequest(
         "Too many redirects while fetching OCR document URL".to_string(),
     ))
 }
 
-fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> CoreResult<()> {
+fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Result<(), Error> {
     if max_bytes == 0 {
-        return Err(CoreError::InvalidRequest(format!(
+        return Err(Error::InvalidRequest(format!(
             "OCR document URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}"
         )));
     }
     if content_length > max_bytes {
         let size_mb = content_length as f64 / (1024.0 * 1024.0);
         let max_size_mb = max_bytes as f64 / (1024.0 * 1024.0);
-        return Err(CoreError::InvalidRequest(format!(
+        return Err(Error::InvalidRequest(format!(
             "OCR document size ({size_mb:.2}MB) exceeds maximum allowed size ({max_size_mb:.2}MB). url={url}"
         )));
     }
@@ -233,7 +232,7 @@ fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Core
 async fn read_response_with_limit(
     mut response: reqwest::Response,
     url: &Url,
-) -> CoreResult<Vec<u8>> {
+) -> Result<Vec<u8>, Error> {
     let max_bytes = max_document_download_bytes();
     if let Some(content_length) = response.content_length() {
         enforce_download_size(content_length, max_bytes, url)?;
@@ -246,7 +245,7 @@ async fn read_response_with_limit(
     while let Some(chunk) = response
         .chunk()
         .await
-        .map_err(|err| CoreError::Network(err.to_string()))?
+        .map_err(|err| Error::Network(err.to_string()))?
     {
         bytes_downloaded += chunk.len() as u64;
         enforce_download_size(bytes_downloaded, max_bytes, url)?;
@@ -255,7 +254,7 @@ async fn read_response_with_limit(
     Ok(bytes)
 }
 
-pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreResult<Value> {
+pub(super) async fn convert_document_url_to_data_uri(document: Value) -> Result<Value, Error> {
     let Some((field, url)) = document_url_field(&document)? else {
         return Ok(document);
     };
@@ -267,7 +266,7 @@ pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreRes
     let status = response.status();
     if !status.is_success() {
         let body = response.text().await.unwrap_or_default();
-        return Err(CoreError::Http {
+        return Err(Error::Http {
             status: status.as_u16(),
             body: truncate_error_body(&body),
         });
@@ -290,7 +289,7 @@ pub(super) async fn convert_document_url_to_data_uri(document: Value) -> CoreRes
     let mut transformed = document
         .as_object()
         .cloned()
-        .ok_or_else(|| CoreError::InvalidRequest("OCR document must be an object".to_string()))?;
+        .ok_or_else(|| Error::InvalidRequest("OCR document must be an object".to_string()))?;
     transformed.insert(field.to_string(), Value::String(data_uri));
     Ok(Value::Object(transformed))
 }
@@ -316,11 +315,11 @@ fn retry_after_secs(response: &reqwest::Response) -> u64 {
         .unwrap_or(2)
 }
 
-fn operation_status(response_json: &Value) -> CoreResult<&str> {
+fn operation_status(response_json: &Value) -> Result<&str, Error> {
     let status = response_json
         .get("status")
         .and_then(Value::as_str)
-        .ok_or(CoreError::MissingField("status"))?;
+        .ok_or(Error::MissingField("status"))?;
     match status {
         "succeeded" => Ok("succeeded"),
         "running" | "notStarted" => Ok("running"),
@@ -330,11 +329,11 @@ fn operation_status(response_json: &Value) -> CoreResult<&str> {
                 .and_then(|error| error.get("message"))
                 .and_then(Value::as_str)
                 .unwrap_or("Unknown error");
-            Err(CoreError::InvalidResponse(format!(
+            Err(Error::InvalidResponse(format!(
                 "Azure Document Intelligence analysis failed: {message}"
             )))
         }
-        other => Err(CoreError::InvalidResponse(format!(
+        other => Err(Error::InvalidResponse(format!(
             "Unknown operation status: {other}"
         ))),
     }
@@ -345,9 +344,9 @@ pub(super) async fn poll_document_intelligence(
     original_url: &str,
     headers: &[(String, String)],
     timeout: Option<Duration>,
-) -> CoreResult<Value> {
+) -> Result<Value, Error> {
     if !same_origin(operation_url, original_url) {
-        return Err(CoreError::InvalidResponse(
+        return Err(Error::InvalidResponse(
             "Azure Document Intelligence: rejected cross-origin polling URL".to_string(),
         ));
     }
@@ -358,7 +357,7 @@ pub(super) async fn poll_document_intelligence(
     ));
     loop {
         if start.elapsed() > timeout {
-            return Err(CoreError::Network(format!(
+            return Err(Error::Network(format!(
                 "Azure Document Intelligence operation polling timed out after {} seconds",
                 timeout.as_secs()
             )));
@@ -373,21 +372,21 @@ pub(super) async fn poll_document_intelligence(
         let response = request_builder
             .send()
             .await
-            .map_err(|err| CoreError::Network(err.to_string()))?;
+            .map_err(|err| Error::Network(err.to_string()))?;
         let retry_after = retry_after_secs(&response);
         let status = response.status();
         let text = response
             .text()
             .await
-            .map_err(|err| CoreError::Network(err.to_string()))?;
+            .map_err(|err| Error::Network(err.to_string()))?;
         if !status.is_success() {
-            return Err(CoreError::Http {
+            return Err(Error::Http {
                 status: status.as_u16(),
                 body: truncate_error_body(&text),
             });
         }
         let response_json: Value = serde_json::from_str(&text).map_err(|err| {
-            CoreError::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}"))
+            Error::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}"))
         })?;
         if operation_status(&response_json)? == "succeeded" {
             return Ok(response_json);
@@ -426,7 +425,7 @@ mod tests {
 
         assert!(matches!(
             error,
-            CoreError::InvalidRequest(message)
+            Error::InvalidRequest(message)
                 if message.contains("SSRF protection")
         ));
     }

litellm-rust/crates/ai-gateway/src/ocr/handler.rs

@@ -1,5 +1,4 @@
-use litellm_core::CoreResult;
-use litellm_core::error::CoreError;
+use litellm_core::error::Error;
 use litellm_core::ocr::transformation::OcrResponseHandling;
 use serde_json::Value;
 
@@ -7,7 +6,7 @@ use super::common_utils::{poll_document_intelligence, truncate_error_body};
 use super::types::ProviderOcrRequest;
 use crate::client::http_client;
 
-pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> CoreResult<Value> {
+pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Result<Value, Error> {
     let mut request_builder = http_client().post(&request.url).json(&request.body);
     for (key, value) in &request.upstream_headers {
         request_builder = request_builder.header(key, value);
@@ -19,7 +18,7 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co
     let response = request_builder
         .send()
         .await
-        .map_err(|err| CoreError::Network(err.to_string()))?;
+        .map_err(|err| Error::Network(err.to_string()))?;
 
     let status = response.status();
     if request.config.response_handling() == OcrResponseHandling::AzureDocumentIntelligencePoll
@@ -31,7 +30,7 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co
             .and_then(|value| value.to_str().ok())
             .map(str::to_string)
             .ok_or_else(|| {
-                CoreError::InvalidResponse(
+                Error::InvalidResponse(
                     "Azure Document Intelligence returned 202 but no Operation-Location header found"
                         .to_string(),
                 )
@@ -52,17 +51,17 @@ pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> Co
     let text = response
         .text()
         .await
-        .map_err(|err| CoreError::Network(err.to_string()))?;
+        .map_err(|err| Error::Network(err.to_string()))?;
 
     if !status.is_success() {
-        return Err(CoreError::Http {
+        return Err(Error::Http {
             status: status.as_u16(),
             body: truncate_error_body(&text),
         });
     }
 
     let response_json: Value = serde_json::from_str(&text)
-        .map_err(|err| CoreError::InvalidResponse(format!("invalid OCR response JSON: {err}")))?;
+        .map_err(|err| Error::InvalidResponse(format!("invalid OCR response JSON: {err}")))?;
 
     Ok(request
         .config

litellm-rust/crates/ai-gateway/src/ocr/hooks.rs

@@ -1,11 +1,9 @@
-use std::future::Future;
-use std::pin::Pin;
-
-use litellm_core::CoreResult;
 use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming};
-use litellm_core::error::CoreError;
+use litellm_core::error::Error;
 use litellm_core::ocr::transformation::OcrAuthStrategy;
 use serde_json::{Map, Value, json};
+use std::future::Future;
+use std::pin::Pin;
 
 use super::common_utils::{
     convert_document_url_to_data_uri, has_header, ocr_provider_config, string_headers,
@@ -27,7 +25,7 @@ pub(crate) struct OcrLifecycleHooks {
     request_metadata: RequestMetadata,
 }
 
-type OcrFuture<'a, T> = Pin<Box<dyn Future<Output = CoreResult<T>> + Send + 'a>>;
+type OcrFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
 type OcrLogFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
 
 impl OcrLifecycleHooks {
@@ -46,7 +44,7 @@ impl OcrLifecycleHooks {
     async fn run_pre_call_guardrails(
         &self,
         request: PreparedOcrRequest,
-    ) -> CoreResult<PreparedOcrRequest> {
+    ) -> Result<PreparedOcrRequest, Error> {
         if self.guardrail_runner.is_empty() {
             return Ok(request);
         }
@@ -74,9 +72,9 @@ impl OcrLifecycleHooks {
     async fn prepare_provider_request(
         &self,
         request: PreparedOcrRequest,
-    ) -> CoreResult<ProviderOcrRequest> {
+    ) -> Result<ProviderOcrRequest, Error> {
         let config = ocr_provider_config(&request.custom_llm_provider, &request.model)
-            .ok_or_else(|| CoreError::InvalidProvider(request.custom_llm_provider.clone()))?;
+            .ok_or_else(|| Error::InvalidProvider(request.custom_llm_provider.clone()))?;
         let env_lookup = |key: &str| std::env::var(key).ok();
         let headers = string_headers(request.extra_headers)?;
         let auth_strategy = config.auth_strategy();
@@ -120,7 +118,7 @@ impl OcrLifecycleHooks {
         custom_llm_provider: &str,
         url: &str,
         body: Value,
-    ) -> CoreResult<Value> {
+    ) -> Result<Value, Error> {
         if self.guardrail_runner.is_empty() {
             return Ok(body);
         }
@@ -217,7 +215,7 @@ impl CallLifecycleHooks<PreparedOcrRequest, ProviderOcrRequest, Value> for OcrLi
     fn async_log_failure_event<'a>(
         &'a self,
         context: &'a CallLifecycleContext,
-        error: &'a CoreError,
+        error: &'a Error,
         timing: &'a CallLifecycleTiming,
     ) -> Self::FailureFuture<'a> {
         Box::pin(async move {
@@ -278,19 +276,19 @@ fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext {
 
 fn parse_ocr_pre_call_guardrail_request(
     request: GuardrailRequest,
-) -> CoreResult<(Value, Map<String, Value>)> {
+) -> Result<(Value, Map<String, Value>), Error> {
     let Value::Object(mut data) = request.data else {
-        return Err(CoreError::InvalidRequest(
+        return Err(Error::InvalidRequest(
             "OCR pre_call guardrail must return an object".to_string(),
         ));
     };
     let document = data.remove("document").ok_or_else(|| {
-        CoreError::InvalidRequest("OCR pre_call guardrail removed document".to_string())
+        Error::InvalidRequest("OCR pre_call guardrail removed document".to_string())
     })?;
     let optional_params = match data.remove("optional_params") {
         Some(Value::Object(params)) => params,
         Some(_) => {
-            return Err(CoreError::InvalidRequest(
+            return Err(Error::InvalidRequest(
                 "OCR pre_call guardrail optional_params must be an object".to_string(),
             ));
         }
@@ -299,33 +297,32 @@ fn parse_ocr_pre_call_guardrail_request(
     Ok((document, optional_params))
 }
 
-fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> CoreResult<Value> {
+fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> Result<Value, Error> {
     let Value::Object(mut data) = request.data else {
-        return Err(CoreError::InvalidRequest(
+        return Err(Error::InvalidRequest(
             "OCR during_call guardrail must return an object".to_string(),
         ));
     };
-    data.remove("body").ok_or_else(|| {
-        CoreError::InvalidRequest("OCR during_call guardrail removed body".to_string())
-    })
+    data.remove("body")
+        .ok_or_else(|| Error::InvalidRequest("OCR during_call guardrail removed body".to_string()))
 }
 
-fn guardrail_error_to_core_error(error: GuardrailError) -> CoreError {
-    CoreError::InvalidRequest(format!("{}: {}", error.kind, error.message))
+fn guardrail_error_to_core_error(error: GuardrailError) -> Error {
+    Error::InvalidRequest(format!("{}: {}", error.kind, error.message))
 }
 
-fn core_error_kind(error: &CoreError) -> &'static str {
+fn core_error_kind(error: &Error) -> &'static str {
     match error {
-        CoreError::Auth(_) => "AuthError",
-        CoreError::InvalidProvider(_) => "InvalidProvider",
-        CoreError::InvalidRequest(_) => "InvalidRequest",
-        CoreError::InvalidType { .. } => "InvalidType",
-        CoreError::MissingField(_) => "MissingField",
-        CoreError::Http { .. } => "HttpError",
-        CoreError::InvalidResponse(_) => "InvalidResponse",
-        CoreError::Network(_) => "NetworkError",
-        CoreError::Connect(_) => "ConnectError",
-        CoreError::Routing(_) => "RoutingError",
-        CoreError::Unsupported(_) => "UnsupportedRequest",
+        Error::Auth(_) => "AuthError",
+        Error::InvalidProvider(_) => "InvalidProvider",
+        Error::InvalidRequest(_) => "InvalidRequest",
+        Error::InvalidType { .. } => "InvalidType",
+        Error::MissingField(_) => "MissingField",
+        Error::Http { .. } => "HttpError",
+        Error::InvalidResponse(_) => "InvalidResponse",
+        Error::Network(_) => "NetworkError",
+        Error::Connect(_) => "ConnectError",
+        Error::Routing(_) => "RoutingError",
+        Error::Unsupported(_) => "UnsupportedRequest",
     }
 }

litellm-rust/crates/ai-gateway/src/ocr/mod.rs

@@ -1,4 +1,4 @@
-use litellm_core::CoreResult;
+use litellm_core::Error;
 use litellm_core::call_lifecycle::CallLifecycle;
 use serde_json::Value;
 
@@ -13,7 +13,7 @@ pub use types::OcrRequest;
 use handler::execute_ocr_provider_call;
 use prepare::{PreparedOcrCall, prepare_ocr_call};
 
-pub async fn ocr(request: OcrRequest<'_>) -> CoreResult<Value> {
+pub async fn ocr(request: OcrRequest<'_>) -> Result<Value, Error> {
     let PreparedOcrCall { request, hooks } = prepare_ocr_call(request);
     CallLifecycle::default()
         .run_request(request, &hooks, execute_ocr_provider_call)

litellm-rust/crates/ai-gateway/src/ocr/tests.rs

@@ -1,7 +1,7 @@
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
-use litellm_core::error::CoreError;
+use litellm_core::error::Error;
 use litellm_core::ocr::transformation::OcrResponseHandling;
 use serde_json::{Map, Value, json};
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
@@ -395,7 +395,7 @@ async fn ocr_lifecycle_runs_failure_hook_on_provider_error() {
     .await
     .expect_err("provider error propagates");
 
-    assert!(matches!(err, CoreError::Http { status: 500, .. }));
+    assert!(matches!(err, Error::Http { status: 500, .. }));
     server.await.expect("server task completes");
     assert_eq!(
         logger.events(),
@@ -439,7 +439,7 @@ async fn ocr_lifecycle_pre_call_block_skips_provider_socket() {
     .await
     .expect_err("guardrail blocks request");
 
-    assert!(matches!(err, CoreError::InvalidRequest(_)));
+    assert!(matches!(err, Error::InvalidRequest(_)));
     assert_eq!(guardrail.events(), vec!["async_pre_call_hook"]);
     assert_eq!(
         logger.events(),
@@ -607,7 +607,7 @@ fn string_headers_rejects_non_string_values() {
     let err = string_headers(Some(headers)).expect_err("non-string header rejected");
     assert_eq!(
         err,
-        CoreError::InvalidRequest(
+        Error::InvalidRequest(
             "OCR extra_headers.x-retry-count must be a string, got number".to_string()
         )
     );

litellm-rust/crates/ai-gateway/src/python/config.rs

@@ -6,33 +6,31 @@
 //! (and recorded in [`crate::gil`]); the realtime hot path never touches Python.
 //!
 //! Compiled only under the `python-config` feature.
-
-use litellm_core::CoreResult;
-use litellm_core::error::CoreError;
+use litellm_core::error::Error;
 use litellm_core::router::{Deployment, Router};
 use pyo3::prelude::*;
 
 use crate::gil;
 
 /// Load the router's `model_list` from `config_path` via the Python reader.
-pub fn load_router_from_config(config_path: &str) -> CoreResult<Router> {
+pub fn load_router_from_config(config_path: &str) -> Result<Router, Error> {
     gil::record_acquisition();
     Python::attach(|py| {
         let model_list = py
             .import("litellm.proxy.read_model_list")
             .and_then(|module| module.getattr("read_model_list"))
             .and_then(|reader| reader.call1((config_path,)))
-            .map_err(|err| CoreError::Routing(format!("read_model_list failed: {err}")))?;
+            .map_err(|err| Error::Routing(format!("read_model_list failed: {err}")))?;
 
         let model_list_json: String = py
             .import("json")
             .and_then(|json| json.getattr("dumps"))
             .and_then(|dumps| dumps.call1((model_list,)))
             .and_then(|encoded| encoded.extract())
-            .map_err(|err| CoreError::Routing(format!("serializing model_list failed: {err}")))?;
+            .map_err(|err| Error::Routing(format!("serializing model_list failed: {err}")))?;
 
         let deployments: Vec<Deployment> = serde_json::from_str(&model_list_json)
-            .map_err(|err| CoreError::Routing(format!("parsing model_list failed: {err}")))?;
+            .map_err(|err| Error::Routing(format!("parsing model_list failed: {err}")))?;
 
         Ok(Router::new(deployments))
     })

litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs

@@ -9,7 +9,7 @@ use axum::http::StatusCode;
 use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE, HeaderMap, HeaderValue};
 use axum::response::{IntoResponse, Response};
 use axum::routing::post;
-use litellm_core::CoreError;
+use litellm_core::Error;
 use serde_json::{Map, Value};
 
 use crate::auth::RequireMasterKey;
@@ -46,7 +46,7 @@ fn stream_response(upstream: reqwest::Response) -> Result<Response, MessagesRout
     let mut response = Response::builder()
         .status(
             StatusCode::from_u16(upstream.status().as_u16()).map_err(|error| {
-                MessagesRouteError(CoreError::InvalidResponse(format!(
+                MessagesRouteError(Error::InvalidResponse(format!(
                     "invalid upstream response status: {error}"
                 )))
             })?,
@@ -58,13 +58,13 @@ fn stream_response(upstream: reqwest::Response) -> Result<Response, MessagesRout
     response
         .body(Body::from_stream(upstream.bytes_stream()))
         .map_err(|error| {
-            MessagesRouteError(CoreError::InvalidResponse(format!(
+            MessagesRouteError(Error::InvalidResponse(format!(
                 "failed to build streaming response: {error}"
             )))
         })
 }
 
-fn forwarded_headers(headers: &HeaderMap) -> Result<Option<Map<String, Value>>, CoreError> {
+fn forwarded_headers(headers: &HeaderMap) -> Result<Option<Map<String, Value>>, Error> {
     let forwarded = headers
         .iter()
         .filter(|(name, _)| {
@@ -74,19 +74,19 @@ fn forwarded_headers(headers: &HeaderMap) -> Result<Option<Map<String, Value>>,
         })
         .map(|(name, value)| {
             let value = value.to_str().map_err(|_| {
-                CoreError::InvalidRequest(format!("invalid value for header {}", name.as_str()))
+                Error::InvalidRequest(format!("invalid value for header {}", name.as_str()))
             })?;
             Ok((name.to_string(), Value::String(value.to_string())))
         })
-        .collect::<Result<Map<_, _>, CoreError>>()?;
+        .collect::<Result<Map<_, _>, Error>>()?;
     Ok((!forwarded.is_empty()).then_some(forwarded))
 }
 
 #[derive(Debug)]
-struct MessagesRouteError(CoreError);
+struct MessagesRouteError(Error);
 
-impl From<CoreError> for MessagesRouteError {
-    fn from(error: CoreError) -> Self {
+impl From<Error> for MessagesRouteError {
+    fn from(error: Error) -> Self {
         Self(error)
     }
 }
@@ -94,28 +94,28 @@ impl From<CoreError> for MessagesRouteError {
 impl IntoResponse for MessagesRouteError {
     fn into_response(self) -> Response {
         let (status, message) = match self.0 {
-            CoreError::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message),
-            CoreError::InvalidProvider(_) | CoreError::Routing(_) => (
+            Error::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message),
+            Error::InvalidProvider(_) | Error::Routing(_) => (
                 StatusCode::NOT_FOUND,
                 "no messages deployment is configured for this model".to_string(),
             ),
-            CoreError::Auth(_) => (
+            Error::Auth(_) => (
                 StatusCode::BAD_GATEWAY,
                 "messages provider authentication failed".to_string(),
             ),
-            CoreError::Http { .. }
-            | CoreError::Network(_)
-            | CoreError::Connect(_)
-            | CoreError::InvalidResponse(_)
-            | CoreError::InvalidType { .. }
-            | CoreError::MissingField(_) => (
+            Error::Http { .. }
+            | Error::Network(_)
+            | Error::Connect(_)
+            | Error::InvalidResponse(_)
+            | Error::InvalidType { .. }
+            | Error::MissingField(_) => (
                 StatusCode::BAD_GATEWAY,
                 "messages provider request failed".to_string(),
             ),
             // The gateway has no Python implementation to decline to, so a
             // request the core cannot serve is reported to the caller. The
             // reason is a fixed internal string, never provider content.
-            CoreError::Unsupported(reason) => (
+            Error::Unsupported(reason) => (
                 StatusCode::BAD_REQUEST,
                 format!("messages request is not supported: {reason}"),
             ),

litellm-rust/crates/ai-gateway/src/routes/messages/service.rs

@@ -1,10 +1,10 @@
 use std::sync::Arc;
 
+use litellm_core::Error;
 use litellm_core::constants::ANTHROPIC_MESSAGES_PROVIDER;
 use litellm_core::messages::types::MessagesRequest;
 use litellm_core::messages::{messages, messages_stream};
 use litellm_core::router::Router;
-use litellm_core::{CoreError, CoreResult};
 use serde_json::{Map, Value};
 
 pub(crate) enum MessagesResponse {
@@ -16,16 +16,16 @@ pub async fn run(
     router: &Arc<Router>,
     body: Value,
     extra_headers: Option<Map<String, Value>>,
-) -> CoreResult<MessagesResponse> {
+) -> Result<MessagesResponse, Error> {
     let model = body
         .get("model")
         .and_then(Value::as_str)
         .map(str::trim)
         .filter(|model| !model.is_empty())
-        .ok_or_else(|| CoreError::InvalidRequest("messages body requires a model".to_string()))?;
-    let deployment = router.get_available_deployment(model).ok_or_else(|| {
-        CoreError::Routing(format!("no deployment available for model '{model}'"))
-    })?;
+        .ok_or_else(|| Error::InvalidRequest("messages body requires a model".to_string()))?;
+    let deployment = router
+        .get_available_deployment(model)
+        .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?;
     let provider_model = deployment.litellm_params.model.as_str();
     let upstream_model = provider_model
         .split_once('/')
@@ -37,7 +37,7 @@ pub async fn run(
     };
     let mut body = body;
     body.as_object_mut()
-        .ok_or_else(|| CoreError::InvalidRequest("messages body must be an object".to_string()))?
+        .ok_or_else(|| Error::InvalidRequest("messages body must be an object".to_string()))?
         .insert(
             "model".to_string(),
             Value::String(upstream_model.to_string()),
@@ -60,6 +60,6 @@ pub async fn run(
     serde_json::to_value(response)
         .map(MessagesResponse::Json)
         .map_err(|err| {
-            CoreError::InvalidResponse(format!("failed to serialize messages response: {err}"))
+            Error::InvalidResponse(format!("failed to serialize messages response: {err}"))
         })
 }

litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs

@@ -11,8 +11,7 @@ use std::time::Duration;
 
 use crate::io::realtime_pool::{RealtimePool, upstream_key};
 use futures_util::{Sink, Stream};
-use litellm_core::CoreResult;
-use litellm_core::error::CoreError;
+use litellm_core::error::Error;
 use litellm_core::realtime::types::RealtimeEvent;
 use litellm_core::router::Router;
 
@@ -29,15 +28,15 @@ pub async fn run<In, Out>(
     observe: impl FnMut(&RealtimeEvent) + Send,
     client_in: In,
     client_out: Out,
-) -> CoreResult<()>
+) -> Result<(), Error>
 where
     In: Stream<Item = RealtimeEvent> + Unpin + Send,
     Out: Sink<RealtimeEvent> + Unpin + Send,
     <Out as Sink<RealtimeEvent>>::Error: std::fmt::Display,
 {
-    let deployment = router.get_available_deployment(model).ok_or_else(|| {
-        CoreError::Routing(format!("no deployment available for model '{model}'"))
-    })?;
+    let deployment = router
+        .get_available_deployment(model)
+        .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?;
     let params = &deployment.litellm_params;
     // Strip a leading `openai/` so the OpenAI-only realtime fn gets the bare model.
     let provider_model = params

litellm-rust/crates/ai-gateway/src/routes/responses/service.rs

@@ -2,13 +2,13 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use futures_util::{Sink, Stream};
+use litellm_core::Error;
 use litellm_core::call_lifecycle::{CallLifecycle, CallLifecycleContext};
 use litellm_core::responses::instrumentation::{
     ResponsesWsCallbackPayload, ResponsesWsInstrumentation, ResponsesWsLogOutcome,
     ResponsesWsMetadata,
 };
 use litellm_core::responses::types::ResponsesWsEvent;
-use litellm_core::{CoreError, CoreResult};
 
 use crate::integrations::custom_logger::{
     CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, ModelCallDetails,
@@ -26,22 +26,22 @@ pub async fn run<In, Out>(
     metadata: RequestMetadata,
     client_in: In,
     client_out: Out,
-) -> CoreResult<()>
+) -> Result<(), Error>
 where
     In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
     Out: Sink<ResponsesWsEvent> + Unpin + Send,
     Out::Error: std::fmt::Display,
 {
-    let deployment = router.get_available_deployment(model).ok_or_else(|| {
-        CoreError::Routing(format!("no deployment available for model '{model}'"))
-    })?;
+    let deployment = router
+        .get_available_deployment(model)
+        .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?;
     let params = &deployment.litellm_params;
     let provider_model = params
         .model
         .strip_prefix("openai/")
         .unwrap_or(&params.model);
     if params.model.contains('/') && !params.model.starts_with("openai/") {
-        return Err(CoreError::InvalidProvider(
+        return Err(Error::InvalidProvider(
             "Responses WebSocket route supports OpenAI deployments only".to_string(),
         ));
     }

litellm-rust/crates/core/src/audio_transcription/transformation.rs

@@ -1,7 +1,6 @@
+use crate::Error;
 use serde_json::{Map, Value};
 
-use crate::CoreResult;
-
 use super::types::{AudioTranscriptionRequestData, AudioTranscriptionResponseData};
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -32,13 +31,13 @@ pub trait AudioTranscriptionProviderConfig: Sync {
         model: &str,
         audio: Value,
         optional_params: Map<String, Value>,
-    ) -> CoreResult<AudioTranscriptionRequestData>;
+    ) -> Result<AudioTranscriptionRequestData, Error>;
 
     fn transform_transcription_response(
         &self,
         model: &str,
         response_json: Value,
-    ) -> CoreResult<AudioTranscriptionResponseData>;
+    ) -> Result<AudioTranscriptionResponseData, Error>;
 
     fn complete_url(
         &self,
@@ -46,12 +45,12 @@ pub trait AudioTranscriptionProviderConfig: Sync {
         model: &str,
         optional_params: &Map<String, Value>,
         env_lookup: &dyn Fn(&str) -> Option<String>,
-    ) -> CoreResult<String>;
+    ) -> Result<String, Error>;
 
     fn auth_strategy(
         &self,
         model: &str,
         optional_params: &Map<String, Value>,
         env_lookup: &dyn Fn(&str) -> Option<String>,
-    ) -> CoreResult<AudioTranscriptionAuth>;
+    ) -> Result<AudioTranscriptionAuth, Error>;
 }