Merge latest main and validate positive MCP search limits

.circleci/config.yml

@@ -2731,7 +2731,7 @@ jobs:
             - ~/.cache/uv
       - restore_cache:
           keys:
-            - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+            - ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
       - run:
           name: Install Node dependencies and Playwright
           # The cimg/python:3.12-browsers image already ships the Chromium system
@@ -2742,11 +2742,14 @@ jobs:
           command: |
             cd ui/litellm-dashboard
             npm ci
+            cd ../../tests/e2e/ui
+            npm ci
             npx playwright install chromium
       - save_cache:
-          key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+          key: ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
           paths:
             - ui/litellm-dashboard/node_modules
+            - tests/e2e/ui/node_modules
             - ~/.cache/ms-playwright
       - run:
           name: Build UI from source
@@ -2777,10 +2780,10 @@ jobs:
           name: Seed database
           command: |
             PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
-              -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
+              -f tests/e2e/ui/fixtures/seed.sql
       - run:
           name: Start mock LLM server
-          command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
+          command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
           background: true
       - run:
           name: Start LiteLLM proxy
@@ -2798,7 +2801,7 @@ jobs:
           command: |
             LITELLM_LICENSE="$LITELLM_LICENSE" \
               uv run --no-sync python -m litellm.proxy.proxy_cli \
-                --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
+                --config tests/e2e/ui/fixtures/config.yml \
                 --port 4000
           background: true
       - run:
@@ -2819,15 +2822,15 @@ jobs:
           # Forward LITELLM_LICENSE so license.spec.ts can detect that the
           # proxy was launched with a license and assert premium_user=true.
           command: |
-            cd ui/litellm-dashboard
+            cd tests/e2e/ui
             LITELLM_LICENSE="$LITELLM_LICENSE" \
-              npx playwright test --config e2e_tests/playwright.config.ts
+              npx playwright test --config playwright.config.ts
           no_output_timeout: 10m
       - store_artifacts:
-          path: ui/litellm-dashboard/test-results
+          path: tests/e2e/ui/test-results
           destination: e2e-test-results
       - store_artifacts:
-          path: ui/litellm-dashboard/playwright-report
+          path: tests/e2e/ui/playwright-report
           destination: e2e-playwright-report
 
   e2e_ui_testing_server_root_path:
@@ -2870,17 +2873,20 @@ jobs:
             - ~/.cache/uv
       - restore_cache:
           keys:
-            - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+            - ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
       - run:
           name: Install Node dependencies and Playwright
           command: |
             cd ui/litellm-dashboard
             npm ci
+            cd ../../tests/e2e/ui
+            npm ci
             npx playwright install chromium
       - save_cache:
-          key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+          key: ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
           paths:
             - ui/litellm-dashboard/node_modules
+            - tests/e2e/ui/node_modules
             - ~/.cache/ms-playwright
       - run:
           name: Build UI from source
@@ -2902,10 +2908,10 @@ jobs:
           name: Seed database
           command: |
             PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
-              -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
+              -f tests/e2e/ui/fixtures/seed.sql
       - run:
           name: Start mock LLM server
-          command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
+          command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
           background: true
       - run:
           name: Start LiteLLM proxy under a server root path
@@ -2918,7 +2924,7 @@ jobs:
           command: |
             LITELLM_LICENSE="$LITELLM_LICENSE" \
               uv run --no-sync python -m litellm.proxy.proxy_cli \
-                --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
+                --config tests/e2e/ui/fixtures/config.yml \
                 --port 4000
           background: true
       - run:
@@ -2937,15 +2943,15 @@ jobs:
       - run:
           name: Run migration smoke under SERVER_ROOT_PATH
           command: |
-            cd ui/litellm-dashboard
+            cd tests/e2e/ui
             LITELLM_LICENSE="$LITELLM_LICENSE" \
-              npx playwright test --config e2e_tests/migration.serverRootPath.config.ts
+              npx playwright test --config migration.serverRootPath.config.ts
           no_output_timeout: 10m
       - store_artifacts:
-          path: ui/litellm-dashboard/test-results
+          path: tests/e2e/ui/test-results
           destination: e2e-server-root-path-test-results
       - store_artifacts:
-          path: ui/litellm-dashboard/playwright-report
+          path: tests/e2e/ui/playwright-report
           destination: e2e-server-root-path-playwright-report
 
   build_docker_database_image:

.circleci/scripts/classify_changes.sh

@@ -8,7 +8,7 @@ has_backend=false
 while IFS= read -r file || [ -n "$file" ]; do
   [ -n "$file" ] || continue
   case "$file" in
-    ui/*) has_client=true ;;
+    ui/* | tests/e2e/ui/*) has_client=true ;;
     docs/* | *.md | *.mdx) : ;;
     *) has_backend=true ;;
   esac

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -30,7 +30,7 @@ body:
     id: steps-to-reproduce
     attributes:
       label: Steps to Reproduce
-      description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
+      description: Please provide a numbered list of the exact steps to reproduce this bug (include a curl/python snippet to reproduce it). Number each step (1., 2., 3., ...) in the order you performed them.
       placeholder: |
         1. config.yaml file/ .env file/ etc.
         2. Run the following code...

.github/pull_request_template.md

@@ -1,3 +1,18 @@
+## TLDR
+
+<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
+     This section must be extremely human parsable, comprehensible, and readable: its target audience is humans, not AI agents -->
+
+Problem this solves:
+
+- <blah>
+- ...
+
+How it solves it:
+
+- <blah>
+- ...
+
 ## Relevant issues
 
 <!-- e.g., "Fixes #000" -->

.github/workflows/_test-unit-base.yml

@@ -80,7 +80,7 @@ jobs:
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
 
       - name: Generate Prisma client
         if: steps.changes.outputs.decision != 'skip'

.github/workflows/codspeed.yml

@@ -5,10 +5,24 @@ on:
     branches:
       - main
       - litellm_internal_staging
+    paths:
+      - "litellm/**"
+      - "tests/benchmarks/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/codspeed.yml"
+      - ".github/actions/setup-uv-with-retries/**"
   pull_request:
     branches:
       - main
       - litellm_internal_staging
+    paths:
+      - "litellm/**"
+      - "tests/benchmarks/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/codspeed.yml"
+      - ".github/actions/setup-uv-with-retries/**"
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:

.github/workflows/create_daily_oss_branch.yml

@@ -1,61 +0,0 @@
-name: Create Daily OSS Branch
-
-on:
-  schedule:
-    - cron: "0 16 * * 1-5" # 9am PT during daylight saving time, weekdays.
-  workflow_dispatch:
-    inputs:
-      date:
-        description: "Branch date in YYYY_MM_DD format. Defaults to today's UTC date."
-        required: false
-        type: string
-
-permissions:
-  contents: write
-
-jobs:
-  create-oss-branch:
-    if: github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - name: Create dated OSS branch
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          REQUESTED_DATE: ${{ inputs.date }}
-        run: |
-          set -euo pipefail
-
-          if [ -n "${REQUESTED_DATE}" ]; then
-            if ! echo "${REQUESTED_DATE}" | grep -Eq '^[0-9]{4}_[0-9]{2}_[0-9]{2}$'; then
-              echo "::error::date must use YYYY_MM_DD format, got '${REQUESTED_DATE}'"
-              exit 1
-            fi
-            BRANCH_DATE="${REQUESTED_DATE}"
-          else
-            BRANCH_DATE="$(date -u +'%Y_%m_%d')"
-          fi
-
-          BRANCH_NAME="litellm_oss_daily_${BRANCH_DATE}"
-          echo "Creating branch: ${BRANCH_NAME}"
-
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          git fetch origin main "${BRANCH_NAME}" || true
-
-          if git show-ref --verify --quiet "refs/remotes/origin/${BRANCH_NAME}"; then
-            echo "Branch ${BRANCH_NAME} already exists. Skipping creation."
-            exit 0
-          fi
-
-          git checkout -b "${BRANCH_NAME}" origin/main
-          git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "${BRANCH_NAME}"
-          echo "Successfully created and pushed branch: ${BRANCH_NAME}"

.github/workflows/guard-main-branch.yml

@@ -31,12 +31,12 @@ jobs:
           echo "PR head repo: $HEAD_REPO"
           echo "PR head branch: $HEAD_REF"
           if [ "$HEAD_REPO" != "$BASE_REPO" ]; then
-            echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the current daily OSS branch (named litellm_oss_daily_YYYY_MM_DD; a fresh one is cut each weekday, so target the most recent) instead."
+            echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against 'litellm_internal_staging' instead."
             exit 1
           fi
           if [ "$HEAD_REF" = "litellm_internal_staging" ] || [[ "$HEAD_REF" == litellm_hotfix_?* ]]; then
             echo "Allowed source branch."
             exit 0
           fi
-          echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against the current daily OSS branch (named litellm_oss_daily_YYYY_MM_DD; a fresh one is cut each weekday, so target the most recent) instead."
+          echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_internal_staging' instead."
           exit 1

.github/workflows/image-scan.yml

@@ -9,6 +9,7 @@ on:
       - "litellm_**"
     paths:
       - docker/Dockerfile.non_root
+      - tests/proxy_migration_tests/test_offline_image_migration.py
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
       - .github/workflows/image-scan.yml
@@ -51,6 +52,23 @@ jobs:
       - name: Build runtime image
         run: docker build -f docker/Dockerfile.non_root -t litellm-image-scan:${{ github.sha }} .
 
+      # The prisma bake must migrate a fresh DB with no egress as an arbitrary
+      # non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyRootFilesystem).
+      # `docker run` as the default uid with network hides a broken bake because
+      # the migration entrypoint exits 0 even when it applied nothing; asserting
+      # the schema was created is what catches it.
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify offline migration as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-image-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+
       # Scans the whole shipped artifact: OS/apk plus every language package
       # baked into the image, including ones no lockfile declares (e.g. prisma's
       # vendored node engine) that osv-scan cannot see. osv-scan stays the fast
@@ -58,6 +76,8 @@ jobs:
       # free OSS, run as a pinned, checksum-verified binary; no GitHub Action
       # dependency and no vendor SaaS callout.
       - name: Scan image for fixable HIGH/CRITICAL CVEs
+        env:
+          GRYPE_MATCH_PYTHON_USING_CPES: "true"
         run: |
           "$RUNNER_TEMP/grype" litellm-image-scan:${{ github.sha }} \
             --only-fixed \

.github/workflows/mutation-test.yml

@@ -55,7 +55,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
 
       - name: Generate Prisma client
         env:

.github/workflows/oss_daily_guardrails.yml

@@ -1,50 +0,0 @@
-name: OSS Daily Guardrails
-
-on:
-  push:
-    branches:
-      - "litellm_oss_daily_20*"
-  pull_request:
-    branches:
-      - "litellm_oss_daily_20*"
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  oss-safe-checks:
-    name: Run OSS daily safe checks
-    if: startsWith(github.ref_name, 'litellm_oss_daily_20') || startsWith(github.head_ref, 'litellm_oss_daily_20') || startsWith(github.base_ref, 'litellm_oss_daily_20')
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Run secret scan test
-        run: |
-          uv run --frozen --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v
-
-      - name: Run Ruff
-        run: |
-          uv sync --frozen
-          cd litellm
-          uv run --no-sync ruff check .

.github/workflows/test-code-quality.yml

@@ -115,6 +115,9 @@ jobs:
       - name: check_fastuuid_usage
         run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
 
+      - name: check_e2e_no_raw_requests
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

.github/workflows/test-linting.yml

@@ -104,6 +104,7 @@ jobs:
       - name: Check basedpyright budget (delta vs base)
         env:
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          NODE_OPTIONS: --max-old-space-size=12288
         run: |
           (uv run --no-sync basedpyright --outputjson || true) | uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
 

.github/workflows/test-litellm-ui-lint.yml

@@ -29,7 +29,15 @@ jobs:
         id: changed
         env:
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
+          # base.sha is the base branch tip from when the PR was opened, while
+          # actions/checkout leaves HEAD on a merge of the PR into the *current*
+          # base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
+          # landed since, so a PR that touches no UI file still gets linted
+          # against hundreds of other people's files. Diff the PR head against its
+          # own merge base instead, which is exactly what this PR changed.
+          merge_base=$(git merge-base "$BASE_SHA" "$HEAD_SHA")
           : > "$RUNNER_TEMP/prettier_files.txt"
           : > "$RUNNER_TEMP/eslint_files.txt"
           while IFS= read -r f; do
@@ -41,7 +49,7 @@ jobs:
               *.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
                 printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
             esac
-          done < <(git diff --name-only --diff-filter=ACMR --relative "$BASE_SHA"...HEAD -- .)
+          done < <(git diff --name-only --diff-filter=ACMR --relative "$merge_base" "$HEAD_SHA" -- .)
           if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
             echo "has_files=true" >> "$GITHUB_OUTPUT"
           else

.github/workflows/test-litellm-ui-unit.yml

@@ -0,0 +1,57 @@
+name: UI Unit Tests
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - litellm_internal_staging
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  ui-unit-tests:
+    runs-on: ubuntu-latest-16-cores
+    timeout-minutes: 20
+    defaults:
+      run:
+        working-directory: ui/litellm-dashboard
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: ui/litellm-dashboard/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run UI unit tests (Vitest)
+        env:
+          CI: "true"
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          if [ -n "$BASE_SHA" ]; then
+            echo "Pull request: running only tests related to changes since $BASE_SHA"
+            npm run test -- --run --changed "$BASE_SHA" --passWithNoTests \
+              --pool forks --poolOptions.forks.maxForks=14
+          else
+            echo "Push to $GITHUB_REF_NAME: running the full suite"
+            npm run test -- --run --pool forks --poolOptions.forks.maxForks=14
+          fi

.github/workflows/test-rust.yml

@@ -61,5 +61,11 @@ jobs:
       - name: Run Clippy
         run: cargo clippy --workspace --all-targets --locked -- -D warnings
 
+      - name: Run Clippy with Bedrock auth
+        run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
+
       - name: Run Rust tests
         run: cargo test --workspace --locked
+
+      - name: Run core tests with Bedrock auth
+        run: cargo test -p litellm-core --features bedrock-auth --locked

.github/workflows/test-unit-proxy-db.yml

@@ -5,6 +5,8 @@ on:
     branches:
       - main
       - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-endpoints.yml

@@ -46,6 +46,7 @@ jobs:
         tests/test_litellm/proxy/rag_endpoints
         tests/test_litellm/proxy/realtime_endpoints
         tests/test_litellm/proxy/ui_crud_endpoints
+        tests/test_litellm/proxy/config_resolvers
         tests/test_litellm/proxy/utils
       workers: 2
       reruns: 2

.github/workflows/test_server_root_path.yml

@@ -1,151 +0,0 @@
-name: Test Proxy SERVER_ROOT_PATH Routing
-permissions:
-  contents: read
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-
-jobs:
-  test-server-root-path:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-
-    strategy:
-      fail-fast: false
-      matrix:
-        root_path: ["/api/v1", "/llmproxy"]
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Free up disk space
-        run: |
-          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/share/boost
-          sudo apt-get clean
-          df -h /
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
-
-      - name: Build Docker image
-        uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 # v6.14.0
-        with:
-          context: .
-          file: ./docker/Dockerfile.non_root
-          tags: litellm-test:${{ github.sha }}
-          load: true
-          push: false
-
-      - name: Start LiteLLM container with SERVER_ROOT_PATH
-        run: |
-          docker run -d \
-            --name litellm-test \
-            -p 4000:4000 \
-            -e SERVER_ROOT_PATH="${{ matrix.root_path }}" \
-            -e LITELLM_MASTER_KEY="sk-1234" \
-            litellm-test:${{ github.sha }} \
-            --detailed_debug
-
-      - name: Wait for container to be healthy
-        run: |
-          echo "Waiting for LiteLLM to start..."
-          max_attempts=30
-          attempt=0
-
-          while [ $attempt -lt $max_attempts ]; do
-            if docker logs litellm-test 2>&1 | grep -q "Uvicorn running"; then
-              echo "LiteLLM started successfully"
-              break
-            fi
-            attempt=$((attempt + 1))
-            echo "Attempt $attempt/$max_attempts - waiting for server to start..."
-            sleep 2
-          done
-
-          if [ $attempt -eq $max_attempts ]; then
-            echo "Server failed to start within timeout"
-            docker logs litellm-test
-            exit 1
-          fi
-
-          sleep 5
-
-      - name: Show container logs
-        if: always()
-        run: docker logs litellm-test
-
-      - name: Test UI endpoint with root path
-        run: |
-          ROOT_PATH="${{ matrix.root_path }}"
-          echo "Testing UI at: http://localhost:4000${ROOT_PATH}/ui/"
-
-          for i in 1 2 3; do
-            content=$(curl -sL --max-time 5 -H "Authorization: Bearer sk-1234" "http://localhost:4000${ROOT_PATH}/ui/")
-            if echo "$content" | grep -q -E "(html|<!DOCTYPE|<head|<body)"; then
-              echo "UI page contains valid HTML content"
-              exit 0
-            fi
-            echo "Attempt $i/3 - no valid HTML, retrying in 5s..."
-            sleep 5
-          done
-          echo "UI page does not contain expected HTML content"
-          echo "Response: $content"
-          docker logs litellm-test
-          exit 1
-
-      - name: Setup Node for Playwright
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
-        with:
-          node-version: "20"
-
-      - name: Install UI deps and Chromium
-        working-directory: ui/litellm-dashboard
-        run: |
-          retry() {
-            local attempt=1
-            local max_attempts=4
-            until "$@"; do
-              if [ "$attempt" -ge "$max_attempts" ]; then
-                echo "Command failed after $attempt attempts: $*"
-                return 1
-              fi
-              echo "Attempt $attempt failed: $*. Retrying in $((attempt * 15))s..."
-              sleep $((attempt * 15))
-              attempt=$((attempt + 1))
-            done
-          }
-
-          npm config set fetch-retries 5
-          npm config set fetch-retry-mintimeout 20000
-          npm config set fetch-retry-maxtimeout 120000
-
-          retry npm ci
-          retry npx playwright install --with-deps chromium
-
-      - name: Run SERVER_ROOT_PATH redirect e2e
-        working-directory: ui/litellm-dashboard
-        env:
-          SERVER_ROOT_PATH: ${{ matrix.root_path }}
-        run: npx playwright test --config=e2e_tests/serverRootPath.config.ts
-
-      - name: Upload Playwright artifacts on failure
-        if: failure()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: playwright-trace-${{ strategy.job-index }}
-          path: ui/litellm-dashboard/test-results/
-          retention-days: 7
-
-      - name: Cleanup
-        if: always()
-        run: |
-          docker stop litellm-test || true
-          docker rm litellm-test || true

.github/workflows/weekly_load_anomaly.yml

@@ -0,0 +1,81 @@
+name: "Weekly Load Anomaly Check"
+
+on:
+  schedule:
+    - cron: "0 12 * * 6"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  weekly-load-anomaly:
+    if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-weekly-anomaly-check
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/load/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Run the weekly session anomaly test
+        env:
+          E2E_WEEKLY_ANOMALY: "1"
+        run: |
+          uv run --no-sync pytest tests/e2e/load/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log

.github/workflows/zizmor.yml

@@ -4,7 +4,11 @@ on:
   push:
     branches: [main, litellm_internal_staging]
   pull_request:
-    branches: [main, litellm_internal_staging]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.gitignore

@@ -15,6 +15,9 @@ litellm/rust_bridge/_native*.so
 litellm/rust_bridge/_native*.pyd
 litellm-rust/target/
 
+# Python package build output
+dist/
+
 bun.lockb
 **/.DS_Store
 .aider*

CLAUDE.md

@@ -19,7 +19,7 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
 
 End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`
 
-When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose for internal contributors; external / OSS contributions target the current daily OSS branch instead, named `litellm_oss_daily_YYYY_MM_DD` (a fresh one is cut each weekday, so use the most recent)
+When creating PRs, don't set base to `main`. `litellm_internal_staging` is the default base branch and serves that purpose for both internal and external / OSS contributions
 
 When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
 

CONTRIBUTING.md

@@ -322,7 +322,7 @@ npm run build
 ## Submitting Your PR
 
 1. **Push your branch**: `git push origin your-feature-branch`
-2. **Create a PR**: Go to GitHub and open a pull request against the current daily OSS branch, named `litellm_oss_daily_YYYY_MM_DD`. A fresh one is cut each weekday, so pick the most recent from the [branch list](https://github.com/BerriAI/litellm/branches/all?query=litellm_oss_daily). Do not target `main`.
+2. **Create a PR**: Go to GitHub and open a pull request against [`litellm_internal_staging`](https://github.com/BerriAI/litellm/tree/litellm_internal_staging), which is the default base branch. Do not target `main`.
 3. **Fill out the PR template**: Provide clear description of changes
 4. **Wait for review**: Maintainers will review and provide feedback
 5. **Address feedback**: Make requested changes and push updates

Dockerfile

@@ -64,6 +64,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
 # Copy full source tree
@@ -84,9 +85,12 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -100,7 +104,11 @@ USER root
 RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
 
 WORKDIR /app
-ENV PATH="/app/.venv/bin:${PATH}"
+ENV PATH="/app/.venv/bin:${PATH}" \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
+    PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
+    PRISMA_OFFLINE_MODE=true
 
 # Copy only what runtime needs. The application is installed inside the venv;
 # the rest of the builder's /app is source and build metadata that must not
@@ -114,16 +122,19 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
 # working directory on sys.path; litellm/proxy/hooks resolves
 # enterprise.enterprise_hooks from it)
 COPY --from=builder /app/enterprise /app/enterprise
-# Prisma binaries live in $HOME/.cache (default prisma-python location),
-# which is /root/.cache here. Copy only the Prisma subdirs — copying the
-# whole /root/.cache drags in the uv build cache (~660 MB, includes a
-# setuptools wheel that surfaces as a CVE finding even though it's not
-# on the runtime sys.path).
-COPY --from=builder /root/.cache/prisma /root/.cache/prisma
-COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
+COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
+# Prisma CLI + engines are baked under /opt/prisma, a fixed path every
+# runtime uid can read and that no cache volume mount shadows. The paths are
+# pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and recorded into the
+# generated client at build time, so `prisma migrate deploy` on a fresh
+# database needs no npm and no network access (#33650, #24554).
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
 
 EXPOSE 4000/tcp
 

backend/routes/allowlist.py

@@ -18,6 +18,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/team/",
     "/v2/team/",
     "/organization/",
+    "/v2/organization/",
     "/customer/",
     "/end_user/",
     "/sso/",
@@ -69,6 +70,10 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/project/",
     "/memory/",
     "/mcp/",
+    # Control plane (see the List Endpoints + Tables standard). Every resource
+    # eventually moves under this prefix, so allowlist it once rather than
+    # per-resource.
+    "/management/v1/",
     # Spend / analytics
     "/spend/",
     "/analytics/",

basedpyright-code-budget.json

@@ -1,9 +1,9 @@
 {
   "reportAny": {
-    "limit": 37484
+    "limit": 34906
   },
   "reportArgumentType": {
-    "limit": 2704
+    "limit": 2701
   },
   "reportAssignmentType": {
     "limit": 330
@@ -12,7 +12,7 @@
     "limit": 516
   },
   "reportCallIssue": {
-    "limit": 124
+    "limit": 123
   },
   "reportConstantRedefinition": {
     "limit": 59
@@ -24,7 +24,7 @@
     "limit": 42
   },
   "reportExplicitAny": {
-    "limit": 10397
+    "limit": 10230
   },
   "reportFunctionMemberAccess": {
     "limit": 11
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5900
+    "limit": 5893
   },
   "reportMissingTypeArgument": {
-    "limit": 15903
+    "limit": 15886
   },
   "reportMissingTypeStubs": {
     "limit": 41
@@ -99,31 +99,31 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 45894
+    "limit": 45870
   },
   "reportUnknownLambdaType": {
     "limit": 113
   },
   "reportUnknownMemberType": {
-    "limit": 40539
+    "limit": 40525
   },
   "reportUnknownParameterType": {
-    "limit": 20403
+    "limit": 20384
   },
   "reportUnknownVariableType": {
-    "limit": 32141
+    "limit": 32099
   },
   "reportUnnecessaryCast": {
     "limit": 177
   },
   "reportUnnecessaryComparison": {
-    "limit": 1025
+    "limit": 1023
   },
   "reportUnnecessaryContains": {
     "limit": 7
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 1209
+    "limit": 1206
   },
   "reportUntypedBaseClass": {
     "limit": 165

db_scripts/backfill_daily_tool_spend.sql

@@ -0,0 +1,46 @@
+-- One-shot backfill of the LiteLLM_DailyToolSpend rollup from the per-request
+-- LiteLLM_SpendLogToolIndex x LiteLLM_SpendLogs tables.
+--
+-- This is an opt-in, manual operation. New deployments do not need it: the
+-- rollup is written at request time from the moment the release is deployed.
+-- Run it only if you want the Cost Optimization "Spend by tool" card to show
+-- history from before the deploy, and only once.
+--
+-- IMPORTANT caveats before running:
+--
+-- 1. Pre-deploy index rows may include tools that were merely DECLARED in a
+--    request body but never invoked (the release this ships with stops
+--    recording those). For agentic clients that declare many tools per
+--    request, backfilled history attributes each request's full spend to
+--    every declared tool, overstating per-tool spend. Post-deploy rows do not
+--    have this problem. If your traffic is mostly such clients, consider not
+--    backfilling.
+--
+-- 2. Coverage is bounded by spend-log retention: rows older than
+--    maximum_spend_logs_retention_period are already gone.
+--
+-- 3. Replace the cutover timestamp below with the time you deployed the
+--    release, so backfilled per-request rows cannot double-count on top of
+--    rollup rows the new writer already created. ON CONFLICT DO NOTHING is a
+--    second guard for (date, tool_name) buckets the writer already touched:
+--    such buckets keep the writer's numbers and skip the backfill's.
+--
+-- Usage:
+--   psql "$DATABASE_URL" -v cutover="'2026-07-25T00:00:00Z'" -f db_scripts/backfill_daily_tool_spend.sql
+
+SET TIME ZONE 'UTC';
+
+INSERT INTO "LiteLLM_DailyToolSpend" (date, tool_name, spend, total_tokens, request_count, created_at, updated_at)
+SELECT
+    to_char(ti.start_time, 'YYYY-MM-DD') AS date,
+    ti.tool_name,
+    COALESCE(SUM(sl.spend), 0) AS spend,
+    COALESCE(SUM(sl.total_tokens), 0) AS total_tokens,
+    COUNT(*) AS request_count,
+    now() AS created_at,
+    now() AS updated_at
+FROM "LiteLLM_SpendLogToolIndex" ti
+JOIN "LiteLLM_SpendLogs" sl ON sl.request_id = ti.request_id
+WHERE ti.start_time < :cutover::timestamptz
+GROUP BY 1, 2
+ON CONFLICT (date, tool_name) DO NOTHING;

docker/Dockerfile.database

@@ -62,6 +62,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
 # Copy full source tree
@@ -82,9 +83,12 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -97,7 +101,11 @@ USER root
 RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
 
 WORKDIR /app
-ENV PATH="/app/.venv/bin:${PATH}"
+ENV PATH="/app/.venv/bin:${PATH}" \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
+    PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
+    PRISMA_OFFLINE_MODE=true
 
 # Copy only what runtime needs. The application is installed inside the venv;
 # the rest of the builder's /app is source and build metadata that must not
@@ -111,16 +119,21 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
 # working directory on sys.path; litellm/proxy/hooks resolves
 # enterprise.enterprise_hooks from it)
 COPY --from=builder /app/enterprise /app/enterprise
-# Prisma binaries live in $HOME/.cache (default prisma-python location),
-# which is /root/.cache here. Copy them from the builder so they survive
-# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
-# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
-# Only the Prisma subdirs: the whole /root/.cache drags in the uv build cache.
-COPY --from=builder /root/.cache/prisma /root/.cache/prisma
-COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
+COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
+# Prisma CLI + engines are baked under /opt/prisma, a fixed path every
+# runtime uid can read and that no cache volume mount shadows (unlike
+# /app/.cache or $HOME/.cache in readOnlyRootFilesystem + emptyDir setups).
+# The paths are pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and
+# recorded into the generated client at build time, so `prisma migrate
+# deploy` on a fresh database needs no npm and no network access
+# (#33650, #24554).
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
 
 EXPOSE 4000/tcp
 

docker/Dockerfile.non_root

@@ -54,7 +54,6 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
     PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
-    PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
     XDG_CACHE_HOME=/app/.cache
 
 # Copy dependency metadata first for layer caching
@@ -69,6 +68,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     --extra proxy-runtime \
     --extra extra_proxy \
     --extra semantic-router \
+    --extra saml \
     --python python3
 
 # Copy full source tree
@@ -95,6 +95,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra saml \
         --python python3 \
         --no-sources-package litellm-proxy-extras; \
     else \
@@ -103,10 +104,13 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra saml \
         --python python3; \
     fi
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -127,8 +131,6 @@ RUN for i in 1 2 3; do \
 # the rest of the builder's /app is source and build metadata that must not
 # ship (manifest-scanning tools attribute everything in it to this image).
 # entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path.
-# Prisma caches live under /app/.cache here (XDG_CACHE_HOME /
-# PRISMA_BINARY_CACHE_DIR) so the runtime prisma generate finds them.
 COPY --from=builder /app/.venv /app/.venv
 COPY --from=builder /app/docker /app/docker
 COPY --from=builder /app/schema.prisma /app/schema.prisma
@@ -137,21 +139,36 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
 # working directory on sys.path; litellm/proxy/hooks resolves
 # enterprise.enterprise_hooks from it)
 COPY --from=builder /app/enterprise /app/enterprise
-COPY --from=builder /app/.cache /app/.cache
+COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
+# Prisma CLI + engines are baked under /opt/prisma, a fixed path every runtime
+# uid can read and that no cache volume mount shadows (unlike /app/.cache or
+# $HOME/.cache under readOnlyRootFilesystem + emptyDir or arbitrary-uid setups).
+# PRISMA_CLI_QUERY_ENGINE_TYPE=binary makes the CLI use the baked binary query
+# engine directly, so `prisma migrate deploy` on a fresh database needs no npm
+# and no network access; without it the CLI looks for the library engine, which
+# prisma stopped baking, and falls back to a download that fails offline or as a
+# non-writable uid (#33650, #24554).
+COPY --from=builder /opt/prisma /opt/prisma
 COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
 COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
 
+# XDG_CACHE_HOME is intentionally left unset so it falls back to $HOME/.cache
+# (/app/.cache, writable by the runtime uid). The prisma bake at the read-only
+# /opt/prisma is anchored by PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH, so
+# nothing needs XDG to point there; pointing it at the read-only bake would
+# deny any XDG-aware library that writes a cache at runtime.
 ENV PATH="/app/.venv/bin:${PATH}" \
-    PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
+    PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
     HOME=/app \
     LITELLM_NON_ROOT=true \
-    XDG_CACHE_HOME=/app/.cache \
     PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
     PRISMA_HIDE_UPDATE_MESSAGE=1 \
     PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
     PRISMA_OFFLINE_MODE=true
 
-RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \
+RUN mkdir -p /nonexistent /app/.cache /var/lib/litellm/assets /var/lib/litellm/ui && \
     chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \
     PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
     chown -R nobody:nogroup "$PRISMA_PATH" && \
@@ -164,12 +181,14 @@ RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \
     [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u "$LITELLM_PROXY_EXTRAS_PATH" || true && \
     chmod -R g+w "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
     [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w "$LITELLM_PROXY_EXTRAS_PATH" || true && \
-    chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets /app/.cache
+    chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
+    chmod -R a+rX /opt/prisma && \
+    test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
+    ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1
 
 USER 65534
 
-RUN prisma generate --schema=./schema.prisma
-
 EXPOSE 4000/tcp
 
 ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]

docker/build_from_pip/Dockerfile.build_from_pip

@@ -36,7 +36,7 @@ RUN uv venv --python python && \
       "opentelemetry-api==1.28.0" \
       "opentelemetry-sdk==1.28.0" \
       "opentelemetry-exporter-otlp==1.28.0" \
-      "ddtrace==2.19.0" \
+      "ddtrace==4.11.0" \
       "sentry-sdk==2.21.0" \
       "mangum==0.17.0" \
       "azure-ai-contentsafety==1.0.0" \

enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py

@@ -7,7 +7,8 @@
 #  Thank you users! We ❤️ you! - Krrish & Ishaan
 ## This provides an LLM Guard Integration for content moderation on the proxy
 
-from typing import Literal, Optional
+import asyncio
+from typing import Optional
 
 import aiohttp
 from fastapi import HTTPException
@@ -18,7 +19,6 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.utils import CallTypesLiteral
-from litellm.utils import get_formatted_prompt
 
 
 class _ENTERPRISE_LLMGuard(CustomLogger):
@@ -46,45 +46,44 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
         except Exception:
             pass
 
-    async def moderation_check(self, text: str):
+    async def moderation_check(self, text: str) -> str:
         """
+        Runs the LLM Guard moderation check on ``text``.
+
+        Raises an HTTPException when the content violates the safety policy;
+        otherwise returns the sanitized prompt from LLM Guard, falling back to
+        the original text when the API does not provide one.
+
         [TODO] make this more performant for high-throughput scenario
         """
         try:
-            async with aiohttp.ClientSession() as session:
-                if self.mock_redacted_text is not None:
-                    redacted_text = self.mock_redacted_text
-                else:
-                    # Make the first request to /analyze
-                    analyze_url = f"{self.llm_guard_api_base}analyze/prompt"
-                    verbose_proxy_logger.debug("Making request to: %s", analyze_url)
-                    analyze_payload = {"prompt": text}
-                    redacted_text = None
+            if self.mock_redacted_text is not None:
+                redacted_text = self.mock_redacted_text
+            else:
+                analyze_url = f"{self.llm_guard_api_base}analyze/prompt"
+                verbose_proxy_logger.debug("Making request to: %s", analyze_url)
+                async with aiohttp.ClientSession() as session:
                     async with session.post(
-                        analyze_url, json=analyze_payload
+                        analyze_url, json={"prompt": text}
                     ) as response:
                         redacted_text = await response.json()
-                verbose_proxy_logger.debug(
-                    f"LLM Guard: Received response - {redacted_text}"
+            verbose_proxy_logger.debug(
+                f"LLM Guard: Received response - {redacted_text}"
+            )
+            if redacted_text is None:
+                raise HTTPException(
+                    status_code=500,
+                    detail={
+                        "error": f"Invalid content moderation response: {redacted_text}"
+                    },
                 )
-                if redacted_text is not None:
-                    if (
-                        redacted_text.get("is_valid", None) is not None
-                        and redacted_text["is_valid"] is False
-                    ):
-                        raise HTTPException(
-                            status_code=400,
-                            detail={"error": "Violated content safety policy"},
-                        )
-                    else:
-                        pass
-                else:
-                    raise HTTPException(
-                        status_code=500,
-                        detail={
-                            "error": f"Invalid content moderation response: {redacted_text}"
-                        },
-                    )
+            if redacted_text.get("is_valid", None) is False:
+                raise HTTPException(
+                    status_code=400,
+                    detail={"error": "Violated content safety policy"},
+                )
+            sanitized_prompt = redacted_text.get("sanitized_prompt")
+            return sanitized_prompt if isinstance(sanitized_prompt, str) else text
         except Exception as e:
             verbose_proxy_logger.exception(
                 "litellm.enterprise.enterprise_hooks.llm_guard::moderation_check - Exception occurred - {}".format(
@@ -138,23 +137,75 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
             return
 
         self.print_verbose("Makes LLM Guard Check")
-        try:
-            assert call_type in [
-                "completion",
-                "embeddings",
-                "image_generation",
-                "moderation",
-                "audio_transcription",
-            ]
-        except Exception:
+        if call_type not in [
+            "completion",
+            "embeddings",
+            "image_generation",
+            "moderation",
+            "audio_transcription",
+        ]:
             self.print_verbose(
                 f"Call Type - {call_type}, not in accepted list - ['completion','embeddings','image_generation','moderation','audio_transcription']"
             )
             return data
 
-        formatted_prompt = get_formatted_prompt(data=data, call_type=call_type)  # type: ignore
-        self.print_verbose(f"LLM Guard, formatted_prompt: {formatted_prompt}")
-        return await self.moderation_check(text=formatted_prompt)
+        return await self._moderate_request(data=data)
+
+    async def _moderate_request(self, data: dict) -> dict:
+        """
+        Sanitizes the request in place using the prompt returned by LLM Guard so
+        the provider-bound request carries the redacted content, then returns it.
+        """
+        messages = data.get("messages")
+        if messages is not None:
+            data["messages"] = list(
+                await asyncio.gather(
+                    *(self._moderate_message(message) for message in messages)
+                )
+            )
+            return data
+
+        input_ = data.get("input")
+        if input_ is not None:
+            data["input"] = await self._moderate_input(input_)
+            return data
+
+        prompt = data.get("prompt")
+        if isinstance(prompt, str):
+            data["prompt"] = await self.moderation_check(text=prompt)
+        return data
+
+    async def _moderate_message(self, message: dict) -> dict:
+        content = message.get("content")
+        if isinstance(content, str):
+            return {**message, "content": await self.moderation_check(text=content)}
+        if isinstance(content, list):
+            return {
+                **message,
+                "content": list(
+                    await asyncio.gather(
+                        *(self._moderate_content_part(part) for part in content)
+                    )
+                ),
+            }
+        return message
+
+    async def _moderate_content_part(self, part: dict) -> dict:
+        if part.get("type") == "text" and isinstance(part.get("text"), str):
+            return {**part, "text": await self.moderation_check(text=part["text"])}
+        return part
+
+    async def _moderate_input(self, input_: object) -> object:
+        if isinstance(input_, str):
+            return await self.moderation_check(text=input_)
+        if isinstance(input_, list):
+            return [
+                await self.moderation_check(text=item)
+                if isinstance(item, str)
+                else item
+                for item in input_
+            ]
+        return input_
 
     async def async_post_call_streaming_hook(
         self, user_api_key_dict: UserAPIKeyAuth, response: str

enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py

@@ -113,6 +113,10 @@ class PagerDutyAlerting(SlackAlerting):
                 user_api_key_spend=_meta.get("user_api_key_spend"),
                 user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
                 user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
+                user_api_key_user_spend=_meta.get("user_api_key_user_spend"),
+                user_api_key_user_max_budget=_meta.get("user_api_key_user_max_budget"),
+                user_api_key_team_spend=_meta.get("user_api_key_team_spend"),
+                user_api_key_team_max_budget=_meta.get("user_api_key_team_max_budget"),
                 user_api_key_org_id=_meta.get("user_api_key_org_id"),
                 user_api_key_org_alias=_meta.get("user_api_key_org_alias"),
                 user_api_key_team_id=_meta.get("user_api_key_team_id"),
@@ -196,6 +200,10 @@ class PagerDutyAlerting(SlackAlerting):
                     if user_api_key_dict.budget_reset_at
                     else None
                 ),
+                user_api_key_user_spend=user_api_key_dict.user_spend,
+                user_api_key_user_max_budget=user_api_key_dict.user_max_budget,
+                user_api_key_team_spend=user_api_key_dict.team_spend,
+                user_api_key_team_max_budget=user_api_key_dict.team_max_budget,
                 user_api_key_org_id=user_api_key_dict.org_id,
                 user_api_key_org_alias=user_api_key_dict.organization_alias,
                 user_api_key_team_id=user_api_key_dict.team_id,

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -316,26 +316,34 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         where_clause: Dict[str, Any] = {"file_purpose": "batch", **owner_filter}
 
         if after:
-            where_clause["id"] = {"gt": after}
+            cursor_row = (
+                await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                    where={**where_clause, "unified_object_id": after}
+                )
+            )
+            if cursor_row is None:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Invalid 'after' cursor: no batch found with id '{after}'.",
+                )
 
-        fetch_limit = limit or 20
-        if target_model_names:
-            # Oversample so post-fetch model-name filtering still has enough rows.
-            fetch_limit = max(fetch_limit * 3, 100)
+        page_size = limit or 20
+        cursor_args: Dict[str, Any] = (
+            {"cursor": {"unified_object_id": after}, "skip": 1} if after else {}
+        )
 
         batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
             where=where_clause,
-            take=fetch_limit,
-            order={"created_at": "desc"},
+            take=page_size + 1,
+            order=[{"created_at": "desc"}, {"unified_object_id": "desc"}],
+            **cursor_args,
         )
 
-        batch_objects: List[LiteLLMBatch] = []
-        for batch in batches:
-            try:
-                # Stop once we have enough after filtering
-                if len(batch_objects) >= (limit or 20):
-                    break
+        has_more = len(batches) > page_size
 
+        batch_objects: List[LiteLLMBatch] = []
+        for batch in batches[:page_size]:
+            try:
                 batch_data = (
                     json.loads(batch.file_object)
                     if isinstance(batch.file_object, str)
@@ -351,9 +359,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 )
                 continue
 
-        return build_list_page(
-            batch_objects, has_more=len(batch_objects) == (limit or 20)
-        )
+        return build_list_page(batch_objects, has_more=has_more)
 
     async def get_user_created_file_ids(
         self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -11,7 +11,8 @@ Endpoints for /project operations
 #### PROJECT MANAGEMENT ####
 
 import json
-from typing import List, Optional, Union
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING
 
 from fastapi import APIRouter, Depends, HTTPException, Request
 
@@ -25,15 +26,24 @@ from litellm.proxy.management_helpers.utils import (
 )
 from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
 
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
+    from prisma.actions import LiteLLM_TeamTableActions
+
 router = APIRouter()
 
 
+def _team_table(prisma_client: PrismaClient) -> "LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable]":
+    team_table: LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable] = prisma_client.db.litellm_teamtable
+    return team_table
+
+
 async def _check_user_permission_for_project(
     user_api_key_dict: UserAPIKeyAuth,
-    team_id: Optional[str],
+    team_id: str | None,
     prisma_client: PrismaClient,
     require_admin: bool = False,
-    team_object: Optional[LiteLLM_TeamTable] = None,
+    team_object: LiteLLM_TeamTable | None = None,
 ) -> bool:
     """
     Check if user has permission to manage a project.
@@ -57,9 +67,7 @@ async def _check_user_permission_for_project(
 
     team = team_object
     if team is None:
-        team = await prisma_client.db.litellm_teamtable.find_unique(
-            where={"team_id": team_id}
-        )
+        team = await _team_table(prisma_client).find_unique(where={"team_id": team_id})
 
     if team and team.admins:
         return user_api_key_dict.user_id in team.admins
@@ -70,9 +78,9 @@ async def _check_user_permission_for_project(
 async def _validate_team_exists(
     team_id: str,
     prisma_client: PrismaClient,
-):
+) -> "prisma_models.LiteLLM_TeamTable":
     """Validate that a team exists. Returns the team row."""
-    team = await prisma_client.db.litellm_teamtable.find_unique(
+    team = await _team_table(prisma_client).find_unique(
         where={"team_id": team_id},
     )
 
@@ -89,7 +97,7 @@ async def _validate_team_exists(
 
 def _check_team_project_limits(
     team_object: LiteLLM_TeamTable,
-    data: Union[NewProjectRequest, UpdateProjectRequest],
+    data: NewProjectRequest | UpdateProjectRequest,
 ) -> None:
     """
     Check that project limits respect its parent Team's limits.
@@ -108,16 +116,12 @@ def _check_team_project_limits(
     if data.max_budget is not None and data.max_budget < 0:
         raise HTTPException(
             status_code=400,
-            detail={
-                "error": f"max_budget cannot be negative. Received: {data.max_budget}"
-            },
+            detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"},
         )
     if data.soft_budget is not None and data.soft_budget < 0:
         raise HTTPException(
             status_code=400,
-            detail={
-                "error": f"soft_budget cannot be negative. Received: {data.soft_budget}"
-            },
+            detail={"error": f"soft_budget cannot be negative. Received: {data.soft_budget}"},
         )
 
     # --- soft_budget < max_budget ---
@@ -131,7 +135,7 @@ def _check_team_project_limits(
             )
 
     # --- Validate project models are a subset of team models ---
-    project_models = getattr(data, "models", None)
+    project_models = data.models
     team_models = team_object.models or []
     if project_models and len(team_models) > 0:
         # If team has 'all-proxy-models', skip validation as it allows all models
@@ -148,11 +152,7 @@ def _check_team_project_limits(
     # --- Validate project max_budget <= team max_budget ---
     # Team stores budget fields directly (max_budget, tpm_limit, rpm_limit)
     # unlike Project which uses a separate LiteLLM_BudgetTable relation
-    if (
-        data.max_budget is not None
-        and team_object.max_budget is not None
-        and data.max_budget > team_object.max_budget
-    ):
+    if data.max_budget is not None and team_object.max_budget is not None and data.max_budget > team_object.max_budget:
         raise HTTPException(
             status_code=400,
             detail={
@@ -161,11 +161,7 @@ def _check_team_project_limits(
         )
 
     # --- Validate project tpm_limit <= team tpm_limit ---
-    if (
-        data.tpm_limit is not None
-        and team_object.tpm_limit is not None
-        and data.tpm_limit > team_object.tpm_limit
-    ):
+    if data.tpm_limit is not None and team_object.tpm_limit is not None and data.tpm_limit > team_object.tpm_limit:
         raise HTTPException(
             status_code=400,
             detail={
@@ -174,11 +170,7 @@ def _check_team_project_limits(
         )
 
     # --- Validate project rpm_limit <= team rpm_limit ---
-    if (
-        data.rpm_limit is not None
-        and team_object.rpm_limit is not None
-        and data.rpm_limit > team_object.rpm_limit
-    ):
+    if data.rpm_limit is not None and team_object.rpm_limit is not None and data.rpm_limit > team_object.rpm_limit:
         raise HTTPException(
             status_code=400,
             detail={
@@ -189,19 +181,19 @@ def _check_team_project_limits(
 
 async def _create_budget_for_project(
     data: NewProjectRequest,
-    user_id: Optional[str],
+    user_id: str | None,
     litellm_proxy_admin_name: str,
     prisma_client: PrismaClient,
 ) -> str:
     """Create a budget for the project and return budget_id."""
     budget_params = LiteLLM_BudgetTable.model_fields.keys()
-    _json_data = data.json(exclude_none=True)
+    _json_data: Mapping[str, object] = data.json(exclude_none=True)
     _budget_data = {k: v for k, v in _json_data.items() if k in budget_params}
-    budget_row = LiteLLM_BudgetTable(**_budget_data)
+    budget_row = LiteLLM_BudgetTable.model_validate(_budget_data)
 
     new_budget = prisma_client.jsonify_object(budget_row.json(exclude_none=True))
 
-    _budget = await prisma_client.db.litellm_budgettable.create(
+    _budget: prisma_models.LiteLLM_BudgetTable = await prisma_client.db.litellm_budgettable.create(
         data={
             **new_budget,
             "created_by": user_id or litellm_proxy_admin_name,
@@ -214,8 +206,8 @@ async def _create_budget_for_project(
 
 async def _set_project_object_permission(
     data: NewProjectRequest,
-    prisma_client: Optional[PrismaClient],
-) -> Optional[str]:
+    prisma_client: PrismaClient | None,
+) -> str | None:
     """
     Creates the LiteLLM_ObjectPermissionTable record for the project.
     Returns the object_permission_id if created, otherwise None.
@@ -224,7 +216,7 @@ async def _set_project_object_permission(
         return None
 
     if data.object_permission is not None:
-        created_object_permission = (
+        created_object_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
             await prisma_client.db.litellm_objectpermissiontable.create(
                 data=data.object_permission.model_dump(exclude_none=True),
             )
@@ -344,8 +336,7 @@ async def new_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Only premium users can add tags to projects. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Only premium users can add tags to projects. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -353,8 +344,7 @@ async def new_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Project management is an enterprise feature. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Project management is an enterprise feature. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -375,13 +365,11 @@ async def new_project(
             )
 
         # Validate team exists and get team object with budget
-        team_object = await _validate_team_exists(
-            team_id=data.team_id, prisma_client=prisma_client
-        )
+        team_object = await _validate_team_exists(team_id=data.team_id, prisma_client=prisma_client)
 
         # Validate project limits against team limits
         _check_team_project_limits(
-            team_object=LiteLLM_TeamTable(**team_object.model_dump()),
+            team_object=LiteLLM_TeamTable.model_validate(team_object.model_dump()),
             data=data,
         )
 
@@ -391,7 +379,7 @@ async def new_project(
             user_api_key_dict=user_api_key_dict,
             team_id=data.team_id,
             prisma_client=prisma_client,
-            team_object=LiteLLM_TeamTable(**team_object.model_dump()),
+            team_object=LiteLLM_TeamTable.model_validate(team_object.model_dump()),
         )
 
         if not has_permission:
@@ -449,17 +437,13 @@ async def new_project(
                     value=getattr(data, field),
                 )
 
-        new_project_row = prisma_client.jsonify_object(
-            project_row.json(exclude_none=True)
-        )
+        new_project_row = prisma_client.jsonify_object(project_row.json(exclude_none=True))
 
         # Remove budget fields (following organization_endpoints.py pattern)
         new_project_row = _remove_budget_fields_from_project_data(new_project_row)
 
-        verbose_proxy_logger.info(
-            f"new_project_row: {json.dumps(new_project_row, indent=2)}"
-        )
-        response = await prisma_client.db.litellm_projecttable.create(
+        verbose_proxy_logger.info(f"new_project_row: {json.dumps(new_project_row, indent=2)}")
+        response: prisma_models.LiteLLM_ProjectTable = await prisma_client.db.litellm_projecttable.create(
             data={
                 **new_project_row,  # type: ignore
             },
@@ -469,9 +453,7 @@ async def new_project(
         return response
     except Exception as e:
         verbose_proxy_logger.exception(
-            "litellm.proxy.management_endpoints.project_endpoints.new_project(): Exception occured - {}".format(
-                str(e)
-            )
+            "litellm.proxy.management_endpoints.project_endpoints.new_project(): Exception occured - {}".format(str(e))
         )
         raise handle_exception_on_proxy(e)
 
@@ -539,8 +521,7 @@ async def update_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Only premium users can add tags to projects. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Only premium users can add tags to projects. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -548,8 +529,7 @@ async def update_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Project management is an enterprise feature. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Project management is an enterprise feature. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -576,9 +556,9 @@ async def update_project(
             )
 
         # Fetch existing project
-        existing_project = await prisma_client.db.litellm_projecttable.find_unique(
-            where={"project_id": data.project_id}
-        )
+        existing_project: (
+            prisma_models.LiteLLM_ProjectTable | None
+        ) = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": data.project_id})
 
         if existing_project is None:
             raise ProxyException(
@@ -595,9 +575,7 @@ async def update_project(
         target_team_id = data.team_id or existing_project.team_id
         target_team_obj = None
         if target_team_id is not None:
-            target_team_obj = await _validate_team_exists(
-                team_id=target_team_id, prisma_client=prisma_client
-            )
+            target_team_obj = await _validate_team_exists(team_id=target_team_id, prisma_client=prisma_client)
 
         has_permission = await _check_user_permission_for_project(
             user_api_key_dict=user_api_key_dict,
@@ -620,32 +598,26 @@ async def update_project(
                 team_id=data.team_id,
                 prisma_client=prisma_client,
                 team_object=(
-                    LiteLLM_TeamTable(**target_team_obj.model_dump())
-                    if target_team_obj
-                    else None
+                    LiteLLM_TeamTable.model_validate(target_team_obj.model_dump()) if target_team_obj else None
                 ),
             )
             if not can_assign_to_target:
                 raise HTTPException(
                     status_code=403,
-                    detail={
-                        "error": "Cannot reassign project to a team you are not an admin of"
-                    },
+                    detail={"error": "Cannot reassign project to a team you are not an admin of"},
                 )
 
         # Validate project limits against team limits
         if target_team_obj is not None:
             _check_team_project_limits(
-                team_object=LiteLLM_TeamTable(**target_team_obj.model_dump()),
+                team_object=LiteLLM_TeamTable.model_validate(target_team_obj.model_dump()),
                 data=data,
             )
 
         # Prepare update data
         update_data = data.json(exclude_none=True, exclude={"project_id"})
         update_data = prisma_client.jsonify_object(update_data)
-        update_data["updated_by"] = (
-            user_api_key_dict.user_id or litellm_proxy_admin_name
-        )
+        update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
 
         # Handle budget updates
         budget_fields = LiteLLM_BudgetTable.model_fields.keys()
@@ -671,21 +643,17 @@ async def update_project(
                 if existing_project.object_permission_id:
                     # Update existing permission
                     await prisma_client.db.litellm_objectpermissiontable.update(
-                        where={
-                            "object_permission_id": existing_project.object_permission_id
-                        },
+                        where={"object_permission_id": existing_project.object_permission_id},
                         data=object_permission_data,
                     )
                 else:
                     # Create new permission
-                    created_permission = (
+                    created_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
                         await prisma_client.db.litellm_objectpermissiontable.create(
                             data=object_permission_data,
                         )
                     )
-                    update_data["object_permission_id"] = (
-                        created_permission.object_permission_id
-                    )
+                    update_data["object_permission_id"] = created_permission.object_permission_id
 
         # Handle metadata fields
         for field in LiteLLM_ManagementEndpoint_MetadataFields:
@@ -698,7 +666,7 @@ async def update_project(
         update_data = _remove_budget_fields_from_project_data(update_data)
 
         # Update project
-        updated_project = await prisma_client.db.litellm_projecttable.update(
+        updated_project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.update(
             where={"project_id": data.project_id},
             data=update_data,
             include={"litellm_budget_table": True, "object_permission": True},
@@ -718,7 +686,7 @@ async def update_project(
     "/project/delete",
     tags=["project management"],
     dependencies=[Depends(user_api_key_auth)],
-    response_model=List[LiteLLM_ProjectTable],
+    response_model=list[LiteLLM_ProjectTable],
 )
 @management_endpoint_wrapper
 async def delete_project(
@@ -749,8 +717,7 @@ async def delete_project(
             raise HTTPException(
                 status_code=403,
                 detail={
-                    "error": "Project management is an enterprise feature. "
-                    + CommonProxyErrors.not_premium_user.value
+                    "error": "Project management is an enterprise feature. " + CommonProxyErrors.not_premium_user.value
                 },
             )
 
@@ -778,9 +745,7 @@ async def delete_project(
 
         for project_id in data.project_ids:
             # Check if project exists
-            existing_project = await prisma_client.db.litellm_projecttable.find_unique(
-                where={"project_id": project_id}
-            )
+            existing_project = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": project_id})
 
             if existing_project is None:
                 raise ProxyException(
@@ -791,11 +756,9 @@ async def delete_project(
                 )
 
             # Check if there are any keys associated with this project
-            associated_keys = (
-                await prisma_client.db.litellm_verificationtoken.find_many(
-                    where={"project_id": project_id}
-                )
-            )
+            associated_keys: Sequence[
+                prisma_models.LiteLLM_VerificationToken
+            ] = await prisma_client.db.litellm_verificationtoken.find_many(where={"project_id": project_id})
 
             if len(associated_keys) > 0:
                 raise ProxyException(
@@ -806,9 +769,9 @@ async def delete_project(
                 )
 
             # Delete the project
-            deleted_project = await prisma_client.db.litellm_projecttable.delete(
-                where={"project_id": project_id}
-            )
+            deleted_project: (
+                prisma_models.LiteLLM_ProjectTable | None
+            ) = await prisma_client.db.litellm_projecttable.delete(where={"project_id": project_id})
 
             deleted_projects.append(deleted_project)
 
@@ -854,7 +817,7 @@ async def project_info(
             )
 
         # Fetch project
-        project = await prisma_client.db.litellm_projecttable.find_unique(
+        project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.find_unique(
             where={"project_id": project_id},
             include={"litellm_budget_table": True, "object_permission": True},
         )
@@ -872,17 +835,11 @@ async def project_info(
         is_team_member = False
 
         if project.team_id and user_api_key_dict.user_id:
-            team = await prisma_client.db.litellm_teamtable.find_unique(
-                where={"team_id": project.team_id}
-            )
+            team = await _team_table(prisma_client).find_unique(where={"team_id": project.team_id})
             if team:
                 caller_user_id = user_api_key_dict.user_id
                 for m in team.members_with_roles or []:
-                    m_user_id = (
-                        m.get("user_id")
-                        if isinstance(m, dict)
-                        else getattr(m, "user_id", None)
-                    )
+                    m_user_id = m.get("user_id") if isinstance(m, dict) else getattr(m, "user_id", None)
                     if m_user_id == caller_user_id:
                         is_team_member = True
                         break
@@ -896,9 +853,7 @@ async def project_info(
         return project
     except Exception as e:
         verbose_proxy_logger.exception(
-            "litellm.proxy.management_endpoints.project_endpoints.project_info(): Exception occured - {}".format(
-                str(e)
-            )
+            "litellm.proxy.management_endpoints.project_endpoints.project_info(): Exception occured - {}".format(str(e))
         )
         raise handle_exception_on_proxy(e)
 
@@ -907,7 +862,7 @@ async def project_info(
     "/project/list",
     tags=["project management"],
     dependencies=[Depends(user_api_key_auth)],
-    response_model=List[LiteLLM_ProjectTable],
+    response_model=list[LiteLLM_ProjectTable],
 )
 async def list_projects(
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@@ -932,21 +887,19 @@ async def list_projects(
 
         # If proxy admin, get all projects
         if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
-            projects = await prisma_client.db.litellm_projecttable.find_many(
+            projects: Sequence[
+                prisma_models.LiteLLM_ProjectTable
+            ] = await prisma_client.db.litellm_projecttable.find_many(
                 include={"litellm_budget_table": True, "object_permission": True}
             )
         else:
             # Look up the user's team memberships via the reverse-index on
             # LiteLLM_UserTable.teams (maintained by team_member_add alongside
             # members_with_roles). This avoids a full scan of all team rows.
-            user_record = await prisma_client.db.litellm_usertable.find_unique(
+            user_record: prisma_models.LiteLLM_UserTable | None = await prisma_client.db.litellm_usertable.find_unique(
                 where={"user_id": user_api_key_dict.user_id},
             )
-            user_team_ids = (
-                user_record.teams
-                if user_record is not None and user_record.teams
-                else []
-            )
+            user_team_ids: Sequence[str] = user_record.teams if user_record is not None and user_record.teams else []
 
             projects = await prisma_client.db.litellm_projecttable.find_many(
                 where={"team_id": {"in": user_team_ids}},

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.50"
+version = "0.1.52"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.50"
+version = "0.1.52"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -46,6 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra bedrock-realtime \
         --python python3
 
 # Stage 2 — copy source and install the project + workspace members.
@@ -57,6 +58,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
+        --extra bedrock-realtime \
         --python python3
 
 RUN mkdir -p /home/nonroot && \

gateway/routes/allowlist.py

@@ -54,6 +54,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/messages",
     "/v1/skills",
     "/v1/a2a/",
+    "/a2a/",
     # LiteLLM-native LLM surface
     "/v1/rerank",
     "/v2/rerank",

helm/litellm/templates/NOTES.txt

@@ -46,4 +46,9 @@ Reminders:
       - gateway.config.proxy_config                            (rendered into a ConfigMap and mounted at
                                                                 /app/config/config.yaml; gateway reads it via
                                                                 CONFIG_FILE_PATH)
+      - {component}.pdb.{enabled,minAvailable,maxUnavailable}  (per-component PodDisruptionBudget; disabled by
+                                                                default — with hpa.minReplicas of 1, minAvailable: 1
+                                                                would block node drains)
+      - {component}.topologySpreadConstraints                  (standard k8s list, e.g. spread replicas across
+                                                                topology.kubernetes.io/zone)
   - Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend.

helm/litellm/templates/_helpers.tpl

@@ -295,6 +295,52 @@ harmless no-op for the Job and authoritative for the app pods.
 {{- end }}
 {{- end -}}
 
+{{/*
+PodDisruptionBudget shared by gateway, backend, and ui.
+
+Invoke with a dict:
+  (dict "root" $ "component" .Values.gateway "componentName" "gateway"
+        "fullname" (include "litellm.gateway.fullname" .)
+        "selectorLabels" (include "litellm.gateway.selectorLabels" .))
+
+Renders nothing unless both the component and its `pdb.enabled` are on.
+Only one of minAvailable / maxUnavailable should be set; if both are,
+minAvailable wins. If neither is set, falls back to `maxUnavailable: 1` so
+an enabled-but-unconfigured PDB still permits node drains.
+
+"Set" means non-nil and non-empty-string, so an explicit 0 (e.g.
+`maxUnavailable: 0` to forbid all voluntary disruptions) is honored rather
+than silently replaced by the fallback.
+*/}}
+{{- define "litellm.pdb" -}}
+{{- $root := .root -}}
+{{- $component := .component -}}
+{{- $min := $component.pdb.minAvailable -}}
+{{- $max := $component.pdb.maxUnavailable -}}
+{{- $minSet := not (or (kindIs "invalid" $min) (eq (printf "%v" $min) "")) -}}
+{{- $maxSet := not (or (kindIs "invalid" $max) (eq (printf "%v" $max) "")) -}}
+{{- if and $component.enabled $component.pdb $component.pdb.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ .fullname }}
+  labels:
+    {{- include "litellm.commonLabels" $root | nindent 4 }}
+    app.kubernetes.io/component: {{ .componentName }}
+spec:
+  selector:
+    matchLabels:
+      {{- .selectorLabels | nindent 6 }}
+  {{- if $minSet }}
+  minAvailable: {{ $min }}
+  {{- else if $maxSet }}
+  maxUnavailable: {{ $max }}
+  {{- else }}
+  maxUnavailable: 1
+  {{- end }}
+{{- end }}
+{{- end -}}
+
 {{/*
 Renders `envFrom:` block for a component's `envConfigMaps` / `envSecrets`
 lists. Each entry is a resource name; the chart wires the whole ConfigMap /

helm/litellm/templates/backend/deployment.yaml

@@ -98,4 +98,8 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.backend.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

helm/litellm/templates/backend/poddisruptionbudget.yaml

@@ -0,0 +1,6 @@
+{{- include "litellm.pdb" (dict
+  "root" $
+  "component" .Values.backend
+  "componentName" "backend"
+  "fullname" (include "litellm.backend.fullname" .)
+  "selectorLabels" (include "litellm.backend.selectorLabels" .)) }}

helm/litellm/templates/gateway/deployment.yaml

@@ -100,4 +100,8 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.gateway.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

helm/litellm/templates/gateway/poddisruptionbudget.yaml

@@ -0,0 +1,6 @@
+{{- include "litellm.pdb" (dict
+  "root" $
+  "component" .Values.gateway
+  "componentName" "gateway"
+  "fullname" (include "litellm.gateway.fullname" .)
+  "selectorLabels" (include "litellm.gateway.selectorLabels" .)) }}

helm/litellm/templates/ingress.yaml

@@ -19,7 +19,7 @@
     "/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads"
     "/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes"
     "/v1/models" "/models" "/openai" "/engines"
-    "/v1/messages" "/messages" "/v1/skills" "/v1/a2a"
+    "/v1/messages" "/messages" "/v1/skills" "/v1/a2a" "/a2a"
     "/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag"
     "/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
     "/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"

helm/litellm/templates/ui/deployment.yaml

@@ -76,4 +76,8 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.ui.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

helm/litellm/templates/ui/poddisruptionbudget.yaml

@@ -0,0 +1,6 @@
+{{- include "litellm.pdb" (dict
+  "root" $
+  "component" .Values.ui
+  "componentName" "ui"
+  "fullname" (include "litellm.ui.fullname" .)
+  "selectorLabels" (include "litellm.ui.selectorLabels" .)) }}

helm/litellm/tests/pdb_topology_spread_tests.yaml

@@ -0,0 +1,188 @@
+suite: test pod disruption budgets and topology spread constraints
+templates:
+  - gateway/poddisruptionbudget.yaml
+  - backend/poddisruptionbudget.yaml
+  - ui/poddisruptionbudget.yaml
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - ui/deployment.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: renders no PDB by default
+    templates:
+      - gateway/poddisruptionbudget.yaml
+      - backend/poddisruptionbudget.yaml
+      - ui/poddisruptionbudget.yaml
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: gateway PDB uses minAvailable and matches the gateway selector labels
+    template: gateway/poddisruptionbudget.yaml
+    set:
+      gateway.pdb.enabled: true
+      gateway.pdb.minAvailable: 1
+    asserts:
+      - isKind:
+          of: PodDisruptionBudget
+      - equal:
+          path: apiVersion
+          value: policy/v1
+      - equal:
+          path: metadata.name
+          value: RELEASE-NAME-litellm-gateway
+      - equal:
+          path: spec.minAvailable
+          value: 1
+      - notExists:
+          path: spec.maxUnavailable
+      - equal:
+          path: spec.selector.matchLabels
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+            app.kubernetes.io/component: gateway
+
+  - it: backend PDB uses maxUnavailable when minAvailable is unset
+    template: backend/poddisruptionbudget.yaml
+    set:
+      backend.pdb.enabled: true
+      backend.pdb.maxUnavailable: 25%
+    asserts:
+      - equal:
+          path: spec.maxUnavailable
+          value: 25%
+      - notExists:
+          path: spec.minAvailable
+      - equal:
+          path: spec.selector.matchLabels
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+            app.kubernetes.io/component: backend
+
+  - it: minAvailable wins when both minAvailable and maxUnavailable are set
+    template: gateway/poddisruptionbudget.yaml
+    set:
+      gateway.pdb.enabled: true
+      gateway.pdb.minAvailable: 2
+      gateway.pdb.maxUnavailable: 1
+    asserts:
+      - equal:
+          path: spec.minAvailable
+          value: 2
+      - notExists:
+          path: spec.maxUnavailable
+
+  - it: an explicit maxUnavailable 0 is honored instead of the fallback
+    template: backend/poddisruptionbudget.yaml
+    set:
+      backend.pdb.enabled: true
+      backend.pdb.maxUnavailable: 0
+    asserts:
+      - equal:
+          path: spec.maxUnavailable
+          value: 0
+      - notExists:
+          path: spec.minAvailable
+
+  - it: an explicit minAvailable 0 is honored and beats a set maxUnavailable
+    template: gateway/poddisruptionbudget.yaml
+    set:
+      gateway.pdb.enabled: true
+      gateway.pdb.minAvailable: 0
+      gateway.pdb.maxUnavailable: 1
+    asserts:
+      - equal:
+          path: spec.minAvailable
+          value: 0
+      - notExists:
+          path: spec.maxUnavailable
+
+  - it: enabled PDB with neither knob set falls back to maxUnavailable 1
+    template: ui/poddisruptionbudget.yaml
+    set:
+      ui.pdb.enabled: true
+    asserts:
+      - equal:
+          path: spec.maxUnavailable
+          value: 1
+      - notExists:
+          path: spec.minAvailable
+      - equal:
+          path: spec.selector.matchLabels
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+            app.kubernetes.io/component: ui
+
+  - it: renders no PDB for a disabled component even when its pdb is enabled
+    template: gateway/poddisruptionbudget.yaml
+    set:
+      gateway.enabled: false
+      gateway.pdb.enabled: true
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: deployments omit topologySpreadConstraints by default
+    templates:
+      - gateway/deployment.yaml
+      - backend/deployment.yaml
+      - ui/deployment.yaml
+    asserts:
+      - notExists:
+          path: spec.template.spec.topologySpreadConstraints
+
+  - it: gateway deployment renders configured topologySpreadConstraints
+    template: gateway/deployment.yaml
+    set:
+      gateway.topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: topology.kubernetes.io/zone
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/component: gateway
+    asserts:
+      - equal:
+          path: spec.template.spec.topologySpreadConstraints
+          value:
+            - maxSkew: 1
+              topologyKey: topology.kubernetes.io/zone
+              whenUnsatisfiable: ScheduleAnyway
+              labelSelector:
+                matchLabels:
+                  app.kubernetes.io/component: gateway
+
+  - it: backend deployment renders configured topologySpreadConstraints
+    template: backend/deployment.yaml
+    set:
+      backend.topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/component: backend
+    asserts:
+      - equal:
+          path: spec.template.spec.topologySpreadConstraints[0].topologyKey
+          value: kubernetes.io/hostname
+      - equal:
+          path: spec.template.spec.topologySpreadConstraints[0].whenUnsatisfiable
+          value: DoNotSchedule
+
+  - it: ui deployment renders configured topologySpreadConstraints
+    template: ui/deployment.yaml
+    set:
+      ui.topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: topology.kubernetes.io/zone
+          whenUnsatisfiable: ScheduleAnyway
+    asserts:
+      - equal:
+          path: spec.template.spec.topologySpreadConstraints[0].topologyKey
+          value: topology.kubernetes.io/zone

helm/litellm/values.yaml

@@ -190,10 +190,28 @@ gateway:
     maxReplicas: 10
     targetCPUUtilizationPercentage: 70
     targetMemoryUtilizationPercentage: 80
+  # PodDisruptionBudget for the gateway pods. Set exactly one of
+  # `minAvailable` / `maxUnavailable` (minAvailable wins if both are set;
+  # enabling without either falls back to `maxUnavailable: 1`). Disabled by
+  # default: with the default hpa.minReplicas of 1, a `minAvailable: 1` PDB
+  # would block node drains entirely.
+  pdb:
+    enabled: false
+    minAvailable: ""
+    maxUnavailable: ""
   podAnnotations: {}
   nodeSelector: {}
   tolerations: []
   affinity: {}
+  # Standard k8s topologySpreadConstraints for the gateway pods, e.g. to
+  # spread replicas across zones:
+  #   - maxSkew: 1
+  #     topologyKey: topology.kubernetes.io/zone
+  #     whenUnsatisfiable: ScheduleAnyway
+  #     labelSelector:
+  #       matchLabels:
+  #         app.kubernetes.io/component: gateway
+  topologySpreadConstraints: []
 
 # ---------- backend (UI / management API) ----------
 backend:
@@ -233,10 +251,17 @@ backend:
     minReplicas: 1
     maxReplicas: 4
     targetCPUUtilizationPercentage: 70
+  # Same shape as gateway.pdb.
+  pdb:
+    enabled: false
+    minAvailable: ""
+    maxUnavailable: ""
   podAnnotations: {}
   nodeSelector: {}
   tolerations: []
   affinity: {}
+  # Same shape as gateway.topologySpreadConstraints.
+  topologySpreadConstraints: []
 
 # ---------- ui (Next.js static dashboard) ----------
 ui:
@@ -279,7 +304,14 @@ ui:
     minReplicas: 1
     maxReplicas: 3
     targetCPUUtilizationPercentage: 80
+  # Same shape as gateway.pdb.
+  pdb:
+    enabled: false
+    minAvailable: ""
+    maxUnavailable: ""
   podAnnotations: {}
   nodeSelector: {}
   tolerations: []
   affinity: {}
+  # Same shape as gateway.topologySpreadConstraints.
+  topologySpreadConstraints: []

litellm-proxy-extras/litellm_proxy_extras/migrations/20260715000000_add_issuer_to_mcp_server_table/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "issuer" TEXT;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_compression_saved_tokens/migration.sql

@@ -0,0 +1,17 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_mcp_server_oauth_client_table/migration.sql

@@ -0,0 +1,9 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_MCPServerOAuthClient" (
+    "server_id" TEXT NOT NULL,
+    "credentials" JSONB,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_MCPServerOAuthClient_pkey" PRIMARY KEY ("server_id")
+);

litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql

@@ -0,0 +1,23 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260721000000_add_sso_identity_assertion/migration.sql

@@ -0,0 +1,9 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_SSOIdentityAssertion" (
+    "user_id" TEXT NOT NULL,
+    "assertion_b64" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_SSOIdentityAssertion_pkey" PRIMARY KEY ("user_id")
+);

litellm-proxy-extras/litellm_proxy_extras/migrations/20260724000000_add_spend_log_tool_index_start_time_idx/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogToolIndex_start_time_idx" ON "LiteLLM_SpendLogToolIndex"("start_time");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260725000000_add_daily_tool_spend/migration.sql

@@ -0,0 +1,12 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_DailyToolSpend" (
+    "date" TEXT NOT NULL,
+    "tool_name" TEXT NOT NULL,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+    "total_tokens" BIGINT NOT NULL DEFAULT 0,
+    "request_count" BIGINT NOT NULL DEFAULT 0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_DailyToolSpend_pkey" PRIMARY KEY ("date","tool_name")
+);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -326,6 +326,7 @@ model LiteLLM_MCPServerTable {
   command      String?
   args         String[]       @default([])
   env          Json?          @default("{}")
+  issuer             String?
   authorization_url  String?
   token_url          String?
   registration_url   String?
@@ -396,6 +397,22 @@ model LiteLLM_MCPUserEnvVars {
   @@index([server_id])
 }
 
+model LiteLLM_MCPServerOAuthClient {
+  server_id   String   @id
+  credentials Json?
+  created_at  DateTime @default(now()) @map("created_at")
+  updated_at  DateTime @default(now()) @updatedAt @map("updated_at")
+}
+
+// The enterprise IdP identity assertion captured at SSO login, one row per user.
+// assertion_b64 is an encrypted JSON payload: {id_token, refresh_token?, issuer?, expires_at?}.
+model LiteLLM_SSOIdentityAssertion {
+  user_id       String   @id
+  assertion_b64 String
+  created_at    DateTime @default(now()) @map("created_at")
+  updated_at    DateTime @default(now()) @updatedAt @map("updated_at")
+}
+
 // Generate Tokens for Proxy
 model LiteLLM_VerificationToken {
     token      String   @id
@@ -729,6 +746,9 @@ model LiteLLM_DailyUserSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
@@ -760,6 +780,9 @@ model LiteLLM_DailyOrganizationSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
@@ -791,6 +814,9 @@ model LiteLLM_DailyEndUserSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
@@ -821,6 +847,9 @@ model LiteLLM_DailyAgentSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
@@ -851,6 +880,9 @@ model LiteLLM_DailyTeamSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
@@ -883,6 +915,9 @@ model LiteLLM_DailyTagSpend {
   completion_tokens   BigInt      @default(0)
   cache_read_input_tokens     BigInt      @default(0)
   cache_creation_input_tokens BigInt      @default(0)
+  compression_saved_tokens    BigInt      @default(0)
+  compression_savings_spend    Float       @default(0.0)
+  prompt_caching_savings_spend Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
@@ -1060,6 +1095,20 @@ model LiteLLM_SpendLogToolIndex {
 
   @@id([request_id, tool_name])
   @@index([tool_name, start_time])
+  @@index([start_time])
+}
+
+// Daily tool spend rollup (one row per tool per day) – the Cost Optimization card reads this, never SpendLogs
+model LiteLLM_DailyToolSpend {
+  date          String
+  tool_name     String
+  spend         Float    @default(0.0)
+  total_tokens  BigInt   @default(0)
+  request_count BigInt   @default(0)
+  created_at    DateTime @default(now())
+  updated_at    DateTime @updatedAt
+
+  @@id([date, tool_name])
 }
 
 // Prompt table for storing prompt configurations

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.77"
+version = "0.4.81"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.77"
+version = "0.4.81"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/ADDING_A_PROVIDER.md

@@ -6,4 +6,23 @@ Three layers, same for every route (see `ocr` and `realtime` as references):
 2. **Provider config (pure)** — `crates/providers/src/<provider>/<route>/transformation.rs`: implement that trait as a `const <PROVIDER>_<ROUTE>_CONFIG`, mirroring the Python provider tree. Add parity unit tests.
 3. **HTTP / transport (the host)** — `crates/providers/src/<route>.rs` (e.g. `ocr.rs`, `realtime.rs`): the callable fn (`run_ocr`, `realtime`). It resolves the key, builds the auth header, builds URL + transforms via the config, then does the network call. This is the only layer allowed to do I/O.
 
+## Coding standards
+
+Before writing new logic, look for an existing base to extend. When a change is
+“the same behavior for one more provider/endpoint/integration”, the codebase
+almost always already has a shared abstraction for it (for example, provider
+`BaseConfig` transformation classes in `litellm/llms/base_llm/`, shared
+helpers in `litellm_core_utils/`, typed request/response models, or factory
+functions). Find it first with a search, then add the new variant by inheriting
+from or composing that base, overriding only what genuinely differs (model
+name, parameter mapping, or auth).
+
+Never copy an existing implementation and edit it in place, and never hand-roll
+a parallel version of logic a base already provides. If you catch yourself
+writing a second copy of a pattern that exists twice already, stop and extract a
+base instead: put the shared shape in one place and make both call sites thin
+variants of it. The test for a good abstraction is that adding the next provider
+is a few declarative lines, not a new file of duplicated flow. Only diverge from
+the base when behavior is genuinely different, and say so explicitly in the PR.
+
 **Calling:** the host invokes the route fn — the Python bridge calls `run_ocr`; the `ai-gateway` server calls `realtime`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.

litellm-rust/AGENTS.md

@@ -15,3 +15,12 @@ Dependency direction (acyclic): litellm-core ← litellm-ai-gateway ← litellm-
 Adding a crate: default to a MODULE. New crate ONLY on a real trigger — separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these.
 
 Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional.
+
+## Style
+
+All Rust in `litellm-rust/` follows the official Rust Style Guide:
+https://doc.rust-lang.org/style-guide/
+
+`rustfmt` implements its formatting by default, so run `cargo fmt` before committing; CI gates every PR on `cargo fmt --check`. Do not hand-format against rustfmt or add a `rustfmt.toml` that diverges from the default style.
+
+Beyond formatting, follow the guide's naming and idiom conventions rustfmt cannot auto-apply: `snake_case` items/functions/modules, `UpperCamelCase` types/traits/variants, `SCREAMING_SNAKE_CASE` constants/statics (acronyms as one word, e.g. `HttpClient`), and the import grouping and item ordering it prescribes. See CLAUDE.md for the detailed version.

litellm-rust/CLAUDE.md

@@ -2,6 +2,25 @@
 
 This file defines the rules for Rust work in LiteLLM.
 
+## Provider Coding Standards
+
+Before writing new logic, look for an existing base to extend. When a change is
+“the same behavior for one more provider/endpoint/integration”, the codebase
+almost always already has a shared abstraction for it (for example, provider
+`BaseConfig` transformation classes in `litellm/llms/base_llm/`, shared
+helpers in `litellm_core_utils/`, typed request/response models, or factory
+functions). Find it first with a search, then add the new variant by inheriting
+from or composing that base, overriding only what genuinely differs (model
+name, parameter mapping, or auth).
+
+Never copy an existing implementation and edit it in place, and never hand-roll
+a parallel version of logic a base already provides. If you catch yourself
+writing a second copy of a pattern that exists twice already, stop and extract a
+base instead: put the shared shape in one place and make both call sites thin
+variants of it. The test for a good abstraction is that adding the next provider
+is a few declarative lines, not a new file of duplicated flow. Only diverge from
+the base when behavior is genuinely different, and say so explicitly in the PR.
+
 ## Crates (exactly three — see AGENTS.md)
 
 `litellm-core` describes work; `litellm-ai-gateway` executes it; `litellm-python-bridge`
@@ -20,6 +39,11 @@ Route-level Rust structure mirrors LiteLLM's Python responsibilities:
 - Network execution lives in the host crate `ai-gateway` (`ai-gateway/src/io/`),
   never inside `core`.
 
+Call-hook and lifecycle instrumentation, including phase timing, usage
+accumulation, and callback payload construction, always lives in `core`.
+Hosts feed observed events into core and dispatch the completed payloads through
+their I/O logger; hosts must not own callback orchestration.
+
 Allowed in `core`:
 - Pure request transforms
 - Pure response transforms
@@ -38,6 +62,13 @@ Not allowed in `core`:
 
 Python owns rollout state and fallback while Rust is being introduced. Rust
 paths must be off by default until parity tests prove equivalence with Python.
+A new provider/route may instead be implemented rust-only with no Python
+reference; then the Python interface is a thin dispatch that calls Rust with no
+fallback, and you state the rust-only choice explicitly in the PR. Either way
+the Python side stays minimal (it only marshals inputs and calls the Rust
+interface), never add a per-route feature flag, and never push provider
+dispatch into `litellm/main.py`; put it in a thin dispatch class under
+`litellm/llms/<provider>/<route>/`.
 
 ## Production Bar
 
@@ -77,6 +108,26 @@ such as `ai-gateway`, router hosts, or standalone servers:
 - Avoid `expect`/`unwrap` in server startup and request paths unless the panic is
   impossible by construction and documented.
 
+## Rust Style Guide
+
+All Rust in `litellm-rust/` follows the official Rust Style Guide:
+https://doc.rust-lang.org/style-guide/
+
+`rustfmt` implements the guide's formatting rules by default, so the mechanical
+side is enforced for you: run `cargo fmt` before committing and CI gates every
+PR on `cargo fmt --check` (see Checks). Do not hand-format against rustfmt or add
+a `rustfmt.toml` that diverges from the default style; the default style *is* the
+guide.
+
+The guide also covers conventions rustfmt cannot auto-apply; follow these too:
+- Naming: `snake_case` for items, functions, and modules; `UpperCamelCase` for
+  types, traits, and enum variants; `SCREAMING_SNAKE_CASE` for constants and
+  statics; acronyms count as one word (`HttpClient`, not `HTTPClient`).
+- Ordering and grouping the guide prescribes: imports grouped std / external /
+  crate-local, derives before other attributes, and consistent item order.
+- Idioms the guide recommends over the formatter fighting you (e.g. prefer
+  restructuring an over-long expression rather than forcing an awkward wrap).
+
 ## Constants
 
 Magic numbers and fixed strings go in a crate-level `constants.rs`, never

litellm-rust/Cargo.toml

@@ -7,7 +7,8 @@ members = [
 resolver = "2"
 
 [workspace.package]
-edition = "2021"
+edition = "2024"
+rust-version = "1.88"
 license = "MIT"
 repository = "https://github.com/BerriAI/litellm"
 
@@ -15,8 +16,8 @@ repository = "https://github.com/BerriAI/litellm"
 litellm-core = { path = "crates/core" }
 litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false }
 axum = "0.7"
-pyo3 = "0.23.5"
-pyo3-async-runtimes = { version = "0.23.0", features = ["tokio-runtime"] }
+pyo3 = "0.29.0"
+pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] }
 rand = "0.8"
 reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] }
 serde = { version = "1.0", features = ["derive"] }

litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md

@@ -0,0 +1,59 @@
+# Provider coding standards (litellm-rust)
+
+Rules for adding or changing an LLM provider/route in `litellm-rust`. OCR (`MISTRAL_OCR_CONFIG`) is the reference; `messages` (`ANTHROPIC_MESSAGES_CONFIG`) is the next port.
+
+## Provider resolution
+
+1. Always resolve the provider/model first with `get_custom_llm_provider` (`core/src/routing_utils/provider.rs`). Nothing downstream may branch on a raw model string.
+2. Model/provider is resolved once, in `prepare.rs`, and passed down as typed fields. Don't re-resolve or re-parse it in transforms or handlers.
+
+## Transforms and the base config
+
+3. Every route defines a base config trait with `transform_request` + `transform_response` (+ `complete_url`, `supported_params`), living in `core/src/<route>/transformation.rs` (e.g. `AnthropicMessagesProviderConfig`, mirroring `OcrProviderConfig`).
+4. Each provider implements that trait as a `const <PROVIDER>_<ROUTE>_CONFIG` in `core/src/providers/<provider>/<route>/transformation.rs`, mirroring the Python provider tree.
+5. Individual configs implement only the request/response transforms. Shared behavior (param filtering, defaults) stays as trait default methods so future providers inherit existing logic instead of reimplementing it.
+6. Prefer composition: a provider that extends another reuses the base trait's defaults or wraps another config; don't copy transform bodies between providers.
+
+## Boundaries
+
+7. Layers never cross: `core` = pure transforms/types (no network, env, secrets, auth, logging, global mutable state); `ai-gateway` = all I/O, auth headers, HTTP/SSE, lifecycle hooks; `python-bridge` = thin PyO3 adapter.
+8. Generic/route files contain zero provider-specific branches. A provider is one module under `core/src/providers/<provider>/<route>/`; a route is a module, never a new crate.
+9. Route entry point stays thin: `<route>()` -> `prepare_*` -> `CallLifecycle::run_request`, which owns the pre_call -> during_call -> provider call -> success/failure order and phase timing. Handlers validate and delegate; no business logic in them.
+10. Constants (URLs, env-var names, API versions, error messages) live in a crate `constants.rs`, never inline. Env reads happen only at the host/config layer, with the `DEFAULT_*` fallback defined in `constants.rs`.
+
+## Types and errors
+
+11. Typed contracts only: no bare `serde_json::Value` / `String` / `Vec<String>` as a transform input or output. Parse wire bytes into typed structs/enums at the host edge; a `type` discriminator is a typed field, not a raw string.
+12. Model failures as values: return typed `CoreError`, don't panic. No `unwrap`/`expect`/`panic!` on user or provider input.
+13. No mutation: build values in one shot (comprehensions/iterators, `collect`), prefer immutable bindings and owned typed structs over seeding-and-mutating.
+14. Early returns over deep nesting; small focused files over god modules.
+15. Preserve Python output shape intentionally. If a field is always serialized as `null` for parity, keep it and pin it with a test.
+
+## Safety and data minimization
+
+16. Never log request/response bodies, base64 payloads, document contents, or secrets. Truncate and bound any upstream body before it crosses a host boundary.
+17. Treat empty/whitespace credentials, URLs, and config values as absent at the host resolution layer.
+18. Host I/O sets connect + request timeouts (no unbounded waits), reuses a shared HTTP client, and prefers rustls TLS.
+
+## Tests and rollout
+
+19. Every provider transform ships tests for: supported-param filtering, request body shape, response normalization, missing/null fields, bad input, and `*_match_python` fixture parity.
+20. Lifecycle/hook tests cover hook order, success + failure callback payloads, pre-call guardrail blocking before any provider I/O, during-call body mutation, and provider-error mapping.
+21. When a route has a Python reference implementation, the Rust path stays off by default and behind Python parity tests (disabled / enabled-equals-Python / bridge-unavailable fallback) until parity is proven. A new provider/route may instead be implemented rust-only with no Python reference; then the Python interface is a thin dispatch to Rust with no fallback, and tests cover the rust-backed path plus the unavailable-bridge error. State the rust-only choice explicitly in the PR.
+
+## Python bridge (SDK side)
+
+22. A Python -> Rust bridge keeps the Python side minimal: the Python interface only marshals inputs and calls the Rust interface, with no transform, handler, or business logic. Aim for well under 100 lines of interface code per route; if the Python grows past that, the logic belongs in Rust.
+23. Do not bloat `litellm/main.py`. A route's provider dispatch lives in a thin dispatch class under `litellm/llms/<provider>/<route>/` that calls the Rust bridge; `main.py` only instantiates it and calls its sync/async method.
+24. Do not add new feature flags unless explicitly requested. Reuse the existing litellm rust rollout mechanism (`use_litellm_rust`); never introduce a per-route env flag such as `LITELLM_USE_RUST_<ROUTE>`.
+
+## Checks before push
+
+25. Run, and keep green:
+    ```bash
+    cd litellm-rust
+    cargo fmt --check
+    cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings
+    cargo clippy -p litellm-core -p litellm-python-bridge --all-targets -- -D warnings
+    cargo test --workspace
+    ```

litellm-rust/crates/ai-gateway/Cargo.toml

@@ -14,7 +14,7 @@ path = "src/main.rs"
 required-features = ["server"]
 
 [dependencies]
-litellm-core.workspace = true
+litellm-core = { workspace = true, features = ["bedrock-auth"] }
 # reqwest (rustls + json) is used by io/ocr and ships realtime logs to the
 # Python proxy callbacks API.
 reqwest.workspace = true
@@ -41,3 +41,4 @@ python-config = ["dep:pyo3"]
 
 [dev-dependencies]
 futures-channel = "0.3"
+tower = { version = "0.5.3", features = ["util"] }

litellm-rust/crates/ai-gateway/src/audio_transcription/common_utils.rs

@@ -0,0 +1,48 @@
+use std::collections::BTreeMap;
+
+use litellm_core::CoreResult;
+use litellm_core::audio_transcription::transformation::AudioTranscriptionProviderConfig;
+use litellm_core::error::CoreError;
+use litellm_core::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG;
+use serde_json::{Map, Value};
+
+pub(super) fn audio_transcription_provider_config(
+    provider: &str,
+) -> Option<&'static dyn AudioTranscriptionProviderConfig> {
+    match provider {
+        "bedrock" => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
+        _ => None,
+    }
+}
+
+pub(super) fn string_headers(
+    headers: Option<Map<String, Value>>,
+) -> CoreResult<BTreeMap<String, String>> {
+    headers
+        .unwrap_or_default()
+        .into_iter()
+        .map(|(key, value)| {
+            value
+                .as_str()
+                .map(|value| (key.clone(), value.to_string()))
+                .ok_or_else(|| {
+                    CoreError::InvalidRequest(format!(
+                        "audio transcription extra_headers.{key} must be a string"
+                    ))
+                })
+        })
+        .collect()
+}
+
+pub(super) fn has_header(headers: &BTreeMap<String, String>, name: &str) -> bool {
+    headers.keys().any(|key| key.eq_ignore_ascii_case(name))
+}
+
+pub(super) fn truncate_error_body(body: &str) -> String {
+    let truncated: String = body.chars().take(256).collect();
+    if truncated.chars().count() == body.chars().count() {
+        truncated
+    } else {
+        format!("{truncated}... (truncated)")
+    }
+}

litellm-rust/crates/ai-gateway/src/audio_transcription/handler.rs

@@ -0,0 +1,89 @@
+use std::time::SystemTime;
+
+use litellm_core::CoreResult;
+use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth;
+use litellm_core::error::CoreError;
+use litellm_core::providers::bedrock::audio_transcription::aws_auth_config;
+use litellm_core::providers::bedrock::aws_base::{resolve_credentials, sign_bedrock_post};
+use serde_json::Value;
+
+use super::common_utils::truncate_error_body;
+use super::types::ProviderAudioTranscriptionRequest;
+use crate::client::http_client;
+
+pub(crate) async fn execute_audio_transcription_provider_call(
+    request: ProviderAudioTranscriptionRequest,
+) -> CoreResult<Value> {
+    let body = serde_json::to_vec(&request.body).map_err(|error| {
+        CoreError::InvalidRequest(format!("invalid audio request body: {error}"))
+    })?;
+    let mut request_builder = http_client().post(&request.url).body(body.clone());
+    for (key, value) in &request.upstream_headers {
+        request_builder = request_builder.header(key, value);
+    }
+    if let Some(duration) = request.timeout {
+        request_builder = request_builder.timeout(duration);
+    }
+    let response = request_builder
+        .send()
+        .await
+        .map_err(|error| CoreError::Network(error.to_string()))?;
+    let status = response.status();
+    let text = response
+        .text()
+        .await
+        .map_err(|error| CoreError::Network(error.to_string()))?;
+    if !status.is_success() {
+        return Err(CoreError::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&text),
+        });
+    }
+    let response_json: Value = serde_json::from_str(&text).map_err(|error| {
+        CoreError::InvalidResponse(format!("invalid audio response JSON: {error}"))
+    })?;
+    Ok(request
+        .config
+        .transform_transcription_response(&request.model, response_json)?
+        .into_json())
+}
+
+pub(crate) async fn sign_request(
+    request: &ProviderAudioTranscriptionRequest,
+    optional_params: &serde_json::Map<String, Value>,
+) -> CoreResult<ProviderAudioTranscriptionRequest> {
+    let env_lookup = environment_lookup;
+    let auth = request
+        .config
+        .auth_strategy(&request.model, optional_params, &env_lookup)?;
+    let body = serde_json::to_vec(&request.body).map_err(|error| {
+        CoreError::InvalidRequest(format!("invalid audio request body: {error}"))
+    })?;
+    let mut headers = super::common_utils::string_headers(None)?;
+    headers.insert("Content-Type".to_string(), "application/json".to_string());
+    headers.extend(request.upstream_headers.iter().cloned());
+    match auth {
+        AudioTranscriptionAuth::Bearer => {}
+        AudioTranscriptionAuth::AwsSigV4 { region, .. } => {
+            let credentials =
+                resolve_credentials(aws_auth_config(optional_params, &env_lookup), &env_lookup)
+                    .await?;
+            headers.extend(sign_bedrock_post(
+                &request.url,
+                &body,
+                &headers,
+                &region,
+                &credentials,
+                SystemTime::now(),
+            )?);
+        }
+    }
+    Ok(ProviderAudioTranscriptionRequest {
+        upstream_headers: headers.into_iter().collect(),
+        ..request.clone()
+    })
+}
+
+pub(super) fn environment_lookup(key: &str) -> Option<String> {
+    std::env::var(key).ok()
+}

litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs

@@ -0,0 +1,300 @@
+use std::future::Future;
+use std::pin::Pin;
+
+use litellm_core::CoreResult;
+use litellm_core::audio_transcription::transformation::AudioTranscriptionAuth;
+use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming};
+use litellm_core::error::CoreError;
+use serde_json::{Map, Value, json};
+
+use super::common_utils::{audio_transcription_provider_config, has_header, string_headers};
+use super::handler::sign_request;
+use super::types::{PreparedAudioTranscriptionRequest, ProviderAudioTranscriptionRequest};
+use crate::integrations::custom_guardrail::{
+    CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest,
+};
+use crate::integrations::custom_logger::{
+    CallType, CallbackTiming, CallbackValue, CustomLoggerRunner, LoggingError, ModelCallDetails,
+};
+use crate::integrations::types::{
+    RequestMetadata, StandardLoggingMetadata, StandardLoggingPayload,
+};
+
+pub(crate) struct AudioTranscriptionLifecycleHooks {
+    logger_runner: CustomLoggerRunner,
+    guardrail_runner: CustomGuardrailRunner,
+    request_metadata: RequestMetadata,
+}
+
+type AudioFuture<'a, T> = Pin<Box<dyn Future<Output = CoreResult<T>> + Send + 'a>>;
+type AudioLogFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
+
+impl AudioTranscriptionLifecycleHooks {
+    pub(crate) fn new(
+        logger_runner: CustomLoggerRunner,
+        guardrail_runner: CustomGuardrailRunner,
+        request_metadata: RequestMetadata,
+    ) -> Self {
+        Self {
+            logger_runner,
+            guardrail_runner,
+            request_metadata,
+        }
+    }
+
+    async fn run_pre_call_guardrails(
+        &self,
+        request: PreparedAudioTranscriptionRequest,
+    ) -> CoreResult<PreparedAudioTranscriptionRequest> {
+        if self.guardrail_runner.is_empty() {
+            return Ok(request);
+        }
+        let (guardrail_request, _) = self
+            .guardrail_runner
+            .run_pre_call(
+                &guardrail_context(&self.request_metadata),
+                GuardrailRequest::new(json!({
+                    "model": request.model,
+                    "custom_llm_provider": request.custom_llm_provider,
+                    "audio": request.audio,
+                    "optional_params": request.optional_params,
+                })),
+            )
+            .await
+            .map_err(guardrail_error_to_core_error)?;
+        let Value::Object(mut data) = guardrail_request.data else {
+            return Err(CoreError::InvalidRequest(
+                "audio transcription pre_call guardrail must return an object".to_string(),
+            ));
+        };
+        let audio = data.remove("audio").ok_or_else(|| {
+            CoreError::InvalidRequest("audio transcription guardrail removed audio".to_string())
+        })?;
+        let optional_params = match data.remove("optional_params") {
+            Some(Value::Object(value)) => value,
+            Some(_) => {
+                return Err(CoreError::InvalidRequest(
+                    "audio transcription optional_params must be an object".to_string(),
+                ));
+            }
+            None => Map::new(),
+        };
+        Ok(PreparedAudioTranscriptionRequest {
+            audio,
+            optional_params,
+            ..request
+        })
+    }
+
+    async fn prepare_provider_request(
+        &self,
+        request: PreparedAudioTranscriptionRequest,
+    ) -> CoreResult<ProviderAudioTranscriptionRequest> {
+        let config = audio_transcription_provider_config(&request.custom_llm_provider)
+            .ok_or_else(|| CoreError::InvalidProvider(request.custom_llm_provider.clone()))?;
+        let env_lookup = super::handler::environment_lookup;
+        let headers = string_headers(request.extra_headers)?;
+        let url = config.complete_url(
+            request.api_base.as_deref(),
+            &request.model,
+            &request.optional_params,
+            &env_lookup,
+        )?;
+        let filtered_params = config.map_transcription_params(&request.optional_params);
+        let body = config.transform_transcription_request(
+            &request.model,
+            request.audio,
+            filtered_params,
+        )?;
+        let auth = config.auth_strategy(&request.model, &request.optional_params, &env_lookup)?;
+        let mut upstream_headers = headers.into_iter().collect::<Vec<_>>();
+        if matches!(auth, AudioTranscriptionAuth::Bearer)
+            && !has_header(
+                &upstream_headers
+                    .iter()
+                    .cloned()
+                    .collect::<std::collections::BTreeMap<_, _>>(),
+                "authorization",
+            )
+            && let Some(api_key) = request.api_key.as_deref()
+        {
+            upstream_headers.push(("Authorization".to_string(), format!("Bearer {api_key}")));
+        }
+        let provider_request = ProviderAudioTranscriptionRequest {
+            model: request.model,
+            config,
+            url,
+            body: body.body,
+            upstream_headers,
+            timeout: request.timeout,
+        };
+        let provider_request = self.run_during_call_guardrails(provider_request).await?;
+        sign_request(&provider_request, &request.optional_params).await
+    }
+
+    async fn run_during_call_guardrails(
+        &self,
+        request: ProviderAudioTranscriptionRequest,
+    ) -> CoreResult<ProviderAudioTranscriptionRequest> {
+        if self.guardrail_runner.is_empty() {
+            return Ok(request);
+        }
+        let (guardrail_request, _) = self
+            .guardrail_runner
+            .run_during_call(
+                &guardrail_context(&self.request_metadata),
+                GuardrailRequest::new(json!({
+                    "model": request.model,
+                    "custom_llm_provider": "bedrock",
+                    "url": request.url,
+                    "body": request.body,
+                })),
+            )
+            .await
+            .map_err(guardrail_error_to_core_error)?;
+        let Value::Object(mut data) = guardrail_request.data else {
+            return Err(CoreError::InvalidRequest(
+                "audio transcription during_call guardrail must return an object".to_string(),
+            ));
+        };
+        let body = data.remove("body").ok_or_else(|| {
+            CoreError::InvalidRequest("audio transcription guardrail removed body".to_string())
+        })?;
+        Ok(ProviderAudioTranscriptionRequest { body, ..request })
+    }
+
+    fn logging_payload(
+        &self,
+        context: &CallLifecycleContext,
+        timing: &CallLifecycleTiming,
+    ) -> StandardLoggingPayload {
+        StandardLoggingPayload {
+            id: context.litellm_call_id.clone(),
+            litellm_call_id: context.litellm_call_id.clone(),
+            call_type: context.call_type.clone(),
+            model: context.model.clone(),
+            custom_llm_provider: context.custom_llm_provider.clone(),
+            response_cost: 0.0,
+            prompt_tokens: 0,
+            completion_tokens: 0,
+            total_tokens: 0,
+            start_time: timing.start_time,
+            end_time: timing.end_time,
+            stream: false,
+            metadata: StandardLoggingMetadata {
+                user_api_key_hash: self.request_metadata.user_api_key_hash.clone(),
+                user_api_key_user_id: self.request_metadata.user_api_key_user_id.clone(),
+                user_api_key_team_id: self.request_metadata.user_api_key_team_id.clone(),
+                ..Default::default()
+            },
+            messages: None,
+        }
+    }
+}
+
+impl CallLifecycleHooks<PreparedAudioTranscriptionRequest, ProviderAudioTranscriptionRequest, Value>
+    for AudioTranscriptionLifecycleHooks
+{
+    type PreCallFuture<'a> = AudioFuture<'a, PreparedAudioTranscriptionRequest>;
+    type DuringCallFuture<'a> = AudioFuture<'a, ProviderAudioTranscriptionRequest>;
+    type SuccessFuture<'a> = AudioLogFuture<'a>;
+    type FailureFuture<'a> = AudioLogFuture<'a>;
+
+    fn async_pre_call_hook<'a>(
+        &'a self,
+        _context: &'a CallLifecycleContext,
+        request: PreparedAudioTranscriptionRequest,
+    ) -> Self::PreCallFuture<'a> {
+        Box::pin(async move { self.run_pre_call_guardrails(request).await })
+    }
+
+    fn async_during_call_hook<'a>(
+        &'a self,
+        _context: &'a CallLifecycleContext,
+        request: PreparedAudioTranscriptionRequest,
+    ) -> Self::DuringCallFuture<'a> {
+        Box::pin(async move { self.prepare_provider_request(request).await })
+    }
+
+    fn async_log_success_event<'a>(
+        &'a self,
+        context: &'a CallLifecycleContext,
+        response: &'a Value,
+        timing: &'a CallLifecycleTiming,
+    ) -> Self::SuccessFuture<'a> {
+        Box::pin(async move {
+            if self.logger_runner.is_empty() {
+                return;
+            }
+            self.logger_runner
+                .async_log_success_event(
+                    &ModelCallDetails::from_standard_logging_payload(
+                        self.logging_payload(context, timing),
+                    ),
+                    &CallbackValue::new("audio_transcription", response.clone()),
+                    CallbackTiming::new(timing.start_time, timing.end_time),
+                )
+                .await;
+        })
+    }
+
+    fn async_log_failure_event<'a>(
+        &'a self,
+        context: &'a CallLifecycleContext,
+        error: &'a CoreError,
+        timing: &'a CallLifecycleTiming,
+    ) -> Self::FailureFuture<'a> {
+        Box::pin(async move {
+            if self.logger_runner.is_empty() {
+                return;
+            }
+            let logging_error = LoggingError {
+                message: error.to_string(),
+                kind: core_error_kind(error).to_string(),
+            };
+            self.logger_runner
+                .async_log_failure_event(
+                    &ModelCallDetails::from_standard_logging_payload(
+                        self.logging_payload(context, timing),
+                    )
+                    .with_failure_error(logging_error.clone()),
+                    Some(&CallbackValue::new(
+                        "error",
+                        json!({"message": logging_error.message, "kind": logging_error.kind}),
+                    )),
+                    CallbackTiming::new(timing.start_time, timing.end_time),
+                )
+                .await;
+        })
+    }
+}
+
+fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext {
+    GuardrailContext {
+        call_type: CallType::Other("audio_transcription".to_string()),
+        selected_guardrails: Vec::new(),
+        metadata: std::collections::HashMap::new(),
+        user_api_key_hash: metadata.user_api_key_hash.clone(),
+        user_api_key_user_id: metadata.user_api_key_user_id.clone(),
+        user_api_key_team_id: metadata.user_api_key_team_id.clone(),
+        trace_parent: None,
+    }
+}
+
+fn guardrail_error_to_core_error(error: GuardrailError) -> CoreError {
+    CoreError::InvalidRequest(format!("{}: {}", error.kind, error.message))
+}
+
+fn core_error_kind(error: &CoreError) -> &'static str {
+    match error {
+        CoreError::Auth(_) => "AuthError",
+        CoreError::InvalidProvider(_) => "InvalidProvider",
+        CoreError::InvalidRequest(_) => "InvalidRequest",
+        CoreError::InvalidType { .. } => "InvalidType",
+        CoreError::MissingField(_) => "MissingField",
+        CoreError::Http { .. } => "HttpError",
+        CoreError::InvalidResponse(_) => "InvalidResponse",
+        CoreError::Network(_) => "NetworkError",
+        CoreError::Routing(_) => "RoutingError",
+    }
+}

litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs

@@ -0,0 +1,25 @@
+use litellm_core::CoreResult;
+use litellm_core::call_lifecycle::CallLifecycle;
+use serde_json::Value;
+
+mod common_utils;
+mod handler;
+mod hooks;
+mod prepare;
+mod types;
+
+pub use types::AudioTranscriptionRequest;
+
+use handler::execute_audio_transcription_provider_call;
+use prepare::{PreparedAudioTranscriptionCall, prepare_audio_transcription_call};
+
+pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> CoreResult<Value> {
+    let PreparedAudioTranscriptionCall { request, hooks } =
+        prepare_audio_transcription_call(request);
+    CallLifecycle::default()
+        .run_request(request, &hooks, execute_audio_transcription_provider_call)
+        .await
+}
+
+#[cfg(test)]
+mod tests;

litellm-rust/crates/ai-gateway/src/audio_transcription/prepare.rs

@@ -0,0 +1,55 @@
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
+
+use super::hooks::AudioTranscriptionLifecycleHooks;
+use super::types::{AudioTranscriptionRequest, PreparedAudioTranscriptionRequest};
+use crate::integrations::custom_guardrail::CustomGuardrailRunner;
+use crate::integrations::custom_logger::CustomLoggerRunner;
+
+pub(crate) struct PreparedAudioTranscriptionCall {
+    pub(crate) request: PreparedAudioTranscriptionRequest,
+    pub(crate) hooks: AudioTranscriptionLifecycleHooks,
+}
+
+pub(crate) fn prepare_audio_transcription_call(
+    request: AudioTranscriptionRequest<'_>,
+) -> PreparedAudioTranscriptionCall {
+    let call_id = request
+        .litellm_call_id
+        .map(str::to_string)
+        .unwrap_or_else(new_audio_transcription_call_id);
+    let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider)
+        .unwrap_or(CustomLlmProvider {
+            model: request.model,
+            custom_llm_provider: "bedrock",
+        });
+    PreparedAudioTranscriptionCall {
+        request: PreparedAudioTranscriptionRequest {
+            model: provider_info.model.to_string(),
+            custom_llm_provider: provider_info.custom_llm_provider.to_string(),
+            litellm_call_id: call_id,
+            audio: request.audio,
+            api_key: request.api_key.map(str::to_string),
+            api_base: request.api_base.map(str::to_string),
+            extra_headers: request.extra_headers,
+            optional_params: request.optional_params,
+            timeout: request.timeout,
+        },
+        hooks: AudioTranscriptionLifecycleHooks::new(
+            CustomLoggerRunner::new(request.callbacks),
+            CustomGuardrailRunner::new(request.guardrails),
+            request.request_metadata,
+        ),
+    }
+}
+
+fn new_audio_transcription_call_id() -> String {
+    static COUNTER: AtomicU64 = AtomicU64::new(1);
+    let sequence = COUNTER.fetch_add(1, Ordering::Relaxed);
+    let timestamp = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_or(0, |duration| duration.as_nanos());
+    format!("audio-transcription-{timestamp}-{sequence}")
+}

litellm-rust/crates/ai-gateway/src/audio_transcription/tests.rs

@@ -0,0 +1,53 @@
+use std::io::{Read, Write};
+use std::net::TcpListener;
+use std::thread;
+
+use serde_json::{Map, json};
+
+use super::{AudioTranscriptionRequest, audio_transcription};
+
+#[tokio::test]
+async fn bedrock_request_is_signed_and_contains_audio() {
+    let listener = TcpListener::bind("127.0.0.1:0").expect("listener");
+    let address = listener.local_addr().expect("address");
+    let server = thread::spawn(move || {
+        let (mut stream, _) = listener.accept().expect("connection");
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 16_384];
+        let count = stream.read(&mut buffer).expect("request");
+        request.extend_from_slice(&buffer[..count]);
+        let request = String::from_utf8_lossy(&request);
+        assert!(request.contains("POST /model/mistral.voxtral-mini-3b-2507/converse"));
+        assert!(request.contains("authorization: AWS4-HMAC-SHA256"));
+        assert!(request.contains("x-amz-date:"));
+        assert!(request.contains("\"bytes\":\"AQI=\""));
+        assert!(request.contains("Transcribe the audio. Respond with only the transcript."));
+        let response = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 53\r\nConnection: close\r\n\r\n{\"output\":{\"message\":{\"content\":[{\"text\":\"hello\"}]}}}";
+        stream.write_all(response).expect("response");
+    });
+
+    let optional_params = Map::from_iter([
+        ("aws_access_key_id".to_string(), json!("access-key")),
+        ("aws_secret_access_key".to_string(), json!("secret-key")),
+        ("aws_region_name".to_string(), json!("us-east-1")),
+    ]);
+    let api_base = format!("http://{address}");
+    let response = audio_transcription(AudioTranscriptionRequest {
+        model: "mistral.voxtral-mini-3b-2507",
+        audio: json!({"data": "AQI=", "format": "wav", "filename": "audio.wav"}),
+        api_key: None,
+        api_base: Some(&api_base),
+        custom_llm_provider: Some("bedrock"),
+        extra_headers: None,
+        optional_params,
+        timeout: None,
+        callbacks: Vec::new(),
+        guardrails: Vec::new(),
+        request_metadata: Default::default(),
+        litellm_call_id: None,
+    })
+    .await
+    .expect("transcription");
+    assert_eq!(response, json!({"text": "hello"}));
+    server.join().expect("server");
+}

litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs

@@ -0,0 +1,58 @@
+use std::sync::Arc;
+use std::time::Duration;
+
+use litellm_core::audio_transcription::transformation::AudioTranscriptionProviderConfig;
+use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleRequest};
+use serde_json::{Map, Value};
+
+use crate::integrations::custom_guardrail::CustomGuardrail;
+use crate::integrations::custom_logger::CustomLogger;
+use crate::integrations::types::RequestMetadata;
+
+pub struct AudioTranscriptionRequest<'a> {
+    pub model: &'a str,
+    pub audio: Value,
+    pub api_key: Option<&'a str>,
+    pub api_base: Option<&'a str>,
+    pub custom_llm_provider: Option<&'a str>,
+    pub extra_headers: Option<Map<String, Value>>,
+    pub optional_params: Map<String, Value>,
+    pub timeout: Option<Duration>,
+    pub callbacks: Vec<Arc<dyn CustomLogger>>,
+    pub guardrails: Vec<Arc<dyn CustomGuardrail>>,
+    pub request_metadata: RequestMetadata,
+    pub litellm_call_id: Option<&'a str>,
+}
+
+pub(crate) struct PreparedAudioTranscriptionRequest {
+    pub(crate) model: String,
+    pub(crate) custom_llm_provider: String,
+    pub(crate) litellm_call_id: String,
+    pub(crate) audio: Value,
+    pub(crate) api_key: Option<String>,
+    pub(crate) api_base: Option<String>,
+    pub(crate) extra_headers: Option<Map<String, Value>>,
+    pub(crate) optional_params: Map<String, Value>,
+    pub(crate) timeout: Option<Duration>,
+}
+
+impl CallLifecycleRequest for PreparedAudioTranscriptionRequest {
+    fn lifecycle_context(&self) -> CallLifecycleContext {
+        CallLifecycleContext::new(
+            "audio_transcription",
+            self.model.clone(),
+            self.custom_llm_provider.clone(),
+            self.litellm_call_id.clone(),
+        )
+    }
+}
+
+#[derive(Clone)]
+pub(crate) struct ProviderAudioTranscriptionRequest {
+    pub(crate) model: String,
+    pub(crate) config: &'static dyn AudioTranscriptionProviderConfig,
+    pub(crate) url: String,
+    pub(crate) body: Value,
+    pub(crate) upstream_headers: Vec<(String, String)>,
+    pub(crate) timeout: Option<Duration>,
+}

litellm-rust/crates/ai-gateway/src/auth/mod.rs

@@ -9,9 +9,9 @@
 //! runs during extraction, before the handler body. Routes never re-implement it.
 
 use axum::extract::FromRequestParts;
+use axum::http::StatusCode;
 use axum::http::header::AUTHORIZATION;
 use axum::http::request::Parts;
-use axum::http::StatusCode;
 use sha2::{Digest, Sha256};
 use subtle::ConstantTimeEq;
 

litellm-rust/crates/ai-gateway/src/client.rs

@@ -1,13 +1,13 @@
 use std::sync::OnceLock;
 use std::time::Duration;
 
-const OCR_TIMEOUT_SECS: u64 = 600;
+const HTTP_CLIENT_TIMEOUT_SECS: u64 = 600;
 
-pub(super) fn http_client() -> &'static reqwest::Client {
+pub(crate) fn http_client() -> &'static reqwest::Client {
     static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
     CLIENT.get_or_init(|| {
         reqwest::Client::builder()
-            .timeout(Duration::from_secs(OCR_TIMEOUT_SECS))
+            .timeout(Duration::from_secs(HTTP_CLIENT_TIMEOUT_SECS))
             .build()
             .expect("failed to build reqwest client")
     })

litellm-rust/crates/ai-gateway/src/constants.rs

@@ -28,3 +28,31 @@ pub(crate) const DEFAULT_FLUSH_INTERVAL_MS: u64 = 500;
 /// Provider attributed to realtime sessions in the logging payload.
 #[cfg(feature = "server")]
 pub(crate) const DEFAULT_PROVIDER: &str = "openai";
+
+/// Full-request timeout ceiling for Anthropic Messages provider calls, in
+/// seconds. Mirrors the Python Anthropic Messages default. The per-request
+/// timeout from `litellm_params` still overrides this on the request builder.
+pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600;
+
+/// Connect timeout for Anthropic Messages provider calls, in seconds.
+pub(crate) const MESSAGES_CONNECT_TIMEOUT_SECS: u64 = 10;
+
+/// Max characters of an upstream error body echoed across the host boundary
+/// before truncation, so provider bodies are bounded and data-minimized.
+pub(crate) const MESSAGES_ERROR_BODY_MAX_CHARS: usize = 256;
+
+pub(crate) const DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS: u64 = 10;
+pub(crate) const DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS: u64 = 300;
+
+/// HTTP path for the non-streaming Anthropic Messages route.
+#[cfg(feature = "server")]
+pub(crate) const MESSAGES_ROUTE_PATH: &str = "/v1/messages";
+
+/// Provider name used by the Anthropic Messages route when a deployment's
+/// provider model does not carry an explicit provider prefix.
+pub(crate) const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic";
+
+/// Request headers owned by the gateway and never forwarded upstream.
+#[cfg(feature = "server")]
+pub(crate) const MESSAGES_HEADERS_NOT_FORWARDED: &[&str] =
+    &["authorization", "connection", "content-length", "host"];

litellm-rust/crates/ai-gateway/src/io/audio_transcription.rs

@@ -0,0 +1 @@
+pub use crate::audio_transcription::{AudioTranscriptionRequest, audio_transcription};

litellm-rust/crates/ai-gateway/src/io/messages.rs

@@ -0,0 +1 @@
+pub use crate::messages::{MessagesRequest, messages};

litellm-rust/crates/ai-gateway/src/io/mod.rs

@@ -1,3 +1,6 @@
+pub mod audio_transcription;
+pub mod messages;
 pub mod ocr;
 pub mod realtime;
 pub mod realtime_pool;
+pub mod responses_ws;

litellm-rust/crates/ai-gateway/src/io/ocr.rs

@@ -1 +1 @@
-pub use crate::ocr::{ocr, OcrRequest};
+pub use crate::ocr::{OcrRequest, ocr};

litellm-rust/crates/ai-gateway/src/io/realtime.rs

@@ -15,16 +15,16 @@ use std::time::Duration;
 
 use futures_util::stream::{SplitSink, SplitStream};
 use futures_util::{Sink, SinkExt, Stream, StreamExt};
+use litellm_core::CoreResult;
 use litellm_core::error::CoreError;
 use litellm_core::realtime::transformation::RealtimeProviderConfig;
 use litellm_core::realtime::types::RealtimeEvent;
-use litellm_core::CoreResult;
 use tokio::net::TcpStream;
-use tokio_tungstenite::tungstenite::client::IntoClientRequest;
-use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION;
-use tokio_tungstenite::tungstenite::http::HeaderValue;
 use tokio_tungstenite::tungstenite::Message;
-use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream};
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::tungstenite::http::HeaderValue;
+use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION;
+use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async};
 
 use litellm_core::providers::openai::realtime::transformation::OPENAI_REALTIME_CONFIG;
 
@@ -113,7 +113,7 @@ pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> CoreResult<Realt
             Message::Close(_) => {
                 return Err(CoreError::Network(
                     "upstream closed before first event".to_string(),
-                ))
+                ));
             }
             _ => continue,
         }

litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs

@@ -28,11 +28,11 @@ use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
 
 use futures_util::StreamExt;
-use litellm_core::realtime::types::RealtimeEvent;
 use litellm_core::CoreResult;
+use litellm_core::realtime::types::RealtimeEvent;
 
 use crate::io::realtime::{
-    dial_upstream, read_event, resolve_api_key, UpstreamRx, UpstreamTx, UpstreamWs,
+    UpstreamRx, UpstreamTx, UpstreamWs, dial_upstream, read_event, resolve_api_key,
 };
 
 /// Default target warm sockets per key when pooling is enabled.
@@ -473,8 +473,8 @@ pub fn upstream_key(
 /// unhealthy — we'd rather discard and fresh-dial than hand over a socket in an
 /// unexpected state. `Pending` (the healthy case) returns `false`.
 fn is_dead(rx: &mut UpstreamRx) -> bool {
-    use futures_util::task::noop_waker_ref;
     use futures_util::Stream;
+    use futures_util::task::noop_waker_ref;
     use std::pin::Pin;
     use std::task::{Context, Poll};
 
@@ -523,15 +523,15 @@ mod tests {
             ))
             .await;
         while let Some(Ok(msg)) = ws.next().await {
-            if let Message::Text(text) = msg {
-                if text.contains("response.create") {
-                    for frame in [
-                        r#"{"type":"response.created"}"#,
-                        r#"{"type":"response.output_audio.delta","delta":"AAAA"}"#,
-                        r#"{"type":"response.done"}"#,
-                    ] {
-                        let _ = ws.send(Message::Text(frame.to_string())).await;
-                    }
+            if let Message::Text(text) = msg
+                && text.contains("response.create")
+            {
+                for frame in [
+                    r#"{"type":"response.created"}"#,
+                    r#"{"type":"response.output_audio.delta","delta":"AAAA"}"#,
+                    r#"{"type":"response.done"}"#,
+                ] {
+                    let _ = ws.send(Message::Text(frame.to_string())).await;
                 }
             }
         }

litellm-rust/crates/ai-gateway/src/io/responses_ws.rs

@@ -0,0 +1,549 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+
+use futures_util::stream::{SplitSink, SplitStream};
+use futures_util::{Sink, SinkExt, Stream, StreamExt};
+use litellm_core::providers::openai::responses::transformation::OPENAI_RESPONSES_WS_CONFIG;
+use litellm_core::responses::types::ResponsesWsEvent;
+use litellm_core::responses::websocket::ResponsesWebSocketProviderConfig;
+use litellm_core::{CoreError, CoreResult};
+use tokio::net::TcpStream;
+use tokio::sync::Mutex;
+use tokio_tungstenite::tungstenite::Message;
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::tungstenite::http::HeaderValue;
+use tokio_tungstenite::tungstenite::http::header::{AUTHORIZATION, HeaderName};
+use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async};
+
+use crate::constants::{
+    DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS, DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS,
+};
+
+const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY";
+const MISSING_KEY_MESSAGE: &str = "Missing OpenAI API Key - a Responses WebSocket call is being made but no key was passed via params or the OPENAI_API_KEY environment variable";
+
+pub type ResponsesUpstreamWs = WebSocketStream<MaybeTlsStream<TcpStream>>;
+type UpstreamTx = SplitSink<ResponsesUpstreamWs, Message>;
+type UpstreamRx = SplitStream<ResponsesUpstreamWs>;
+
+#[derive(Clone)]
+pub struct ResponsesWebSocketConnection {
+    socket: Arc<Mutex<Option<ResponsesUpstreamWs>>>,
+}
+
+impl ResponsesWebSocketConnection {
+    pub async fn connect_url(
+        url: &str,
+        headers: &HashMap<String, String>,
+        timeout: Option<Duration>,
+    ) -> CoreResult<Self> {
+        let mut request = url
+            .into_client_request()
+            .map_err(|error| CoreError::Network(error.to_string()))?;
+        for (name, value) in headers {
+            let header_name = name
+                .parse::<HeaderName>()
+                .map_err(|error| CoreError::InvalidRequest(error.to_string()))?;
+            let header_value = HeaderValue::from_str(value)
+                .map_err(|error| CoreError::InvalidRequest(error.to_string()))?;
+            request.headers_mut().insert(header_name, header_value);
+        }
+        let connect = connect_async(request);
+        let result = match timeout {
+            Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| {
+                CoreError::Network("Responses WebSocket connection timed out".to_string())
+            })?,
+            None => connect.await,
+        };
+        let (socket, _) = result.map_err(|error| match error {
+            tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http {
+                status: response.status().as_u16(),
+                body: String::new(),
+            },
+            other => CoreError::Network(other.to_string()),
+        })?;
+        Ok(Self {
+            socket: Arc::new(Mutex::new(Some(socket))),
+        })
+    }
+
+    pub async fn send_text(&self, text: String) -> CoreResult<()> {
+        let mut socket = self.socket.lock().await;
+        let Some(socket) = socket.as_mut() else {
+            return Err(CoreError::Network(
+                "Responses WebSocket is closed".to_string(),
+            ));
+        };
+        socket
+            .send(Message::Text(text))
+            .await
+            .map_err(|error| CoreError::Network(error.to_string()))
+    }
+
+    pub async fn recv_text(&self) -> CoreResult<Option<String>> {
+        let mut socket_guard = self.socket.lock().await;
+        let Some(socket) = socket_guard.as_mut() else {
+            return Ok(None);
+        };
+        match socket.next().await {
+            Some(Ok(Message::Text(text))) => Ok(Some(text)),
+            Some(Ok(Message::Binary(bytes))) => String::from_utf8(bytes.to_vec())
+                .map(Some)
+                .map_err(|error| CoreError::InvalidResponse(error.to_string())),
+            Some(Ok(Message::Close(_))) | None => Ok(None),
+            Some(Ok(_)) => Ok(None),
+            Some(Err(error)) => Err(CoreError::Network(error.to_string())),
+        }
+    }
+
+    pub async fn close(&self) -> CoreResult<()> {
+        let mut socket = self.socket.lock().await;
+        if let Some(socket) = socket.as_mut() {
+            socket
+                .close(None)
+                .await
+                .map_err(|error| CoreError::Network(error.to_string()))?;
+        }
+        *socket = None;
+        Ok(())
+    }
+}
+
+pub(crate) fn resolve_api_key(api_key: Option<&str>) -> CoreResult<String> {
+    api_key
+        .map(str::trim)
+        .filter(|value| !value.is_empty())
+        .map(str::to_string)
+        .or_else(|| {
+            std::env::var(OPENAI_API_KEY_ENV)
+                .ok()
+                .filter(|value| !value.trim().is_empty())
+        })
+        .ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string()))
+}
+
+async fn dial_upstream(
+    model: &str,
+    api_key: &str,
+    api_base: Option<&str>,
+) -> CoreResult<ResponsesUpstreamWs> {
+    let url = OPENAI_RESPONSES_WS_CONFIG.complete_websocket_url(api_base, model);
+    let mut request = url
+        .as_str()
+        .into_client_request()
+        .map_err(|error| CoreError::Network(error.to_string()))?;
+    request.headers_mut().insert(
+        AUTHORIZATION,
+        HeaderValue::from_str(&format!("Bearer {api_key}"))
+            .map_err(|error| CoreError::Auth(error.to_string()))?,
+    );
+    let result = tokio::time::timeout(
+        Duration::from_secs(DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS),
+        connect_async(request),
+    )
+    .await
+    .map_err(|_| CoreError::Network("Responses WebSocket connection timed out".to_string()))?;
+    result
+        .map(|(socket, _)| socket)
+        .map_err(|error| match error {
+            tokio_tungstenite::tungstenite::Error::Http(response) => CoreError::Http {
+                status: response.status().as_u16(),
+                body: String::new(),
+            },
+            other => CoreError::Network(other.to_string()),
+        })
+}
+
+pub struct ResponsesWebSocketStreaming;
+
+impl ResponsesWebSocketStreaming {
+    pub async fn bidirectional_forward<In, Out>(
+        model: &str,
+        upstream_tx: UpstreamTx,
+        upstream_rx: UpstreamRx,
+        idle_timeout: Option<Duration>,
+        observe: impl FnMut(&ResponsesWsEvent) + Send,
+        client_in: In,
+        client_out: Out,
+    ) -> CoreResult<()>
+    where
+        In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
+        Out: Sink<ResponsesWsEvent> + Unpin + Send,
+        Out::Error: std::fmt::Display,
+    {
+        splice(
+            model,
+            upstream_tx,
+            upstream_rx,
+            idle_timeout,
+            observe,
+            client_in,
+            client_out,
+        )
+        .await
+    }
+}
+
+pub(crate) async fn splice<In, Out>(
+    model: &str,
+    mut upstream_tx: UpstreamTx,
+    mut upstream_rx: UpstreamRx,
+    idle_timeout: Option<Duration>,
+    mut observe: impl FnMut(&ResponsesWsEvent) + Send,
+    mut client_in: In,
+    mut client_out: Out,
+) -> CoreResult<()>
+where
+    In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
+    Out: Sink<ResponsesWsEvent> + Unpin + Send,
+    Out::Error: std::fmt::Display,
+{
+    let idle =
+        idle_timeout.unwrap_or_else(|| Duration::from_secs(DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS));
+    loop {
+        tokio::select! {
+            event = client_in.next() => {
+                let Some(event) = event else { break };
+                for outbound in OPENAI_RESPONSES_WS_CONFIG
+                    .transform_ws_request(&event, model)?
+                    .events
+                {
+                    let payload = serde_json::to_string(&outbound)
+                        .map_err(|error| CoreError::InvalidResponse(error.to_string()))?;
+                    upstream_tx.send(Message::Text(payload))
+                        .await
+                        .map_err(|error| CoreError::Network(error.to_string()))?;
+                }
+            }
+            message = upstream_rx.next() => {
+                let Some(message) = message else { break };
+                match message.map_err(|error| CoreError::Network(error.to_string()))? {
+                    Message::Text(text) => {
+                        let event = serde_json::from_str::<ResponsesWsEvent>(&text)
+                            .map_err(|error| CoreError::InvalidResponse(error.to_string()))?;
+                        observe(&event);
+                        for outbound in OPENAI_RESPONSES_WS_CONFIG
+                            .transform_ws_response(&event, model)?
+                            .events
+                        {
+                            client_out.send(outbound)
+                                .await
+                                .map_err(|error| CoreError::Network(error.to_string()))?;
+                        }
+                    }
+                    Message::Close(_) => break,
+                    _ => {}
+                }
+            }
+            _ = tokio::time::sleep(idle) => break,
+        }
+    }
+    Ok(())
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn async_responses_websocket<In, Out>(
+    model: &str,
+    api_key: Option<&str>,
+    api_base: Option<&str>,
+    first_frame: Option<ResponsesWsEvent>,
+    idle_timeout: Option<Duration>,
+    mut observe: impl FnMut(&ResponsesWsEvent) + Send,
+    client_in: In,
+    client_out: Out,
+) -> CoreResult<()>
+where
+    In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
+    Out: Sink<ResponsesWsEvent> + Unpin + Send,
+    Out::Error: std::fmt::Display,
+{
+    let key = resolve_api_key(api_key)?;
+    let upstream = dial_upstream(model, &key, api_base).await?;
+    let (mut upstream_tx, upstream_rx) = upstream.split();
+    if let Some(first_frame) = first_frame {
+        for outbound in OPENAI_RESPONSES_WS_CONFIG
+            .transform_ws_request(&first_frame, model)?
+            .events
+        {
+            let payload = serde_json::to_string(&outbound)
+                .map_err(|error| CoreError::InvalidResponse(error.to_string()))?;
+            upstream_tx
+                .send(Message::Text(payload))
+                .await
+                .map_err(|error| CoreError::Network(error.to_string()))?;
+        }
+    }
+    ResponsesWebSocketStreaming::bidirectional_forward(
+        model,
+        upstream_tx,
+        upstream_rx,
+        idle_timeout,
+        &mut observe,
+        client_in,
+        client_out,
+    )
+    .await
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn responses_ws<In, Out>(
+    model: &str,
+    api_key: Option<&str>,
+    api_base: Option<&str>,
+    first_frame: Option<ResponsesWsEvent>,
+    idle_timeout: Option<Duration>,
+    observe: impl FnMut(&ResponsesWsEvent) + Send,
+    client_in: In,
+    client_out: Out,
+) -> CoreResult<()>
+where
+    In: Stream<Item = ResponsesWsEvent> + Unpin + Send,
+    Out: Sink<ResponsesWsEvent> + Unpin + Send,
+    Out::Error: std::fmt::Display,
+{
+    async_responses_websocket(
+        model,
+        api_key,
+        api_base,
+        first_frame,
+        idle_timeout,
+        observe,
+        client_in,
+        client_out,
+    )
+    .await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use futures_channel::mpsc;
+    use futures_util::{SinkExt, StreamExt};
+    use litellm_core::responses::types::ResponsesWsEventType;
+    use serde_json::json;
+    use tokio::io::AsyncWriteExt;
+    use tokio::net::TcpListener;
+    use tokio_tungstenite::accept_async;
+
+    async fn websocket_base() -> (String, tokio::task::JoinHandle<()>) {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
+        let address = listener.local_addr().expect("local address");
+        let task = tokio::spawn(async move {
+            let (stream, _) = listener.accept().await.expect("accept");
+            let mut socket = accept_async(stream).await.expect("websocket handshake");
+            while let Some(Ok(Message::Text(text))) = socket.next().await {
+                let request: serde_json::Value = serde_json::from_str(&text).expect("request json");
+                let model = request
+                    .get("model")
+                    .and_then(serde_json::Value::as_str)
+                    .or_else(|| {
+                        request
+                            .get("response")
+                            .and_then(serde_json::Value::as_object)
+                            .and_then(|response| {
+                                response.get("model").and_then(serde_json::Value::as_str)
+                            })
+                    })
+                    .expect("enforced model");
+                socket
+                    .send(Message::Text(
+                        json!({
+                            "type": "response.created",
+                            "response": {
+                                "id": format!("resp-{model}"),
+                                "model": model,
+                                "extra": "preserved"
+                            }
+                        })
+                        .to_string(),
+                    ))
+                    .await
+                    .expect("created event");
+                socket
+                    .send(Message::Text(
+                        json!({
+                            "type": "response.completed",
+                            "response": {
+                                "id": format!("resp-{model}"),
+                                "model": model,
+                                "usage": {
+                                    "input_tokens": 1,
+                                    "output_tokens": 2,
+                                    "total_tokens": 3
+                                }
+                            }
+                        })
+                        .to_string(),
+                    ))
+                    .await
+                    .expect("completed event");
+            }
+        });
+        (format!("http://{address}"), task)
+    }
+
+    fn event(value: serde_json::Value) -> ResponsesWsEvent {
+        serde_json::from_value(value).expect("event")
+    }
+
+    #[test]
+    fn explicit_nonblank_key_wins() {
+        assert_eq!(
+            resolve_api_key(Some(" explicit ")).expect("key"),
+            "explicit"
+        );
+    }
+
+    #[test]
+    fn blank_key_is_not_accepted_without_environment_key() {
+        if std::env::var(OPENAI_API_KEY_ENV).is_err() {
+            assert!(resolve_api_key(Some("  ")).is_err());
+        }
+    }
+
+    #[tokio::test]
+    async fn forwards_events_sequentially_and_enforces_model() {
+        let (api_base, server) = websocket_base().await;
+        let (client_tx, client_rx) = mpsc::unbounded();
+        let (output_tx, mut output_rx) = mpsc::unbounded();
+        let (observed_tx, observed_rx) = mpsc::unbounded();
+        client_tx
+            .unbounded_send(event(json!({
+                "type": "response.create",
+                "model": "wrong"
+            })))
+            .expect("first request");
+        client_tx
+            .unbounded_send(event(json!({
+                "type": "response.create",
+                "response": {"model": "also-wrong"}
+            })))
+            .expect("second request");
+
+        let task = tokio::spawn(async move {
+            responses_ws(
+                "authorized-model",
+                Some("test-key"),
+                Some(&api_base),
+                None,
+                Some(Duration::from_secs(1)),
+                move |event| {
+                    observed_tx
+                        .unbounded_send(event.clone())
+                        .expect("observe event");
+                },
+                client_rx,
+                output_tx,
+            )
+            .await
+        });
+
+        let first = output_rx.next().await.expect("first output");
+        let second = output_rx.next().await.expect("second output");
+        let third = output_rx.next().await.expect("third output");
+        let fourth = output_rx.next().await.expect("fourth output");
+        drop(client_tx);
+        task.await.expect("splice task").expect("successful splice");
+        server.await.expect("server task");
+
+        assert_eq!(first.event_type, ResponsesWsEventType::ResponseCreated);
+        assert_eq!(first.model(), Some("authorized-model"));
+        assert_eq!(first.data["response"]["extra"], "preserved");
+        assert_eq!(second.event_type, ResponsesWsEventType::ResponseCompleted);
+        assert_eq!(third.event_type, ResponsesWsEventType::ResponseCreated);
+        assert_eq!(fourth.event_type, ResponsesWsEventType::ResponseCompleted);
+        let observed: Vec<_> = observed_rx.collect().await;
+        assert_eq!(observed.len(), 4);
+        assert!(
+            observed
+                .iter()
+                .all(|event| event.event_type != ResponsesWsEventType::ResponseCreate)
+        );
+    }
+
+    #[tokio::test]
+    async fn idle_timeout_ends_without_upstream_events() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
+        let address = listener.local_addr().expect("address");
+        let server = tokio::spawn(async move {
+            let (stream, _) = listener.accept().await.expect("accept");
+            let _socket = accept_async(stream).await.expect("handshake");
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        });
+        let (_client_tx, client_rx) = mpsc::unbounded::<ResponsesWsEvent>();
+        let (output_tx, mut output_rx) = mpsc::unbounded();
+        let result = responses_ws(
+            "model",
+            Some("key"),
+            Some(&format!("http://{address}")),
+            None,
+            Some(Duration::from_millis(20)),
+            |_| {},
+            client_rx,
+            output_tx,
+        )
+        .await;
+        assert!(result.is_ok());
+        assert!(output_rx.next().await.is_none());
+        server.abort();
+    }
+
+    #[tokio::test]
+    async fn dial_http_status_is_preserved() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
+        let address = listener.local_addr().expect("address");
+        let server = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("accept");
+            stream
+                .write_all(b"HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\n\r\n")
+                .await
+                .expect("response");
+        });
+        let (_client_tx, client_rx) = mpsc::unbounded::<ResponsesWsEvent>();
+        let (output_tx, _output_rx) = mpsc::unbounded();
+        let error = responses_ws(
+            "model",
+            Some("key"),
+            Some(&format!("http://{address}")),
+            None,
+            Some(Duration::from_millis(20)),
+            |_| {},
+            client_rx,
+            output_tx,
+        )
+        .await
+        .expect_err("status error");
+        assert!(matches!(error, CoreError::Http { status: 401, .. }));
+        server.await.expect("server task");
+    }
+
+    #[tokio::test]
+    async fn dial_http_500_status_is_preserved() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
+        let address = listener.local_addr().expect("address");
+        let server = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("accept");
+            stream
+                .write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n")
+                .await
+                .expect("response");
+        });
+        let (_client_tx, client_rx) = mpsc::unbounded::<ResponsesWsEvent>();
+        let (output_tx, _output_rx) = mpsc::unbounded();
+        let error = responses_ws(
+            "model",
+            Some("key"),
+            Some(&format!("http://{address}")),
+            None,
+            Some(Duration::from_millis(20)),
+            |_| {},
+            client_rx,
+            output_tx,
+        )
+        .await
+        .expect_err("status error");
+        assert!(matches!(error, CoreError::Http { status: 500, .. }));
+        server.await.expect("server task");
+    }
+}

litellm-rust/crates/ai-gateway/src/lib.rs

@@ -11,7 +11,10 @@
 //!   binary turns on. The `python-config` feature additionally pulls in [`python`]
 //!   for the load-time config reader.
 
+pub mod audio_transcription;
+mod client;
 pub mod io;
+pub mod messages;
 pub mod ocr;
 
 /// GIL-activity tracking. Pure (atomics only); shared by the `server` routes and
@@ -25,9 +28,6 @@ pub mod routes;
 #[cfg(feature = "server")]
 pub mod state;
 
-// Realtime request logging. Only the server serves realtime, so these are
-// `server`-gated; `io::realtime` exposes the generic `observe` hook while the
-// collector and callback fan-out live here.
 mod constants;
 pub mod integrations;
 #[cfg(feature = "server")]

litellm-rust/crates/ai-gateway/src/main.rs

@@ -11,7 +11,7 @@
 
 use std::sync::Arc;
 
-use litellm_ai_gateway::io::realtime_pool::{upstream_key, PoolConfig, RealtimePool};
+use litellm_ai_gateway::io::realtime_pool::{PoolConfig, RealtimePool, upstream_key};
 use litellm_ai_gateway::routes;
 use litellm_ai_gateway::state::AppState;
 use litellm_core::router::{Deployment, LiteLLMParams, Router};

litellm-rust/crates/ai-gateway/src/messages/client.rs

@@ -0,0 +1,15 @@
+use std::sync::OnceLock;
+use std::time::Duration;
+
+use crate::constants::{MESSAGES_CONNECT_TIMEOUT_SECS, MESSAGES_TIMEOUT_SECS};
+
+pub(super) fn http_client() -> &'static reqwest::Client {
+    static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
+    CLIENT.get_or_init(|| {
+        reqwest::Client::builder()
+            .timeout(Duration::from_secs(MESSAGES_TIMEOUT_SECS))
+            .connect_timeout(Duration::from_secs(MESSAGES_CONNECT_TIMEOUT_SECS))
+            .build()
+            .unwrap_or_else(|_| reqwest::Client::new())
+    })
+}

litellm-rust/crates/ai-gateway/src/messages/common_utils.rs

@@ -0,0 +1,64 @@
+use litellm_core::CoreResult;
+use litellm_core::error::{CoreError, json_type_name};
+use litellm_core::messages::transformation::AnthropicMessagesProviderConfig;
+use litellm_core::providers::anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG;
+use litellm_core::providers::azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG;
+use serde_json::{Map, Value};
+
+use crate::constants::MESSAGES_ERROR_BODY_MAX_CHARS;
+
+pub(super) fn truncate_error_body(body: &str) -> String {
+    if body.chars().count() <= MESSAGES_ERROR_BODY_MAX_CHARS {
+        return body.to_string();
+    }
+    let truncated: String = body.chars().take(MESSAGES_ERROR_BODY_MAX_CHARS).collect();
+    format!("{truncated}... (truncated)")
+}
+
+pub(super) fn messages_provider_config(
+    provider: &str,
+) -> Option<&'static dyn AnthropicMessagesProviderConfig> {
+    match provider {
+        "anthropic" => Some(&ANTHROPIC_MESSAGES_CONFIG),
+        "azure_ai" => Some(&AZURE_ANTHROPIC_MESSAGES_CONFIG),
+        _ => None,
+    }
+}
+
+pub(super) fn string_headers(
+    extra_headers: Option<Map<String, Value>>,
+) -> CoreResult<Vec<(String, String)>> {
+    extra_headers
+        .unwrap_or_default()
+        .into_iter()
+        .map(|(key, value)| {
+            value
+                .as_str()
+                .map(|value| (key.clone(), value.to_string()))
+                .ok_or_else(|| {
+                    CoreError::InvalidRequest(format!(
+                        "messages extra_headers.{key} must be a string, got {}",
+                        json_type_name(&value)
+                    ))
+                })
+        })
+        .collect()
+}
+
+pub(super) fn has_header(headers: &[(String, String)], name: &str) -> bool {
+    headers
+        .iter()
+        .any(|(key, _)| key.eq_ignore_ascii_case(name))
+}
+
+pub(super) fn has_bearer_auth(headers: &[(String, String)]) -> bool {
+    headers.iter().any(|(name, value)| {
+        if !name.eq_ignore_ascii_case("authorization") {
+            return false;
+        }
+        let value = value.trim();
+        value.len() > 7
+            && value[..7].eq_ignore_ascii_case("bearer ")
+            && !value[7..].trim().is_empty()
+    })
+}

litellm-rust/crates/ai-gateway/src/messages/handler.rs

@@ -0,0 +1,83 @@
+use litellm_core::CoreResult;
+use litellm_core::error::CoreError;
+use serde_json::Value;
+
+use super::client::http_client;
+use super::common_utils::truncate_error_body;
+use super::types::ProviderMessagesRequest;
+use crate::constants::ANTHROPIC_MESSAGES_PROVIDER;
+
+pub(super) async fn execute_messages_provider_call(
+    request: ProviderMessagesRequest,
+) -> CoreResult<Value> {
+    let mut request_builder = http_client().post(&request.url).json(&request.body);
+    for (key, value) in &request.upstream_headers {
+        request_builder = request_builder.header(key, value);
+    }
+    if let Some(duration) = request.timeout {
+        request_builder = request_builder.timeout(duration);
+    }
+
+    let response = request_builder
+        .send()
+        .await
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+
+    let status = response.status();
+    let text = response
+        .text()
+        .await
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+
+    if !status.is_success() {
+        return Err(CoreError::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&text),
+        });
+    }
+
+    let response = serde_json::from_str(&text).map_err(|err| {
+        CoreError::InvalidResponse(format!("invalid messages response JSON: {err}"))
+    })?;
+    let transformed = request
+        .config
+        .transform_response(&request.model, response)?;
+    serde_json::to_value(transformed).map_err(|err| {
+        CoreError::InvalidResponse(format!("failed to serialize messages response: {err}"))
+    })
+}
+
+pub(super) async fn execute_messages_provider_stream(
+    request: ProviderMessagesRequest,
+) -> CoreResult<reqwest::Response> {
+    if request.provider != ANTHROPIC_MESSAGES_PROVIDER {
+        return Err(CoreError::InvalidRequest(
+            "streaming messages is not supported for this provider".to_string(),
+        ));
+    }
+
+    let mut request_builder = http_client().post(&request.url).json(&request.body);
+    for (key, value) in &request.upstream_headers {
+        request_builder = request_builder.header(key, value);
+    }
+    if let Some(duration) = request.timeout {
+        request_builder = request_builder.timeout(duration);
+    }
+
+    let response = request_builder
+        .send()
+        .await
+        .map_err(|err| CoreError::Network(err.to_string()))?;
+    let status = response.status();
+    if !status.is_success() {
+        let text = response
+            .text()
+            .await
+            .map_err(|err| CoreError::Network(err.to_string()))?;
+        return Err(CoreError::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&text),
+        });
+    }
+    Ok(response)
+}

litellm-rust/crates/ai-gateway/src/messages/mod.rs

@@ -0,0 +1,49 @@
+use litellm_core::CoreResult;
+use serde_json::Value;
+
+mod client;
+mod common_utils;
+mod handler;
+mod prepare;
+mod types;
+
+pub use types::MessagesRequest;
+
+use handler::{execute_messages_provider_call, execute_messages_provider_stream};
+use prepare::prepare_messages_call;
+
+pub async fn messages(request: MessagesRequest<'_>) -> CoreResult<Value> {
+    match execute_messages(request, false).await? {
+        MessagesResponse::Json(body) => Ok(body),
+        MessagesResponse::Stream(response) => {
+            drop(response);
+            Err(litellm_core::CoreError::InvalidResponse(
+                "non-streaming messages execution returned a stream".to_string(),
+            ))
+        }
+    }
+}
+
+pub(crate) enum MessagesResponse {
+    Json(Value),
+    Stream(reqwest::Response),
+}
+
+pub(crate) async fn execute_messages(
+    request: MessagesRequest<'_>,
+    stream: bool,
+) -> CoreResult<MessagesResponse> {
+    let prepared = prepare_messages_call(request)?;
+    if stream {
+        execute_messages_provider_stream(prepared)
+            .await
+            .map(MessagesResponse::Stream)
+    } else {
+        execute_messages_provider_call(prepared)
+            .await
+            .map(MessagesResponse::Json)
+    }
+}
+
+#[cfg(test)]
+mod tests;

litellm-rust/crates/ai-gateway/src/messages/prepare.rs

@@ -0,0 +1,75 @@
+use litellm_core::CoreError;
+use litellm_core::CoreResult;
+use litellm_core::messages::transformation::MessagesAuthStrategy;
+use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
+
+use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers};
+use super::types::{MessagesRequest, ProviderMessagesRequest};
+
+pub(super) fn prepare_messages_call(
+    request: MessagesRequest<'_>,
+) -> CoreResult<ProviderMessagesRequest> {
+    let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider)
+        .or_else(|| {
+            request
+                .custom_llm_provider
+                .map(|provider| CustomLlmProvider {
+                    model: request.model,
+                    custom_llm_provider: provider,
+                })
+        })
+        .ok_or_else(|| {
+            CoreError::InvalidProvider(
+                "unable to resolve custom_llm_provider for messages request".to_string(),
+            )
+        })?;
+    let model = provider_info.model.to_string();
+    let provider = provider_info.custom_llm_provider;
+
+    let config = messages_provider_config(provider)
+        .ok_or_else(|| CoreError::InvalidProvider(provider.to_string()))?;
+    let env_lookup = |key: &str| std::env::var(key).ok();
+
+    let mut headers = string_headers(request.extra_headers)?;
+
+    let auth_strategy = config.auth_strategy();
+    let already_authorized = has_header(&headers, auth_strategy.header_name())
+        || (config.accepts_bearer_auth() && has_bearer_auth(&headers));
+    if !already_authorized {
+        let api_key = config.resolve_api_key(request.api_key, &env_lookup)?;
+        let auth_header = match auth_strategy {
+            MessagesAuthStrategy::Bearer => {
+                ("authorization".to_string(), format!("Bearer {api_key}"))
+            }
+            MessagesAuthStrategy::Header(name) => (name.to_string(), api_key),
+        };
+        headers.push(auth_header);
+    }
+
+    for (name, value) in config.default_headers() {
+        if !has_header(&headers, name) {
+            headers.push((name.to_string(), value.to_string()));
+        }
+    }
+
+    let url = config.complete_url(request.api_base, &model, &env_lookup)?;
+    let typed_request = serde_json::from_value(request.body).map_err(|err| {
+        CoreError::InvalidRequest(format!("invalid Anthropic messages request: {err}"))
+    })?;
+    let transformed = config.transform_request(typed_request)?;
+    let body = serde_json::to_value(transformed).map_err(|err| {
+        CoreError::InvalidRequest(format!(
+            "failed to serialize Anthropic messages request: {err}"
+        ))
+    })?;
+
+    Ok(ProviderMessagesRequest {
+        provider: provider.to_string(),
+        model,
+        config,
+        url,
+        body,
+        upstream_headers: headers,
+        timeout: request.timeout,
+    })
+}

litellm-rust/crates/ai-gateway/src/messages/tests.rs

@@ -0,0 +1,439 @@
+use std::time::Duration;
+
+use litellm_core::error::CoreError;
+use serde_json::{Map, Value, json};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio::net::{TcpListener, TcpStream};
+
+use super::common_utils::{
+    has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body,
+};
+use super::{MessagesRequest, messages};
+
+async fn read_http_request(socket: &mut TcpStream) -> String {
+    let mut request = Vec::new();
+    let mut buffer = [0_u8; 1024];
+    let header_end = loop {
+        let n = socket.read(&mut buffer).await.expect("reads request");
+        if n == 0 {
+            break request.len();
+        }
+        request.extend_from_slice(&buffer[..n]);
+        if let Some(position) = request.windows(4).position(|window| window == b"\r\n\r\n") {
+            break position + 4;
+        }
+    };
+    let headers = String::from_utf8_lossy(&request[..header_end]);
+    let content_length = headers
+        .lines()
+        .find_map(|line| {
+            let (name, value) = line.split_once(':')?;
+            name.eq_ignore_ascii_case("content-length")
+                .then(|| value.trim().parse::<usize>().ok())
+                .flatten()
+        })
+        .unwrap_or(0);
+    while request.len().saturating_sub(header_end) < content_length {
+        let n = socket.read(&mut buffer).await.expect("reads body");
+        if n == 0 {
+            break;
+        }
+        request.extend_from_slice(&buffer[..n]);
+    }
+    String::from_utf8(request).expect("request is utf8")
+}
+
+fn write_response(body: &str) -> String {
+    format!(
+        "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+        body.len(),
+        body
+    )
+}
+
+#[test]
+fn provider_config_resolves_anthropic_and_azure_ai() {
+    assert!(messages_provider_config("anthropic").is_some());
+    assert!(messages_provider_config("azure_ai").is_some());
+    assert!(messages_provider_config("openai").is_none());
+}
+
+#[test]
+fn truncate_error_body_caps_long_payloads() {
+    let body = "x".repeat(400);
+    let truncated = truncate_error_body(&body);
+    assert!(truncated.ends_with("... (truncated)"));
+    let prefix_chars = truncated
+        .strip_suffix("... (truncated)")
+        .expect("truncated marker present")
+        .chars()
+        .count();
+    assert_eq!(prefix_chars, 256);
+}
+
+#[test]
+fn string_headers_rejects_non_string_values() {
+    let headers = json!({"x-count": 3}).as_object().unwrap().clone();
+    let err = string_headers(Some(headers)).expect_err("non-string header rejected");
+    assert!(matches!(err, CoreError::InvalidRequest(_)));
+}
+
+#[test]
+fn has_header_is_case_insensitive() {
+    let headers = vec![("X-Api-Key".to_string(), "secret".to_string())];
+    assert!(has_header(&headers, "x-api-key"));
+    assert!(!has_header(&headers, "authorization"));
+}
+
+#[test]
+fn has_bearer_auth_requires_a_nonempty_bearer_token() {
+    assert!(has_bearer_auth(&[(
+        "Authorization".to_string(),
+        "Bearer tok".to_string()
+    )]));
+    assert!(has_bearer_auth(&[(
+        "authorization".to_string(),
+        "bearer tok".to_string()
+    )]));
+    assert!(!has_bearer_auth(&[(
+        "authorization".to_string(),
+        "Bearer ".to_string()
+    )]));
+    assert!(!has_bearer_auth(&[(
+        "authorization".to_string(),
+        String::new()
+    )]));
+    assert!(!has_bearer_auth(&[(
+        "authorization".to_string(),
+        "Basic abc".to_string()
+    )]));
+    assert!(!has_bearer_auth(&[(
+        "x-api-key".to_string(),
+        "sk".to_string()
+    )]));
+}
+
+#[tokio::test]
+async fn messages_round_trip_builds_azure_request_and_passes_response_through() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let request = read_http_request(&mut socket).await;
+        let response_body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"hi"}],"model":"claude-sonnet-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":2}}"#;
+        socket
+            .write_all(write_response(response_body).as_bytes())
+            .await
+            .expect("writes response");
+        request
+    });
+
+    let response = messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 1024,
+            "messages": [{
+                "role": "user",
+                "content": [{
+                    "type": "text",
+                    "text": "hi",
+                    "cache_control": {"type": "ephemeral", "scope": "global"}
+                }]
+            }]
+        }),
+        api_key: Some("sk-azure"),
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: None,
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect("messages request succeeds");
+
+    assert_eq!(response["content"][0]["text"], "hi");
+    assert_eq!(response["stop_reason"], "end_turn");
+
+    let request = server.await.expect("server task completes");
+    let (head, body) = request.split_once("\r\n\r\n").expect("has body");
+    assert!(head.starts_with("POST /anthropic/v1/messages "), "{head}");
+    let head_lower = head.to_ascii_lowercase();
+    assert!(head_lower.contains("x-api-key: sk-azure"), "{head}");
+    assert!(
+        head_lower.contains("anthropic-version: 2023-06-01"),
+        "{head}"
+    );
+    assert!(
+        head_lower.contains("content-type: application/json"),
+        "{head}"
+    );
+
+    let sent_body: Value = serde_json::from_str(body).expect("body is json");
+    assert_eq!(
+        sent_body["messages"][0]["content"][0]["cache_control"],
+        json!({"type": "ephemeral"})
+    );
+}
+
+#[tokio::test]
+async fn messages_round_trip_builds_native_anthropic_request() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let request = read_http_request(&mut socket).await;
+        let response_body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"hi"}],"model":"claude-sonnet-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":2}}"#;
+        socket
+            .write_all(write_response(response_body).as_bytes())
+            .await
+            .expect("writes response");
+        request
+    });
+
+    let response = messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({
+            "model": "claude-sonnet-4-5",
+            "max_tokens": 1024,
+            "messages": [{"role": "user", "content": "hi"}]
+        }),
+        api_key: Some("sk-ant"),
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("anthropic"),
+        extra_headers: None,
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect("messages request succeeds");
+
+    assert_eq!(response["content"][0]["text"], "hi");
+    assert_eq!(response["stop_reason"], "end_turn");
+
+    let request = server.await.expect("server task completes");
+    let (head, _) = request.split_once("\r\n\r\n").expect("has body");
+    assert!(head.starts_with("POST /v1/messages "), "{head}");
+    let head_lower = head.to_ascii_lowercase();
+    assert!(head_lower.contains("x-api-key: sk-ant"), "{head}");
+    assert!(
+        head_lower.contains("anthropic-version: 2023-06-01"),
+        "{head}"
+    );
+}
+
+#[tokio::test]
+async fn messages_does_not_duplicate_auth_when_x_api_key_supplied() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let request = read_http_request(&mut socket).await;
+        let response_body =
+            r#"{"id":"msg_2","type":"message","role":"assistant","content":[],"model":"m"}"#;
+        socket
+            .write_all(write_response(response_body).as_bytes())
+            .await
+            .expect("writes response");
+        request
+    });
+
+    let mut headers = Map::new();
+    headers.insert(
+        "x-api-key".to_string(),
+        Value::String("from-python".to_string()),
+    );
+    headers.insert(
+        "anthropic-beta".to_string(),
+        Value::String("token-efficient-tools-2025-02-19".to_string()),
+    );
+
+    messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
+        api_key: Some("rust-fallback-key"),
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: Some(headers),
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect("messages request succeeds");
+
+    let request = server.await.expect("server task completes");
+    let head = request
+        .split_once("\r\n\r\n")
+        .expect("has body")
+        .0
+        .to_ascii_lowercase();
+    let api_key_count = head
+        .lines()
+        .filter(|line| line.starts_with("x-api-key:"))
+        .count();
+    assert_eq!(api_key_count, 1, "{head}");
+    assert!(head.contains("x-api-key: from-python"), "{head}");
+    assert!(
+        head.contains("anthropic-beta: token-efficient-tools-2025-02-19"),
+        "{head}"
+    );
+    assert!(!head.contains("rust-fallback-key"), "{head}");
+}
+
+#[tokio::test]
+async fn messages_forwards_entra_id_bearer_without_requiring_api_key() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let request = read_http_request(&mut socket).await;
+        let response_body =
+            r#"{"id":"msg_3","type":"message","role":"assistant","content":[],"model":"m"}"#;
+        socket
+            .write_all(write_response(response_body).as_bytes())
+            .await
+            .expect("writes response");
+        request
+    });
+
+    let mut headers = Map::new();
+    headers.insert(
+        "Authorization".to_string(),
+        Value::String("Bearer entra-token".to_string()),
+    );
+
+    messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
+        api_key: None,
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: Some(headers),
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect("entra id request succeeds without api key");
+
+    let request = server.await.expect("server task completes");
+    let head = request
+        .split_once("\r\n\r\n")
+        .expect("has body")
+        .0
+        .to_ascii_lowercase();
+    assert!(head.contains("authorization: bearer entra-token"), "{head}");
+    assert!(!head.contains("x-api-key"), "{head}");
+}
+
+#[tokio::test]
+async fn messages_requires_auth_when_no_key_and_no_header() {
+    let err = messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
+        api_key: None,
+        api_base: Some("http://127.0.0.1:1"),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: None,
+        timeout: Some(Duration::from_millis(50)),
+    })
+    .await
+    .expect_err("missing auth errors");
+
+    assert!(matches!(err, CoreError::Auth(_)));
+}
+
+#[tokio::test]
+async fn messages_ignores_malformed_authorization_and_uses_api_key() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let request = read_http_request(&mut socket).await;
+        let response_body =
+            r#"{"id":"msg_4","type":"message","role":"assistant","content":[],"model":"m"}"#;
+        socket
+            .write_all(write_response(response_body).as_bytes())
+            .await
+            .expect("writes response");
+        request
+    });
+
+    let mut headers = Map::new();
+    headers.insert(
+        "Authorization".to_string(),
+        Value::String("Bearer ".to_string()),
+    );
+
+    messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
+        api_key: Some("sk-azure"),
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: Some(headers),
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect("falls back to api key");
+
+    let request = server.await.expect("server task completes");
+    let head = request
+        .split_once("\r\n\r\n")
+        .expect("has body")
+        .0
+        .to_ascii_lowercase();
+    assert!(head.contains("x-api-key: sk-azure"), "{head}");
+}
+
+#[tokio::test]
+async fn messages_maps_provider_error_status_to_http_error() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let _ = read_http_request(&mut socket).await;
+        let body = "unauthorized";
+        let response = format!(
+            "HTTP/1.1 401 Unauthorized\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+            body.len(),
+            body
+        );
+        socket
+            .write_all(response.as_bytes())
+            .await
+            .expect("writes response");
+    });
+
+    let err = messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
+        api_key: Some("sk-azure"),
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: None,
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect_err("provider error propagates");
+
+    assert!(matches!(err, CoreError::Http { status: 401, .. }));
+}
+
+#[tokio::test]
+async fn messages_rejects_unsupported_provider() {
+    let err = messages(MessagesRequest {
+        model: "claude-3-5-sonnet",
+        body: json!({"model": "claude-3-5-sonnet", "max_tokens": 8, "messages": []}),
+        api_key: Some("sk"),
+        api_base: Some("http://127.0.0.1:1"),
+        custom_llm_provider: Some("openai"),
+        extra_headers: None,
+        timeout: Some(Duration::from_millis(50)),
+    })
+    .await
+    .expect_err("unsupported provider errors");
+
+    assert!(matches!(err, CoreError::InvalidProvider(provider) if provider == "openai"));
+}

litellm-rust/crates/ai-gateway/src/messages/types.rs

@@ -0,0 +1,24 @@
+use std::time::Duration;
+
+use litellm_core::messages::transformation::AnthropicMessagesProviderConfig;
+use serde_json::{Map, Value};
+
+pub struct MessagesRequest<'a> {
+    pub model: &'a str,
+    pub body: Value,
+    pub api_key: Option<&'a str>,
+    pub api_base: Option<&'a str>,
+    pub custom_llm_provider: Option<&'a str>,
+    pub extra_headers: Option<Map<String, Value>>,
+    pub timeout: Option<Duration>,
+}
+
+pub(crate) struct ProviderMessagesRequest {
+    pub(crate) provider: String,
+    pub(crate) model: String,
+    pub(crate) config: &'static dyn AnthropicMessagesProviderConfig,
+    pub(crate) url: String,
+    pub(crate) body: Value,
+    pub(crate) upstream_headers: Vec<(String, String)>,
+    pub(crate) timeout: Option<Duration>,
+}

litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs

@@ -1,11 +1,11 @@
 use std::net::IpAddr;
 use std::time::{Duration, Instant};
 
-use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
 use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
+use litellm_core::CoreResult;
 use litellm_core::error::CoreError;
 use litellm_core::ocr::transformation::OcrProviderConfig;
-use litellm_core::CoreResult;
 use reqwest::Url;
 use serde_json::{Map, Value};
 
@@ -18,7 +18,7 @@ use litellm_core::providers::vertex_ai::ocr::transformation::{
     VERTEX_AI_DEEPSEEK_OCR_CONFIG, VERTEX_AI_OCR_CONFIG,
 };
 
-use super::client::http_client;
+use crate::client::http_client;
 
 const ERROR_BODY_MAX_CHARS: usize = 256;
 const AZURE_DOCUMENT_INTELLIGENCE_POLL_TIMEOUT_SECS: u64 = 120;

litellm-rust/crates/ai-gateway/src/ocr/handler.rs

@@ -1,11 +1,11 @@
+use litellm_core::CoreResult;
 use litellm_core::error::CoreError;
 use litellm_core::ocr::transformation::OcrResponseHandling;
-use litellm_core::CoreResult;
 use serde_json::Value;
 
-use super::client::http_client;
 use super::common_utils::{poll_document_intelligence, truncate_error_body};
 use super::types::ProviderOcrRequest;
+use crate::client::http_client;
 
 pub(crate) async fn execute_ocr_provider_call(request: ProviderOcrRequest) -> CoreResult<Value> {
     let mut request_builder = http_client().post(&request.url).json(&request.body);

litellm-rust/crates/ai-gateway/src/ocr/hooks.rs

@@ -1,11 +1,11 @@
 use std::future::Future;
 use std::pin::Pin;
 
+use litellm_core::CoreResult;
 use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming};
 use litellm_core::error::CoreError;
 use litellm_core::ocr::transformation::OcrAuthStrategy;
-use litellm_core::CoreResult;
-use serde_json::{json, Map, Value};
+use serde_json::{Map, Value, json};
 
 use super::common_utils::{
     convert_document_url_to_data_uri, has_header, ocr_provider_config, string_headers,
@@ -292,7 +292,7 @@ fn parse_ocr_pre_call_guardrail_request(
         Some(_) => {
             return Err(CoreError::InvalidRequest(
                 "OCR pre_call guardrail optional_params must be an object".to_string(),
-            ))
+            ));
         }
         None => Map::new(),
     };

litellm-rust/crates/ai-gateway/src/ocr/mod.rs

@@ -1,8 +1,7 @@
-use litellm_core::call_lifecycle::CallLifecycle;
 use litellm_core::CoreResult;
+use litellm_core::call_lifecycle::CallLifecycle;
 use serde_json::Value;
 
-mod client;
 mod common_utils;
 mod handler;
 mod hooks;
@@ -12,7 +11,7 @@ mod types;
 pub use types::OcrRequest;
 
 use handler::execute_ocr_provider_call;
-use prepare::{prepare_ocr_call, PreparedOcrCall};
+use prepare::{PreparedOcrCall, prepare_ocr_call};
 
 pub async fn ocr(request: OcrRequest<'_>) -> CoreResult<Value> {
     let PreparedOcrCall { request, hooks } = prepare_ocr_call(request);

litellm-rust/crates/ai-gateway/src/ocr/prepare.rs

@@ -1,7 +1,7 @@
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{SystemTime, UNIX_EPOCH};
 
-use litellm_core::routing_utils::provider::{get_custom_llm_provider, CustomLlmProvider};
+use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
 
 use super::hooks::OcrLifecycleHooks;
 use super::types::{OcrRequest, PreparedOcrRequest};

litellm-rust/crates/ai-gateway/src/ocr/tests.rs

@@ -3,12 +3,12 @@ use std::time::Duration;
 
 use litellm_core::error::CoreError;
 use litellm_core::ocr::transformation::OcrResponseHandling;
-use serde_json::{json, Map, Value};
+use serde_json::{Map, Value, json};
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::{TcpListener, TcpStream};
 
 use super::common_utils::{has_header, ocr_provider_config, string_headers, truncate_error_body};
-use super::{ocr, OcrRequest};
+use super::{OcrRequest, ocr};
 use crate::integrations::custom_guardrail::{
     CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailError, GuardrailEventHook,
     GuardrailFuture, GuardrailRequest,
@@ -228,19 +228,23 @@ fn truncate_error_body_does_not_split_multibyte_chars() {
 #[test]
 fn ocr_dispatch_supports_migrated_providers() {
     assert!(ocr_provider_config("mistral", "mistral-ocr-latest").is_some());
-    assert!(ocr_provider_config("azure_ai", "pixtral-12b-2409")
-        .expect("azure ai config resolves")
-        .requires_data_uri_document());
+    assert!(
+        ocr_provider_config("azure_ai", "pixtral-12b-2409")
+            .expect("azure ai config resolves")
+            .requires_data_uri_document()
+    );
     assert_eq!(
         ocr_provider_config("azure_ai", "doc-intelligence/prebuilt-read")
             .expect("document intelligence config resolves")
             .response_handling(),
         OcrResponseHandling::AzureDocumentIntelligencePoll
     );
-    assert!(ocr_provider_config("vertex_ai", "deepseek-ocr-maas")
-        .expect("vertex deepseek config resolves")
-        .supported_ocr_params()
-        .contains(&"temperature"));
+    assert!(
+        ocr_provider_config("vertex_ai", "deepseek-ocr-maas")
+            .expect("vertex deepseek config resolves")
+            .supported_ocr_params()
+            .contains(&"temperature")
+    );
     assert!(ocr_provider_config("openai", "gpt-4o").is_none());
 }
 

litellm-rust/crates/ai-gateway/src/python/config.rs

@@ -7,9 +7,9 @@
 //!
 //! Compiled only under the `python-config` feature.
 
+use litellm_core::CoreResult;
 use litellm_core::error::CoreError;
 use litellm_core::router::{Deployment, Router};
-use litellm_core::CoreResult;
 use pyo3::prelude::*;
 
 use crate::gil;
@@ -17,7 +17,7 @@ use crate::gil;
 /// Load the router's `model_list` from `config_path` via the Python reader.
 pub fn load_router_from_config(config_path: &str) -> CoreResult<Router> {
     gil::record_acquisition();
-    Python::with_gil(|py| {
+    Python::attach(|py| {
         let model_list = py
             .import("litellm.proxy.read_model_list")
             .and_then(|module| module.getattr("read_model_list"))

litellm-rust/crates/ai-gateway/src/routes/health.rs

@@ -1,8 +1,8 @@
 //! Health probes. Simple-route template: a `router()` plus its handlers, in one file.
 
+use axum::Router;
 use axum::http::StatusCode;
 use axum::routing::get;
-use axum::Router;
 
 use crate::state::AppState;