Merge branch 'litellm_internal_staging' into bedrock-embedding-aws-external-id

.circleci/config.yml

@@ -228,7 +228,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=litellm \
+                --cov=./litellm \
                 --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
@@ -293,7 +293,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=litellm \
+                --cov=./litellm \
                 --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
@@ -409,14 +409,25 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -v -x \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
           no_output_timeout: 15m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml auth_ui_unit_tests_coverage.xml
+            mv .coverage auth_ui_unit_tests_coverage
 
       # Store test results
       - store_test_results:
           path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - auth_ui_unit_tests_coverage.xml
+            - auth_ui_unit_tests_coverage
 
   litellm_router_testing: # Runs all tests with the "router" keyword
     docker:
@@ -493,13 +504,24 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -v -x \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
           no_output_timeout: 15m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml router_unit_tests_coverage.xml
+            mv .coverage router_unit_tests_coverage
       # Store test results
       - store_test_results:
           path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - router_unit_tests_coverage.xml
+            - router_unit_tests_coverage
   litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
     docker:
       - *python312_image
@@ -603,7 +625,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
                 -n 4 \
@@ -646,7 +668,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -688,7 +710,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2 \
@@ -732,7 +754,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 --retries 3 --retry-delay 5"
@@ -814,7 +836,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -856,7 +878,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -930,7 +952,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -972,7 +994,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -1015,7 +1037,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1090,7 +1112,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 -n 4 \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
@@ -1133,7 +1155,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -1184,7 +1206,7 @@ jobs:
               --verbose \
               --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=litellm --cov-report=xml \
+                --cov=./litellm --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 -n 2 \
                 --reruns 2 --reruns-delay 1"
@@ -2280,10 +2302,11 @@ jobs:
       - run:
           name: Combine Coverage
           command: |
-            uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage
+            uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
             uv tool run --from 'coverage[toml]==7.10.6' coverage xml
       - codecov/upload:
           file: ./coverage.xml
+          flags: circleci
 
   ui_build:
     docker:
@@ -2669,6 +2692,8 @@ workflows:
             - local_testing_part1
             - local_testing_part2
             - litellm_assistants_api_testing
+            - litellm_router_unit_testing
+            - auth_ui_unit_tests
       - db_migration_disable_update_check:
           requires:
             - build_docker_database_image

.github/actions/helm-oci-chart-releaser/action.yml

@@ -1,94 +0,0 @@
-name: Helm OCI Chart Releaser
-description: Push Helm charts to OCI-based (Docker) registries
-author: sergeyshaykhullin
-branding:
-  color: yellow
-  icon: upload-cloud
-inputs:
-  name:
-    required: true
-    description: Chart name
-  repository:
-    required: true
-    description: Chart repository name
-  tag:
-    required: true
-    description: Chart version
-  app_version:
-    required: true
-    description: App version
-  path:
-    required: false
-    description: Chart path (Default 'charts/{name}')
-  registry:
-    required: true
-    description: OCI registry
-  registry_username:
-    required: true
-    description: OCI registry username
-  registry_password:
-    required: true
-    description: OCI registry password
-  update_dependencies:
-    required: false
-    default: 'false'
-    description: Update chart dependencies before packaging (Default 'false')
-outputs:
-  image:
-    value: ${{ steps.output.outputs.image }}
-    description: Chart image (Default '{registry}/{repository}/{image}:{tag}')
-runs:
-  using: composite
-  steps:
-    - name: Helm | Setup
-      uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
-      with:
-        version: v3.20.0
-
-    - name: Helm | Login
-      shell: bash
-      env:
-        REGISTRY_PASSWORD: ${{ inputs.registry_password }}
-        REGISTRY_USERNAME: ${{ inputs.registry_username }}
-        REGISTRY: ${{ inputs.registry }}
-      run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY"
-
-    - name: Helm | Dependency
-      if: inputs.update_dependencies == 'true'
-      shell: bash
-      env:
-        CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
-      run: helm dependency update "$CHART_PATH"
-
-    - name: Helm | Package
-      shell: bash
-      env:
-        CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
-        TAG: ${{ inputs.tag }}
-        APP_VERSION: ${{ inputs.app_version }}
-      run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION"
-
-    - name: Helm | Push
-      shell: bash
-      env:
-        NAME: ${{ inputs.name }}
-        TAG: ${{ inputs.tag }}
-        REGISTRY: ${{ inputs.registry }}
-        REPOSITORY: ${{ inputs.repository }}
-      run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}"
-
-    - name: Helm | Logout
-      shell: bash
-      env:
-        REGISTRY: ${{ inputs.registry }}
-      run: helm registry logout "$REGISTRY"
-
-    - name: Helm | Output
-      id: output
-      shell: bash
-      env:
-        REGISTRY: ${{ inputs.registry }}
-        REPOSITORY: ${{ inputs.repository }}
-        NAME: ${{ inputs.name }}
-        TAG: ${{ inputs.tag }}
-      run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT

.github/workflows/README.md

@@ -1,35 +0,0 @@
-# Simple PyPI Publishing
-
-A GitHub workflow to manually publish LiteLLM packages to PyPI with a specified version.
-
-## How to Use
-
-1. Go to the **Actions** tab in the GitHub repository
-2. Select **Simple PyPI Publish** from the workflow list
-3. Click **Run workflow**
-4. Enter the version to publish (e.g., `1.74.10`)
-
-## What the Workflow Does
-
-1. **Updates** the version in `pyproject.toml`
-2. **Copies** the model prices backup file
-3. **Builds** the Python package
-4. **Publishes** to PyPI
-
-## Prerequisites
-
-Make sure the following secret is configured in the repository:
-- `PYPI_PUBLISH_PASSWORD`: PyPI API token for authentication
-
-## Example Usage
-
-- Version: `1.74.11` → Publishes as v1.74.11
-- Version: `1.74.10-hotfix1` → Publishes as v1.74.10-hotfix1
-
-## Features
-
-- ✅ Manual trigger with version input
-- ✅ Automatic version updates in `pyproject.toml`
-- ✅ Repository safety check (only runs on official repo)
-- ✅ Clean package building and publishing
-- ✅ Success confirmation with PyPI package link 

.github/workflows/_test-unit-base.yml

@@ -91,7 +91,7 @@ jobs:
             --reruns-delay 1 \
             --dist=loadscope \
             --durations=20 \
-            --cov=litellm \
+            --cov=./litellm \
             --cov-report=xml:coverage.xml \
             --cov-config=pyproject.toml
 
@@ -132,4 +132,5 @@ jobs:
           use_oidc: true
           directory: coverage-reports
           root_dir: ${{ github.workspace }}
+          flags: ${{ inputs.artifact-name }}
           fail_ci_if_error: false

.github/workflows/_test-unit-services-base.yml

@@ -132,7 +132,7 @@ jobs:
               --reruns "${RERUNS}" \
               --reruns-delay 1 \
               --durations=20 \
-              --cov=litellm \
+              --cov=./litellm \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           else
@@ -144,7 +144,7 @@ jobs:
               --reruns-delay 1 \
               --dist="${DIST}" \
               --durations=20 \
-              --cov=litellm \
+              --cov=./litellm \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           fi
@@ -186,4 +186,5 @@ jobs:
           use_oidc: true
           directory: coverage-reports
           root_dir: ${{ github.workspace }}
+          flags: ${{ inputs.artifact-name }}
           fail_ci_if_error: false

.github/workflows/llm-translation-testing.yml

@@ -1,92 +0,0 @@
-name: LLM Translation Tests
-
-on:
-  workflow_dispatch:
-    inputs:
-      release_candidate_tag:
-        description: "Release candidate tag/version"
-        required: true
-        type: string
-  push:
-    tags:
-      - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1
-
-permissions:
-  contents: read
-
-jobs:
-  run-llm-translation-tests:
-    runs-on: ubuntu-latest
-    timeout-minutes: 90
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-          ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
-        with:
-          version: "0.10.9"
-          enable-cache: false
-
-      - name: Restore uv dependencies cache
-        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: |
-            ~/.cache/uv
-            .venv
-          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-uv-
-
-      - name: Install dependencies
-        run: |
-          uv sync --frozen
-
-      - name: Create test results directory
-        run: mkdir -p test-results
-
-      - name: Run LLM Translation Tests
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
-          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
-          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
-          AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
-          RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }}
-          COMMIT_SHA: ${{ github.sha }}
-        run: |
-          python .github/workflows/run_llm_translation_tests.py \
-            --tag "$RC_TAG" \
-            --commit "$COMMIT_SHA" \
-            || true  # Continue even if tests fail
-
-      - name: Display test summary
-        if: always()
-        run: |
-          if [ -f "test-results/llm_translation_report.md" ]; then
-            echo "Test report generated successfully!"
-            echo "Artifact will contain:"
-            echo "- test-results/junit.xml (JUnit XML results)"
-            echo "- test-results/llm_translation_report.md (Beautiful markdown report)"
-          else
-            echo "Warning: Test report was not generated"
-          fi
-
-      - name: Upload test artifacts
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        if: always()
-        with:
-          name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
-          path: test-results/
-          retention-days: 30

.github/workflows/publish_to_pypi.yml

@@ -1,153 +0,0 @@
-name: Publish to PyPI
-
-on:
-  workflow_dispatch:
-
-jobs:
-  preflight-checks:
-    name: Preflight Checks
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-    # No environment — read-only checks, no approval needed
-    outputs:
-      needs_publish: ${{ steps.check-litellm.outputs.needs_publish }}
-      version: ${{ steps.check-litellm.outputs.version }}
-
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
-        with:
-          version: "0.10.9"
-          enable-cache: false
-
-      - name: Check litellm version on PyPI
-        id: check-litellm
-        run: |
-          VERSION=$(python - <<'PY'
-          import tomllib
-
-          with open("pyproject.toml", "rb") as f:
-              print(tomllib.load(f)["project"]["version"])
-          PY
-          )
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-          echo "Checking if litellm $VERSION exists on PyPI..."
-
-          HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json")
-          if [ "$HTTP_STATUS" = "200" ]; then
-            echo "litellm $VERSION already exists on PyPI. Skipping publish."
-            echo "needs_publish=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "litellm $VERSION not found on PyPI. Publish needed."
-            echo "needs_publish=true" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Sanity check proxy-extras version
-        run: |
-          # Read pinned version from project optional dependencies
-          PYPROJECT_VERSION=$(python3 - <<'PY'
-          import sys
-          import tomllib
-
-          with open("pyproject.toml", "rb") as f:
-              proxy_requirements = tomllib.load(f)["project"]["optional-dependencies"]["proxy"]
-
-          version = None
-          for requirement in proxy_requirements:
-              normalized = requirement.split(";", 1)[0].strip()
-              if not normalized.startswith("litellm-proxy-extras"):
-                  continue
-              parts = normalized.split("==", 1)
-              if len(parts) == 2 and parts[0].strip() == "litellm-proxy-extras":
-                  candidate = parts[1].strip()
-                  if candidate:
-                      version = candidate
-                      break
-
-          if version is None:
-              print(
-                  "::error::Could not find an exact litellm-proxy-extras pin in project.optional-dependencies.proxy",
-                  file=sys.stderr,
-              )
-              sys.exit(1)
-
-          print(version)
-          PY
-          )
-          echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION"
-
-          # Check that the pinned version exists on PyPI
-          echo "Checking if litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI..."
-          HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$PYPROJECT_VERSION/json")
-          if [ "$HTTP_STATUS" != "200" ]; then
-            echo "::error::litellm-proxy-extras $PYPROJECT_VERSION is not published on PyPI yet. Publish it before releasing litellm."
-            exit 1
-          fi
-          echo "litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI. Sanity check passed."
-
-  publish-litellm:
-    name: Publish litellm to PyPI
-    needs: preflight-checks
-    if: needs.preflight-checks.outputs.needs_publish == 'true'
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      id-token: write
-      contents: read
-    environment: pypi-publish
-
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
-        with:
-          version: "0.10.9"
-          enable-cache: false
-
-      - name: Copy model prices backup
-        run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
-
-      - name: Build package
-        run: |
-          rm -rf build dist
-          uv build
-
-      - name: Verify build artifacts
-        env:
-          EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }}
-        run: |
-          echo "Contents of dist/:"
-          ls -la dist/
-          # Ensure we have both sdist and wheel
-          ls dist/*.tar.gz
-          ls dist/*.whl
-          # Verify built version matches expected
-          ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || {
-            echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION"
-            ls dist/
-            exit 1
-          }
-
-      - name: Validate package metadata
-        run: |
-          uv tool run --from 'twine==6.2.0' twine check dist/*
-
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0

.github/workflows/read_pyproject_version.yml

@@ -1,28 +0,0 @@
-name: Read Version from pyproject.toml
-
-on:
-  push:
-    branches:
-      - main # Change this to the default branch of your repository
-
-permissions:
-  contents: read
-
-jobs:
-  read-version:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Read version from pyproject.toml
-        id: read-version
-        run: |
-          version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
-          printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
-
-      - name: Display version
-        run: echo "Current version is $LITELLM_VERSION"

.github/workflows/results_stats.csv

@@ -1,27 +0,0 @@
-Date,"Ben 
-Ashley",Tom Brooks,Jimmy Cooney,"Sue 
-Daniels",Berlinda Fong,Terry Jones,Angelina Little,Linda Smith
-10/1,FALSE,TRUE,TRUE,TRUE,TRUE,TRUE,FALSE,TRUE
-10/2,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/3,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/4,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/5,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/6,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/7,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/8,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/9,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/10,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/11,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/12,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/13,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/14,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/15,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/16,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/17,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/18,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/19,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/20,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/21,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/22,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/23,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-Total,0,1,1,1,1,1,0,1

.github/workflows/run_observatory_tests.yml

@@ -1,229 +0,0 @@
-name: Run Observatory Tests
-on:
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: "Docker image tag to test (e.g. v1.61.0.rc1)"
-        required: true
-        type: string
-      commit_hash:
-        description: "Commit hash (defaults to HEAD of current branch)"
-        required: false
-        type: string
-  workflow_call:
-    inputs:
-      tag:
-        description: "Docker image tag to test"
-        required: true
-        type: string
-      commit_hash:
-        description: "Commit hash of the release"
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-env:
-  LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
-
-jobs:
-  observatory-tests:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Validate tag input
-        env:
-          TAG: ${{ inputs.tag }}
-        run: |
-          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
-            echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
-            exit 1
-          fi
-
-      - name: Start LiteLLM container
-        env:
-          TAG: ${{ inputs.tag }}
-          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
-          WORKSPACE: ${{ github.workspace }}
-        run: |
-          docker run -d \
-            --name litellm-rc \
-            -p 4000:4000 \
-            -v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
-            -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
-            -e AZURE_API_KEY="${AZURE_API_KEY}" \
-            -e AZURE_API_BASE="${AZURE_API_BASE}" \
-            "litellm/litellm:${TAG}" \
-            --config /app/config.yaml --port 4000
-
-      - name: Wait for LiteLLM health check
-        run: |
-          echo "Waiting for LiteLLM to be ready..."
-          for i in $(seq 1 30); do
-            if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
-              echo "LiteLLM is healthy"
-              exit 0
-            fi
-            echo "Attempt $i/30 - not ready yet, waiting 10s..."
-            sleep 10
-          done
-          echo "LiteLLM failed to start within 5 minutes"
-          docker logs litellm-rc
-          exit 1
-
-      - name: Start cloudflared tunnel
-        run: |
-          # Install cloudflared (pinned version + checksum)
-          curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
-          echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8  /usr/local/bin/cloudflared" | sha256sum -c -
-          chmod +x /usr/local/bin/cloudflared
-
-          # Start a quick tunnel (no account needed) and capture the URL
-          cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
-          CLOUDFLARED_PID=$!
-          echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
-
-          # Wait for tunnel URL to appear in logs
-          echo "Waiting for tunnel URL..."
-          for i in $(seq 1 30); do
-            TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
-            if [ -n "$TUNNEL_URL" ]; then
-              echo "Tunnel URL: $TUNNEL_URL"
-              echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
-              exit 0
-            fi
-            sleep 2
-          done
-          echo "Failed to get tunnel URL"
-          cat /tmp/cloudflared.log
-          exit 1
-
-      - name: Verify tunnel connectivity
-        run: |
-          echo "Testing tunnel at ${TUNNEL_URL}..."
-          # Quick tunnels need time for DNS propagation; retry to avoid
-          # transient NXDOMAIN (curl exit code 6) on first attempt.
-          for i in $(seq 1 10); do
-            if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then
-              echo "Tunnel is working (attempt $i)"
-              exit 0
-            fi
-            echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
-            sleep 5
-          done
-          echo "Tunnel failed to become reachable after 50s"
-          cat /tmp/cloudflared.log
-          exit 1
-
-      - name: Trigger observatory test run
-        id: trigger
-        env:
-          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
-          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
-        run: |
-          PAYLOAD=$(jq -n \
-            --arg url "${TUNNEL_URL}" \
-            --arg key "${LITELLM_MASTER_KEY}" \
-            '{
-              deployment_url: $url,
-              api_key: $key,
-              test_suite: "TestOAIAzureRelease",
-              models: ["gpt-4o-mini", "gpt-4o"]
-            }')
-          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
-            -H "Content-Type: application/json" \
-            -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
-            -d "$PAYLOAD")
-          HTTP_CODE=$(echo "$RESPONSE" | tail -1)
-          BODY=$(echo "$RESPONSE" | head -n -1)
-          echo "Response ($HTTP_CODE): $BODY"
-          if [ "$HTTP_CODE" -ge 400 ]; then
-            echo "Failed to trigger test run"
-            exit 1
-          fi
-
-          # Extract request_id for polling this specific run
-          REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
-          if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
-            echo "Failed to extract request_id from response"
-            exit 1
-          fi
-          echo "Request ID: $REQUEST_ID"
-          echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
-
-      - name: Poll for test completion
-        id: poll
-        env:
-          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
-          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
-          REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
-        run: |
-          TIMEOUT=900  # 15 minutes
-          INTERVAL=30
-          ELAPSED=0
-          while [ $ELAPSED -lt $TIMEOUT ]; do
-            STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
-              -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
-            RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
-            echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
-
-            if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
-              echo "Test finished with status: $RUN_STATUS"
-              echo "$STATUS" > /tmp/observatory_result.json
-              exit 0
-            fi
-
-            sleep $INTERVAL
-            ELAPSED=$((ELAPSED + INTERVAL))
-          done
-          echo "Timed out waiting for test to complete after ${TIMEOUT}s"
-          exit 1
-
-      - name: Verify test results
-        run: |
-          RESULT=$(cat /tmp/observatory_result.json)
-          echo "Full result: $RESULT"
-
-          STATUS=$(echo "$RESULT" | jq -r '.status')
-          TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
-          FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
-          ERROR=$(echo "$RESULT" | jq -r '.error // empty')
-
-          echo "Status: $STATUS"
-          echo "Test passed: $TEST_PASSED"
-          echo "Failure rate: $FAILURE_RATE"
-
-          if [ -n "$ERROR" ]; then
-            echo "Error: $ERROR"
-          fi
-
-          if [ "$STATUS" = "failed" ]; then
-            echo "Test run failed"
-            exit 1
-          fi
-
-          if [ "$TEST_PASSED" != "true" ]; then
-            echo "Tests did not pass (failure rate: $FAILURE_RATE)"
-            exit 1
-          fi
-
-          echo "All tests passed!"
-
-      - name: Print LiteLLM logs on failure
-        if: failure()
-        run: |
-          docker logs litellm-rc 2>/dev/null || true
-          cat /tmp/cloudflared.log 2>/dev/null || true
-
-      - name: Cleanup
-        if: always()
-        run: |
-          kill "$CLOUDFLARED_PID" 2>/dev/null || true
-          docker rm -f litellm-rc 2>/dev/null || true

.github/workflows/scan_duplicate_issues.yml

@@ -1,48 +0,0 @@
-name: Scan Duplicate Issues (One-Time)
-
-on:
-  workflow_dispatch:
-    inputs:
-      threshold:
-        description: "Similarity threshold (0-1)"
-        required: false
-        default: "0.85"
-      close:
-        description: "Actually close duplicates (false = dry run)"
-        required: false
-        type: boolean
-        default: false
-
-jobs:
-  scan:
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      contents: read
-    steps:
-      - name: Checkout scripts
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          sparse-checkout: .github/scripts
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Scan for duplicate issues
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          INPUT_THRESHOLD: ${{ inputs.threshold }}
-          INPUT_CLOSE: ${{ inputs.close }}
-        run: |
-          CLOSE_FLAG=""
-          if [ "$INPUT_CLOSE" = "true" ]; then
-            CLOSE_FLAG="--close"
-          fi
-          python3 .github/scripts/close_duplicate_issues.py \
-            --scan \
-            --repo ${{ github.repository }} \
-            --threshold "$INPUT_THRESHOLD" \
-            $CLOSE_FLAG

.github/workflows/test-litellm.yml

@@ -1,45 +0,0 @@
-name: LiteLLM Mock Tests (folder - tests/test_litellm)
-
-# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs
-# the same tests in parallel across 10 jobs for faster CI times.
-# Kept for manual debugging only.
-on:
-  workflow_dispatch: # Manual trigger only
-  # pull_request:
-  #   branches: [ main ]
-
-permissions:
-  contents: read
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 25
-
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Thank You Message
-        run: |
-          echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
-          echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
-        with:
-          version: "0.10.9"
-
-      - name: Install dependencies
-        run: |
-          uv lock --check
-          uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
-      - name: Run tests
-        run: |
-          uv run --no-sync pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

.github/workflows/test-mcp.yml

@@ -43,4 +43,4 @@ jobs:
 
       - name: Run MCP tests
         run: |
-          uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5
+          uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5

.github/workflows/test-unit-caching-redis.yml

@@ -1,38 +0,0 @@
-name: "Unit Tests: Caching (Redis)"
-
-# Uses cloud Redis credentials — only runs on trusted branches, not PRs.
-# This prevents external PRs from accessing Redis credentials.
-on:
-  push:
-    branches: [main, "litellm_*"]
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  caching-redis:
-    uses: ./.github/workflows/_test-unit-services-base.yml
-    with:
-      # Redis-only tests that do NOT require provider API keys.
-      # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py,
-      # test_router_caching.py) are in Phase 3 integration workflows.
-      test-path: >-
-        tests/local_testing/test_dual_cache.py
-        tests/local_testing/test_redis_batch_optimizations.py
-        tests/local_testing/test_router_utils.py
-      workers: 2
-      reruns: 2
-      timeout-minutes: 20
-      enable-redis: true
-      enable-postgres: false
-    secrets:
-      REDIS_HOST: ${{ secrets.REDIS_HOST }}
-      REDIS_PORT: ${{ secrets.REDIS_PORT }}
-      REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
-      DATABASE_URL: ${{ secrets.DATABASE_URL }}
-      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test-unit-proxy-db.yml

@@ -143,6 +143,7 @@ jobs:
               tests/proxy_unit_tests/test_proxy_pass_user_config.py
               tests/proxy_unit_tests/test_proxy_token_counter.py
               tests/proxy_unit_tests/test_request_size_limit_middleware.py
+              tests/proxy_unit_tests/test_multipart_bypass_repro.py
             workers: 4
             dist: loadscope
             timeout: 15

.github/workflows/update_release.py

@@ -1,54 +0,0 @@
-import os
-import requests
-from datetime import datetime
-
-# GitHub API endpoints
-GITHUB_API_URL = "https://api.github.com"
-REPO_OWNER = "BerriAI"
-REPO_NAME = "litellm"
-
-# GitHub personal access token (required for uploading release assets)
-GITHUB_ACCESS_TOKEN = os.environ.get("GITHUB_ACCESS_TOKEN")
-
-# Headers for GitHub API requests
-headers = {
-    "Accept": "application/vnd.github+json",
-    "Authorization": f"Bearer {GITHUB_ACCESS_TOKEN}",
-    "X-GitHub-Api-Version": "2022-11-28",
-}
-
-# Get the latest release
-releases_url = f"{GITHUB_API_URL}/repos/{REPO_OWNER}/{REPO_NAME}/releases/latest"
-response = requests.get(releases_url, headers=headers)
-latest_release = response.json()
-print("Latest release:", latest_release)
-
-# Upload an asset to the latest release
-upload_url = latest_release["upload_url"].split("{?")[0]
-asset_name = "results_stats.csv"
-asset_path = os.path.join(os.getcwd(), asset_name)
-print("upload_url:", upload_url)
-
-with open(asset_path, "rb") as asset_file:
-    asset_data = asset_file.read()
-
-upload_payload = {
-    "name": asset_name,
-    "label": "Load test results",
-    "created_at": datetime.utcnow().isoformat() + "Z",
-}
-
-upload_headers = headers.copy()
-upload_headers["Content-Type"] = "application/octet-stream"
-
-upload_response = requests.post(
-    upload_url,
-    headers=upload_headers,
-    data=asset_data,
-    params=upload_payload,
-)
-
-if upload_response.status_code == 201:
-    print(f"Asset '{asset_name}' uploaded successfully to the latest release.")
-else:
-    print(f"Failed to upload asset. Response: {upload_response.text}")

.gitignore

@@ -101,4 +101,23 @@ STABILIZATION_TODO.md
 **/*.storageState.json
 **/coverage
 test-config
+
+# ---------- Terraform ----------
+# Provider binaries + module cache — regenerated by `terraform init`.
+**/.terraform/
+# State files often contain secrets (DB passwords, API keys snapshotted from
+# data sources). Keep state in a remote backend, never in git.
+*.tfstate
+*.tfstate.*
+*.tfstate.backup
+# Plan files can also contain sensitive values (variables in plaintext).
+*.tfplan
+# User-specific variable inputs — example files (terraform.tfvars.example) are
+# tracked because they end in .example, which doesn't match the glob below.
+*.tfvars
+*.auto.tfvars
+crash.log
+crash.*.log
+# .terraform.lock.hcl is intentionally NOT ignored — it pins provider versions
+# and should be committed.
 .vscode

CLAUDE.md

@@ -117,6 +117,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only `<span>`, `<p>`, `<h*>` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
 
 ### MCP OAuth / OpenAPI Transport Mapping
+- **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive — not `client_credentials`)** — LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database.
 - `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
 - FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
 - When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.

README.md

@@ -292,7 +292,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 | [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ | ✅ | ✅ |  |
-| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |
 | [Databricks (`databricks`)](https://docs.litellm.ai/docs/providers/databricks) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [DataRobot (`datarobot`)](https://docs.litellm.ai/docs/providers/datarobot) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ |  |  | ✅ |  |  |  |  |

backend/Dockerfile

@@ -0,0 +1,83 @@
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
+
+FROM $UV_IMAGE AS uvbin
+
+# ---------- Builder ----------
+FROM $LITELLM_BUILD_IMAGE AS builder
+
+WORKDIR /app
+USER root
+
+COPY --from=uvbin /uv /uvx /usr/local/bin/
+
+RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile
+
+# UV_COMPILE_BYTECODE=1   precompiles .pyc at install time → faster cold start.
+# UV_LINK_MODE=copy       avoids hardlink warnings when uv installs from a
+#                         BuildKit cache mount (different filesystem).
+# UV_PYTHON_DOWNLOADS=0   force uv to use the apk-installed CPython instead of
+#                         silently pulling a managed interpreter.
+ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
+    UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=0 \
+    PATH="/app/.venv/bin:${PATH}"
+
+# Stage 1 — install dependencies only.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
+    --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
+    uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
+        --extra proxy \
+        --extra proxy-runtime \
+        --extra extra_proxy \
+        --extra semantic-router \
+        --python python3
+
+# Stage 2 — copy source and install the project + workspace members.
+COPY . .
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-default-groups --no-editable \
+        --extra proxy \
+        --extra proxy-runtime \
+        --extra extra_proxy \
+        --extra semantic-router \
+        --python python3
+
+RUN mkdir -p /home/nonroot && \
+    HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
+    chown -R nonroot:nonroot /home/nonroot/.cache
+
+# ---------- Runtime ----------
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+
+USER root
+
+RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic
+
+# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
+# /home/nonroot. We run the backend as that user
+WORKDIR /app
+ENV HOME=/home/nonroot \
+    PATH="/app/.venv/bin:${PATH}" \
+    PYTHONPATH="/app" \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+COPY --from=builder --chown=nonroot:nonroot /app /app
+COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
+
+RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
+    find /app/.venv -type d -path "*/tornado/test" -delete
+
+USER nonroot
+
+EXPOSE 4001/tcp
+
+ENTRYPOINT ["uvicorn", "backend.main:app"]
+CMD ["--host", "0.0.0.0", "--port", "4001"]

backend/main.py

@@ -0,0 +1,51 @@
+"""UI backend entrypoint.
+
+Reuses the existing FastAPI app from `litellm.proxy.proxy_server` and trims its
+route table to just the management/admin surface used by the dashboard. Purely
+additive — no existing module is modified.
+
+Run with:
+    uvicorn backend.main:app --host 0.0.0.0 --port 4001
+"""
+
+from contextlib import asynccontextmanager
+
+from fastapi.routing import Mount
+
+# See gateway/main.py for why we assemble DATABASE_URL(s) here before
+# importing proxy_server.
+from litellm.proxy.db.db_url_settings import DatabaseURLSettings
+
+DatabaseURLSettings.from_env().apply_to_env()
+
+from litellm.proxy.proxy_server import app
+
+from backend.routes.allowlist import BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES
+
+
+def _is_backend_route(route) -> bool:
+    """Keep the route on the backend if its path is in the management surface."""
+    path = getattr(route, "path", None)
+    if path is None:
+        return False
+    if isinstance(route, Mount):
+        # Static UI mounts are served by the dedicated UI container, not here.
+        return False
+    if path in BACKEND_EXACT_PATHS:
+        return True
+    return any(path.startswith(prefix) for prefix in BACKEND_PATH_PREFIXES)
+
+
+# See gateway/main.py for why the trim runs inside the lifespan instead of at
+# module scope.
+_proxy_lifespan = app.router.lifespan_context
+
+
+@asynccontextmanager
+async def _backend_lifespan(app_):
+    async with _proxy_lifespan(app_):
+        app_.router.routes = [r for r in app_.router.routes if _is_backend_route(r)]
+        yield
+
+
+app.router.lifespan_context = _backend_lifespan

backend/routes/allowlist.py

@@ -0,0 +1,135 @@
+"""Path allowlist for the UI backend (control plane) component.
+
+The backend exposes management/admin endpoints consumed by the UI: keys, users,
+teams, orgs, customers, budgets, tags, workflows, model management, spend &
+analytics, settings (router/cache/cost-tracking/fallbacks), SSO/onboarding,
+audit logs, debug, enterprise admin, and UI bootstrap helpers (logo, favicon,
+.well-known config).
+
+Anything LLM data-plane is dropped — those run on the gateway component.
+"""
+
+BACKEND_PATH_PREFIXES: tuple[str, ...] = (
+    # Identity / access
+    "/key/",
+    "/v2/key/",
+    "/user/",
+    "/v2/user/",
+    "/team/",
+    "/v2/team/",
+    "/organization/",
+    "/customer/",
+    "/end_user/",
+    "/sso/",
+    "/login",
+    "/v2/login",
+    "/v3/login",
+    "/logout",
+    "/token",
+    "/onboarding/",
+    "/audit",
+    "/oauth/",
+    "/invitation/",
+    "/jwt/",
+    # Models & routing config
+    "/model/",
+    "/v1/model/info",
+    "/v2/model/",
+    "/model_group",
+    "/model_access_group/",
+    "/model_hub/",
+    "/v1/access_group",
+    "/access_group/",
+    "/router/",
+    "/router_settings",
+    "/adaptive_router/",
+    "/fallback",
+    "/fallbacks",
+    "/cache_settings",
+    "/cost_tracking",
+    "/cost/",
+    "/credentials",
+    "/credential",
+    "/provider/budgets",
+    # Tools / agents (registry & policy admin)
+    "/v1/tool/",
+    "/v1/agents",
+    # Guardrails admin
+    "/v2/guardrails/",
+    # MCP server admin + BYOK OAuth flow (UI-initiated) + dynamic per-server endpoints
+    "/v1/mcp/",
+    "/test/",
+    "/{mcp_server_name}/",
+    # Budgets / tags / workflows / memory mgmt
+    "/budget/",
+    "/tag/",
+    "/workflow/",
+    "/v1/workflows/",
+    "/project/",
+    "/memory/",
+    "/mcp/",
+    # Spend / analytics
+    "/spend/",
+    "/analytics/",
+    "/global/",
+    "/user_agent",
+    "/usage/",
+    "/daily/",
+    # CloudZero cost-export admin (init / settings / export / dry-run / delete)
+    "/cloudzero/",
+    # Caching admin
+    "/cache/",
+    "/caching/",
+    # Callbacks / hooks
+    "/active/callbacks",
+    "/callbacks",
+    "/team_callback",
+    # Alerting / email / IP allowlist
+    "/alerting/",
+    "/email/",
+    "/add/allowed_ip",
+    "/delete/allowed_ip",
+    "/get/",
+    # Enterprise admin
+    "/enterprise/",
+    # Debug / config / profiling
+    "/debug/",
+    "/config/",
+    "/memory-usage-in-mem-cache",
+    "/otel-spans",
+    "/lazy/",
+    "/in_product_nudges",
+    # Admin reload / schedule
+    "/reload/",
+    "/schedule/",
+    "/settings",
+    "/update/",
+    "/upload/",
+    # Dev / admin utilities
+    "/utils/",
+    # UI bootstrap helpers (assets the dashboard fetches)
+    "/get_logo_url",
+    "/get_image",
+    "/get_favicon",
+    "/.well-known/",
+    "/litellm/.well-known/",
+    "/ui_discovery/",
+    "/ui-config",
+    "/sso_settings",
+    "/public/",
+    "/robots.txt",
+    # Health (k8s probes)
+    "/health",
+)
+
+BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
+    {
+        "/",
+        "/routes",
+        "/openapi.json",
+        "/docs",
+        "/docs/oauth2-redirect",
+        "/redoc",
+        "/fallback/login",
+    }
+)

codecov.yaml

@@ -3,6 +3,16 @@ codecov:
   notify:
     wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI
 
+# Uploads are flagged per workflow/shard (GHA) or "circleci". carryforward makes
+# a re-upload of a flag replace its prior session instead of accumulating a
+# conflicting one, and lets a commit reuse a flag from its parent when that flag
+# was not re-uploaded. Required because the same commit can receive the
+# push-triggered workflows more than once (re-runs / branches cut at the same
+# SHA); flagless overlapping sessions made Codecov drop the largest files.
+flag_management:
+  default_rules:
+    carryforward: true
+
 component_management:
   individual_components:
     - component_id: "Router"

deploy/charts/litellm-helm/templates/hpa.yaml

@@ -12,6 +12,10 @@ spec:
     name: {{ include "litellm.fullname" . }}
   minReplicas: {{ .Values.autoscaling.minReplicas }}
   maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+  {{- if .Values.autoscaling.behavior }}
+  behavior:
+    {{- toYaml .Values.autoscaling.behavior | nindent 4 }}
+  {{- end }}
   metrics:
     {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
     - type: Resource

deploy/charts/litellm-helm/tests/hpa_tests.yaml

@@ -0,0 +1,36 @@
+suite: "hpa with behavior"
+templates:
+  - hpa.yaml
+tests:
+  - it: "renders behavior when set"
+    set:
+      autoscaling.enabled: true
+      autoscaling.behavior:
+        scaleUp:
+          stabilizationWindowSeconds: 60
+          policies:
+            - type: Pods
+              value: 2
+              periodSeconds: 60
+        scaleDown:
+          stabilizationWindowSeconds: 90
+          policies:
+            - type: Pods
+              value: 1
+              periodSeconds: 60
+    asserts:
+      - isKind: { of: HorizontalPodAutoscaler }
+      - equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 }
+      - equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 }
+
+---
+suite: "hpa without behavior"
+templates:
+  - hpa.yaml
+tests:
+  - it: "does not render behavior when not set"
+    set:
+      autoscaling.enabled: true
+    asserts:
+      - isKind: { of: HorizontalPodAutoscaler }
+      - isNull: { path: spec.behavior }

deploy/charts/litellm-helm/values.yaml

@@ -184,6 +184,7 @@ autoscaling:
   maxReplicas: 100
   targetCPUUtilizationPercentage: 80
   # targetMemoryUtilizationPercentage: 80
+  # behavior: {}
 
 # Autoscaling with keda is mutually exclusive with hpa
 keda:

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -300,6 +300,42 @@ class CheckBatchCost:
                     custom_llm_provider=custom_llm_provider,
                 )
 
+                # CheckBatchCost bypasses async_post_call_success_hook, so convert raw
+                # output/error file IDs to managed base64 IDs before the DB write here.
+                managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
+                if managed_files_hook is not None:
+                    from litellm.proxy._types import UserAPIKeyAuth
+                    _minimal_auth = UserAPIKeyAuth(
+                        user_id=job.created_by or "default-user-id",
+                        team_id=getattr(job, "team_id", None),
+                    )
+                    for _file_attr in ["output_file_id", "error_file_id"]:
+                        _raw_file_id = getattr(response, _file_attr, None)
+                        if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
+                            try:
+                                _unified_file_id = managed_files_hook.get_unified_output_file_id(
+                                    output_file_id=_raw_file_id,
+                                    model_id=model_id,
+                                    model_name=str(model_name) if model_name else deployment_info.model_name or None,
+                                )
+                                await managed_files_hook.store_unified_file_id(
+                                    file_id=_unified_file_id,
+                                    file_object=None,
+                                    litellm_parent_otel_span=None,
+                                    model_mappings={model_id: _raw_file_id},
+                                    user_api_key_dict=_minimal_auth,
+                                )
+                                setattr(response, _file_attr, _unified_file_id)
+                                verbose_proxy_logger.info(
+                                    f"CheckBatchCost: converted {_file_attr} "
+                                    f"{_raw_file_id!r} -> managed ID for batch {batch_id}"
+                                )
+                            except Exception as _e:
+                                verbose_proxy_logger.warning(
+                                    f"CheckBatchCost: failed to create managed file ID for "
+                                    f"{_file_attr}={_raw_file_id!r}: {_e}"
+                                )
+
                 # Pass deployment model_info so custom batch pricing
                 # (input_cost_per_token_batches etc.) is used for cost calc
                 deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.40"
+version = "0.1.41"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.40"
+version = "0.1.41"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -0,0 +1,83 @@
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
+
+FROM $UV_IMAGE AS uvbin
+
+# ---------- Builder ----------
+FROM $LITELLM_BUILD_IMAGE AS builder
+
+WORKDIR /app
+USER root
+
+COPY --from=uvbin /uv /uvx /usr/local/bin/
+
+RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile
+
+# UV_COMPILE_BYTECODE=1   precompiles .pyc at install time → faster cold start.
+# UV_LINK_MODE=copy       avoids hardlink warnings when uv installs from a
+#                         BuildKit cache mount (different filesystem).
+# UV_PYTHON_DOWNLOADS=0   force uv to use the apk-installed CPython instead of
+#                         silently pulling a managed interpreter.
+ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
+    UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=0 \
+    PATH="/app/.venv/bin:${PATH}"
+
+# Stage 1 — install dependencies only.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
+    --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
+    uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
+        --extra proxy \
+        --extra proxy-runtime \
+        --extra extra_proxy \
+        --extra semantic-router \
+        --python python3
+
+# Stage 2 — copy source and install the project + workspace members.
+COPY . .
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-default-groups --no-editable \
+        --extra proxy \
+        --extra proxy-runtime \
+        --extra extra_proxy \
+        --extra semantic-router \
+        --python python3
+
+RUN mkdir -p /home/nonroot && \
+    HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
+    chown -R nonroot:nonroot /home/nonroot/.cache
+
+# ---------- Runtime ----------
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+
+USER root
+
+RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic
+
+# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
+# /home/nonroot. We run the proxy as that user.
+WORKDIR /app
+ENV HOME=/home/nonroot \
+    PATH="/app/.venv/bin:${PATH}" \
+    PYTHONPATH="/app" \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+COPY --from=builder --chown=nonroot:nonroot /app /app
+COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
+
+RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
+    find /app/.venv -type d -path "*/tornado/test" -delete
+
+USER nonroot
+
+EXPOSE 4000/tcp
+
+ENTRYPOINT ["sh", "-c", "exec uvicorn gateway.main:app --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
+CMD ["--host", "0.0.0.0", "--port", "4000"]

gateway/main.py

@@ -0,0 +1,59 @@
+"""Gateway entrypoint.
+
+Reuses the existing FastAPI app from `litellm.proxy.proxy_server` and trims its
+route table to just the LLM data-plane surface. The trim is purely additive —
+no existing module is modified, the full app continues to work via the legacy
+entrypoint (`litellm.proxy.proxy_server:app`).
+
+Run with:
+    uvicorn gateway.main:app --host 0.0.0.0 --port 4000
+"""
+
+from contextlib import asynccontextmanager
+
+from fastapi.routing import Mount
+
+# Assemble DATABASE_URL (+ DATABASE_URL_READ_REPLICA) from the discrete
+# DATABASE_* env vars before proxy_server imports spin up Prisma. Handles
+# both IAM (mint a token) and password auth, writer and reader. The standard
+# CLI flow does this in proxy_cli.py; we bypass proxy_cli by uvicorn'ing the
+# app directly, so without this Prisma initializes with the placeholder URL
+# and every DB-needing endpoint returns "Database not connected".
+from litellm.proxy.db.db_url_settings import DatabaseURLSettings
+
+DatabaseURLSettings.from_env().apply_to_env()
+
+from litellm.proxy.proxy_server import app
+
+from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES
+
+
+def _is_gateway_route(route) -> bool:
+    """Keep the route on the gateway if its path is in the LLM data-plane surface."""
+    path = getattr(route, "path", None)
+    if path is None:
+        return False
+    if isinstance(route, Mount):
+        # Gateway never serves the static UI or its asset bundles.
+        return False
+    if path in GATEWAY_EXACT_PATHS:
+        return True
+    return any(path.startswith(prefix) for prefix in GATEWAY_PATH_PREFIXES)
+
+
+# Wrap proxy_server's existing lifespan so the route trim runs *after* its
+# startup hooks (and any plugin code those hooks load) have had a chance to
+# register routes. A module-load filter would miss routes added during
+# startup; running inside the lifespan, after the inner __aenter__, catches
+# them while still completing before uvicorn opens the listener.
+_proxy_lifespan = app.router.lifespan_context
+
+
+@asynccontextmanager
+async def _gateway_lifespan(app_):
+    async with _proxy_lifespan(app_):
+        app_.router.routes = [r for r in app_.router.routes if _is_gateway_route(r)]
+        yield
+
+
+app.router.lifespan_context = _gateway_lifespan

gateway/routes/allowlist.py

@@ -0,0 +1,121 @@
+"""Path allowlist for the gateway component.
+
+The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
+audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image,
+responses, vector stores, passthrough providers, realtime websockets, MCP
+tool-call endpoints, and operational endpoints (/health, /metrics).
+
+Any path not listed here is dropped from the gateway process so management/UI
+endpoints don't ride on the same pods.
+
+Versioned data-plane paths are enumerated explicitly rather than allowing a
+blanket `/v1/` or `/v2/` prefix — those broad prefixes would otherwise also
+match management routes like `/v1/access_group`, `/v1/tool/{tool_name}/logs`,
+`/v2/key/info`, etc.
+"""
+
+GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
+    # OpenAI-compatible data-plane surface (versioned + unversioned)
+    "/v1/chat/",
+    "/chat/",
+    "/v1/completions",
+    "/completions",
+    "/v1/embeddings",
+    "/embeddings",
+    "/v1/moderations",
+    "/moderations",
+    "/v1/audio/",
+    "/audio/",
+    "/v1/images/",
+    "/images/",
+    "/v1/files",
+    "/files",
+    "/v1/batches",
+    "/batches",
+    "/v1/fine_tuning/",
+    "/fine_tuning/",
+    "/v1/fine-tuning/",
+    "/fine-tuning/",
+    "/v1/responses",
+    "/responses",
+    "/v1/threads",
+    "/threads",
+    "/v1/assistants",
+    "/assistants",
+    "/v1/vector_stores",
+    "/vector_stores",
+    "/v1/indexes",
+    "/v1/models",
+    "/models",
+    "/openai/",
+    "/engines/",
+    # Anthropic / agentic data-plane surface
+    "/v1/messages",
+    "/messages",
+    "/v1/skills",
+    "/v1/a2a/",
+    # LiteLLM-native LLM surface
+    "/v1/rerank",
+    "/v2/rerank",
+    "/rerank",
+    "/v1/ocr",
+    "/ocr",
+    "/v1/rag/",
+    "/rag/",
+    "/v1/video",
+    "/v1/videos",
+    "/video/",
+    "/videos",
+    "/v1/search",
+    "/search",
+    "/v1/containers",
+    "/containers",
+    "/v1/evals",
+    "/v1/memory",
+    "/queue/chat/",
+    # Google data plane (v1beta is the Google AI Studio version)
+    "/v1beta/",
+    "/interactions",
+    # Provider passthrough
+    "/anthropic/",
+    "/azure/",
+    "/azure_ai/",
+    "/aws/",
+    "/bedrock/",
+    "/cohere/",
+    "/gemini/",
+    "/google/",
+    "/vertex_ai/",
+    "/vertex-ai/",
+    "/assemblyai/",
+    "/eu.assemblyai/",
+    "/langfuse/",
+    "/vllm/",
+    "/mistral/",
+    "/groq/",
+    "/voyage/",
+    "/cursor/",
+    "/milvus/",
+    "/openai_passthrough/",
+    # Dynamic provider / toolset passthrough (path templates)
+    "/{provider}/",
+    "/toolset/",
+    # Realtime / streaming
+    "/v1/realtime",
+    "/realtime",
+    # Health & ops
+    "/health",
+    "/metrics",
+)
+
+GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
+    {
+        "/",
+        "/routes",
+        "/openapi.json",
+        "/docs",
+        "/docs/oauth2-redirect",
+        "/redoc",
+        "/test",
+    }
+)

helm/litellm/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+name: litellm
+description: LiteLLM componentized — gateway, UI backend, and UI as separate services
+type: application
+version: 0.1.0
+appVersion: "0.1.0"
+annotations:
+  org.opencontainers.image.source: "https://github.com/BerriAI/litellm"

helm/litellm/templates/NOTES.txt

@@ -0,0 +1,49 @@
+LiteLLM componentized — release {{ .Release.Name }} in namespace {{ .Release.Namespace }}.
+
+Components:
+{{- if .Values.gateway.enabled }}
+  - gateway  : Service {{ include "litellm.gateway.fullname" . }} on port {{ .Values.gateway.service.port }}
+{{- end }}
+{{- if .Values.backend.enabled }}
+  - backend  : Service {{ include "litellm.backend.fullname" . }} on port {{ .Values.backend.service.port }}
+{{- end }}
+{{- if .Values.ui.enabled }}
+  - ui       : Service {{ include "litellm.ui.fullname" . }} on port {{ .Values.ui.service.port }}
+{{- end }}
+
+Port-forward examples:
+  kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.gateway.fullname" . }} {{ .Values.gateway.service.port }}
+  kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.backend.fullname" . }} {{ .Values.backend.service.port }}
+  kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.ui.fullname" . }} {{ .Values.ui.service.port }}
+
+Reminders:
+  - Sensitive values come from Secret references only. Before installing, set:
+      - masterKey.secretName                                   (Secret with the proxy master key)
+      - database.writer.{host,port,dbname}                     (writer connection pieces)
+      - database.writer.passwordSecret.{name,usernameKey,passwordKey}
+                                                               (Secret holding the writer DB username + password)
+      - database.writer.useIAMAuth: true                       (optional — chart sets IAM_TOKEN_DB_AUTH=true and
+                                                                omits DATABASE_PASSWORD / DATABASE_URL so the proxy
+                                                                mints the URL from an IAM token at startup)
+      - database.reader.host                                   (optional — enables read-replica routing; reader
+                                                                .passwordSecret.name is required when set, unless
+                                                                .useIAMAuth is true)
+      - database.reader.useIAMAuth: true                       (optional, requires database.writer.useIAMAuth: true —
+                                                                chart emits DATABASE_*_READ_REPLICA env vars and
+                                                                omits DATABASE_PASSWORD_READ_REPLICA /
+                                                                DATABASE_URL_READ_REPLICA so the proxy mints the
+                                                                reader URL from an IAM token at startup)
+      - redis.passwordSecret.name                              (optional — set when redis.host is provided and the
+                                                                cache requires auth)
+      - redis.cluster: true                                    (optional — chart sets REDIS_CLUSTER_NODES from
+                                                                redis.host / redis.port so the proxy's Cache()
+                                                                constructs a RedisClusterCache; the cluster client
+                                                                discovers remaining nodes from CLUSTER SLOTS)
+  - Per-component extras (gateway / backend / ui):
+      - {component}.extraEnv / envConfigMaps / envSecrets      (the latter two are lists of resource names →
+                                                                envFrom configMapRef / secretRef)
+      - {component}.logLevel                                   (renders as LITELLM_LOG)
+      - gateway.config.proxy_config                            (rendered into a ConfigMap and mounted at
+                                                                /app/config/config.yaml; gateway reads it via
+                                                                CONFIG_FILE_PATH)
+  - Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend.

helm/litellm/templates/_helpers.tpl

@@ -0,0 +1,245 @@
+{{/*
+Common naming + label helpers shared by gateway, backend, and ui templates.
+*/}}
+
+{{- define "litellm.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "litellm.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "litellm.gateway.fullname" -}}
+{{- printf "%s-gateway" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "litellm.backend.fullname" -}}
+{{- printf "%s-backend" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "litellm.ui.fullname" -}}
+{{- printf "%s-ui" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "litellm.commonLabels" -}}
+app.kubernetes.io/name: {{ include "litellm.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
+{{- end -}}
+
+{{/*
+Per-component selector labels — used in both Service selectors and Deployment matchLabels.
+*/}}
+{{- define "litellm.gateway.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "litellm.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: gateway
+{{- end -}}
+
+{{- define "litellm.backend.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "litellm.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: backend
+{{- end -}}
+
+{{- define "litellm.ui.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "litellm.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/component: ui
+{{- end -}}
+
+{{/*
+Shared ServiceAccount name used by all three component Deployments. When
+`serviceAccount.create` is true and `serviceAccount.name` is empty, default
+to the chart fullname. When `create` is false, fall back to the provided
+name or the namespace's `default` SA.
+*/}}
+{{- define "litellm.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+{{ default (include "litellm.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+{{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Master-key + database + redis env block — shared by gateway, backend, and the
+migrations Job.
+
+Invoke with a dict: `(dict "root" $ "component" .Values.gateway)`. `root` is
+the chart context (needed for .Values), `component` selects which component's
+`extraEnv` / `logLevel` to render.
+
+Sensitive values (master key, DB username + password, Redis password) come
+only from referenced Secrets; the chart never accepts inline values for them.
+
+The chart never assembles DATABASE_URL itself. It emits only the discrete
+DATABASE_HOST/PORT/USER/NAME/SCHEMA (+ DATABASE_PASSWORD for password auth)
+vars; the proxy's entrypoint (DatabaseURLSettings in
+litellm/proxy/db/db_url_settings.py) builds the URL from them and
+percent-encodes the credentials. Assembling the URL here via Kubernetes
+`$(VAR)` substitution would embed the raw secret value, corrupting the URL
+whenever the password contains a URL-reserved character (@, /, ?, %, +,
+...) — as AWS RDS auto-generated passwords routinely do.
+
+When `database.writer.useIAMAuth: true`, the chart injects
+IAM_TOKEN_DB_AUTH=true and omits DATABASE_PASSWORD — the entrypoint mints
+the URL from DATABASE_HOST/PORT/USER/NAME plus a short-lived IAM token
+instead of a static password.
+
+The read replica is opt-in via `database.reader.host`. The chart emits
+DATABASE_HOST_READ_REPLICA / DATABASE_PORT_READ_REPLICA /
+DATABASE_NAME_READ_REPLICA (+ DATABASE_SCHEMA_READ_REPLICA) for both auth
+modes, plus DATABASE_USER_READ_REPLICA / DATABASE_PASSWORD_READ_REPLICA for
+password auth. When `database.reader.useIAMAuth: true` it omits
+DATABASE_PASSWORD_READ_REPLICA and the entrypoint mints the reader URL the
+same way. Reader IAM only takes effect when the writer also uses IAM auth
+(the proxy gates URL minting on IAM_TOKEN_DB_AUTH, which only the writer
+sets).
+*/}}
+{{- define "litellm.serverEnv" -}}
+{{- $root := .root -}}
+{{- $component := .component -}}
+- name: LITELLM_MASTER_KEY
+  valueFrom:
+    secretKeyRef:
+      name: {{ required "masterKey.secretName is required (the chart no longer accepts an inline master key)" $root.Values.masterKey.secretName }}
+      key: {{ $root.Values.masterKey.secretKey | default "master-key" }}
+{{- if $component.logLevel }}
+- name: LITELLM_LOG
+  value: {{ $component.logLevel | quote }}
+{{- end }}
+{{- with $root.Values.database.writer }}
+- name: DATABASE_HOST
+  value: {{ required "database.writer.host is required" .host | quote }}
+- name: DATABASE_PORT
+  value: {{ .port | default 5432 | quote }}
+- name: DATABASE_USER
+  valueFrom:
+    secretKeyRef:
+      name: {{ required "database.writer.passwordSecret.name is required" .passwordSecret.name }}
+      key: {{ .passwordSecret.usernameKey | default "username" }}
+- name: DATABASE_NAME
+  value: {{ required "database.writer.dbname is required" .dbname | quote }}
+{{- if .schema }}
+- name: DATABASE_SCHEMA
+  value: {{ .schema | quote }}
+{{- end }}
+{{- if .useIAMAuth }}
+- name: IAM_TOKEN_DB_AUTH
+  value: "true"
+{{- else }}
+- name: DATABASE_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      name: {{ .passwordSecret.name }}
+      key: {{ .passwordSecret.passwordKey | default "password" }}
+{{- end }}
+{{- end }}
+{{- with $root.Values.database.reader }}
+{{- if .host }}
+{{- if and .useIAMAuth (not $root.Values.database.writer.useIAMAuth) }}
+{{- fail "database.reader.useIAMAuth requires database.writer.useIAMAuth: true (the proxy gates IAM URL minting on IAM_TOKEN_DB_AUTH, which is only set by the writer)" }}
+{{- end }}
+- name: DATABASE_HOST_READ_REPLICA
+  value: {{ .host | quote }}
+- name: DATABASE_PORT_READ_REPLICA
+  value: {{ .port | default 5432 | quote }}
+- name: DATABASE_NAME_READ_REPLICA
+  value: {{ required "database.reader.dbname is required when database.reader.host is set" .dbname | quote }}
+{{- if .schema }}
+- name: DATABASE_SCHEMA_READ_REPLICA
+  value: {{ .schema | quote }}
+{{- end }}
+{{- if .useIAMAuth }}
+{{- if .passwordSecret.name }}
+- name: DATABASE_USER_READ_REPLICA
+  valueFrom:
+    secretKeyRef:
+      name: {{ .passwordSecret.name }}
+      key: {{ .passwordSecret.usernameKey | default "username" }}
+{{- end }}
+{{- else }}
+{{- if not .passwordSecret.name }}
+{{- fail "database.reader.passwordSecret.name is required when database.reader.host is set" }}
+{{- end }}
+- name: DATABASE_USER_READ_REPLICA
+  valueFrom:
+    secretKeyRef:
+      name: {{ .passwordSecret.name }}
+      key: {{ .passwordSecret.usernameKey | default "username" }}
+- name: DATABASE_PASSWORD_READ_REPLICA
+  valueFrom:
+    secretKeyRef:
+      name: {{ .passwordSecret.name }}
+      key: {{ .passwordSecret.passwordKey | default "password" }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{/*
+The migrations Job (helm.sh/hook: pre-upgrade) is the single owner of
+`prisma migrate deploy`. Without this, every gateway/backend pod also runs
+Prisma schema-update on startup and contends with the Job — and with each
+other — for Prisma's Postgres advisory lock on the writer, which makes the
+Job's `migrate deploy` intermittently block until its per-attempt timeout
+and retry-exhaust. The Job's entrypoint (migrations/run.py) does not import
+proxy_server and never reads DISABLE_SCHEMA_UPDATE, so emitting it here is a
+harmless no-op for the Job and authoritative for the app pods.
+*/}}
+- name: DISABLE_SCHEMA_UPDATE
+  value: "true"
+{{- if $root.Values.redis.host }}
+- name: REDIS_HOST
+  value: {{ $root.Values.redis.host | quote }}
+- name: REDIS_PORT
+  value: {{ $root.Values.redis.port | quote }}
+{{- if $root.Values.redis.passwordSecret.name }}
+- name: REDIS_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      name: {{ $root.Values.redis.passwordSecret.name }}
+      key: {{ $root.Values.redis.passwordSecret.passwordKey | default "password" }}
+{{- end }}
+{{- if $root.Values.redis.cluster }}
+{{/* The proxy's Cache() reads REDIS_CLUSTER_NODES as JSON and constructs a
+     RedisClusterCache when it's set (litellm/caching/caching.py:169-192).
+     We seed with the single configured endpoint — the cluster client
+     discovers the remaining nodes from CLUSTER SLOTS at startup. */}}
+- name: REDIS_CLUSTER_NODES
+  value: {{ printf "[{\"host\":%q,\"port\":%v}]" $root.Values.redis.host (int $root.Values.redis.port) | quote }}
+{{- end }}
+{{- end }}
+{{- with $component.extraEnv }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Renders `envFrom:` block for a component's `envConfigMaps` / `envSecrets`
+lists. Each entry is a resource name; the chart wires the whole ConfigMap /
+Secret into the container's env via configMapRef / secretRef.
+
+Invoke with just the component dict, e.g. `.Values.gateway`. Emits nothing
+when both lists are empty so the container spec stays clean.
+*/}}
+{{- define "litellm.envFrom" -}}
+{{- $component := . -}}
+{{- if or $component.envConfigMaps $component.envSecrets }}
+envFrom:
+{{- range $component.envConfigMaps }}
+  - configMapRef:
+      name: {{ . }}
+{{- end }}
+{{- range $component.envSecrets }}
+  - secretRef:
+      name: {{ . }}
+{{- end }}
+{{- end }}
+{{- end -}}

helm/litellm/templates/backend/deployment.yaml

@@ -0,0 +1,60 @@
+{{- if .Values.backend.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "litellm.backend.fullname" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: backend
+spec:
+  selector:
+    matchLabels:
+      {{- include "litellm.backend.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.backend.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "litellm.backend.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "litellm.serviceAccountName" . }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: backend
+          image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.backend.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 4001
+              protocol: TCP
+          env:
+            {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}
+          {{- include "litellm.envFrom" .Values.backend | nindent 10 }}
+          {{- with .Values.backend.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.backend.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          resources:
+            {{- toYaml .Values.backend.resources | nindent 12 }}
+      {{- with .Values.backend.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.backend.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.backend.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/litellm/templates/backend/hpa.yaml

@@ -0,0 +1,33 @@
+{{- if and .Values.backend.enabled .Values.backend.hpa.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "litellm.backend.fullname" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: backend
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "litellm.backend.fullname" . }}
+  minReplicas: {{ .Values.backend.hpa.minReplicas }}
+  maxReplicas: {{ .Values.backend.hpa.maxReplicas }}
+  metrics:
+    {{- if .Values.backend.hpa.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.backend.hpa.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.backend.hpa.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.backend.hpa.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

helm/litellm/templates/backend/service.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.backend.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "litellm.backend.fullname" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: backend
+spec:
+  type: {{ .Values.backend.service.type }}
+  ports:
+    - port: {{ .Values.backend.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "litellm.backend.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/litellm/templates/gateway/configmap.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.gateway.config.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "litellm.gateway.fullname" . }}-config
+data:
+  config.yaml: |
+{{ .Values.gateway.config.proxy_config | toYaml | indent 6 }}
+{{- end }}

helm/litellm/templates/gateway/deployment.yaml

@@ -0,0 +1,83 @@
+{{- if .Values.gateway.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "litellm.gateway.fullname" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: gateway
+spec:
+  selector:
+    matchLabels:
+      {{- include "litellm.gateway.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- if .Values.gateway.config.create }}
+        checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }}
+        {{- end }}
+        {{- with .Values.gateway.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "litellm.gateway.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "litellm.serviceAccountName" . }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: gateway
+          image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 4000
+              protocol: TCP
+          env:
+            {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
+            {{- if .Values.gateway.config.create }}
+            - name: CONFIG_FILE_PATH
+              value: /app/config/config.yaml
+            {{- end }}
+            {{- if .Values.gateway.numWorkers }}
+            - name: NUM_WORKERS
+              value: {{ .Values.gateway.numWorkers | quote }}
+            {{- end }}
+          {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
+          {{- if .Values.gateway.config.create }}
+          volumeMounts:
+            - name: gateway-config
+              mountPath: /app/config/config.yaml
+              subPath: config.yaml
+          {{- end }}
+          {{- with .Values.gateway.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.gateway.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          resources:
+            {{- toYaml .Values.gateway.resources | nindent 12 }}
+      {{- if .Values.gateway.config.create }}
+      volumes:
+        - name: gateway-config
+          configMap:
+            name: {{ include "litellm.gateway.fullname" . }}-config
+      {{- end }}
+      {{- with .Values.gateway.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.gateway.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.gateway.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/litellm/templates/gateway/hpa.yaml

@@ -0,0 +1,33 @@
+{{- if and .Values.gateway.enabled .Values.gateway.hpa.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "litellm.gateway.fullname" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: gateway
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "litellm.gateway.fullname" . }}
+  minReplicas: {{ .Values.gateway.hpa.minReplicas }}
+  maxReplicas: {{ .Values.gateway.hpa.maxReplicas }}
+  metrics:
+    {{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

helm/litellm/templates/gateway/service.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.gateway.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "litellm.gateway.fullname" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: gateway
+spec:
+  type: {{ .Values.gateway.service.type }}
+  ports:
+    - port: {{ .Values.gateway.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "litellm.gateway.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/litellm/templates/ingress.yaml

@@ -0,0 +1,153 @@
+{{- if .Values.ingress.enabled -}}
+{{- $gatewayName := include "litellm.gateway.fullname" . -}}
+{{- $backendName := include "litellm.backend.fullname" . -}}
+{{- $uiName := include "litellm.ui.fullname" . -}}
+{{- $gatewayPort := .Values.gateway.service.port -}}
+{{- $backendPort := .Values.backend.service.port -}}
+{{- $uiPort := .Values.ui.service.port -}}
+{{/*
+  Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
+  Versioned paths are listed explicitly to avoid routing management routes
+  (e.g. /v1/access_group, /v2/key/info, /v1/tool/*, /v1/agents, /v1/workflows,
+  /v2/user/info, /v2/team/list, /v2/model/info, /v2/login, /v2/guardrails/*,
+  /v1/mcp/*) onto the gateway via a broad /v1 or /v2 prefix.
+*/}}
+{{- $gatewayPrefixes := list
+    "/v1/chat" "/chat" "/v1/completions" "/completions" "/v1/embeddings" "/embeddings"
+    "/v1/moderations" "/moderations" "/v1/audio" "/audio" "/v1/images" "/images"
+    "/v1/files" "/files" "/v1/batches" "/batches" "/v1/fine_tuning" "/fine_tuning"
+    "/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads"
+    "/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes"
+    "/v1/models" "/models" "/openai" "/engines"
+    "/v1/messages" "/messages" "/v1/skills" "/v1/a2a"
+    "/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag"
+    "/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
+    "/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"
+    "/v1beta" "/interactions"
+    "/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/cohere" "/gemini" "/google"
+    "/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm"
+    "/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough"
+    "/toolset"
+    "/v1/realtime" "/realtime"
+    "/health" "/metrics"
+-}}
+{{/*
+  /test is gateway-only as an EXACT path (GATEWAY_EXACT_PATHS), but its
+  children /test/connection and /test/tools/list are MCP-server management
+  endpoints kept only on the backend ("/test/" in BACKEND_PATH_PREFIXES).
+  A Prefix match here would route /test/* to the gateway, which trims those
+  routes at startup -> 404. So /test is rendered as a standalone Exact path
+  and /test/* falls through to the backend catch-all.
+*/}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "litellm.fullname" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- with .Values.ingress.className }}
+  ingressClassName: {{ . | quote }}
+  {{- end }}
+  {{- with .Values.ingress.tls }}
+  tls:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  rules:
+    - {{- with .Values.ingress.host }}
+      host: {{ . | quote }}
+      {{- end }}
+      http:
+        paths:
+          # --- UI (Next.js static export) ---
+          - path: /
+            pathType: Exact
+            backend:
+              service:
+                name: {{ $uiName }}
+                port:
+                  number: {{ $uiPort }}
+          - path: /favicon.ico
+            pathType: Exact
+            backend:
+              service:
+                name: {{ $uiName }}
+                port:
+                  number: {{ $uiPort }}
+          - path: /litellm-asset-prefix
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ $uiName }}
+                port:
+                  number: {{ $uiPort }}
+          - path: /_next
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ $uiName }}
+                port:
+                  number: {{ $uiPort }}
+          # /ui/* is where the Next.js SPA serves its login + dashboard
+          # routes (e.g. /ui/login). Without this, /ui/* falls into the
+          # catch-all → backend → 404.
+          - path: /ui
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ $uiName }}
+                port:
+                  number: {{ $uiPort }}
+          # Next.js App Router (output: "export", basePath: "") emits the
+          # RSC/flight payload for every route as a ROOT-level <route>.txt
+          # (/index.txt, /teams.txt, /__next._tree.txt, ...). The client
+          # router fetches these on every soft navigation / prefetch as
+          # <route>.txt?_rsc=<hash> (the query string is irrelevant to path
+          # matching). They are not under /ui, /_next, or
+          # /litellm-asset-prefix, so without this rule they fall to the
+          # backend catch-all → 404 → client-side navigation never settles
+          # and the login flow spins in an infinite redirect loop
+          # (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt from the
+          # export; this rule only routes the request to it. Needs an
+          # ingress controller whose ImplementationSpecific path is a
+          # wildcard pattern (AWS ALB: `*` = 0+ chars); this chart targets
+          # the AWS Load Balancer Controller.
+          - path: /*.txt
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: {{ $uiName }}
+                port:
+                  number: {{ $uiPort }}
+          # --- Gateway data plane ---
+          # Exact /test only (see the $gatewayPrefixes comment above);
+          # /test/* MCP management endpoints fall to the backend catch-all.
+          - path: /test
+            pathType: Exact
+            backend:
+              service:
+                name: {{ $gatewayName }}
+                port:
+                  number: {{ $gatewayPort }}
+          {{- range $gatewayPrefixes }}
+          - path: {{ . }}
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ $gatewayName }}
+                port:
+                  number: {{ $gatewayPort }}
+          {{- end }}
+          # --- Catch-all → backend (management API: /key/*, /user/*, /team/*, ...) ---
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ $backendName }}
+                port:
+                  number: {{ $backendPort }}
+{{- end }}

helm/litellm/templates/migrations-job.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.migrationJob.enabled -}}
+# Pre-install / pre-upgrade hook that runs `prisma migrate deploy` against
+# the writer database before the gateway and backend Deployments are rolled
+# out. Required because the gateway and backend both spin up Prisma at
+# startup and assume the LiteLLM schema (LiteLLM_Config,
+# LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...) already exists.
+#
+# Running this pre-upgrade closes the window where new application pods would
+# otherwise serve traffic against the previous release's unmigrated schema.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "litellm.fullname" . }}-migrations
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: migrations
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation
+    helm.sh/hook-weight: "0"
+spec:
+  backoffLimit: {{ .Values.migrationJob.backoffLimit }}
+  ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
+  template:
+    metadata:
+      labels:
+        {{- include "litellm.commonLabels" . | nindent 8 }}
+        app.kubernetes.io/component: migrations
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ include "litellm.serviceAccountName" . }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: prisma-migrations
+          image: "{{ .Values.migrationJob.image.repository }}:{{ .Values.migrationJob.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.migrationJob.image.pullPolicy }}
+          env:
+            {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.migrationJob) | nindent 12 }}
+          {{- with .Values.migrationJob.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+{{- end }}

helm/litellm/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "litellm.serviceAccountName" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
+{{- end }}

helm/litellm/templates/ui/deployment.yaml

@@ -0,0 +1,70 @@
+{{- if .Values.ui.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "litellm.ui.fullname" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: ui
+spec:
+  selector:
+    matchLabels:
+      {{- include "litellm.ui.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.ui.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "litellm.ui.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "litellm.serviceAccountName" . }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: ui
+          image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.ui.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 3000
+              protocol: TCP
+          env:
+            {{- if .Values.ui.logLevel }}
+            - name: LITELLM_LOG
+              value: {{ .Values.ui.logLevel | quote }}
+            {{- end }}
+            {{- if .Values.ui.backendUrl }}
+            - name: LITELLM_BACKEND_URL
+              value: {{ .Values.ui.backendUrl | quote }}
+            {{- end }}
+            {{- with .Values.ui.extraEnv }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- include "litellm.envFrom" .Values.ui | nindent 10 }}
+          {{- with .Values.ui.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.ui.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          resources:
+            {{- toYaml .Values.ui.resources | nindent 12 }}
+      {{- with .Values.ui.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.ui.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.ui.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/litellm/templates/ui/hpa.yaml

@@ -0,0 +1,33 @@
+{{- if and .Values.ui.enabled .Values.ui.hpa.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "litellm.ui.fullname" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: ui
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "litellm.ui.fullname" . }}
+  minReplicas: {{ .Values.ui.hpa.minReplicas }}
+  maxReplicas: {{ .Values.ui.hpa.maxReplicas }}
+  metrics:
+    {{- if .Values.ui.hpa.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.ui.hpa.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.ui.hpa.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.ui.hpa.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

helm/litellm/templates/ui/service.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.ui.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "litellm.ui.fullname" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: ui
+spec:
+  type: {{ .Values.ui.service.type }}
+  ports:
+    - port: {{ .Values.ui.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "litellm.ui.selectorLabels" . | nindent 4 }}
+{{- end }}

helm/litellm/values.yaml

@@ -0,0 +1,225 @@
+# LiteLLM helm chart values
+
+nameOverride: ""
+fullnameOverride: ""
+
+imagePullSecrets: []
+
+# Optional Ingress wiring the three component Services behind a single L7
+# entrypoint. Required when serving the static UI bundle over the network.
+ingress:
+  enabled: false
+  className: ""
+  annotations: {}
+  host: ""        # optional; if set, becomes the rule's host
+  tls: []
+
+# Shared ServiceAccount used by all three component Deployments. Set
+# `create: true` to have the chart provision it (e.g. when wiring an EKS
+# Pod Identity association by SA name). Set `name` to use an existing SA
+# (chart-created or out-of-band). When both are empty / false, pods run
+# with the namespace's `default` SA.
+serviceAccount:
+  create: false
+  automount: true
+  annotations: {}
+  name: ""
+
+# Pre-install / pre-upgrade Helm hook that runs `prisma migrate deploy`
+# against the writer database, creating the LiteLLM schema (tables that
+# gateway + backend assume exist at startup: LiteLLM_Config,
+# LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...). Disable if your
+# pipeline runs migrations out-of-band.
+#
+# Uses a dedicated `litellm-migrations` image (prisma CLI + the migration
+# files from `litellm-proxy-extras`) instead of the backend image, so the
+# Job doesn't drag in the rest of the proxy and doesn't run `prisma
+# generate` — the migration engine doesn't need the generated client.
+migrationJob:
+  enabled: true
+  backoffLimit: 4
+  ttlSecondsAfterFinished: 120
+  resources: {}
+  image:
+    repository: ghcr.io/berriai/litellm-migrations
+    tag: ""              # defaults to .Chart.AppVersion
+    pullPolicy: IfNotPresent
+  # Extra env appended to the migration container. The migration entrypoint
+  # uses the v2 resolver by default (no diff-and-force recovery — avoids the
+  # schema thrashing seen during rolling deploys). To opt back into the v1
+  # resolver, append `- name: USE_V2_MIGRATION_RESOLVER` / `value: "false"`.
+  extraEnv: []
+
+# Required: a master key used by gateway + backend to mint/verify proxy tokens.
+# Must reference an existing Secret.
+masterKey:
+  secretName: litellm-master-key-secret              # name of a Secret containing the master key
+  secretKey: master-key
+
+# External Postgres connection. 
+database:
+  writer:
+    host: ""
+    port: 5432
+    dbname: ""
+    schema: ""
+    useIAMAuth: false
+    passwordSecret:
+      name: litellm-writer-secret
+      usernameKey: username
+      passwordKey: password
+
+  # Optional read-replica routing. When `reader.host` is set, the proxy routes
+  # reads (find_*, count, group_by, query_raw/_first) to this endpoint while
+  # writes stay on the writer. Leave `reader.host` empty to disable.
+  reader:
+    host: ""
+    port: 5432
+    dbname: ""
+    schema: ""
+    useIAMAuth: false
+    passwordSecret:
+      name: litellm-reader-secret
+      usernameKey: username
+      passwordKey: password
+
+# Optional Redis (caching, rate limiting). Leave host empty to disable.
+#
+# Set `cluster: true` for Redis Cluster mode (e.g. AWS ElastiCache Cluster,
+# self-hosted Redis Cluster). The chart emits REDIS_CLUSTER_NODES from
+# `host` / `port` as the single seed; the cluster client discovers the
+# remaining nodes from CLUSTER SLOTS at startup.
+redis:
+  cluster: false
+  host: ""
+  port: 6379
+  passwordSecret:
+    name: "" # Leave empty for auth-less Redis
+    passwordKey: password
+
+# ---------- gateway (LLM data plane) ----------
+gateway:
+  enabled: true
+  logLevel: INFO
+  # Number of uvicorn worker processes per gateway pod. Sets NUM_WORKERS,
+  # consumed by the gateway image entrypoint. Default is 1.
+  numWorkers: 1
+  extraEnv: [] # Add extra environment variables to the gateway
+  envConfigMaps: [] # Add extra environment variables to the gateway from config maps
+  envSecrets: [] # Add extra environment variables to the gateway from secrets
+  config:
+    create: true
+    proxy_config: {}
+  image:
+    repository: ghcr.io/berriai/litellm-gateway
+    tag: ""              # defaults to .Chart.AppVersion
+    pullPolicy: IfNotPresent
+  service:
+    type: ClusterIP
+    port: 4000
+  resources:
+    requests:
+      cpu: "1"
+      memory: 4Gi
+    limits:
+      cpu: "2"
+      memory: 4Gi
+  livenessProbe:
+    httpGet: { path: /health/liveliness, port: http }
+    initialDelaySeconds: 10
+    periodSeconds: 15
+  readinessProbe:
+    httpGet: { path: /health/readiness, port: http }
+    initialDelaySeconds: 5
+    periodSeconds: 10
+  hpa:
+    enabled: true
+    minReplicas: 1
+    maxReplicas: 10
+    targetCPUUtilizationPercentage: 70
+    targetMemoryUtilizationPercentage: 80
+  podAnnotations: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+
+# ---------- backend (UI / management API) ----------
+backend:
+  enabled: true
+  logLevel: INFO
+  extraEnv: []
+  envConfigMaps: []
+  envSecrets: []
+  image:
+    repository: ghcr.io/berriai/litellm-backend
+    tag: ""
+    pullPolicy: IfNotPresent
+  service:
+    type: ClusterIP
+    port: 4001
+  resources:
+    requests:
+      cpu: "1"
+      memory: 4Gi
+    limits:
+      cpu: "2"
+      memory: 4Gi
+  livenessProbe:
+    httpGet: { path: /health/liveliness, port: http }
+    initialDelaySeconds: 10
+    periodSeconds: 15
+  readinessProbe:
+    httpGet: { path: /health/readiness, port: http }
+    initialDelaySeconds: 5
+    periodSeconds: 10
+  hpa:
+    enabled: true
+    minReplicas: 1
+    maxReplicas: 4
+    targetCPUUtilizationPercentage: 70
+  podAnnotations: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+
+# ---------- ui (Next.js static dashboard) ----------
+ui:
+  enabled: true
+  logLevel: INFO
+  extraEnv: []
+  envConfigMaps: []
+  envSecrets: []
+  image:
+    repository: ghcr.io/berriai/litellm-ui
+    tag: ""
+    pullPolicy: IfNotPresent
+  service:
+    type: ClusterIP
+    port: 3000
+  # The dashboard expects to know where to reach the backend API. Set this to
+  # the externally-routable URL (typically the ingress host + /api or similar).
+  backendUrl: ""
+  resources:
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    limits:
+      cpu: "1"
+      memory: 1Gi
+  livenessProbe:
+    httpGet: { path: /, port: http }
+    initialDelaySeconds: 5
+    periodSeconds: 20
+  readinessProbe:
+    httpGet: { path: /, port: http }
+    initialDelaySeconds: 2
+    periodSeconds: 10
+  hpa:
+    enabled: false
+    minReplicas: 1
+    maxReplicas: 3
+    targetCPUUtilizationPercentage: 80
+  podAnnotations: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}

litellm-proxy-extras/litellm_proxy_extras/migrations/20260421135425_add_team_membership_total_spend/migration.sql

@@ -1,3 +1,3 @@
 -- AlterTable
-ALTER TABLE "LiteLLM_TeamMembership" ADD COLUMN     "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_TeamMembership" ADD COLUMN IF NOT EXISTS "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
 

litellm-proxy-extras/litellm_proxy_extras/migrations/20260513120000_add_delegate_auth_to_upstream_to_mcp_servers/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "delegate_auth_to_upstream" BOOLEAN NOT NULL DEFAULT false;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260514120000_add_blocked_to_proxy_model_table/migration.sql

@@ -0,0 +1,4 @@
+-- AlterTable
+-- Adds the admin-toggleable pause flag used by the router's blocked filter and the
+-- credential lookup helpers; defaults to false so existing rows behave unchanged.
+ALTER TABLE "LiteLLM_ProxyModelTable" ADD COLUMN IF NOT EXISTS "blocked" BOOLEAN NOT NULL DEFAULT false;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -48,9 +48,10 @@ model LiteLLM_CredentialsTable {
 // Models on proxy
 model LiteLLM_ProxyModelTable {
   model_id String @id @default(uuid())
-  model_name String 
+  model_name String
   litellm_params Json
-  model_info Json? 
+  model_info Json?
+  blocked Boolean @default(false)
   created_at    DateTime               @default(now()) @map("created_at")
   created_by String
   updated_at    DateTime               @default(now()) @updatedAt @map("updated_at")
@@ -323,6 +324,7 @@ model LiteLLM_MCPServerTable {
   registration_url   String?
   allow_all_keys Boolean @default(false)
   available_on_public_internet Boolean @default(true)
+  delegate_auth_to_upstream Boolean @default(false)
   is_byok               Boolean  @default(false)
   byok_description      String[] @default([])
   byok_api_key_help_url String?

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.71"
+version = "0.4.72"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.71"
+version = "0.4.72"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/__init__.py

@@ -416,6 +416,7 @@ custom_prometheus_metadata_labels: List[str] = []
 custom_prometheus_tags: List[str] = []
 prometheus_metrics_config: Optional[List] = None
 prometheus_emit_stream_label: bool = False
+prometheus_user_budget_label_include_email_alias: bool = False
 prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
 prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
 prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0
@@ -1880,6 +1881,12 @@ if TYPE_CHECKING:
     from .llms.dashscope.chat.transformation import (
         DashScopeChatConfig as DashScopeChatConfig,
     )
+    from .llms.dashscope.embed.transformation import (
+        DashScopeEmbeddingConfig as DashScopeEmbeddingConfig,
+    )
+    from .llms.dashscope.rerank.transformation import (
+        DashScopeRerankConfig as DashScopeRerankConfig,
+    )
     from .llms.moonshot.chat.transformation import (
         MoonshotChatConfig as MoonshotChatConfig,
     )

litellm/_lazy_imports_registry.py

@@ -376,7 +376,6 @@ UTILS_MODULE_NAMES = (
     "HTTPHandler",
     "get_num_retries_from_retry_policy",
     "reset_retry_policy",
-    "get_secret",
     "get_coroutine_checker",
     "get_litellm_logging_class",
     "get_set_callbacks",
@@ -1284,7 +1283,6 @@ _UTILS_MODULE_IMPORT_MAP = {
         "litellm.router_utils.get_retry_from_policy",
         "reset_retry_policy",
     ),
-    "get_secret": ("litellm.secret_managers.main", "get_secret"),
     "get_coroutine_checker": (
         "litellm.litellm_core_utils.cached_imports",
         "get_coroutine_checker",

litellm/_logging.py

@@ -404,6 +404,7 @@ def _turn_on_debug():
 
 
 def _disable_debugging():
+    """Disable the package, router, and proxy verbose loggers."""
     verbose_logger.disabled = True
     verbose_router_logger.disabled = True
     verbose_proxy_logger.disabled = True

litellm/_redis.py

@@ -41,7 +41,7 @@ def _get_redis_kwargs():
         "retry",
     }
 
-    include_args = [
+    include_args = {
         "url",
         "redis_connect_func",
         "gcp_service_account",
@@ -50,9 +50,9 @@ def _get_redis_kwargs():
         "azure_client_id",
         "azure_tenant_id",
         "azure_client_secret",
-    ]
+    }
 
-    available_args = [x for x in arg_spec.args if x not in exclude_args] + include_args
+    available_args = {x for x in arg_spec.args if x not in exclude_args} | include_args
 
     return available_args
 
@@ -84,23 +84,25 @@ def _get_redis_cluster_kwargs(client=None):
     # Only allow primitive arguments
     exclude_args = {"self", "connection_pool", "retry", "host", "port", "startup_nodes"}
 
-    available_args = [x for x in arg_spec.args if x not in exclude_args]
-    available_args.append("password")
-    available_args.append("username")
-    available_args.append("ssl")
-    available_args.append("ssl_cert_reqs")
-    available_args.append("ssl_check_hostname")
-    available_args.append("ssl_ca_certs")
-    available_args.append(
-        "redis_connect_func"
-    )  # Needed for sync clusters and IAM detection
-    available_args.append("gcp_service_account")
-    available_args.append("gcp_ssl_ca_certs")
-    available_args.append("azure_redis_ad_token")
-    available_args.append("azure_client_id")
-    available_args.append("azure_tenant_id")
-    available_args.append("azure_client_secret")
-    available_args.append("max_connections")
+    available_args = {x for x in arg_spec.args if x not in exclude_args}
+    available_args |= {
+        "password",
+        "username",
+        "ssl",
+        "ssl_cert_reqs",
+        "ssl_check_hostname",
+        "ssl_ca_certs",
+        "redis_connect_func",  # Needed for sync clusters and IAM detection
+        "gcp_service_account",
+        "gcp_ssl_ca_certs",
+        "azure_redis_ad_token",
+        "azure_client_id",
+        "azure_tenant_id",
+        "azure_client_secret",
+        "max_connections",
+        "socket_timeout",
+        "socket_connect_timeout",
+    }
 
     return available_args
 
@@ -479,10 +481,24 @@ def init_redis_cluster(redis_kwargs) -> redis.RedisCluster:
     return redis.RedisCluster(startup_nodes=new_startup_nodes, **cluster_kwargs)  # type: ignore
 
 
+def _get_redis_sentinel_connection_kwargs(redis_kwargs: dict) -> dict:
+    connection_kwargs = {}
+    args = _get_redis_kwargs()
+    for arg in redis_kwargs:
+        if arg in args:
+            connection_kwargs[arg] = redis_kwargs[arg]
+
+    return connection_kwargs
+
+
 def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
     sentinel_nodes = redis_kwargs.get("sentinel_nodes")
     sentinel_password = redis_kwargs.get("sentinel_password")
     service_name = redis_kwargs.get("service_name")
+    connection_kwargs = _get_redis_sentinel_connection_kwargs(redis_kwargs)
+    connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
+    sentinel_kwargs = dict(connection_kwargs)
+    sentinel_kwargs["password"] = sentinel_password
 
     if not sentinel_nodes or not service_name:
         raise ValueError(
@@ -494,19 +510,22 @@ def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
     # Set up the Sentinel client
     sentinel = redis.Sentinel(
         sentinel_nodes,
-        socket_timeout=REDIS_SOCKET_TIMEOUT,
-        password=sentinel_password,
+        sentinel_kwargs=sentinel_kwargs,
     )
 
     # Return the master instance for the given service
 
-    return sentinel.master_for(service_name)
+    return sentinel.master_for(service_name, **connection_kwargs)
 
 
 def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
     sentinel_nodes = redis_kwargs.get("sentinel_nodes")
     sentinel_password = redis_kwargs.get("sentinel_password")
     service_name = redis_kwargs.get("service_name")
+    connection_kwargs = _get_redis_sentinel_connection_kwargs(redis_kwargs)
+    connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
+    sentinel_kwargs = dict(connection_kwargs)
+    sentinel_kwargs["password"] = sentinel_password
 
     if not sentinel_nodes or not service_name:
         raise ValueError(
@@ -518,13 +537,12 @@ def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
     # Set up the Sentinel client
     sentinel = async_redis.Sentinel(
         sentinel_nodes,
-        socket_timeout=REDIS_SOCKET_TIMEOUT,
-        password=sentinel_password,
+        sentinel_kwargs=sentinel_kwargs,
     )
 
     # Return the master instance for the given service
 
-    return sentinel.master_for(service_name)
+    return sentinel.master_for(service_name, **connection_kwargs)
 
 
 def get_redis_client(**env_overrides):

litellm/batches/batch_utils.py

@@ -113,8 +113,11 @@ def _batch_cost_calculator(
     """
     Calculate the cost of a batch based on the output file id
     """
-    # Handle Vertex AI with specialized method
-    if custom_llm_provider == "vertex_ai" and model_name:
+    if (
+        custom_llm_provider == "vertex_ai"
+        and model_name
+        and getattr(litellm, "disable_vertex_batch_output_transformation", False)
+    ):
         batch_cost, _ = calculate_vertex_ai_batch_cost_and_usage(
             file_content_dictionary, model_name
         )
@@ -136,10 +139,13 @@ def calculate_vertex_ai_batch_cost_and_usage(
     model_name: Optional[str] = None,
 ) -> Tuple[float, Usage]:
     """
-    Calculate both cost and usage from Vertex AI batch responses.
+    Calculate both cost and usage from raw Vertex AI batch responses.
 
-    Vertex AI batch output lines have format:
-      {"request": ..., "status": "", "response": {"candidates": [...], "usageMetadata": {...}}}
+    Used only when ``litellm.disable_vertex_batch_output_transformation = True``.
+    In that case the GCS predictions.jsonl is returned as-is, with each line in
+    the native Vertex format:
+
+      {"request": ..., "response": {"candidates": [...], "usageMetadata": {...}}}
 
     usageMetadata contains promptTokenCount, candidatesTokenCount, totalTokenCount.
     """
@@ -362,8 +368,11 @@ def _get_batch_job_total_usage_from_file_content(
     """
     Get the tokens of a batch job from the file content
     """
-    # Handle Vertex AI with specialized method
-    if custom_llm_provider == "vertex_ai" and model_name:
+    if (
+        custom_llm_provider == "vertex_ai"
+        and model_name
+        and getattr(litellm, "disable_vertex_batch_output_transformation", False)
+    ):
         _, batch_usage = calculate_vertex_ai_batch_cost_and_usage(
             file_content_dictionary, model_name
         )

litellm/budget_manager.py

@@ -178,6 +178,18 @@ class BudgetManager:
         return list(self.user_dict.keys())
 
     def reset_cost(self, user):
+        """
+        Reset the tracked spend for a user back to zero.
+
+        Clears both the aggregate ``current_cost`` and the per-model
+        ``model_cost`` breakdown stored for the given user.
+
+        Args:
+            user: The user identifier whose cost should be reset.
+
+        Returns:
+            dict: ``{"user": <updated user record>}`` reflecting the reset state.
+        """
         self.user_dict[user]["current_cost"] = 0
         self.user_dict[user]["model_cost"] = {}
         return {"user": self.user_dict[user]}

litellm/caching/caching_handler.py

@@ -87,6 +87,16 @@ class CachingHandlerResponse(BaseModel):
 in_memory_cache_obj = InMemoryCache()
 
 
+def _is_chat_completion_cached_dict(cached_result: dict) -> bool:
+    cached_id = cached_result.get("id")
+    if isinstance(cached_id, str) and cached_id.startswith("chatcmpl"):
+        return True
+    obj = cached_result.get("object")
+    if isinstance(obj, str):
+        return obj.startswith("chat.completion")
+    return "choices" in cached_result
+
+
 def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bool:
     """
     When stream=True, do not run success callbacks at cache-hit time.
@@ -861,27 +871,47 @@ class LLMCachingHandler:
         elif (call_type == "aresponses" or call_type == "responses") and isinstance(
             cached_result, dict
         ):
-            from litellm.responses.streaming_iterator import (
-                CachedResponsesAPIStreamingIterator,
-            )
-
-            response_obj = ResponsesAPIResponse(**cached_result)
-            if (
-                hasattr(response_obj, "_hidden_params")
-                and response_obj._hidden_params is not None
-                and isinstance(response_obj._hidden_params, dict)
-            ):
-                response_obj._hidden_params["cache_hit"] = True
-
-            if kwargs.get("stream", False) is True:
-                cached_result = CachedResponsesAPIStreamingIterator(
-                    response=response_obj,
-                    logging_obj=logging_obj,
-                    request_data=kwargs,
-                    call_type=call_type,
-                )
+            use_chat_completion_cache = _is_chat_completion_cached_dict(cached_result)
+            if use_chat_completion_cache:
+                if kwargs.get("stream", False) is True:
+                    bridge_call_type = (
+                        CallTypes.acompletion.value
+                        if call_type == "aresponses"
+                        else CallTypes.completion.value
+                    )
+                    cached_result = self._convert_cached_stream_response(
+                        cached_result=cached_result,
+                        call_type=bridge_call_type,
+                        logging_obj=logging_obj,
+                        model=model,
+                    )
+                else:
+                    cached_result = convert_to_model_response_object(
+                        response_object=cached_result,
+                        model_response_object=ModelResponse(),
+                    )
             else:
-                cached_result = response_obj
+                from litellm.responses.streaming_iterator import (
+                    CachedResponsesAPIStreamingIterator,
+                )
+
+                response_obj = ResponsesAPIResponse(**cached_result)
+                if (
+                    hasattr(response_obj, "_hidden_params")
+                    and response_obj._hidden_params is not None
+                    and isinstance(response_obj._hidden_params, dict)
+                ):
+                    response_obj._hidden_params["cache_hit"] = True
+
+                if kwargs.get("stream", False) is True:
+                    cached_result = CachedResponsesAPIStreamingIterator(
+                        response=response_obj,
+                        logging_obj=logging_obj,
+                        request_data=kwargs,
+                        call_type=call_type,
+                    )
+                else:
+                    cached_result = response_obj
 
         if (
             hasattr(cached_result, "_hidden_params")

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -37,6 +37,15 @@ class ResponsesToCompletionBridgeHandler:
             stream = litellm_params.get("stream", False)
         return bool(stream)
 
+    @staticmethod
+    def _is_preformatted_cached_chat_stream(result: Any) -> bool:
+        from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
+
+        return (
+            isinstance(result, CustomStreamWrapper)
+            and result.custom_llm_provider == "cached_response"
+        )
+
     @staticmethod
     def _coerce_response_object(
         response_obj: Any,
@@ -177,6 +186,8 @@ class ResponsesToCompletionBridgeHandler:
             **request_data,
         )
 
+        from litellm.types.utils import ModelResponse
+
         stream = self._resolve_stream_flag(optional_params, litellm_params)
         if isinstance(result, ResponsesAPIResponse):
             return self.transformation_handler.transform_response(
@@ -192,6 +203,8 @@ class ResponsesToCompletionBridgeHandler:
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
+        elif isinstance(result, ModelResponse):
+            return result
         elif not stream:
             responses_api_response = self._collect_response_from_stream(result)
             return self.transformation_handler.transform_response(
@@ -208,6 +221,10 @@ class ResponsesToCompletionBridgeHandler:
                 json_mode=kwargs.get("json_mode"),
             )
         else:
+            if self._is_preformatted_cached_chat_stream(result):
+                return self._apply_post_stream_processing(
+                    result, model, custom_llm_provider
+                )
             completion_stream = self.transformation_handler.get_model_response_iterator(
                 streaming_response=result,  # type: ignore
                 sync_stream=True,
@@ -256,6 +273,8 @@ class ResponsesToCompletionBridgeHandler:
             aresponses=True,
         )
 
+        from litellm.types.utils import ModelResponse
+
         stream = self._resolve_stream_flag(optional_params, litellm_params)
         if isinstance(result, ResponsesAPIResponse):
             return self.transformation_handler.transform_response(
@@ -271,6 +290,8 @@ class ResponsesToCompletionBridgeHandler:
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
+        elif isinstance(result, ModelResponse):
+            return result
         elif not stream:
             responses_api_response = await self._collect_response_from_stream_async(
                 result
@@ -289,6 +310,10 @@ class ResponsesToCompletionBridgeHandler:
                 json_mode=kwargs.get("json_mode"),
             )
         else:
+            if self._is_preformatted_cached_chat_stream(result):
+                return self._apply_post_stream_processing(
+                    result, model, custom_llm_provider
+                )
             completion_stream = self.transformation_handler.get_model_response_iterator(
                 streaming_response=result,  # type: ignore
                 sync_stream=False,

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -1141,6 +1141,14 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         event_type = parsed_chunk.get("type")
         if isinstance(event_type, ResponsesAPIStreamEvents):
             event_type = event_type.value
+
+        if parsed_chunk.get("object") == "chat.completion.chunk" or (
+            event_type is None
+            and isinstance(parsed_chunk.get("choices"), list)
+            and parsed_chunk.get("choices")
+        ):
+            return ModelResponseStream(**parsed_chunk)
+
         verbose_logger.debug(f"Chat provider: Processing event type: {event_type}")
 
         if event_type == "response.created":

litellm/constants.py

@@ -1569,6 +1569,15 @@ DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(
     os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)
 )
 DEFAULT_ACCESS_GROUP_CACHE_TTL = int(os.getenv("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600))
+# Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated
+# callers from forcing a DB query per request for unknown names, while bounding
+# staleness so a transient DB error (which surfaces as an empty list) cannot
+# hide a real group for long.
+DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL = 10
+# Maximum number of comma-separated MCP server / access-group tokens accepted
+# in a single ``/{name1,name2,...}/mcp`` URL. Bounds the per-request DB / cache
+# fan-out an authenticated caller can trigger by stuffing the path with tokens.
+DEFAULT_MCP_NAMESPACE_CSV_MAX_TOKENS = 16
 
 # Sentry Scrubbing Configuration
 SENTRY_DENYLIST = [

litellm/cost_calculator.py

@@ -173,17 +173,45 @@ def _cost_per_token_custom_pricing_helper(
     prompt_tokens: float = 0,
     completion_tokens: float = 0,
     response_time_ms: Optional[float] = 0.0,
+    cached_tokens: float = 0,
+    cache_creation_tokens: float = 0,
     ### CUSTOM PRICING ###
     custom_cost_per_token: Optional[CostPerToken] = None,
     custom_cost_per_second: Optional[float] = None,
 ) -> Optional[Tuple[float, float]]:
-    """Internal helper function for calculating cost, if custom pricing given"""
+    """Internal helper function for calculating cost, if custom pricing given.
+
+    prompt_tokens is assumed to include both cached_tokens and cache_creation_tokens
+    (OpenAI-compatible convention). Anthropic-style usage where prompt_tokens excludes
+    cache tokens is handled at the caller (cost_per_token) before invoking this helper.
+    """
     if custom_cost_per_token is None and custom_cost_per_second is None:
         return None
 
     if custom_cost_per_token is not None:
-        input_cost = custom_cost_per_token["input_cost_per_token"] * prompt_tokens
-        output_cost = custom_cost_per_token["output_cost_per_token"] * completion_tokens
+        input_cost_per_token = custom_cost_per_token["input_cost_per_token"]
+        output_cost_per_token = custom_cost_per_token["output_cost_per_token"]
+
+        cache_read_input_token_cost = custom_cost_per_token.get(
+            "cache_read_input_token_cost",
+            input_cost_per_token,
+        )
+        cache_creation_input_token_cost = custom_cost_per_token.get(
+            "cache_creation_input_token_cost",
+            input_cost_per_token,
+        )
+
+        regular_prompt_tokens = max(
+            prompt_tokens - cached_tokens - cache_creation_tokens,
+            0,
+        )
+
+        input_cost = (
+            regular_prompt_tokens * input_cost_per_token
+            + cached_tokens * cache_read_input_token_cost
+            + cache_creation_tokens * cache_creation_input_token_cost
+        )
+        output_cost = completion_tokens * output_cost_per_token
         return input_cost, output_cost
     elif custom_cost_per_second is not None:
         output_cost = custom_cost_per_second * response_time_ms / 1000  # type: ignore
@@ -323,10 +351,56 @@ def cost_per_token(  # noqa: PLR0915
         )
 
     ## CUSTOM PRICING ##
+    # Normalize cache token counts across providers:
+    #   - OpenAI-compatible: usage.prompt_tokens_details.cached_tokens
+    #     (prompt_tokens already INCLUDES cached_tokens)
+    #   - Anthropic: usage.cache_read_input_tokens / cache_creation_input_tokens
+    #     (prompt_tokens does NOT include these — adjust before calling helper)
+    _cache_read_tokens: float = 0
+    _cache_creation_tokens: float = 0
+    _is_anthropic_style = False
+
+    if usage_object is not None:
+        _pt_details = getattr(usage_object, "prompt_tokens_details", None)
+        if _pt_details is not None:
+            _cache_read_tokens = float(getattr(_pt_details, "cached_tokens", 0) or 0)
+            # OpenAI-compatible providers report cache-write tokens under
+            # either `cache_write_tokens` (kimi-k2) or `cache_creation_tokens`.
+            # Mirror db_spend_update_writer to stay symmetric.
+            _cache_creation_tokens = float(
+                getattr(_pt_details, "cache_write_tokens", 0)
+                or getattr(_pt_details, "cache_creation_tokens", 0)
+                or 0
+            )
+
+        _anthropic_read = getattr(usage_object, "cache_read_input_tokens", None)
+        _anthropic_create = getattr(usage_object, "cache_creation_input_tokens", None)
+        if _anthropic_read is not None or _anthropic_create is not None:
+            _is_anthropic_style = True
+            if _anthropic_read is not None:
+                _cache_read_tokens = float(_anthropic_read)
+            if _anthropic_create is not None:
+                _cache_creation_tokens = float(_anthropic_create)
+
+    if not _cache_read_tokens and cache_read_input_tokens:
+        _cache_read_tokens = float(cache_read_input_tokens)
+        _is_anthropic_style = True
+    if not _cache_creation_tokens and cache_creation_input_tokens:
+        _cache_creation_tokens = float(cache_creation_input_tokens)
+        _is_anthropic_style = True
+
+    # Anthropic reports prompt_tokens as input_tokens (excluding cache tokens).
+    # Adjust so the helper's "prompt_tokens includes cache tokens" invariant holds.
+    _normalized_prompt_tokens = float(prompt_tokens)
+    if _is_anthropic_style:
+        _normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens
+
     response_cost = _cost_per_token_custom_pricing_helper(
-        prompt_tokens=prompt_tokens,
+        prompt_tokens=_normalized_prompt_tokens,
         completion_tokens=completion_tokens,
         response_time_ms=response_time_ms,
+        cached_tokens=_cache_read_tokens,
+        cache_creation_tokens=_cache_creation_tokens,
         custom_cost_per_second=custom_cost_per_second,
         custom_cost_per_token=custom_cost_per_token,
     )
@@ -2120,6 +2194,26 @@ def batch_cost_calculator(
             )
         except Exception:
             model_info = None
+    elif not any(
+        model_info.get(k) is not None
+        for k in (
+            "input_cost_per_token_batches",
+            "input_cost_per_token",
+            "output_cost_per_token_batches",
+            "output_cost_per_token",
+        )
+    ):
+        # model_info was provided (e.g. deployment metadata with only id/db_model)
+        # but carries no pricing fields. Fall back to the global pricing table so
+        # that standard model pricing is used instead of silently returning $0.
+        try:
+            global_info = litellm.get_model_info(
+                model=model, custom_llm_provider=custom_llm_provider
+            )
+            if global_info:
+                model_info = global_info
+        except Exception:
+            pass
 
     if not model_info:
         return 0.0, 0.0

litellm/exceptions.py

@@ -918,9 +918,11 @@ class GuardrailRaisedException(Exception):
         guardrail_name: Optional[str] = None,
         message: str = "",
         should_wrap_with_default_message: bool = True,
+        status_code: int = 400,
     ):
         default_message = f"Guardrail raised an exception, Guardrail: {guardrail_name}, Message: {message}"
         self.guardrail_name = guardrail_name
+        self.status_code = status_code
         self.message = default_message if should_wrap_with_default_message else message
         super().__init__(self.message)
 
@@ -930,12 +932,14 @@ class BlockedPiiEntityError(Exception):
         self,
         entity_type: str,
         guardrail_name: Optional[str] = None,
+        status_code: int = 400,
     ):
         """
         Raised when a blocked entity is detected by a guardrail.
         """
         self.entity_type = entity_type
         self.guardrail_name = guardrail_name
+        self.status_code = status_code
         self.message = f"Blocked entity detected: {entity_type} by Guardrail: {guardrail_name}. This entity is not allowed to be used in this request."
         super().__init__(self.message)
 

litellm/integrations/custom_guardrail.py

@@ -43,7 +43,11 @@ if TYPE_CHECKING:
 dc = DualCache()
 
 
-from litellm.exceptions import ModifyResponseException as ModifyResponseException
+from litellm.exceptions import (
+    BlockedPiiEntityError,
+    GuardrailRaisedException,
+    ModifyResponseException,
+)
 
 
 class CustomGuardrail(CustomLogger):
@@ -737,12 +741,15 @@ class CustomGuardrail(CustomLogger):
         (this was logged previously as an API failure - guardrail_failed_to_respond).
 
         Guardrails signal intentional blocks by raising:
+        - GuardrailRaisedException (generic guardrail API, tool permission)
+        - BlockedPiiEntityError (Presidio PII detection)
         - HTTPException with status 400 (content policy violation)
         - ModifyResponseException (passthrough mode violation)
         """
-
         if isinstance(e, ModifyResponseException):
             return True
+        if isinstance(e, (GuardrailRaisedException, BlockedPiiEntityError)):
+            return True
         if (
             HTTPException is not None
             and isinstance(e, HTTPException)

litellm/integrations/custom_logger.py

@@ -697,6 +697,27 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         """
         return AgenticLoopPlan(run_agentic_loop=False)
 
+    async def async_post_agentic_loop_response_hook(
+        self,
+        response: Any,
+        plan: AgenticLoopPlan,
+        kwargs: Dict,
+    ) -> Any:
+        """
+        Post-process the response returned by the agentic-loop follow-up call.
+
+        Called after BaseLLMHTTPHandler executes ``AgenticLoopPlan.request_patch``
+        and receives the final response from the provider. Lets callbacks shape
+        what the client sees without bypassing the loop's safety / observability
+        machinery (depth tracking, fingerprinting, etc.).
+
+        Use ``plan.metadata`` to carry whatever the build step decided to expose
+        for post-processing (e.g. native tool_result blocks to inject).
+
+        Default returns ``response`` unchanged.
+        """
+        return response
+
     async def async_should_run_chat_completion_agentic_loop(
         self,
         response: Any,

litellm/integrations/opentelemetry.py

@@ -1,7 +1,7 @@
 import os
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union, cast
 
 import litellm
 from litellm._logging import verbose_logger
@@ -10,6 +10,12 @@ from litellm.integrations._types.open_inference import (
     SpanAttributes,
 )
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
+    OTEL_SEMCONV_STABILITY_OPT_IN_ENV,
+    OTELGenAISemconvMixin,
+    OTELSemconvCategory,
+    parse_semconv_opt_in,
+)
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.secret_managers.main import get_secret_bool, str_to_bool
 from litellm.types.services import ServiceLoggerPayload
@@ -53,6 +59,11 @@ LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm")
 LITELLM_METER_NAME = os.getenv("LITELLM_METER_NAME", "litellm")
 LITELLM_LOGGER_NAME = os.getenv("LITELLM_LOGGER_NAME", "litellm")
 LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
+# OTel-standard names. status is also kept under error.code for back compat.
+HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE = "http.response.status_code"
+HTTP_ROUTE_ATTRIBUTE = "http.route"
+URL_PATH_ATTRIBUTE = "url.path"
+PREPROCESSING_DURATION_MS_ATTRIBUTE = "litellm.preprocessing.duration_ms"
 # Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later
 RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
 LITELLM_REQUEST_SPAN_NAME = "litellm_request"
@@ -85,6 +96,7 @@ class OpenTelemetryConfig:
     # Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT.
     # One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias).
     capture_message_content: Optional[str] = None
+    semconv_stability_opt_in: Set[OTELSemconvCategory] = field(default_factory=set)
 
     def __post_init__(self) -> None:
         # If endpoint is specified but exporter is still the default "console",
@@ -110,6 +122,11 @@ class OpenTelemetryConfig:
             self.ignore_context_propagation = str_to_bool(
                 os.getenv("OTEL_IGNORE_CONTEXT_PROPAGATION")
             )
+        # Resolve the env opt-in once here so self.semconv_stability_opt_in is the
+        # single source of truth: the union of programmatic and env categories.
+        self.semconv_stability_opt_in |= parse_semconv_opt_in(
+            os.getenv(OTEL_SEMCONV_STABILITY_OPT_IN_ENV)
+        )
 
     @classmethod
     def from_env(cls):
@@ -157,7 +174,7 @@ class OpenTelemetryConfig:
         )
 
 
-class OpenTelemetry(CustomLogger):
+class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
     def __init__(
         self,
         config: Optional[OpenTelemetryConfig] = None,
@@ -655,6 +672,31 @@ class OpenTelemetry(CustomLogger):
         parent_otel_span = user_api_key_dict.parent_otel_span
         if parent_otel_span is not None:
             parent_otel_span.set_status(Status(StatusCode.ERROR))
+
+            # Stamp structured error attrs on the SERVER span itself; the
+            # failure path otherwise only sets its status (_handle_failure
+            # records on the litellm_request child span). Inline import:
+            # litellm_logging <-> integrations is circular.
+            from litellm.litellm_core_utils.litellm_logging import (
+                StandardLoggingPayloadSetup,
+            )
+
+            error_information = StandardLoggingPayloadSetup.get_error_information(
+                original_exception=original_exception,
+                traceback_str=traceback_str,
+            )
+            self._record_exception_on_span(
+                span=parent_otel_span,
+                kwargs={
+                    "exception": original_exception,
+                    "standard_logging_object": {"error_information": error_information},
+                },
+            )
+
+            # Pre-request latency (request_data carries the propagated
+            # metadata on the failure path; omitted if it failed before handoff).
+            self.set_preprocessing_duration_attribute(parent_otel_span, request_data)
+
             _span_name = "Failed Proxy Server Request"
 
             # Exception Logging Child Span
@@ -691,6 +733,14 @@ class OpenTelemetry(CustomLogger):
 
             ctx, _ = self._get_span_context(kwargs, default_span=parent_span)
 
+            # Pre-request latency on the SERVER span (success path).
+            self.set_preprocessing_duration_attribute(parent_span, kwargs)
+
+            # http.response.status_code on the SERVER span (success path).
+            # A successful proxy response is HTTP 200; the failure path sets
+            # this from the error code in _record_exception_on_span.
+            self.set_response_status_code_attribute(parent_span, 200)
+
             # 3. Guardrail span
             self._create_guardrail_span(kwargs=kwargs, context=ctx)
 
@@ -979,13 +1029,14 @@ class OpenTelemetry(CustomLogger):
 
         otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
 
-        # Always create a new span
-        # The parent relationship is preserved through the context parameter
-        span = otel_tracer.start_span(
-            name=self._get_span_name(kwargs),
-            start_time=self._to_ns(start_time),
-            context=context,
-        )
+        span_kwargs: Dict[str, Any] = {
+            "name": self._get_span_name(kwargs),
+            "start_time": self._to_ns(start_time),
+            "context": context,
+        }
+        if self._gen_ai_semconv_latest_experimental:
+            span_kwargs["kind"] = self.span_kind.CLIENT
+        span = otel_tracer.start_span(**span_kwargs)
 
         span.set_status(Status(StatusCode.OK))
         self.set_attributes(span, kwargs, response_obj)
@@ -998,6 +1049,10 @@ class OpenTelemetry(CustomLogger):
         from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
+        # raw_gen_ai_request is non-standard in semconv mode.
+        if self._gen_ai_semconv_latest_experimental:
+            return
+
         if not self._capture_in_span():
             return
 
@@ -1023,7 +1078,11 @@ class OpenTelemetry(CustomLogger):
         provider = params.get("custom_llm_provider", "Unknown")
 
         common_attrs = {
-            "gen_ai.operation.name": "chat",
+            "gen_ai.operation.name": (
+                self._gen_ai_operation_name(kwargs)
+                if self._gen_ai_semconv_latest_experimental
+                else "chat"
+            ),
             "gen_ai.system": provider,
             "gen_ai.request.model": kwargs.get("model"),
             "gen_ai.framework": "litellm",
@@ -1048,8 +1107,13 @@ class OpenTelemetry(CustomLogger):
             "mcp_tool_call_metadata",
             "vector_store_request_metadata",
         ]:
-            if md.get(key) is not None:
-                common_attrs[f"metadata.{key}"] = str(md[key])
+            value = md.get(key)
+            if value is None:
+                continue
+            if isinstance(value, (dict, list)):
+                common_attrs[f"metadata.{key}"] = safe_dumps(value)
+            else:
+                common_attrs[f"metadata.{key}"] = str(value)
 
         # get hidden params
         hidden_params = getattr(std_log, "hidden_params", None) or (std_log or {}).get(
@@ -1246,6 +1310,24 @@ class OpenTelemetry(CustomLogger):
                 response_duration_seconds, attributes=common_attrs
             )
 
+    @staticmethod
+    def _otel_log_types():
+        """Resolve ``(LogRecord, SeverityNumber)`` across OTEL SDK versions.
+
+        ``LogRecord`` moved out of ``opentelemetry.sdk._logs`` in OTEL >= 1.39.0
+        (open-telemetry/opentelemetry-python#4676). Imports stay function-local
+        because the SDK is an optional dependency.
+        """
+        from opentelemetry._logs import SeverityNumber
+
+        try:
+            from opentelemetry.sdk._logs import LogRecord  # OTEL < 1.39.0
+        except ImportError:
+            from opentelemetry.sdk._logs._internal import (  # OTEL >= 1.39.0
+                LogRecord,
+            )
+        return LogRecord, SeverityNumber
+
     def _emit_semantic_logs(self, kwargs, response_obj, span: Span):
         if not self.config.enable_events:
             return
@@ -1259,16 +1341,7 @@ class OpenTelemetry(CustomLogger):
         # See: https://github.com/open-telemetry/opentelemetry-python/pull/4676
         # TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
 
-        from opentelemetry._logs import SeverityNumber
-
-        try:
-            from opentelemetry.sdk._logs import (  # type: ignore[attr-defined]  # OTEL < 1.39.0
-                LogRecord as SdkLogRecord,
-            )
-        except ImportError:
-            from opentelemetry.sdk._logs._internal import (
-                LogRecord as SdkLogRecord,  # type: ignore[attr-defined]  # OTEL >= 1.39.0
-            )
+        SdkLogRecord, SeverityNumber = self._otel_log_types()
 
         # Resolve through the handler's own LoggerProvider (which may be a
         # private one when skip_set_global=True) rather than the module-level
@@ -1280,6 +1353,16 @@ class OpenTelemetry(CustomLogger):
             "custom_llm_provider", "Unknown"
         )
 
+        if self._gen_ai_semconv_latest_experimental:
+            self._emit_inference_details_event(
+                kwargs=kwargs,
+                response_obj=response_obj,
+                provider=provider,
+                otel_logger=otel_logger,
+                parent_ctx=parent_ctx,
+            )
+            return
+
         # per-message events
         for msg in kwargs.get("messages", []):
             role = msg.get("role", "user")
@@ -1496,11 +1579,14 @@ class OpenTelemetry(CustomLogger):
         if should_create_primary_span:
             # Span 1: Request sent to litellm SDK
             otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
-            span = otel_tracer.start_span(
-                name=self._get_span_name(kwargs),
-                start_time=self._to_ns(start_time),
-                context=_parent_context,
-            )
+            span_kwargs: Dict[str, Any] = {
+                "name": self._get_span_name(kwargs),
+                "start_time": self._to_ns(start_time),
+                "context": _parent_context,
+            }
+            if self._gen_ai_semconv_latest_experimental:
+                span_kwargs["kind"] = self.span_kind.CLIENT
+            span = otel_tracer.start_span(**span_kwargs)
             span.set_status(Status(StatusCode.ERROR))
             self.set_attributes(span, kwargs, response_obj)
 
@@ -1584,6 +1670,19 @@ class OpenTelemetry(CustomLogger):
                     value=error_information["error_code"],
                 )
 
+                # Also expose under the OTel-standard name as an int
+                # (error_code is a str, may be non-numeric).
+                _error_code_val = error_information["error_code"]
+                if _error_code_val is not None:
+                    try:
+                        self.safe_set_attribute(
+                            span=span,
+                            key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
+                            value=int(_error_code_val),
+                        )
+                    except (ValueError, TypeError):
+                        pass
+
             if error_information.get("error_class"):
                 self.safe_set_attribute(
                     span=span,
@@ -1782,11 +1881,21 @@ class OpenTelemetry(CustomLogger):
             )
 
             # The Generative AI Provider: Azure, OpenAI, etc.
-            self.safe_set_attribute(
-                span=span,
-                key=SpanAttributes.LLM_SYSTEM.value,
-                value=litellm_params.get("custom_llm_provider", "Unknown"),
-            )
+            provider_name = litellm_params.get("custom_llm_provider", "Unknown")
+            # Latest-experimental semconv replaced gen_ai.system with
+            # gen_ai.provider.name; emit only the conformant key in that mode.
+            if self._gen_ai_semconv_latest_experimental:
+                self.safe_set_attribute(
+                    span=span,
+                    key="gen_ai.provider.name",
+                    value=provider_name,
+                )
+            else:
+                self.safe_set_attribute(
+                    span=span,
+                    key=SpanAttributes.LLM_SYSTEM.value,
+                    value=provider_name,
+                )
 
             # The maximum number of tokens the LLM generates for a request.
             if optional_params.get("max_tokens"):
@@ -1812,11 +1921,17 @@ class OpenTelemetry(CustomLogger):
                     value=optional_params.get("top_p"),
                 )
 
-            self.safe_set_attribute(
-                span=span,
-                key=SpanAttributes.LLM_IS_STREAMING.value,
-                value=str(optional_params.get("stream", False)),
-            )
+            if self._gen_ai_semconv_latest_experimental:
+                # Semconv emits gen_ai.request.stream (only when streaming) via
+                # _set_semconv_request_attributes; skip the legacy llm.is_streaming.
+                self._set_semconv_request_attributes(span, optional_params)
+                self._set_semconv_cache_token_attributes(span, standard_logging_payload)
+            else:
+                self.safe_set_attribute(
+                    span=span,
+                    key=SpanAttributes.LLM_IS_STREAMING.value,
+                    value=str(optional_params.get("stream", False)),
+                )
 
             if optional_params.get("user"):
                 self.safe_set_attribute(
@@ -1937,14 +2052,18 @@ class OpenTelemetry(CustomLogger):
                         value=safe_dumps(transformed_system_instructions),
                     )
 
-            self.safe_set_attribute(
-                span=span,
-                key=SpanAttributes.GEN_AI_OPERATION_NAME.value,
-                value=(
+            if self._gen_ai_semconv_latest_experimental:
+                operation_name = self._gen_ai_operation_name(kwargs)
+            else:
+                operation_name = (
                     "chat"
                     if standard_logging_payload.get("call_type") == "completion"
                     else standard_logging_payload.get("call_type") or "chat"
-                ),
+                )
+            self.safe_set_attribute(
+                span=span,
+                key=SpanAttributes.GEN_AI_OPERATION_NAME.value,
+                value=operation_name,
             )
 
             if standard_logging_payload.get("request_id"):
@@ -2281,6 +2400,10 @@ class OpenTelemetry(CustomLogger):
         if generation_name:
             return generation_name
 
+        if self._gen_ai_semconv_latest_experimental:
+            model = kwargs.get("model") or "unknown"
+            return f"{self._gen_ai_operation_name(kwargs)} {model}"
+
         return LITELLM_REQUEST_SPAN_NAME
 
     def get_traceparent_from_header(self, headers):
@@ -2822,3 +2945,86 @@ class OpenTelemetry(CustomLogger):
             context=self.get_traceparent_from_header(headers=headers),
             kind=self.span_kind.SERVER,
         )
+
+    def set_proxy_request_route_attributes(
+        self,
+        span: Optional[Span],
+        *,
+        url_path: Optional[str] = None,
+        http_route: Optional[str] = None,
+    ) -> None:
+        """
+        Set OTel-standard ``http.route`` / ``url.path`` on the proxy SERVER
+        span. Called from the auth path, the only point where both the
+        SERVER span and the request are in hand. No-op if span/value missing.
+        """
+        if span is None:
+            return
+        if url_path:
+            self.safe_set_attribute(span=span, key=URL_PATH_ATTRIBUTE, value=url_path)
+        if http_route:
+            self.safe_set_attribute(
+                span=span, key=HTTP_ROUTE_ATTRIBUTE, value=http_route
+            )
+
+    def set_response_status_code_attribute(
+        self, span: Optional[Span], status_code: Optional[int]
+    ) -> None:
+        """
+        Set OTel-standard ``http.response.status_code`` (int) on the proxy
+        SERVER span. The failure path sets this from the error code in
+        ``_record_exception_on_span``; this is the success-path counterpart
+        so the attribute is present on every SERVER span regardless of
+        outcome (required by the HTTP semconv, and needed for error-ratio /
+        status-breakdown dashboards). No-op if span/value missing.
+        """
+        if span is None or status_code is None:
+            return
+        self.safe_set_attribute(
+            span=span,
+            key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
+            value=int(status_code),
+        )
+
+    def set_preprocessing_duration_attribute(
+        self, span: Optional[Span], container: Any
+    ) -> None:
+        """
+        Set ``litellm.preprocessing.duration_ms`` (proxy-receive -> first
+        provider handoff) on the proxy SERVER span. ``litellm_received_at``
+        rides request metadata; ``first_api_call_start_time`` is the
+        set-once first-handoff instant (retries/backoff excluded). Works
+        uniformly for the success (model_call_details) and failure
+        (request_data) containers. No-op if span/either anchor is missing.
+        """
+        if span is None or not isinstance(container, dict):
+            return
+        received_at = None
+        # first_api_call_start_time is top-level (never in user metadata).
+        first_handoff = container.get("first_api_call_start_time")
+        _lp = container.get("litellm_params")
+        for _md in (
+            (_lp or {}).get("metadata") if isinstance(_lp, dict) else None,
+            container.get("metadata"),
+            container.get("litellm_metadata"),
+        ):
+            if isinstance(_md, dict):
+                received_at = received_at or _md.get("litellm_received_at")
+        if received_at is None or first_handoff is None:
+            return
+        try:
+            start_ts = self._to_timestamp(received_at)
+            end_ts = self._to_timestamp(first_handoff)
+        except Exception:
+            return
+        if start_ts is None or end_ts is None:
+            return
+        duration_ms = (end_ts - start_ts) * 1000.0
+        # Clock skew → omit rather than emit a negative latency.
+        if duration_ms < 0:
+            return
+        self.safe_set_attribute(
+            span=span,
+            key=PREPROCESSING_DURATION_MS_ATTRIBUTE,
+            value=duration_ms,
+        )

litellm/integrations/opentelemetry_utils/gen_ai_semconv.py

@@ -0,0 +1,271 @@
+"""OTEL GenAI ``gen_ai_latest_experimental`` semantic conventions.
+
+Setting ``OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental`` switches the
+emitted traces to the experimental OTEL GenAI conventions
+(https://opentelemetry.io/docs/specs/semconv/gen-ai/). Concretely, versus the
+default LiteLLM output:
+
+Request span:
+
+- name is ``{operation} {model}`` (e.g. ``chat gpt-4``) instead of
+  ``litellm_request``; span kind is ``CLIENT``.
+- ``gen_ai.operation.name`` is the actual operation (``chat`` /
+  ``text_completion`` / ``embeddings``) instead of always ``chat``.
+- the provider is reported as ``gen_ai.provider.name``; the superseded
+  ``gen_ai.system`` and the legacy ``llm.is_streaming`` are dropped.
+- adds ``gen_ai.request.{frequency_penalty,presence_penalty,top_k,seed}``,
+  ``gen_ai.request.stop_sequences`` (a string array),
+  ``gen_ai.request.stream`` (only when streaming),
+  ``gen_ai.request.choice.count`` (only when n > 1), and
+  ``gen_ai.usage.cache_{creation,read}.input_tokens``.
+- the non-standard ``raw_gen_ai_request`` child span is no longer created.
+
+Events:
+
+- the per-message ``gen_ai.content.prompt`` / per-choice
+  ``gen_ai.content.completion`` log events are replaced by a single
+  ``gen_ai.client.inference.operation.details`` log event carrying
+  ``gen_ai.input.messages`` / ``gen_ai.output.messages`` (message content
+  included only when content capture is enabled).
+"""
+
+from datetime import datetime
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union
+
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+
+if TYPE_CHECKING:
+    from opentelemetry.trace import Span as _Span
+
+    from litellm.integrations.opentelemetry import OpenTelemetryConfig
+
+    Span = Union[_Span, Any]
+else:
+    Span = Any
+
+
+# OTEL_SEMCONV_STABILITY_OPT_IN is a comma-separated list of category-specific
+# opt-in values. See https://opentelemetry.io/docs/specs/semconv/gen-ai/
+OTEL_SEMCONV_STABILITY_OPT_IN_ENV = "OTEL_SEMCONV_STABILITY_OPT_IN"
+
+
+class OTELSemconvCategory(Enum):
+    GEN_AI_LATEST_EXPERIMENTAL = "gen_ai_latest_experimental"
+
+
+# Reverse lookup: opt-in token string -> OTELSemconvCategory.
+_SEMCONV_CATEGORY_BY_VALUE = {
+    category.value: category for category in OTELSemconvCategory
+}
+
+
+# LiteLLM optional_params key -> OTEL gen_ai semconv span attribute.
+_SEMCONV_REQUEST_ATTRIBUTES = {
+    "frequency_penalty": "gen_ai.request.frequency_penalty",
+    "presence_penalty": "gen_ai.request.presence_penalty",
+    "top_k": "gen_ai.request.top_k",
+    "seed": "gen_ai.request.seed",
+}
+
+# usage_object key -> OTEL gen_ai semconv cache-token span attribute.
+_SEMCONV_CACHE_TOKEN_ATTRIBUTES = {
+    "cache_creation_input_tokens": "gen_ai.usage.cache_creation.input_tokens",
+    "cache_read_input_tokens": "gen_ai.usage.cache_read.input_tokens",
+}
+
+# Name of the consolidated GenAI inference event (replaces the legacy
+# per-message gen_ai.content.prompt / per-choice gen_ai.content.completion).
+_INFERENCE_DETAILS_EVENT_NAME = "gen_ai.client.inference.operation.details"
+
+
+def parse_semconv_opt_in(raw: Optional[str]) -> Set[OTELSemconvCategory]:
+    """Parse the comma-separated OTEL_SEMCONV_STABILITY_OPT_IN value into the
+    set of recognized categories. Unknown tokens are ignored per the spec."""
+    if not raw:
+        return set()
+    return {
+        _SEMCONV_CATEGORY_BY_VALUE[token]
+        for token in (part.strip() for part in raw.split(","))
+        if token in _SEMCONV_CATEGORY_BY_VALUE
+    }
+
+
+class OTELGenAISemconvMixin:
+    """OTEL GenAI ``gen_ai_latest_experimental`` semantic-convention behavior.
+
+    Mixed into ``OpenTelemetry`` (its only host). Every member is internal to
+    the OTEL integration; the leading underscore marks "subsystem-internal",
+    not "class-private" (the host lives in a sibling module).
+
+    Members the host calls (the mixin -> host contract):
+
+    - ``_gen_ai_semconv_latest_experimental`` -- opt-in gate; guards every
+      semconv code path in ``opentelemetry.py``.
+    - ``_gen_ai_operation_name`` -- LiteLLM ``call_type`` -> spec
+      ``gen_ai.operation.name``.
+    - ``_set_semconv_request_attributes`` /
+      ``_set_semconv_cache_token_attributes`` -- add the ``gen_ai.request.*``
+      / ``gen_ai.usage.cache_*`` span attributes.
+    - ``_emit_inference_details_event`` -- emit the consolidated event.
+
+    Helpers the host must provide (declared under ``TYPE_CHECKING`` below):
+    ``config``, ``safe_set_attribute``, ``_capture_in_event``,
+    ``_transform_messages_to_otel_semantic_conventions``,
+    ``_transform_choices_to_otel_semantic_conventions``, ``_to_ns``,
+    ``_otel_log_types``.
+    """
+
+    if TYPE_CHECKING:
+        config: "OpenTelemetryConfig"
+
+        def safe_set_attribute(self, span: Span, key: str, value: Any) -> None: ...
+
+        def _capture_in_event(self) -> bool: ...
+
+        def _transform_messages_to_otel_semantic_conventions(
+            self, messages: Union[List[dict], str]
+        ) -> List[dict]: ...
+
+        def _transform_choices_to_otel_semantic_conventions(
+            self, choices: List[dict]
+        ) -> List[dict]: ...
+
+        def _to_ns(self, dt: datetime) -> int: ...
+
+        def _otel_log_types(self) -> Tuple[Any, Any]: ...
+
+    @property
+    def _gen_ai_semconv_latest_experimental(self) -> bool:
+        """Whether the ``gen_ai_latest_experimental`` opt-in is active.
+
+        Every semconv behavior is gated on this; ``False`` => legacy output.
+        """
+        return (
+            OTELSemconvCategory.GEN_AI_LATEST_EXPERIMENTAL
+            in self.config.semconv_stability_opt_in
+        )
+
+    @staticmethod
+    def _gen_ai_operation_name(kwargs: dict) -> str:
+        """Map a LiteLLM ``call_type`` to spec ``gen_ai.operation.name``.
+
+        Substring match (e.g. ``aembedding`` -> ``embeddings``); defaults to
+        ``chat``.
+        """
+        call_type = kwargs.get("call_type", "") or ""
+        match call_type:
+            case s if "embedding" in s:
+                return "embeddings"
+            case s if "text_completion" in s:
+                return "text_completion"
+            case _:
+                return "chat"
+
+    def _set_semconv_request_attributes(
+        self, span: Span, optional_params: dict
+    ) -> None:
+        """Add ``gen_ai.request.*`` span attributes from ``optional_params``.
+
+        Covers the sampling params plus the conditionally-required
+        ``stop_sequences`` / ``stream`` / ``choice.count`` per the spec.
+        """
+        for source_key, semconv_key in _SEMCONV_REQUEST_ATTRIBUTES.items():
+            value = optional_params.get(source_key)
+            if value is not None:
+                self.safe_set_attribute(span=span, key=semconv_key, value=value)
+
+        stop = optional_params.get("stop")
+        if stop is not None:
+            # Spec types this as string[]. safe_set_attribute coerces to a
+            # primitive, so set the array directly via the span API.
+            stop_list = stop if isinstance(stop, list) else [stop]
+            span.set_attribute(
+                "gen_ai.request.stop_sequences", [str(s) for s in stop_list]
+            )
+
+        # Conditionally required: set only when the request is streaming.
+        if optional_params.get("stream"):
+            self.safe_set_attribute(span=span, key="gen_ai.request.stream", value=True)
+
+        # Conditionally required per spec ("if available and != 1"). Valid n is
+        # an int >= 1, so n > 1 is equivalent for conformant input while
+        # suppressing nonsensical values (0, negative, non-int).
+        n = optional_params.get("n")
+        if isinstance(n, int) and n > 1:
+            self.safe_set_attribute(
+                span=span, key="gen_ai.request.choice.count", value=n
+            )
+
+    def _set_semconv_cache_token_attributes(
+        self, span: Span, standard_logging_payload
+    ) -> None:
+        """Add ``gen_ai.usage.cache_*.input_tokens`` from the usage object.
+
+        No-op when the payload or the usage values are missing/zero.
+        """
+        if not standard_logging_payload:
+            return
+        usage = (standard_logging_payload.get("metadata") or {}).get(
+            "usage_object"
+        ) or {}
+        for source_key, semconv_key in _SEMCONV_CACHE_TOKEN_ATTRIBUTES.items():
+            value = usage.get(source_key)
+            if value:
+                self.safe_set_attribute(span=span, key=semconv_key, value=value)
+
+    def _build_inference_details_attrs(
+        self, kwargs: dict, response_obj: dict, provider: str
+    ) -> Dict[str, Any]:
+        """Build the attribute payload for the inference-details event.
+
+        Always includes provider/operation; input/output messages are added
+        only when content capture is enabled and non-empty. Mixin-internal.
+        """
+        attrs: Dict[str, Any] = {
+            "event_name": _INFERENCE_DETAILS_EVENT_NAME,
+            "gen_ai.provider.name": provider,
+            "gen_ai.operation.name": self._gen_ai_operation_name(kwargs),
+        }
+        if not self._capture_in_event():
+            return attrs
+
+        input_messages = self._transform_messages_to_otel_semantic_conventions(
+            kwargs.get("messages") or []
+        )
+        output_messages = self._transform_choices_to_otel_semantic_conventions(
+            response_obj.get("choices", [])
+        )
+        if input_messages:
+            attrs["gen_ai.input.messages"] = safe_dumps(input_messages)
+        if output_messages:
+            attrs["gen_ai.output.messages"] = safe_dumps(output_messages)
+        return attrs
+
+    def _emit_inference_details_event(
+        self,
+        kwargs: dict,
+        response_obj: dict,
+        provider: str,
+        otel_logger,
+        parent_ctx,
+    ) -> None:
+        """Emit the consolidated ``gen_ai.client.inference.operation.details``
+        log event, correlated to the request span via ``parent_ctx``.
+
+        Replaces the legacy per-message / per-choice content events.
+        """
+        LogRecord, SeverityNumber = self._otel_log_types()
+        log_record = LogRecord(
+            timestamp=self._to_ns(datetime.now()),
+            trace_id=parent_ctx.trace_id,
+            span_id=parent_ctx.span_id,
+            trace_flags=parent_ctx.trace_flags,
+            severity_number=SeverityNumber.INFO,
+            severity_text="INFO",
+            body=None,
+            attributes=self._build_inference_details_attrs(
+                kwargs, response_obj, provider
+            ),
+        )
+        otel_logger.emit(log_record)

litellm/integrations/prometheus.py

@@ -1226,6 +1226,17 @@ class PrometheusLogger(CustomLogger):
             label_context=label_context,
         )
 
+        # Provider-agnostic fallback: providers like Bedrock and Vertex don't return
+        # x-ratelimit-remaining-* headers, so the gauges above only fire for OpenAI /
+        # Anthropic / Azure. When the proxy router has tpm/rpm configured for the
+        # model_group, derive remaining from configured-limit minus current usage so
+        # the same metric is populated for any provider.
+        await self._async_set_router_remaining_metrics(
+            standard_logging_payload=standard_logging_payload,  # type: ignore
+            enum_values=enum_values,
+            label_context=label_context,
+        )
+
         # cache metrics
         self._increment_cache_metrics(
             standard_logging_payload=standard_logging_payload,  # type: ignore
@@ -2199,6 +2210,99 @@ class PrometheusLogger(CustomLogger):
             )
             self.litellm_deployment_rpm_limit.labels(**_labels).set(rpm)
 
+    async def _async_set_router_remaining_metrics(
+        self,
+        standard_logging_payload: StandardLoggingPayload,
+        enum_values: UserAPIKeyLabelValues,
+        label_context: Optional[PrometheusLabelFactoryContext] = None,
+    ) -> None:
+        """
+        Populate ``litellm_remaining_tokens_metric`` /
+        ``litellm_remaining_requests_metric`` from the router's internal usage
+        counters when the upstream provider did not return
+        ``x-ratelimit-remaining-*`` response headers.
+
+        OpenAI / Anthropic / Azure return remaining tokens/requests in response
+        headers, but Bedrock and Vertex AI do not. This fallback computes
+        ``configured_limit - current_usage`` via
+        ``Router.get_remaining_model_group_usage`` so the same gauges are
+        emitted for every provider when tpm/rpm is configured on the
+        deployment.
+        """
+        try:
+            additional_headers = (
+                standard_logging_payload.get("hidden_params", {}) or {}
+            ).get("additional_headers") or {}
+
+            already_have_tokens = (
+                additional_headers.get("x_ratelimit_remaining_tokens") is not None
+            )
+            already_have_requests = (
+                additional_headers.get("x_ratelimit_remaining_requests") is not None
+            )
+            if already_have_tokens and already_have_requests:
+                return
+
+            model_group = standard_logging_payload.get("model_group")
+            if not model_group:
+                return
+
+            try:
+                from litellm.proxy.proxy_server import llm_router
+            except ImportError:
+                llm_router = None
+
+            if llm_router is None:
+                return
+
+            try:
+                remaining_usage = await llm_router.get_remaining_model_group_usage(
+                    model_group
+                )
+            except Exception as e:
+                verbose_logger.exception(
+                    "Prometheus: get_remaining_model_group_usage failed for "
+                    "model_group=%s: %s",
+                    model_group,
+                    e,
+                )
+                return
+
+            if not remaining_usage:
+                return
+
+            remaining_tokens = remaining_usage.get("x-ratelimit-remaining-tokens")
+            remaining_requests = remaining_usage.get("x-ratelimit-remaining-requests")
+
+            if not already_have_tokens and remaining_tokens is not None:
+                _labels = prometheus_label_factory(
+                    supported_enum_labels=self.get_labels_for_metric(
+                        metric_name="litellm_remaining_tokens_metric"
+                    ),
+                    enum_values=enum_values,
+                    label_context=label_context,
+                )
+                self.litellm_remaining_tokens_metric.labels(**_labels).set(
+                    remaining_tokens
+                )
+
+            if not already_have_requests and remaining_requests is not None:
+                _labels = prometheus_label_factory(
+                    supported_enum_labels=self.get_labels_for_metric(
+                        metric_name="litellm_remaining_requests_metric"
+                    ),
+                    enum_values=enum_values,
+                    label_context=label_context,
+                )
+                self.litellm_remaining_requests_metric.labels(**_labels).set(
+                    remaining_requests
+                )
+        except Exception as e:
+            verbose_logger.exception(
+                "Prometheus Error: _async_set_router_remaining_metrics. "
+                "Exception occured - {}".format(str(e))
+            )
+
     def set_llm_deployment_success_metrics(
         self,
         request_kwargs: dict,
@@ -3436,6 +3540,10 @@ class PrometheusLogger(CustomLogger):
             user_object.budget_reset_at = user_info.budget_reset_at
             if user_object.max_budget is None and user_info.max_budget is not None:
                 user_object.max_budget = user_info.max_budget
+            if user_info.user_email is not None:
+                user_object.user_email = user_info.user_email
+            if user_info.user_alias is not None:
+                user_object.user_alias = user_info.user_alias
 
         return user_object
 
@@ -3452,6 +3560,8 @@ class PrometheusLogger(CustomLogger):
         """
         enum_values = UserAPIKeyLabelValues(
             user=user.user_id,
+            user_email=user.user_email or "",
+            user_alias=user.user_alias or "",
         )
 
         _labels = prometheus_label_factory(

litellm/integrations/websearch_interception/handler.py

@@ -19,12 +19,14 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.websearch_interception.tools import (
     get_litellm_web_search_tool,
     get_litellm_web_search_tool_openai,
+    is_anthropic_native_web_search_tool,
     is_web_search_tool,
     is_web_search_tool_chat_completion,
 )
 from litellm.integrations.websearch_interception.transformation import (
     WebSearchTransformation,
 )
+from litellm.llms.base_llm.search.transformation import SearchResponse
 from litellm.types.integrations.websearch_interception import (
     WebSearchInterceptionConfig,
 )
@@ -36,6 +38,16 @@ from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import LlmProviders
 from litellm.utils import ProviderConfigManager
 
+# Key used to flag, on per-request kwargs, that the originating client sent
+# an Anthropic-native ``web_search_*`` tool — meaning the final response
+# should include ``web_search_tool_result`` content blocks so the client
+# (e.g. Claude Desktop's citations panel) can render sources.
+WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY = "_websearch_interception_emit_native_blocks"
+
+# Key on ``AgenticLoopPlan.metadata`` carrying the list of pre-built
+# ``web_search_tool_result`` blocks to inject into the final response.
+WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY = "websearch_native_blocks"
+
 
 class WebSearchInterceptionLogger(CustomLogger):
     """
@@ -152,22 +164,55 @@ class WebSearchInterceptionLogger(CustomLogger):
             f"(provider={provider_str}, query='{query}')"
         )
 
-        # Execute search
+        # Native clients (Claude Desktop / Cowork / Anthropic SDK) make a
+        # standalone /v1/messages sub-request just for the search, and they
+        # expect the response in native shape with server_tool_use +
+        # web_search_tool_result content blocks so the citations panel can
+        # render. The agentic-loop post-hook never fires on this path because
+        # there is no model call — emit the native blocks here instead.
+        native_tool = next(
+            (t for t in tools if is_anthropic_native_web_search_tool(t)),
+            None,
+        )
+
+        # Execute search — keep the structured SearchResponse so the native
+        # block can carry per-result url/title/page_age.
         try:
-            search_result_text = await self._execute_search(query)
+            search_result_text, structured = await self._execute_search(query)
         except Exception as e:
             verbose_logger.error(
                 f"WebSearchInterception: Short-circuit search failed: {e}"
             )
-            search_result_text = f"Search failed: {e}"
+            search_result_text, structured = f"Search failed: {e}", None
+
+        content: List[Dict[str, Any]] = []
+        if native_tool is not None:
+            tool_use_id = f"srvtoolu_{uuid.uuid4().hex}"
+            tool_name = native_tool.get("name") or "web_search"
+            content.append(
+                {
+                    "type": "server_tool_use",
+                    "id": tool_use_id,
+                    "name": tool_name,
+                    "input": {"query": query},
+                }
+            )
+            content.append(
+                WebSearchTransformation.build_web_search_tool_result_block(
+                    tool_use_id=tool_use_id,
+                    search_response=structured,
+                )
+            )
+        # Keep the text block so non-native short-circuit callers (Claude Code,
+        # github_copilot, etc.) see the same payload they always have.
+        content.append({"type": "text", "text": search_result_text})
 
-        # Build synthetic Anthropic response
         response: Dict[str, Any] = {
             "id": f"msg_{str(uuid.uuid4())}",
             "type": "message",
             "role": "assistant",
             "model": model,
-            "content": [{"type": "text", "text": search_result_text}],
+            "content": content,
             "stop_reason": "end_turn",
             "stop_sequence": None,
             "usage": {"input_tokens": 0, "output_tokens": 0},
@@ -175,7 +220,8 @@ class WebSearchInterceptionLogger(CustomLogger):
 
         verbose_logger.debug(
             "WebSearchInterception: Short-circuit search completed, "
-            f"returning synthetic response ({len(search_result_text)} chars)"
+            f"returning synthetic response ({len(search_result_text)} chars, "
+            f"native_blocks={native_tool is not None})"
         )
         return response
 
@@ -219,6 +265,14 @@ class WebSearchInterceptionLogger(CustomLogger):
             "WebSearchInterception: Converting native web_search tools to LiteLLM standard"
         )
 
+        # If the client sent an Anthropic-native web_search_* tool, mark the
+        # request so the agentic loop emits native web_search_tool_result
+        # blocks in the final response (matches async_pre_request_hook). This
+        # deployment hook fires before async_pre_request_hook on some paths,
+        # so flagging here ensures the signal isn't lost regardless of order.
+        if any(is_anthropic_native_web_search_tool(t) for t in tools):
+            kwargs[WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY] = True
+
         # Convert native/custom web_search tools to LiteLLM standard
         converted_tools = []
         for tool in tools:
@@ -342,6 +396,14 @@ class WebSearchInterceptionLogger(CustomLogger):
             f"WebSearchInterception: Pre-request hook triggered for provider={custom_llm_provider}"
         )
 
+        # If the client sent an Anthropic-native web_search_* tool, mark the
+        # request so the agentic loop emits native web_search_tool_result
+        # blocks in the final response (for citations panels, etc.). The flag
+        # is read by async_build_agentic_loop_plan; the leading underscore
+        # prefix ensures it is stripped before the follow-up call kwargs.
+        if any(is_anthropic_native_web_search_tool(t) for t in tools):
+            kwargs[WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY] = True
+
         # Convert native web search tools to LiteLLM standard
         converted_tools = []
         for tool in tools:
@@ -591,7 +653,7 @@ class WebSearchInterceptionLogger(CustomLogger):
     ) -> AgenticLoopPlan:
         tool_calls = tools["tool_calls"]
         thinking_blocks = tools.get("thinking_blocks", [])
-        request_patch = await self._build_anthropic_request_patch(
+        request_patch, structured_results = await self._build_anthropic_request_patch(
             model=model,
             messages=messages,
             tool_calls=tool_calls,
@@ -600,12 +662,92 @@ class WebSearchInterceptionLogger(CustomLogger):
             logging_obj=logging_obj,
             kwargs=kwargs,
         )
+
+        metadata: Dict[str, Any] = {
+            "tool_type": "websearch",
+            "response_format": "anthropic",
+        }
+
+        # If the client request originally carried a native web_search_* tool,
+        # pre-build the Anthropic-native ``web_search_tool_result`` blocks now
+        # (while we still have the structured SearchResponse list) and stash
+        # them on plan metadata for the post-hook to inject.
+        if kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY):
+            metadata[WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY] = (
+                self._build_native_result_blocks(
+                    tool_calls=tool_calls,
+                    structured_results=structured_results,
+                )
+            )
+
         return AgenticLoopPlan(
             run_agentic_loop=True,
             request_patch=request_patch,
-            metadata={"tool_type": "websearch", "response_format": "anthropic"},
+            metadata=metadata,
         )
 
+    async def async_post_agentic_loop_response_hook(
+        self,
+        response: Any,
+        plan: AgenticLoopPlan,
+        kwargs: Dict,
+    ) -> Any:
+        """
+        Inject Anthropic-native ``web_search_tool_result`` blocks into the
+        final response when the originating client used a native
+        ``web_search_*`` tool.
+
+        See ``WebSearchTransformation.build_web_search_tool_result_block`` for
+        the block shape. The blocks are prepended to ``response.content`` so
+        Anthropic-native clients (Claude Desktop, the Anthropic SDK) can
+        render citations / sources alongside the model's textual reply.
+        """
+        native_blocks = plan.metadata.get(WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY)
+        if not native_blocks:
+            return response
+        return self._inject_native_blocks(response, native_blocks)
+
+    @staticmethod
+    def _build_native_result_blocks(
+        tool_calls: List[Dict],
+        structured_results: List[Optional[SearchResponse]],
+    ) -> List[Dict[str, Any]]:
+        """Build one ``web_search_tool_result`` block per tool_call."""
+        blocks: List[Dict[str, Any]] = []
+        for i, tool_call in enumerate(tool_calls):
+            tool_use_id = tool_call.get("id") or ""
+            structured = structured_results[i] if i < len(structured_results) else None
+            blocks.append(
+                WebSearchTransformation.build_web_search_tool_result_block(
+                    tool_use_id=tool_use_id,
+                    search_response=structured,
+                )
+            )
+        return blocks
+
+    @staticmethod
+    def _inject_native_blocks(
+        response: Any, native_blocks: List[Dict[str, Any]]
+    ) -> Any:
+        """Prepend native blocks to response content, dict or object form."""
+        if not native_blocks:
+            return response
+        if isinstance(response, dict):
+            existing = response.get("content") or []
+            response["content"] = list(native_blocks) + list(existing)
+            return response
+        existing = getattr(response, "content", None) or []
+        try:
+            response.content = list(native_blocks) + list(existing)
+        except (AttributeError, TypeError):
+            # Object refused write — fall through and leave the response
+            # untouched rather than crash the request.
+            verbose_logger.debug(
+                "WebSearchInterception: could not inject native blocks into "
+                f"response of type {type(response).__name__}"
+            )
+        return response
+
     async def async_run_chat_completion_agentic_loop(
         self,
         tools: Dict,
@@ -733,7 +875,7 @@ class WebSearchInterceptionLogger(CustomLogger):
         kwargs: Dict,
     ) -> Any:
         """Legacy path: execute search + build patch + run follow-up call."""
-        request_patch = await self._build_anthropic_request_patch(
+        request_patch, structured_results = await self._build_anthropic_request_patch(
             model=model,
             messages=messages,
             tool_calls=tool_calls,
@@ -755,7 +897,7 @@ class WebSearchInterceptionLogger(CustomLogger):
         if max_tokens is None:
             max_tokens = cast(int, kwargs.get("max_tokens", 1024))
 
-        return await anthropic_messages.acreate(
+        response = await anthropic_messages.acreate(
             max_tokens=max_tokens,
             messages=request_patch.messages,
             model=request_patch.model or model,
@@ -763,6 +905,18 @@ class WebSearchInterceptionLogger(CustomLogger):
             **request_patch.kwargs,
         )
 
+        # Legacy path: the new path goes through the typed plan + core
+        # dispatcher which runs the post-hook automatically. Mirror the
+        # native-block injection here so both paths behave identically.
+        if kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY):
+            native_blocks = self._build_native_result_blocks(
+                tool_calls=tool_calls,
+                structured_results=structured_results,
+            )
+            response = self._inject_native_blocks(response, native_blocks)
+
+        return response
+
     async def _build_anthropic_request_patch(
         self,
         model: str,
@@ -772,8 +926,16 @@ class WebSearchInterceptionLogger(CustomLogger):
         anthropic_messages_optional_request_params: Dict,
         logging_obj: Any,
         kwargs: Dict,
-    ) -> AgenticLoopRequestPatch:
-        """Execute litellm.search() and build follow-up request patch."""
+    ) -> Tuple[AgenticLoopRequestPatch, List[Optional[SearchResponse]]]:
+        """
+        Execute litellm.search() and build follow-up request patch.
+
+        Returns the patch alongside the parallel list of structured
+        ``SearchResponse`` objects (one per tool_call, ``None`` when the
+        search failed or the tool_call had no query). The caller uses these
+        to optionally build Anthropic-native ``web_search_tool_result``
+        content blocks for the final response.
+        """
 
         # Extract search queries from tool_use blocks
         search_tasks = []
@@ -797,23 +959,38 @@ class WebSearchInterceptionLogger(CustomLogger):
         )
         search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
 
-        # Handle any exceptions in search results
+        # Split the gathered (text, structured) tuples into two parallel lists.
+        # The text list feeds the follow-up model call; the structured list
+        # is returned to the caller for native-block emission.
         final_search_results: List[str] = []
+        structured_results: List[Optional[SearchResponse]] = []
         for i, result in enumerate(search_results):
             if isinstance(result, Exception):
                 verbose_logger.error(
                     f"WebSearchInterception: Search {i} failed with error: {str(result)}"
                 )
                 final_search_results.append(f"Search failed: {str(result)}")
-            elif isinstance(result, str):
-                # Explicitly cast to str for type checker
-                final_search_results.append(cast(str, result))
+                structured_results.append(None)
+            elif isinstance(result, tuple) and len(result) == 2:
+                text_value, structured_value = result
+                final_search_results.append(
+                    cast(str, text_value)
+                    if isinstance(text_value, str)
+                    else str(text_value)
+                )
+                structured_results.append(
+                    structured_value
+                    if isinstance(structured_value, SearchResponse)
+                    else None
+                )
             else:
-                # Should never happen, but handle for type safety
+                # Defensive: legacy callers / unexpected shape — preserve text,
+                # drop structure.
                 verbose_logger.debug(
                     f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
                 )
                 final_search_results.append(str(result))
+                structured_results.append(None)
 
         # Build assistant and user messages using transformation
         assistant_message, user_message = WebSearchTransformation.transform_response(
@@ -859,16 +1036,26 @@ class WebSearchInterceptionLogger(CustomLogger):
             len(follow_up_messages),
             len(final_search_results),
         )
-        return AgenticLoopRequestPatch(
+        patch = AgenticLoopRequestPatch(
             model=full_model_name,
             messages=follow_up_messages,
             max_tokens=max_tokens,
             optional_params=optional_params_without_max_tokens,
             kwargs=kwargs_for_followup,
         )
+        return patch, structured_results
 
-    async def _execute_search(self, query: str) -> str:
-        """Execute a single web search using router's search tools"""
+    async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchResponse]]:
+        """
+        Execute a single web search using router's search tools.
+
+        Returns both the formatted text (fed back to the model in the follow-up
+        call) and the structured ``SearchResponse`` (preserved so callers can
+        build Anthropic-native ``web_search_tool_result`` blocks for clients
+        that requested a native ``web_search_*`` tool). The structured value
+        is None on the failure path so callers can still emit an empty result
+        block rather than dropping the search entirely.
+        """
         try:
             # Import router from proxy_server
             try:
@@ -934,7 +1121,7 @@ class WebSearchInterceptionLogger(CustomLogger):
             verbose_logger.debug(
                 f"WebSearchInterception: Search completed for '{query}', got {len(search_result_text)} chars"
             )
-            return search_result_text
+            return search_result_text, result
         except Exception as e:
             verbose_logger.error(
                 f"WebSearchInterception: Search failed for '{query}': {str(e)}"
@@ -1015,7 +1202,8 @@ class WebSearchInterceptionLogger(CustomLogger):
         )
         search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
 
-        # Handle any exceptions in search results
+        # Chat-completion path only needs text — OpenAI tool_result format
+        # has no equivalent of Anthropic's web_search_tool_result block.
         final_search_results: List[str] = []
         for i, result in enumerate(search_results):
             if isinstance(result, Exception):
@@ -1023,8 +1211,13 @@ class WebSearchInterceptionLogger(CustomLogger):
                     f"WebSearchInterception: Search {i} failed with error: {str(result)}"
                 )
                 final_search_results.append(f"Search failed: {str(result)}")
-            elif isinstance(result, str):
-                final_search_results.append(cast(str, result))
+            elif isinstance(result, tuple) and len(result) == 2:
+                text_value, _ = result
+                final_search_results.append(
+                    cast(str, text_value)
+                    if isinstance(text_value, str)
+                    else str(text_value)
+                )
             else:
                 verbose_logger.debug(
                     f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
@@ -1112,9 +1305,11 @@ class WebSearchInterceptionLogger(CustomLogger):
             kwargs=kwargs_for_followup,
         )
 
-    async def _create_empty_search_result(self) -> str:
+    async def _create_empty_search_result(
+        self,
+    ) -> Tuple[str, Optional[SearchResponse]]:
         """Create an empty search result for tool calls without queries"""
-        return "No search query provided"
+        return "No search query provided", None
 
     @staticmethod
     def initialize_from_proxy_config(

litellm/integrations/websearch_interception/tools.py

@@ -126,6 +126,27 @@ def is_web_search_tool_chat_completion(tool: Dict[str, Any]) -> bool:
     return False
 
 
+def is_anthropic_native_web_search_tool(tool: Dict[str, Any]) -> bool:
+    """
+    Check if a tool is an Anthropic-native ``web_search_*`` tool.
+
+    Native clients (Anthropic SDK, Claude Desktop, Anthropic Console) send
+    tools like ``{"type": "web_search_20250305", "name": "web_search"}`` and
+    expect the response to contain ``web_search_tool_result`` content blocks
+    so that citations can be rendered. This helper identifies that contract
+    so the agentic loop can emit native-format blocks for those clients
+    without affecting clients that send the LiteLLM standard tool.
+
+    Returns False for the LiteLLM standard tool (``litellm_web_search``),
+    the OpenAI-shaped variant, the bare ``WebSearch`` legacy name, and the
+    bare ``web_search`` name (Claude Code style).
+    """
+    tool_type = tool.get("type", "")
+    if not isinstance(tool_type, str):
+        return False
+    return tool_type.startswith("web_search_") and tool_type != "function"
+
+
 def is_web_search_tool(tool: Dict[str, Any]) -> bool:
     """
     Check if a tool is a web search tool (native or LiteLLM standard).
@@ -135,7 +156,22 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
     - OpenAI format: type == "function" with function.name == "litellm_web_search"
     - Anthropic native: type starts with "web_search_" (e.g., "web_search_20250305")
     - Claude Code: name == "web_search" with a type field
-    - Custom: name == "WebSearch" (legacy format)
+    - Custom: name == "WebSearch" (legacy interception marker — only matched
+      when input_schema is absent; see note below)
+
+    Note on the legacy ``WebSearch`` name:
+        Clients like Claude Desktop / Cowork ship a *client-side* tool called
+        ``WebSearch`` (a fully-formed Anthropic client tool with its own
+        ``input_schema``) that they handle themselves. Treating that as our
+        interception marker hijacks it server-side and the client's own tool
+        handler never fires — which means Cowork's separate native
+        ``web_search_20250305`` sub-request (where citation data actually
+        flows) never gets made.
+
+        Real Anthropic client tools always carry an ``input_schema`` (the API
+        rejects them otherwise), so a bare ``{name: "WebSearch"}`` with no
+        schema is the only thing that could be a legacy interception marker.
+        Gate the match on schema absence to keep both groups working.
 
     Args:
         tool: Tool dictionary to check
@@ -152,6 +188,10 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
         True
         >>> is_web_search_tool({"name": "calculator"})
         False
+        >>> is_web_search_tool({"name": "WebSearch"})  # legacy interception marker
+        True
+        >>> is_web_search_tool({"name": "WebSearch", "input_schema": {"type": "object"}})  # Cowork client tool
+        False
     """
     tool_name = tool.get("name", "")
     tool_type = tool.get("type", "")
@@ -175,8 +215,9 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
     if tool_name == "web_search" and tool_type:
         return True
 
-    # Check for legacy WebSearch format
-    if tool_name == "WebSearch":
+    # Legacy "WebSearch" interception marker — only when no schema is
+    # present, so real client-side WebSearch tools (Cowork) pass through.
+    if tool_name == "WebSearch" and "input_schema" not in tool:
         return True
 
     return False

litellm/integrations/websearch_interception/transformation.py

@@ -100,11 +100,14 @@ class WebSearchTransformation:
                 block_id = getattr(block, "id", None)
                 block_input = getattr(block, "input", {})
 
-            # Check for LiteLLM standard or legacy web search tools
-            # Handles: litellm_web_search, WebSearch, web_search
+            # Detect tool_use blocks that came from interception. After
+            # pre-request conversion the model always sees
+            # ``litellm_web_search``; the bare ``web_search`` entry handles
+            # callers that bypass our pre-request hooks (e.g. direct
+            # litellm.acompletion). "WebSearch" is intentionally omitted —
+            # see is_web_search_tool for the Cowork rationale.
             if block_type == "tool_use" and block_name in (
                 LITELLM_WEB_SEARCH_TOOL_NAME,
-                "WebSearch",
                 "web_search",
             ):
                 # Convert to dict for easier handling
@@ -190,10 +193,12 @@ class WebSearchTransformation:
                     getattr(function, "arguments", None) if function else None
                 )
 
-            # Check for LiteLLM standard or legacy web search tools
+            # Detect function-style web search tool_calls. ``WebSearch`` is
+            # intentionally omitted — see is_web_search_tool for the Cowork
+            # rationale (clients ship their own client-side ``WebSearch`` and
+            # we must not hijack it).
             if tool_type == "function" and function_name in (
                 LITELLM_WEB_SEARCH_TOOL_NAME,
-                "WebSearch",
                 "web_search",
             ):
                 # Parse arguments (might be JSON string)
@@ -350,6 +355,57 @@ class WebSearchTransformation:
 
         return assistant_message, tool_messages
 
+    @staticmethod
+    def build_web_search_tool_result_block(
+        tool_use_id: str,
+        search_response: Optional[SearchResponse],
+    ) -> Dict[str, Any]:
+        """
+        Build an Anthropic-native ``web_search_tool_result`` content block.
+
+        Native Anthropic clients (Claude Desktop, the Anthropic SDK, the
+        Anthropic Console) expect search-tool results to be returned as
+        structured ``web_search_tool_result`` blocks so that citations and
+        source links can be rendered. The agentic loop currently feeds the
+        model a flat text blob in the follow-up call (which is correct — the
+        model needs readable evidence). This helper produces the *additional*
+        block that should accompany the model's text reply when the original
+        request used a native ``web_search_*`` tool.
+
+        Spec reference:
+        https://docs.anthropic.com/en/api/web-search-tool
+
+        Args:
+            tool_use_id: The ``tool_use_id`` the model emitted on the first
+                turn. Must match exactly so the client can pair the result
+                with its tool_use block.
+            search_response: Structured ``SearchResponse`` from
+                ``litellm.asearch()``. If None or empty, the block is still
+                emitted with an empty result list (signals "search ran, no
+                results" rather than "search did not run").
+        """
+        items: List[Dict[str, Any]] = []
+        if search_response is not None:
+            results = getattr(search_response, "results", None) or []
+            for r in results:
+                url = getattr(r, "url", "") or ""
+                title = getattr(r, "title", "") or ""
+                page_age = getattr(r, "date", None) or getattr(r, "last_updated", None)
+                items.append(
+                    {
+                        "type": "web_search_result",
+                        "url": url,
+                        "title": title,
+                        "page_age": page_age,
+                        "encrypted_content": "",
+                    }
+                )
+        return {
+            "type": "web_search_tool_result",
+            "tool_use_id": tool_use_id,
+            "content": items,
+        }
+
     @staticmethod
     def format_search_response(result: SearchResponse) -> str:
         """

litellm/interactions/litellm_responses_transformation/streaming_iterator.py

@@ -2,7 +2,7 @@
 Streaming iterator for transforming Responses API stream to Interactions API stream.
 """
 
-from typing import Any, AsyncIterator, Dict, Iterator, Optional, cast
+from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, cast
 
 from litellm.responses.streaming_iterator import (
     BaseResponsesAPIStreamingIterator,
@@ -15,6 +15,7 @@ from litellm.types.interactions import (
     InteractionsAPIStreamingResponse,
 )
 from litellm.types.llms.openai import (
+    ContentPartAddedEvent,
     OutputTextDeltaEvent,
     ResponseCompletedEvent,
     ResponseCreatedEvent,
@@ -51,6 +52,7 @@ class LiteLLMResponsesInteractionsStreamingIterator:
         self.collected_text = ""
         self.sent_interaction_start = False
         self.sent_content_start = False
+        self._pending_events: List[InteractionsAPIStreamingResponse] = []
 
     def _transform_responses_chunk_to_interactions_chunk(
         self,
@@ -80,7 +82,49 @@ class LiteLLMResponsesInteractionsStreamingIterator:
             )
             self.collected_text += delta_text
 
-            # Send interaction.start if not sent
+            # Fallback: emit interaction.start, and queue content.start carrying this
+            # delta so the first token is preserved in the stream.
+            if not self.sent_interaction_start:
+                self.sent_interaction_start = True
+                self.sent_content_start = True
+                self._pending_events.append(
+                    InteractionsAPIStreamingResponse(
+                        event_type="content.start",
+                        id=getattr(responses_chunk, "item_id", None),
+                        object="content",
+                        delta={"type": "text", "text": delta_text},
+                    )
+                )
+                return InteractionsAPIStreamingResponse(
+                    event_type="interaction.start",
+                    id=getattr(responses_chunk, "item_id", None)
+                    or f"interaction_{id(self)}",
+                    object="interaction",
+                    status="in_progress",
+                    model=self.model,
+                )
+
+            # Fallback: emit content.start if ContentPartAddedEvent never arrived
+            if not self.sent_content_start:
+                self.sent_content_start = True
+                return InteractionsAPIStreamingResponse(
+                    event_type="content.start",
+                    id=getattr(responses_chunk, "item_id", None),
+                    object="content",
+                    delta={"type": "text", "text": delta_text},
+                )
+
+            # Normal path: emit content.delta with type field
+            return InteractionsAPIStreamingResponse(
+                event_type="content.delta",
+                id=getattr(responses_chunk, "item_id", None),
+                object="content",
+                delta={"type": "text", "text": delta_text},
+            )
+
+        # Handle ContentPartAddedEvent -> content.start (arrives before text deltas)
+        if isinstance(responses_chunk, ContentPartAddedEvent):
+            # Fallback: emit interaction.start if ResponseCreatedEvent never arrived
             if not self.sent_interaction_start:
                 self.sent_interaction_start = True
                 return InteractionsAPIStreamingResponse(
@@ -91,8 +135,6 @@ class LiteLLMResponsesInteractionsStreamingIterator:
                     status="in_progress",
                     model=self.model,
                 )
-
-            # Send content.start if not sent
             if not self.sent_content_start:
                 self.sent_content_start = True
                 return InteractionsAPIStreamingResponse(
@@ -101,14 +143,7 @@ class LiteLLMResponsesInteractionsStreamingIterator:
                     object="content",
                     delta={"type": "text", "text": ""},
                 )
-
-            # Send content.delta
-            return InteractionsAPIStreamingResponse(
-                event_type="content.delta",
-                id=getattr(responses_chunk, "item_id", None),
-                object="content",
-                delta={"text": delta_text},
-            )
+            return None
 
         # Handle ResponseCreatedEvent or ResponseInProgressEvent -> interaction.start
         if isinstance(responses_chunk, (ResponseCreatedEvent, ResponseInProgressEvent)):
@@ -172,6 +207,10 @@ class LiteLLMResponsesInteractionsStreamingIterator:
             delattr(self, "_pending_interaction_complete")
             return pending
 
+        # Drain events queued from a prior chunk (e.g. content.start emitted alongside
+        # the interaction.start fallback for the first OutputTextDeltaEvent).
+        if self._pending_events:
+            return self._pending_events.pop(0)
         # Use a loop instead of recursion to avoid stack overflow
         sync_iterator = cast(
             SyncResponsesAPIStreamingIterator, self.responses_stream_iterator
@@ -237,6 +276,10 @@ class LiteLLMResponsesInteractionsStreamingIterator:
             delattr(self, "_pending_interaction_complete")
             return pending
 
+        # Drain events queued from a prior chunk (e.g. content.start emitted alongside
+        # the interaction.start fallback for the first OutputTextDeltaEvent).
+        if self._pending_events:
+            return self._pending_events.pop(0)
         # Use a loop instead of recursion to avoid stack overflow
         async_iterator = cast(
             ResponsesAPIStreamingIterator, self.responses_stream_iterator

litellm/litellm_core_utils/litellm_logging.py

@@ -1050,6 +1050,16 @@ class Logging(LiteLLMLoggingBaseClass):
                     )
 
             self.model_call_details["api_call_start_time"] = datetime.datetime.now()
+            # Set-once first provider-handoff instant. api_call_start_time
+            # is overwritten on every retry, so it can't measure one-time
+            # preprocessing; pinning the first attempt excludes retry loops
+            # + backoff. Logging object only — must NOT go into
+            # litellm_params["metadata"] (caller request metadata, typed
+            # Dict[str, str], echoed downstream; a datetime breaks it).
+            if self.model_call_details.get("first_api_call_start_time") is None:
+                self.model_call_details["first_api_call_start_time"] = (
+                    self.model_call_details["api_call_start_time"]
+                )
             # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
             callbacks = litellm.input_callback + (self.dynamic_input_callbacks or [])
             for callback in callbacks:

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -20,6 +20,7 @@ from typing import (
     cast,
 )
 
+import litellm
 from litellm import verbose_logger
 from litellm.router_utils.batch_utils import InMemoryFile
 from litellm.types.llms.openai import (
@@ -1170,9 +1171,16 @@ def migrate_file_to_image_url(
         ChatCompletionImageUrlObject,
     )
 
-    file_id = message["file"].get("file_id")
-    file_data = message["file"].get("file_data")
-    format = message["file"].get("format")
+    file_sub = message.get("file")
+    if file_sub is None:
+        raise litellm.BadRequestError(
+            message="Content block has type='file' but is missing the required 'file' field",
+            model=None,
+            llm_provider=None,
+        )
+    file_id = file_sub.get("file_id")
+    file_data = file_sub.get("file_data")
+    format = file_sub.get("format")
     if not file_id and not file_data:
         raise ValueError("file_id and file_data are both None")
     image_url_object = ChatCompletionImageObject(

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -2057,9 +2057,16 @@ def anthropic_process_openai_file_message(
     AnthropicMessagesContainerUploadParam,
 ]:
     file_message = cast(ChatCompletionFileObject, message)
-    file_data = file_message["file"].get("file_data")
-    file_id = file_message["file"].get("file_id")
-    format = file_message["file"].get("format")
+    file_sub = file_message.get("file")
+    if file_sub is None:
+        raise litellm.BadRequestError(
+            message="Content block has type='file' but is missing the required 'file' field",
+            model=None,
+            llm_provider="anthropic",
+        )
+    file_data = file_sub.get("file_data")
+    file_id = file_sub.get("file_id")
+    format = file_sub.get("format")
     if file_data:
         image_chunk = convert_to_anthropic_image_obj(
             openai_image_url=file_data,
@@ -4879,7 +4886,13 @@ class BedrockConverseMessagesProcessor:
 
     @staticmethod
     def _process_file_message(message: ChatCompletionFileObject) -> BedrockContentBlock:
-        file_message = message["file"]
+        file_message = message.get("file")
+        if file_message is None:
+            raise litellm.BadRequestError(
+                message="Content block has type='file' but is missing the required 'file' field",
+                model=None,
+                llm_provider="bedrock",
+            )
         file_data = file_message.get("file_data")
         file_id = file_message.get("file_id")
 
@@ -4900,7 +4913,13 @@ class BedrockConverseMessagesProcessor:
     async def _async_process_file_message(
         message: ChatCompletionFileObject,
     ) -> BedrockContentBlock:
-        file_message = message["file"]
+        file_message = message.get("file")
+        if file_message is None:
+            raise litellm.BadRequestError(
+                message="Content block has type='file' but is missing the required 'file' field",
+                model=None,
+                llm_provider="bedrock",
+            )
         file_data = file_message.get("file_data")
         file_id = file_message.get("file_id")
         format = file_message.get("format")
@@ -4977,8 +4996,9 @@ class BedrockConverseMessagesProcessor:
             )
             if reasoning_text and not reasoning_text.get("signature"):
                 reasoning_text_text = reasoning_text["text"]
-                assistants_part = BedrockContentBlock(text=reasoning_text_text)
-                assistant_parts.append(assistants_part)
+                if reasoning_text_text.strip():
+                    assistants_part = BedrockContentBlock(text=reasoning_text_text)
+                    assistant_parts.append(assistants_part)
             else:
                 filtered_thinking_blocks.append(block)
         if len(filtered_thinking_blocks) > 0:

litellm/llms/anthropic/common_utils.py

@@ -832,6 +832,49 @@ def strip_thinking_blocks_from_anthropic_messages_request_dict(
     data.pop("thinking", None)
 
 
+def strip_empty_text_blocks_from_anthropic_messages(
+    messages: List[Any],
+) -> List[Any]:
+    """
+    Return a new message list with empty or whitespace-only ``{"type": "text"}``
+    content blocks removed.
+
+    Anthropic's API rejects requests containing such blocks with
+    ``"messages: text content blocks must be non-empty"``, but assistant
+    messages from Anthropic routinely arrive with ``{"type": "text", "text": ""}``
+    alongside ``tool_use`` blocks (see anthropics/anthropic-sdk-python#461).
+    Multi-turn tool-use clients (e.g. Claude Code) loop these prior responses
+    back as conversation history, which then causes the next request to 400
+    on the unified ``/v1/messages`` path.  ``/v1/chat/completions`` already
+    handles this in ``anthropic_messages_pt``; this helper provides the
+    equivalent guarantee for the native Anthropic Messages path.
+
+    Messages whose content is a list and becomes empty after stripping are
+    omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`.
+    The caller's list and its content blocks are never mutated; modified
+    messages are returned as shallow copies with a fresh content list.
+    """
+    out: List[Any] = []
+    for m in messages:
+        if not isinstance(m, dict) or not isinstance(m.get("content"), list):
+            out.append(m)
+            continue
+        content = m["content"]
+        filtered = [b for b in content if not _is_empty_text_block(b)]
+        if len(filtered) == len(content):
+            out.append(m)
+        elif filtered:
+            out.append({**m, "content": filtered})
+    return out
+
+
+def _is_empty_text_block(block: Any) -> bool:
+    if not isinstance(block, dict) or block.get("type") != "text":
+        return False
+    text = block.get("text")
+    return not isinstance(text, str) or not text.strip()
+
+
 def process_anthropic_headers(headers: Union[httpx.Headers, dict]) -> dict:
     openai_headers = {}
     if "anthropic-ratelimit-requests-limit" in headers:

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -1299,9 +1299,18 @@ class LiteLLMAnthropicMessagesAdapter:
                         else truncated_name
                     )
 
+                    # Strip Gemini thought-signature suffix from id (mirrors streaming
+                    # path below); base64 chars (+ / =) violate Anthropic's
+                    # `^[a-zA-Z0-9_-]+$` tool_use.id pattern when replayed.
+                    raw_id = tool_call.id or ""
+                    base_id = (
+                        raw_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0]
+                        if THOUGHT_SIGNATURE_SEPARATOR in raw_id
+                        else raw_id
+                    )
                     tool_use_block = AnthropicResponseContentBlockToolUse(
                         type="tool_use",
-                        id=tool_call.id,
+                        id=base_id,
                         name=original_name,
                         input=parse_tool_call_arguments(
                             tool_call.function.arguments,

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -12,6 +12,9 @@ from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union, c
 
 import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.anthropic.common_utils import (
+    strip_empty_text_blocks_from_anthropic_messages,
+)
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
 )
@@ -188,8 +191,20 @@ async def anthropic_messages(
     **kwargs,
 ) -> Union[AnthropicMessagesResponse, AsyncIterator]:
     """
-    Async: Make llm api request in Anthropic /messages API spec
+    Async: Make llm api request in Anthropic /messages API spec.
+
+    Runs the empty-text-block sanitizer before any backend dispatch.
     """
+    # Anthropic's API rejects requests containing empty / whitespace-only
+    # text content blocks with "messages: text content blocks must be
+    # non-empty".  Multi-turn tool-use clients (e.g. Claude Code) routinely
+    # loop assistant responses that contain {"type": "text", "text": ""}
+    # alongside tool_use blocks back as conversation history, which then
+    # causes the next /v1/messages call to 400.  /v1/chat/completions
+    # already handles this in anthropic_messages_pt; sanitize the native
+    # Anthropic Messages path here for the same guarantee.  See #22930.
+    messages = strip_empty_text_blocks_from_anthropic_messages(messages)
+
     original_stream = stream or kwargs.get(
         "_websearch_interception_converted_stream", False
     )
@@ -336,6 +351,11 @@ def anthropic_messages_handler(
     """
     from litellm.types.utils import LlmProviders
 
+    # Sanitize empty text blocks here too so the sync entry point
+    # (litellm.messages.create -> anthropic_messages_handler) gets the same
+    # protection as the async wrapper.  Idempotent when called twice.
+    messages = strip_empty_text_blocks_from_anthropic_messages(messages)
+
     metadata = validate_anthropic_api_metadata(metadata)
 
     local_vars = locals()

litellm/llms/azure/image_edit/transformation.py

@@ -3,8 +3,10 @@ from typing import Optional, cast
 import httpx
 
 import litellm
+from litellm.llms.azure.common_utils import BaseAzureLLM
 from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
 from litellm.secret_managers.main import get_secret_str
+from litellm.types.router import GenericLiteLLMParams
 from litellm.utils import _add_path_to_api_base
 
 
@@ -30,20 +32,42 @@ class AzureImageEditConfig(OpenAIImageEditConfig):
         litellm_params: Optional[dict] = None,
         api_base: Optional[str] = None,
     ) -> dict:
-        api_key = (
-            api_key
-            or litellm.api_key
-            or litellm.azure_key
-            or get_secret_str("AZURE_OPENAI_API_KEY")
-            or get_secret_str("AZURE_API_KEY")
-        )
+        """
+        Validate Azure environment and set up authentication headers.
 
-        headers.update(
-            {
-                "Authorization": f"Bearer {api_key}",
-            }
+        Delegates to ``BaseAzureLLM._base_validate_azure_environment`` so the
+        Azure image-edit route uses the same auth resolution as every other
+        Azure provider (videos, vector_stores, responses, containers, ...):
+
+        - prefers the Azure-style ``api-key`` header when an API key is available
+        - falls back to ``Authorization: Bearer <azure_ad_token>`` only when AAD
+          auth is configured
+
+        The previous implementation unconditionally set
+        ``Authorization: Bearer <api_key>``, which is correct for OpenAI direct
+        but not for Azure OpenAI / API Management gateways that expect the
+        ``api-key`` header. Subscription-key-based deployments (e.g., behind
+        Azure APIM) responded with ``401 "Access denied due to missing
+        subscription key"``.
+
+        API-key precedence (matches ``AzureVideosConfig``):
+
+        - ``litellm_params["api_key"]`` is the source of truth.
+        - The positional ``api_key`` kwarg only fills in when
+          ``litellm_params["api_key"]`` is empty.
+        - This is a deliberate change from the old ``or`` chain (where the
+          positional ``api_key`` argument won) so behavior matches every other
+          Azure ``validate_environment`` implementation. In production the only
+          caller (``llm_http_handler.image_edit``) sources both values from
+          the same ``litellm_params.api_key``, so the precedence only matters
+          for direct callers of this method.
+        """
+        params = GenericLiteLLMParams(**(litellm_params or {}))
+        if api_key is not None and params.api_key is None:
+            params.api_key = api_key
+        return BaseAzureLLM._base_validate_azure_environment(
+            headers=headers, litellm_params=params
         )
-        return headers
 
     def get_complete_url(
         self,

litellm/llms/bedrock/batches/transformation.py

@@ -5,6 +5,7 @@ from typing import Any, Dict, List, Literal, Optional, Union, cast
 
 from httpx import Headers, Response
 
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.secret_managers.main import get_secret_str
@@ -263,9 +264,32 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
             cancelling_at=None,
             cancelled_at=None,
             request_counts=None,
-            metadata=original_request.get("metadata", {}),
+            metadata=self._get_openai_compatible_batch_metadata(
+                original_request.get("metadata", {})
+            ),
         )
 
+    @staticmethod
+    def _get_openai_compatible_batch_metadata(metadata: Any) -> Dict[str, str]:
+        """
+        OpenAI Batch metadata only accepts string values.
+        """
+        if not isinstance(metadata, dict):
+            return {}
+
+        sanitized_metadata: Dict[str, str] = {}
+        for key, value in metadata.items():
+            if key == "standard_logging_guardrail_information" or value is None:
+                continue
+
+            str_key = str(key)
+            if isinstance(value, str):
+                sanitized_metadata[str_key] = value
+            else:
+                sanitized_metadata[str_key] = safe_dumps(value)
+
+        return sanitized_metadata
+
     def transform_retrieve_batch_request(
         self,
         batch_id: str,

litellm/llms/bedrock/chat/mantle/transformation.py

@@ -21,7 +21,9 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
-MANTLE_ENDPOINT_TEMPLATE = "https://bedrock-mantle.{region}.api.aws/v1/messages"
+MANTLE_ENDPOINT_TEMPLATE = (
+    "https://bedrock-mantle.{region}.api.aws/anthropic/v1/messages"
+)
 
 
 class AmazonMantleConfig(AmazonAnthropicClaudeConfig):

litellm/llms/bedrock/embed/cohere_transformation.py

@@ -22,7 +22,7 @@ class BedrockCohereEmbeddingConfig:
     ) -> dict:
         for k, v in non_default_params.items():
             if k == "encoding_format":
-                optional_params["embedding_types"] = v
+                optional_params["embedding_types"] = v if isinstance(v, list) else [v]
             elif k == "dimensions":
                 optional_params["output_dimension"] = v
         return optional_params

litellm/llms/bedrock/messages/mantle_transformation.py

@@ -20,7 +20,9 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
-MANTLE_ENDPOINT_TEMPLATE = "https://bedrock-mantle.{region}.api.aws/v1/messages"
+MANTLE_ENDPOINT_TEMPLATE = (
+    "https://bedrock-mantle.{region}.api.aws/anthropic/v1/messages"
+)
 
 
 class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):

litellm/llms/custom_httpx/http_handler.py

@@ -485,11 +485,16 @@ class MaskedHTTPStatusError(httpx.HTTPStatusError):
             if k.lower() not in ("content-encoding", "content-length")
         }
 
+        try:
+            request_content = original_error.request.content
+        except httpx.RequestNotRead:
+            request_content = b""
+
         masked_request = httpx.Request(
             method=original_error.request.method,
             url=masked_url,
             headers=original_error.request.headers,
-            content=original_error.request.content,
+            content=request_content,
         )
 
         super().__init__(

litellm/llms/custom_httpx/llm_http_handler.py

@@ -4634,6 +4634,7 @@ class BaseLLMHTTPHandler:
         fingerprints: List[str],
         fingerprint: str,
         stream: bool = False,
+        callback: Optional[Any] = None,
     ) -> Any:
         from litellm.anthropic_interface import messages as anthropic_messages
 
@@ -4675,7 +4676,7 @@ class BaseLLMHTTPHandler:
         kwargs_for_followup["max_agentic_loops"] = max_loops
         kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint]
 
-        return await anthropic_messages.acreate(
+        response = await anthropic_messages.acreate(
             **{
                 "max_tokens": max_tokens,
                 "messages": patch.messages,
@@ -4686,6 +4687,23 @@ class BaseLLMHTTPHandler:
             }
         )
 
+        if callback is not None:
+            try:
+                response = await callback.async_post_agentic_loop_response_hook(
+                    response=response, plan=plan, kwargs=kwargs
+                )
+            except Exception as e:
+                _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
+                verbose_logger.exception(
+                    "LiteLLM.AgenticHookError: Exception in "
+                    "async_post_agentic_loop_response_hook [call_id=%s model=%s]: %s",
+                    _call_id,
+                    model,
+                    str(e),
+                )
+
+        return response
+
     async def _execute_chat_completion_agentic_plan(
         self,
         plan: AgenticLoopPlan,
@@ -4869,6 +4887,7 @@ class BaseLLMHTTPHandler:
                     fingerprints=fingerprints,
                     fingerprint=fingerprint,
                     stream=stream,
+                    callback=callback,
                 )
             except Exception as e:
                 _call_id = getattr(logging_obj, "litellm_call_id", "unknown")

litellm/llms/dashscope/common_utils.py

@@ -0,0 +1,28 @@
+"""
+Common utilities for the DashScope LLM provider.
+"""
+
+from typing import Optional
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+
+class DashScopeError(BaseLLMException):
+    """Exception class for DashScope provider errors."""
+
+    def __init__(
+        self,
+        status_code: int,
+        message: str,
+        headers: Optional[httpx.Headers] = None,
+    ):
+        self.status_code = status_code
+        self.message = message
+        self.headers = headers or httpx.Headers()
+        super().__init__(
+            status_code=status_code,
+            message=message,
+            headers=dict(self.headers),
+        )

litellm/llms/dashscope/embed/__init__.py

@@ -0,0 +1,7 @@
+"""
+DashScope Embedding Module
+"""
+
+from .transformation import DashScopeEmbeddingConfig
+
+__all__ = ["DashScopeEmbeddingConfig"]

litellm/llms/dashscope/embed/transformation.py

@@ -0,0 +1,191 @@
+"""
+Transformation logic from OpenAI /v1/embeddings format to DashScope's /v1/embeddings format.
+
+Supports
+- text-embedding-v4
+- text-embedding-v3
+
+Endpoint
+- https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings
+
+Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
+"""
+
+from typing import List, Optional, Union
+
+import httpx
+
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
+from litellm.types.utils import EmbeddingResponse, Usage
+
+from ..common_utils import DashScopeError
+
+DEFAULT_API_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1"
+
+
+class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
+    """
+    Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
+
+    DashScope exposes an OpenAI-compatible /v1/embeddings endpoint, so the
+    request and response shapes are nearly identical to OpenAI's.
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    def get_supported_openai_params(self, model: str) -> List[str]:
+        # DashScope's compatible-mode embeddings API accepts the same params as OpenAI.
+        # `dimensions` / `encoding_format` are only honored by text-embedding-v3 / v4;
+        # earlier versions silently ignore them server-side.
+        return ["dimensions", "encoding_format", "user"]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool = False,
+    ) -> dict:
+        supported = self.get_supported_openai_params(model)
+        for k, v in non_default_params.items():
+            if v is None:
+                continue
+            if k in supported:
+                optional_params[k] = v
+            # unsupported params are dropped when drop_params=True;
+            # the upstream _check_valid_arg already raised UnsupportedParamsError
+            # for drop_params=False before this method is called.
+        return optional_params
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        if api_key is None:
+            api_key = get_secret_str("DASHSCOPE_API_KEY")
+        if api_key is None:
+            raise ValueError(
+                "DashScope API key is required. Set 'DASHSCOPE_API_KEY' env var or pass api_key explicitly."
+            )
+        default_headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        }
+        return {**default_headers, **headers}
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        base = api_base or get_secret_str("DASHSCOPE_API_BASE") or DEFAULT_API_BASE
+        base = base.rstrip("/")
+        if base.endswith("/embeddings"):
+            return base
+        return f"{base}/embeddings"
+
+    def transform_embedding_request(
+        self,
+        model: str,
+        input: AllEmbeddingInputValues,
+        optional_params: dict,
+        headers: dict,
+    ) -> dict:
+        data: dict = {
+            "model": model,
+            "input": input,
+        }
+        for key in ("dimensions", "encoding_format", "user"):
+            value = optional_params.get(key)
+            if value is not None:
+                data[key] = value
+        return data
+
+    def transform_embedding_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        model_response: EmbeddingResponse,
+        logging_obj: LiteLLMLoggingObj,
+        api_key: Optional[str],
+        request_data: dict,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> EmbeddingResponse:
+        try:
+            response_json = raw_response.json()
+        except Exception as e:
+            raise DashScopeError(
+                status_code=raw_response.status_code,
+                message=f"Failed to parse DashScope response as JSON: {str(e)}",
+            )
+
+        logging_obj.post_call(
+            input=request_data.get("input"),
+            api_key=api_key,
+            additional_args={"complete_input_dict": request_data},
+            original_response=response_json,
+        )
+
+        if "error" in response_json:
+            error = response_json["error"]
+            message = (
+                error.get("message", str(error))
+                if isinstance(error, dict)
+                else str(error)
+            )
+            raise DashScopeError(
+                status_code=raw_response.status_code,
+                message=message,
+            )
+
+        model_response.object = "list"
+        model_response.data = response_json.get("data", [])
+        model_response.model = response_json.get("model", model)
+
+        usage = response_json.get("usage") or {}
+        prompt_tokens = usage.get("prompt_tokens", 0)
+        total_tokens = usage.get("total_tokens", prompt_tokens)
+        setattr(
+            model_response,
+            "usage",
+            Usage(
+                prompt_tokens=prompt_tokens,
+                completion_tokens=0,
+                total_tokens=total_tokens,
+            ),
+        )
+
+        if "id" in response_json:
+            setattr(model_response, "id", response_json["id"])
+
+        return model_response
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: Union[dict, httpx.Headers],
+    ) -> BaseLLMException:
+        if isinstance(headers, dict):
+            headers = httpx.Headers(headers)
+        return DashScopeError(
+            status_code=status_code,
+            message=error_message,
+            headers=headers,
+        )

litellm/llms/dashscope/rerank/__init__.py

@@ -0,0 +1,7 @@
+"""
+DashScope Rerank Module
+"""
+
+from .transformation import DashScopeRerankConfig
+
+__all__ = ["DashScopeRerankConfig"]

litellm/llms/dashscope/rerank/transformation.py

@@ -0,0 +1,241 @@
+"""
+Transformation logic for DashScope's OpenAI-compatible /v1/reranks API.
+
+Supports
+- qwen3-rerank
+
+(Other DashScope rerankers — gte-rerank-v2 / qwen3-vl-rerank — share the same
+endpoint but have not been validated against this transformer. Behavior with
+those models is undefined.)
+
+Endpoint
+- https://dashscope.aliyuncs.com/compatible-api/v1/reranks
+
+Note: chat/embed live under `/compatible-mode/v1/`, but DashScope's rerank
+route is exposed under `/compatible-api/v1/reranks` per the docs. Override
+with `DASHSCOPE_API_BASE_RERANK` to point at a different host or path.
+
+Empirically, qwen3-rerank accepts `return_documents=true` and echoes
+`results[].document.text` back, even though the public docs list the flag
+as supported only for gte-rerank-v2 / qwen3-vl-rerank.
+
+Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api
+"""
+
+from typing import Any, Dict, List, Optional, Union
+
+import httpx
+
+from litellm._uuid import uuid
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.rerank import (
+    OptionalRerankParams,
+    RerankBilledUnits,
+    RerankResponse,
+    RerankResponseMeta,
+    RerankTokens,
+)
+
+from ..common_utils import DashScopeError
+
+DEFAULT_RERANK_URL = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
+
+
+class DashScopeRerankConfig(BaseRerankConfig):
+    """
+    Reference: https://help.aliyun.com/zh/model-studio/text-rerank-api
+
+    Targets DashScope's qwen3-rerank model. Request fields: model, query,
+    documents, top_n, return_documents. Response: results[].index,
+    results[].relevance_score, optionally results[].document.text (when
+    return_documents=true), plus a top-level usage.total_tokens counter.
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        model: str,
+        optional_params: Optional[dict] = None,
+    ) -> str:
+        if api_base is None:
+            api_base = get_secret_str("DASHSCOPE_API_BASE_RERANK") or DEFAULT_RERANK_URL
+
+        if api_base == DEFAULT_RERANK_URL:
+            return DEFAULT_RERANK_URL
+
+        cleaned = api_base.rstrip("/")
+        if cleaned.endswith("/reranks") or cleaned.endswith("/rerank"):
+            return cleaned
+
+        if cleaned.endswith("/v1"):
+            return f"{cleaned}/reranks"
+
+        # Unknown base: append /reranks rather than silently ignoring the caller's api_base.
+        return f"{cleaned}/reranks"
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        api_key: Optional[str] = None,
+        optional_params: Optional[dict] = None,
+    ) -> dict:
+        if api_key is None:
+            api_key = get_secret_str("DASHSCOPE_API_KEY")
+        if api_key is None:
+            raise ValueError(
+                "DashScope API key is required. Set 'DASHSCOPE_API_KEY' env var or pass api_key explicitly."
+            )
+
+        default_headers = {
+            "Authorization": f"Bearer {api_key}",
+            "accept": "application/json",
+            "content-type": "application/json",
+        }
+        return {**default_headers, **headers}
+
+    def get_supported_cohere_rerank_params(self, model: str) -> list:
+        return ["query", "documents", "top_n", "return_documents"]
+
+    def map_cohere_rerank_params(
+        self,
+        non_default_params: Optional[dict],
+        model: str,
+        drop_params: bool,
+        query: str,
+        documents: List[Union[str, Dict[str, Any]]],
+        custom_llm_provider: Optional[str] = None,
+        top_n: Optional[int] = None,
+        rank_fields: Optional[List[str]] = None,
+        return_documents: Optional[bool] = True,
+        max_chunks_per_doc: Optional[int] = None,
+        max_tokens_per_doc: Optional[int] = None,
+    ) -> Dict:
+        # qwen3-rerank accepts query/documents/top_n/return_documents. The
+        # rest (rank_fields, max_*_per_doc) are silently dropped.
+        params: OptionalRerankParams = OptionalRerankParams(
+            query=query,
+            documents=documents,
+        )
+        if top_n is not None:
+            params["top_n"] = top_n
+        if return_documents is not None:
+            params["return_documents"] = return_documents
+        return dict(params)
+
+    def transform_rerank_request(
+        self,
+        model: str,
+        optional_rerank_params: Dict,
+        headers: dict,
+        litellm_params: Optional[dict] = None,
+    ) -> dict:
+        if "query" not in optional_rerank_params:
+            raise ValueError("query is required for DashScope rerank")
+        if "documents" not in optional_rerank_params:
+            raise ValueError("documents is required for DashScope rerank")
+
+        request: Dict[str, Any] = {
+            "model": model,
+            "query": optional_rerank_params["query"],
+            "documents": optional_rerank_params["documents"],
+        }
+        if optional_rerank_params.get("top_n") is not None:
+            request["top_n"] = optional_rerank_params["top_n"]
+        if optional_rerank_params.get("return_documents") is not None:
+            request["return_documents"] = optional_rerank_params["return_documents"]
+        return request
+
+    def transform_rerank_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        model_response: RerankResponse,
+        logging_obj: LiteLLMLoggingObj,
+        api_key: Optional[str] = None,
+        request_data: Optional[dict] = None,
+        optional_params: Optional[dict] = None,
+        litellm_params: Optional[dict] = None,
+    ) -> RerankResponse:
+        request_data = request_data or {}
+        optional_params = optional_params or {}
+        litellm_params = litellm_params or {}
+        try:
+            response_json = raw_response.json()
+        except Exception:
+            raise DashScopeError(
+                status_code=raw_response.status_code,
+                message=raw_response.text,
+            )
+
+        logging_obj.post_call(
+            input=request_data.get("query"),
+            api_key=api_key,
+            additional_args={"complete_input_dict": request_data},
+            original_response=response_json,
+        )
+
+        # DashScope error envelope: {"code": "...", "message": "...", "request_id": "..."}
+        if "code" in response_json and "results" not in response_json:
+            raise DashScopeError(
+                status_code=raw_response.status_code,
+                message=response_json.get("message", str(response_json)),
+            )
+
+        results = response_json.get("results")
+        if results is None:
+            raise DashScopeError(
+                status_code=raw_response.status_code,
+                message=f"No results in DashScope rerank response: {response_json}",
+            )
+
+        # qwen3-rerank returns:
+        #   {"index": int, "relevance_score": float}
+        # plus, when return_documents=true was sent:
+        #   "document": {"text": "..."}
+        # which already matches LiteLLM's RerankResponseDocument shape.
+        transformed_results: List[dict] = []
+        for r in results:
+            item: Dict[str, Any] = {
+                "index": r["index"],
+                "relevance_score": r["relevance_score"],
+            }
+            doc = r.get("document")
+            if isinstance(doc, dict):
+                item["document"] = doc
+            elif isinstance(doc, str):
+                # Defensive: spec says dict, but normalize string-shaped echoes.
+                item["document"] = {"text": doc}
+            transformed_results.append(item)
+
+        usage = response_json.get("usage") or {}
+        total_tokens = usage.get("total_tokens")
+        billed_units = RerankBilledUnits(total_tokens=total_tokens)
+        tokens = RerankTokens(input_tokens=total_tokens)
+        meta = RerankResponseMeta(billed_units=billed_units, tokens=tokens)
+
+        return RerankResponse(
+            id=response_json.get("id") or str(uuid.uuid4()),
+            results=transformed_results,  # type: ignore
+            meta=meta,
+        )
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: Union[dict, httpx.Headers],
+    ) -> BaseLLMException:
+        if isinstance(headers, dict):
+            headers = httpx.Headers(headers)
+        return DashScopeError(
+            status_code=status_code,
+            message=error_message,
+            headers=headers,
+        )

litellm/llms/deepseek/chat/transformation.py

@@ -2,13 +2,15 @@
 Translates from OpenAI's `/v1/chat/completions` to DeepSeek's `/v1/chat/completions`
 """
 
-from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload
+from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
 
+import litellm
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     handle_messages_with_content_list_to_str_conversion,
 )
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
+from litellm.utils import supports_reasoning
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
@@ -62,6 +64,48 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
 
         return optional_params
 
+    def _fill_reasoning_content(
+        self, messages: List[AllMessageValues]
+    ) -> List[AllMessageValues]:
+        """
+        DeepSeek thinking mode requires `reasoning_content` to be passed back on
+        every assistant message in multi-turn conversations. If it is missing,
+        the API returns:
+          "The reasoning_content in the thinking mode must be passed back to the API."
+
+        For each assistant message that is missing `reasoning_content`:
+          1. Promote it from `provider_specific_fields["reasoning_content"]` if present
+             (LiteLLM stores provider-specific response fields there).
+          2. Otherwise inject a single space — the minimum value the API accepts.
+        """
+        result: List[AllMessageValues] = []
+        for msg in messages:
+            if msg.get("role") == "assistant" and not msg.get("reasoning_content"):
+                patched = dict(cast(dict, msg))
+                provider_fields = patched.get("provider_specific_fields") or {}
+                stored = provider_fields.get("reasoning_content")
+                if stored:
+                    patched["reasoning_content"] = stored
+                    cleaned = dict(provider_fields)
+                    cleaned.pop("reasoning_content", None)
+                    patched["provider_specific_fields"] = cleaned
+                else:
+                    litellm.verbose_logger.warning(
+                        "DeepSeek thinking mode: assistant message is missing "
+                        "`reasoning_content` and none was saved in "
+                        "`provider_specific_fields`. A single-space placeholder "
+                        "is being injected to satisfy API validation, but the "
+                        "model will receive a blank reasoning chain for this turn, "
+                        "which may silently degrade multi-turn response quality. "
+                        "Preserve `reasoning_content` from the original assistant "
+                        "response when building multi-turn conversation history."
+                    )
+                    patched["reasoning_content"] = " "
+                result.append(cast(AllMessageValues, patched))
+            else:
+                result.append(msg)
+        return result
+
     @overload
     def _transform_messages(
         self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
@@ -91,6 +135,66 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
                 messages=messages, model=model, is_async=False
             )
 
+    def _thinking_mode_active(self, model: str, optional_params: dict) -> bool:
+        """
+        Returns True only when thinking mode is actually active for this request:
+          - model supports reasoning (capability check)
+          - user explicitly passed thinking={"type": "enabled"} (opt-in check)
+        """
+        return (
+            supports_reasoning(model=model, custom_llm_provider="deepseek")
+            and (optional_params.get("thinking") or {}).get("type") == "enabled"
+        )
+
+    def transform_request(
+        self,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        """
+        Ensures `reasoning_content` is forwarded on assistant messages for
+        multi-turn thinking-mode conversations (issue #28045).
+
+        Only runs when thinking mode is actually active - guarded by both
+        supports_reasoning() (model capability) and optional_params["thinking"]
+        (user explicitly enabled it), preventing spurious injection on models
+        like deepseek-v3.2 that support thinking as opt-in but not always-on.
+        """
+        if self._thinking_mode_active(model=model, optional_params=optional_params):
+            messages = self._fill_reasoning_content(messages)
+        return super().transform_request(
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
+    async def async_transform_request(
+        self,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        """
+        Async equivalent of transform_request — applies the same reasoning_content
+        fix for multi-turn thinking-mode conversations.
+        """
+        if self._thinking_mode_active(model=model, optional_params=optional_params):
+            messages = self._fill_reasoning_content(messages)
+        return await super().async_transform_request(
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
     def _get_openai_compatible_provider_info(
         self, api_base: Optional[str], api_key: Optional[str]
     ) -> Tuple[Optional[str], Optional[str]]:

litellm/llms/deepseek/messages/transformation.py

@@ -0,0 +1,133 @@
+"""
+DeepSeek Anthropic-compatible messages transformation config.
+"""
+
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+    AnthropicMessagesConfig,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.router import GenericLiteLLMParams
+
+
+class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
+    """
+    DeepSeek exposes an Anthropic-compatible Messages API at
+    https://api.deepseek.com/anthropic.
+
+    It accepts the native Anthropic Messages conversation shape, including
+    thinking blocks in assistant history, but rejects Anthropic's explicit
+    custom-tool discriminator (`{"type": "custom"}`).
+    """
+
+    @property
+    def custom_llm_provider(self) -> Optional[str]:
+        return "deepseek"
+
+    @staticmethod
+    def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
+        return api_key or get_secret_str("DEEPSEEK_API_KEY") or litellm.api_key
+
+    @staticmethod
+    def get_api_base(api_base: Optional[str] = None) -> str:
+        return (
+            api_base
+            or get_secret_str("DEEPSEEK_ANTHROPIC_API_BASE")
+            or get_secret_str("DEEPSEEK_API_BASE")
+            or "https://api.deepseek.com/anthropic"
+        )
+
+    def validate_anthropic_messages_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[Any],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> Tuple[dict, Optional[str]]:
+        dynamic_api_key = self.get_api_key(api_key=api_key)
+
+        if (
+            "x-api-key" not in headers
+            and "authorization" not in headers
+            and dynamic_api_key is not None
+        ):
+            headers["x-api-key"] = dynamic_api_key
+
+        if "anthropic-version" not in headers:
+            headers["anthropic-version"] = "2023-06-01"
+        if "content-type" not in headers:
+            headers["content-type"] = "application/json"
+
+        headers = self._update_headers_with_anthropic_beta(
+            headers=headers,
+            optional_params=optional_params,
+            custom_llm_provider=self.custom_llm_provider or "deepseek",
+        )
+
+        return headers, api_base
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        base_url = self.get_api_base(api_base=api_base).rstrip("/")
+
+        if base_url.endswith("/v1/messages") and "/anthropic/" in base_url:
+            return base_url
+        if base_url.endswith("/v1/messages"):
+            base_url = base_url[: -len("/v1/messages")]
+        if base_url.endswith("/v1"):
+            base_url = base_url[: -len("/v1")]
+        if base_url.endswith("/beta"):
+            base_url = base_url[: -len("/beta")]
+
+        if not base_url.endswith("/anthropic") and "/anthropic/" not in base_url:
+            base_url = f"{base_url}/anthropic"
+
+        return f"{base_url}/v1/messages"
+
+    @staticmethod
+    def _sanitize_tools_for_deepseek(tools: Any) -> Any:
+        if not isinstance(tools, list):
+            return tools
+
+        sanitized_tools = []
+        for tool in tools:
+            if isinstance(tool, dict) and tool.get("type") == "custom":
+                sanitized_tool = dict(tool)
+                sanitized_tool.pop("type", None)
+                sanitized_tools.append(sanitized_tool)
+            else:
+                sanitized_tools.append(tool)
+        return sanitized_tools
+
+    def transform_anthropic_messages_request(
+        self,
+        model: str,
+        messages: List[Dict],
+        anthropic_messages_optional_request_params: Dict,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Dict:
+        anthropic_messages_request = super().transform_anthropic_messages_request(
+            model=model,
+            messages=messages,
+            anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+        if "tools" in anthropic_messages_request:
+            anthropic_messages_request["tools"] = self._sanitize_tools_for_deepseek(
+                anthropic_messages_request["tools"]
+            )
+        return anthropic_messages_request

litellm/llms/fireworks_ai/chat/transformation.py

@@ -241,10 +241,13 @@ class FireworksAIConfig(OpenAIGPTConfig):
                                 disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
                             )
             filter_value_from_dict(cast(dict, message), "cache_control")
-            # Remove fields not permitted by FireworksAI that may cause:
-            # "Not permitted, field: 'messages[n].provider_specific_fields'"
-            if isinstance(message, dict) and "provider_specific_fields" in message:
-                cast(dict, message).pop("provider_specific_fields", None)
+            # Remove fields not permitted by FireworksAI (additionalProperties: false
+            # on their ChatMessage schema) that may cause:
+            # "Extra inputs are not permitted, field: 'messages[n].<field>'"
+            if isinstance(message, dict):
+                m = cast(dict, message)
+                m.pop("provider_specific_fields", None)
+                m.pop("thinking_blocks", None)
 
         return messages
 

litellm/llms/gemini/chat/transformation.py

@@ -1,5 +1,7 @@
 from typing import List, Optional, cast
 
+import litellm
+
 from litellm.litellm_core_utils.prompt_templates.factory import (
     convert_generic_image_chunk_to_openai_image_obj,
     convert_to_anthropic_image_obj,
@@ -101,7 +103,10 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
         return supported_params
 
     def _transform_messages(
-        self, messages: List[AllMessageValues], model: Optional[str] = None
+        self,
+        messages: List[AllMessageValues],
+        model: Optional[str] = None,
+        litellm_params: Optional[dict] = None,
     ) -> List[ContentType]:
         """
         Google AI Studio Gemini does not support HTTP/HTTPS URLs for files.
@@ -141,14 +146,23 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
                                 img_element["image_url"] = converted_image_url  # type: ignore
                     elif element.get("type") == "file":
                         file_element = cast(ChatCompletionFileObject, element)
-                        file_id = file_element["file"].get("file_id")
+                        _file_field = file_element.get("file")
+                        if _file_field is None:
+                            raise litellm.BadRequestError(
+                                message="Content block has type='file' but is missing the required 'file' field",
+                                model=model,
+                                llm_provider="gemini",
+                            )
+                        file_id = _file_field.get("file_id")
                         if file_id and ("http://" in file_id or "https://" in file_id):
                             # Convert HTTP/HTTPS file URL to base64 data
                             try:
                                 base64_data = convert_url_to_base64(file_id)
-                                file_element["file"]["file_data"] = base64_data  # type: ignore
-                                file_element["file"].pop("file_id", None)  # type: ignore
+                                _file_field["file_data"] = base64_data  # type: ignore
+                                _file_field.pop("file_id", None)  # type: ignore
                             except Exception:
                                 # If conversion fails, leave as is and let the API handle it
                                 pass
-        return _gemini_convert_messages_with_history(messages=messages, model=model)
+        return _gemini_convert_messages_with_history(
+            messages=messages, model=model, litellm_params=litellm_params
+        )

litellm/llms/ollama/chat/transformation.py

@@ -507,10 +507,10 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
             # PROCESS REASONING CONTENT
             reasoning_content: Optional[str] = None
             content: Optional[str] = None
-            if chunk["message"].get("thinking") is not None:
+            if chunk["message"].get("thinking"):
                 reasoning_content = chunk["message"].get("thinking")
                 self.started_reasoning_content = True
-            elif chunk["message"].get("content") is not None:
+            if chunk["message"].get("content"):
                 if (
                     self.started_reasoning_content
                     and not self.finished_reasoning_content