Merge branch 'litellm_internal_staging' into litellm_metrics_auth

.github/pull_request_template.md

@@ -2,6 +2,10 @@
 
 <!-- e.g. "Fixes #000" -->
 
+## Linear ticket
+
+<!-- if you are an internal contributor, add the Linear ticket e.g. "Resolves LIT-1234" to magically link the Linear ticket to the GitHub PR -->
+
 ## Pre-Submission checklist
 
 **Please complete all items before asking a LiteLLM maintainer to review your PR**

.github/workflows/_test-unit-services-base.yml

@@ -32,41 +32,39 @@ on:
         required: false
         type: boolean
         default: false
+      dist:
+        description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)"
+        required: false
+        type: string
+        default: "loadscope"
       artifact-name:
         description: "Unique name for the coverage artifact (must be unique per run)"
         required: false
         type: string
         default: "run"
-    secrets:
-      DATABASE_URL:
-        required: false
-      POSTGRES_USER:
-        required: false
-      POSTGRES_PASSWORD:
-        required: false
 
 permissions:
   contents: read
 
+# The postgres service container below is spawned per-job on localhost and
+# destroyed with the job. Nothing outside the runner can reach it. The
+# user/password/database here are not secrets — they're bootstrap values
+# for a throwaway container — so we hardcode them instead of attaching
+# every matrix shard to a GHA environment just to read three "secrets"
+# (which also produces a "temporarily deployed to …" notification on the
+# PR timeline per shard per push).
 jobs:
   run:
     name: Run tests
     runs-on: ubuntu-latest
     timeout-minutes: ${{ inputs.timeout-minutes }}
-    # Environment is derived from the enable-* flags, not caller-controllable.
-    # This prevents callers from passing arbitrary environment names to bypass secret scoping.
-    environment: >-
-      ${{
-        inputs.enable-postgres && 'integration-postgres' ||
-        ''
-      }}
 
     services:
       postgres:
         image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14
         env:
-          POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+          POSTGRES_USER: litellm
+          POSTGRES_PASSWORD: litellm
           POSTGRES_DB: litellm_test
         ports:
           - 5432:5432
@@ -114,7 +112,7 @@ jobs:
       - name: Run Prisma migrations
         if: ${{ inputs.enable-postgres }}
         env:
-          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          DATABASE_URL: "postgresql://litellm:litellm@localhost:5432/litellm_test"
         run: |
           uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
 
@@ -124,7 +122,8 @@ jobs:
           MAX_FAILURES: ${{ inputs.max-failures }}
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
-          DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }}
+          DIST: ${{ inputs.dist }}
+          DATABASE_URL: ${{ inputs.enable-postgres && 'postgresql://litellm:litellm@localhost:5432/litellm_test' || '' }}
         run: |
           if [ "${WORKERS}" = "0" ]; then
             uv run --no-sync pytest ${TEST_PATH:?} \
@@ -143,7 +142,7 @@ jobs:
               -n "${WORKERS}" \
               --reruns "${RERUNS}" \
               --reruns-delay 1 \
-              --dist=loadscope \
+              --dist="${DIST}" \
               --durations=20 \
               --cov=litellm \
               --cov-report=xml:coverage.xml \

.github/workflows/check-lazy-openapi-snapshot.yml

@@ -0,0 +1,75 @@
+name: Check Lazy OpenAPI Snapshot
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - "litellm_**"
+
+permissions:
+  contents: read
+  checks: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
+      - name: Install dependencies
+        run: uv sync --frozen --all-groups --all-extras
+
+      - name: Regenerate snapshot to /tmp
+        id: regen
+        run: |
+          cp litellm/proxy/_lazy_openapi_snapshot.json /tmp/snapshot.committed.json
+          uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
+          mv litellm/proxy/_lazy_openapi_snapshot.json /tmp/snapshot.fresh.json
+          mv /tmp/snapshot.committed.json litellm/proxy/_lazy_openapi_snapshot.json
+
+      - name: Compare
+        id: diff
+        continue-on-error: true
+        run: |
+          diff -q /tmp/snapshot.fresh.json litellm/proxy/_lazy_openapi_snapshot.json
+
+      - name: Mark neutral if drift
+        if: steps.diff.outcome == 'failure'
+        uses: LouisBrunner/checks-action@6b626ffbad7cc56fd58627f774b9067e6118af23 # v2.0.0
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: lazy-openapi-snapshot
+          conclusion: neutral
+          output: |
+            {
+              "title": "Lazy openapi snapshot is stale",
+              "summary": "Run `python -m litellm.proxy._lazy_openapi_snapshot` and commit the regenerated `litellm/proxy/_lazy_openapi_snapshot.json`. Not blocking — the snapshot will regenerate at release if not committed."
+            }

.github/workflows/check_duplicate_issues.yml

@@ -39,7 +39,7 @@ jobs:
         if: github.event.action == 'opened'
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Auto-close if high-confidence duplicate
         if: github.event.action == 'opened'

.github/workflows/create-release-branch.yml

@@ -0,0 +1,65 @@
+name: Create Release Branch
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted) — branch will be named release/<tag>"
+        required: true
+        type: string
+      commit_hash:
+        description: "Full 40-char commit SHA the branch should point to"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      tag:
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
+        required: true
+        type: string
+      commit_hash:
+        description: "Full 40-char commit SHA the branch should point to"
+        required: true
+        type: string
+
+permissions: {}
+
+jobs:
+  create-branch:
+    name: Create Release Branch
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Validate inputs
+        env:
+          TAG: ${{ inputs.tag }}
+          COMMIT_HASH: ${{ inputs.commit_hash }}
+        run: |
+          if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then
+            echo "::error::commit_hash must be a full 40-character commit SHA"
+            exit 1
+          fi
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
+            exit 1
+          fi
+
+      - name: Create release branch
+        env:
+          TAG: ${{ inputs.tag }}
+          COMMIT_HASH: ${{ inputs.commit_hash }}
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const tag = process.env.TAG;
+            const commitHash = process.env.COMMIT_HASH;
+            const branchName = `release/${tag}`;
+
+            await github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/heads/${branchName}`,
+              sha: commitHash,
+            });
+            core.info(`Created branch ${branchName} at ${commitHash}`);

.github/workflows/create-release.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag (e.g. v1.83.0-stable)"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
         required: true
         type: string
       commit_hash:
@@ -30,8 +30,8 @@ jobs:
             echo "::error::commit_hash must be a full 40-character commit SHA"
             exit 1
           fi
-          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
-            echo "::error::tag must start with vX.Y.Z"
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
             exit 1
           fi
 
@@ -45,6 +45,11 @@ jobs:
             const tag = process.env.TAG;
             const commitHash = process.env.COMMIT_HASH;
 
+            // Mark RC / dev / nightly / alpha / beta tags as GitHub pre-releases.
+            // PEP 440 post-releases (e.g. `1.84.0.post1`) and legacy `-stable[.patch.N]`
+            // are stable maintenance releases, not pre-releases.
+            const isPrerelease = /(?:rc|nightly|alpha|beta|\.dev)/i.test(tag);
+
             const cosignSection = [
               `## Verify Docker Image Signature`,
               ``,
@@ -89,7 +94,7 @@ jobs:
                 target_commitish: commitHash,
                 name: tag,
                 owner: context.repo.owner,
-                prerelease: false,
+                prerelease: isPrerelease,
                 repo: context.repo.repo,
                 tag_name: tag,
               });
@@ -102,6 +107,17 @@ jobs:
                 body: updatedBody,
                 draft: false,
               });
+
             } catch (error) {
               core.setFailed(error.message);
             }
+
+  create-branch:
+    name: Create Release Branch
+    needs: release
+    permissions:
+      contents: write
+    uses: ./.github/workflows/create-release-branch.yml
+    with:
+      tag: ${{ inputs.tag }}
+      commit_hash: ${{ inputs.commit_hash }}

.github/workflows/guard-fork-dependencies.yml

@@ -0,0 +1,140 @@
+name: Guard fork dependency changes
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
+    paths:
+      - "uv.lock"
+      - "pyproject.toml"
+      - "litellm-proxy-extras/pyproject.toml"
+      - "enterprise/pyproject.toml"
+
+permissions: {}
+
+jobs:
+  guard:
+    name: Block fork dependency changes
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    if: github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - name: Checkout base branch
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          ref: ${{ github.base_ref }}
+          persist-credentials: false
+          path: base
+
+      - name: Checkout PR head (read-only)
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+          path: pr
+
+      - name: Reject uv.lock changes
+        run: |
+          if ! diff -q base/uv.lock pr/uv.lock >/dev/null 2>&1; then
+            echo "::error::Fork PRs must not modify uv.lock. Dependency lockfile changes must come from a branch in the canonical repository."
+            exit 1
+          fi
+          echo "uv.lock is unchanged."
+
+      - name: Reject new dependencies in pyproject.toml files
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          # Write the checker script to a temp file to avoid shell quoting issues.
+          cat > /tmp/extract_deps.py << 'SCRIPT'
+          import re
+          import sys
+          import tomllib
+
+
+          def normalize(name: str) -> str:
+              return re.sub(r"[-_.]+", "-", name).lower()
+
+
+          def extract_dep_names(path: str) -> set[str]:
+              with open(path, "rb") as f:
+                  data = tomllib.load(f)
+
+              deps: set[str] = set()
+              pep508 = re.compile(r"^([A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?)")
+
+              # [project].dependencies
+              for spec in data.get("project", {}).get("dependencies", []):
+                  m = pep508.match(spec)
+                  if m:
+                      deps.add(normalize(m.group(1)))
+
+              # [project.optional-dependencies]
+              for group in data.get("project", {}).get("optional-dependencies", {}).values():
+                  for spec in group:
+                      m = pep508.match(spec)
+                      if m:
+                          deps.add(normalize(m.group(1)))
+
+              # [dependency-groups]
+              for group in data.get("dependency-groups", {}).values():
+                  for item in group:
+                      if isinstance(item, str):
+                          m = pep508.match(item)
+                          if m:
+                              deps.add(normalize(m.group(1)))
+
+              return deps
+
+
+          if __name__ == "__main__":
+              for name in sorted(extract_dep_names(sys.argv[1])):
+                  print(name)
+          SCRIPT
+
+          had_error=0
+
+          check_deps() {
+            local base_file="$1"
+            local pr_file="$2"
+            local label="$3"
+
+            if [ ! -f "$base_file" ] && [ ! -f "$pr_file" ]; then
+              return 0
+            fi
+
+            if [ ! -f "$base_file" ] && [ -f "$pr_file" ]; then
+              echo "::error::Fork PR introduces a new $label that does not exist on the base branch."
+              return 1
+            fi
+
+            base_deps=$(python3 /tmp/extract_deps.py "$base_file")
+            pr_deps=$(python3 /tmp/extract_deps.py "$pr_file")
+
+            new_deps=$(comm -13 <(echo "$base_deps") <(echo "$pr_deps"))
+
+            if [ -n "$new_deps" ]; then
+              echo "::error::Fork PR adds new dependencies in $label: $(echo $new_deps | tr '\n' ', ')"
+              echo "New packages detected:"
+              echo "$new_deps"
+              return 1
+            fi
+
+            echo "$label: no new dependencies."
+            return 0
+          }
+
+          check_deps base/pyproject.toml pr/pyproject.toml "pyproject.toml" || had_error=1
+          check_deps base/litellm-proxy-extras/pyproject.toml pr/litellm-proxy-extras/pyproject.toml "litellm-proxy-extras/pyproject.toml" || had_error=1
+          check_deps base/enterprise/pyproject.toml pr/enterprise/pyproject.toml "enterprise/pyproject.toml" || had_error=1
+
+          if [ "$had_error" -ne 0 ]; then
+            echo ""
+            echo "::error::Fork PRs must not add new dependencies. Please open an issue or coordinate with a maintainer to update dependencies from within the canonical repository."
+            exit 1
+          fi
+
+          echo "All pyproject.toml files passed dependency check."

.github/workflows/llm-translation-testing.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Set up uv
         uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7

.github/workflows/scan_duplicate_issues.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: "3.13"
+          python-version: "3.12"
 
       - name: Scan for duplicate issues
         env:

.github/workflows/test-code-quality.yml

@@ -0,0 +1,128 @@
+name: Code Quality Checks
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  code-quality:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Checkout litellm-docs into docs/my-website (for documentation_tests)
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          repository: BerriAI/litellm-docs
+          path: docs/my-website
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
+      - name: Install dependencies
+        run: uv sync --frozen --all-groups --all-extras
+
+      - name: check_licenses
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py
+
+      - name: check_provider_folders_documented
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
+
+      - name: router_code_coverage
+        run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
+
+      - name: test_chat_completion_imports
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
+
+      - name: info_log_check
+        run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py
+
+      - name: check_guardrail_apply_decorator
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
+
+      - name: test_ban_set_verbose
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py
+
+      - name: code_qa_check_tests
+        run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py
+
+      - name: check_get_model_cost_key_performance
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
+
+      - name: test_proxy_types_import
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py
+
+      - name: callback_manager_test
+        run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py
+
+      - name: recursive_detector
+        run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py
+
+      - name: test_router_strategy_async
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py
+
+      - name: litellm_logging_code_coverage
+        run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
+
+      - name: ensure_async_clients_test
+        run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py
+
+      - name: enforce_llms_folder_style
+        run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py
+
+      - name: prevent_key_leaks_in_exceptions
+        run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
+
+      - name: check_unsafe_enterprise_import
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
+
+      - name: ban_copy_deepcopy_kwargs
+        run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
+
+      - name: check_fastuuid_usage
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
+
+      - name: memory_test
+        run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
+
+      - name: documentation_test_env_keys
+        run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
+
+      - name: documentation_test_router_settings
+        run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
+
+      - name: documentation_test_api_docs
+        run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py

.github/workflows/test-semgrep.yml

@@ -0,0 +1,39 @@
+name: Semgrep
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  semgrep:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Run Semgrep (custom rules)
+        run: uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error

.github/workflows/test-unit-caching-redis.yml

@@ -0,0 +1,38 @@
+name: "Unit Tests: Caching (Redis)"
+
+# Uses cloud Redis credentials — only runs on trusted branches, not PRs.
+# This prevents external PRs from accessing Redis credentials.
+on:
+  push:
+    branches: [main, "litellm_*"]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  caching-redis:
+    uses: ./.github/workflows/_test-unit-services-base.yml
+    with:
+      # Redis-only tests that do NOT require provider API keys.
+      # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py,
+      # test_router_caching.py) are in Phase 3 integration workflows.
+      test-path: >-
+        tests/local_testing/test_dual_cache.py
+        tests/local_testing/test_redis_batch_optimizations.py
+        tests/local_testing/test_router_utils.py
+      workers: 2
+      reruns: 2
+      timeout-minutes: 20
+      enable-redis: true
+      enable-postgres: false
+    secrets:
+      REDIS_HOST: ${{ secrets.REDIS_HOST }}
+      REDIS_PORT: ${{ secrets.REDIS_PORT }}
+      REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
+      DATABASE_URL: ${{ secrets.DATABASE_URL }}
+      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test-unit-documentation.yml

@@ -25,6 +25,13 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Checkout litellm-docs into docs/my-website (for documentation_tests)
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          repository: BerriAI/litellm-docs
+          path: docs/my-website
+          persist-credentials: false
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:

.github/workflows/test-unit-proxy-db.yml

@@ -12,8 +12,74 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+# Semantic matrix: each shard groups tests by concern (auth, server, logging, …)
+# rather than alphabetical letter ranges. Adding a new test file means adding it
+# to whichever group it belongs to, not reshuffling slices.
+#
+# Design targets:
+#   * Every shard runs in <= 7 minutes of wall-clock on the default runner.
+#     Most of a shard's time is pytest plugin load + xdist worker imports +
+#     pytest-cov instrumentation, not the tests themselves. Keeping per-shard
+#     work low and matching worker count to runner cores is what controls it.
+#   * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores
+#     oversubscribes 2x and workers fight for CPU during their cold-start
+#     imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective).
+#   * test_key_generate_prisma.py stays serial (workers=0) — it has event-loop
+#     conflicts with the logging worker when run in parallel.
+#   * test_proxy_utils.py runs as a single shard with --dist=worksteal so
+#     xdist balances its 188 parametrized cases across workers instead of
+#     pinning the whole file to one worker (the default --dist=loadscope
+#     behavior for single-file targets).
+#   * test_db_schema_migration.py is isolated because one test in it
+#     (test_aaaasschema_migration_check) takes ~170s — by itself it
+#     determines the shard's wall-clock floor.
 jobs:
+  # Fast guard — fails the workflow if a test_*.py file under
+  # tests/proxy_unit_tests/ is not referenced by any matrix entry below.
+  # The semantic-shard design (no catch-all "remaining" bucket) relies on
+  # every test file being explicitly assigned; this guard prevents a new
+  # file from silently dropping out of CI.
+  assert-shard-coverage:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+      - name: Assert every test_*.py is in a matrix shard
+        run: |
+          python3 - <<'PY'
+          import pathlib, sys, yaml
+          wf = yaml.safe_load(open(".github/workflows/test-unit-proxy-db.yml"))
+          matrix = wf["jobs"]["proxy-db"]["strategy"]["matrix"]["include"]
+          referenced = set()
+          for entry in matrix:
+              for token in entry["test-path"].split():
+                  if token.startswith("tests/proxy_unit_tests/"):
+                      referenced.add(pathlib.PurePosixPath(token).name)
+          actual = {p.name for p in pathlib.Path("tests/proxy_unit_tests").iterdir()
+                    if p.name.startswith("test_") and (p.suffix == ".py" or p.is_dir())
+                    and p.name != "test_configs"}
+          orphans = sorted(actual - referenced)
+          if orphans:
+              print("ERROR: the following files/dirs under tests/proxy_unit_tests/")
+              print("       are not assigned to any shard in test-unit-proxy-db.yml:")
+              for o in orphans:
+                  print(f"  - {o}")
+              print()
+              print("Add each to whichever semantic shard it belongs to.")
+              sys.exit(1)
+          print(f"OK: all {len(actual)} files assigned to a shard.")
+          PY
+
   proxy-db:
+    needs: assert-shard-coverage
+    # Display only the semantic shard name in the checks UI instead of GHA's
+    # default "proxy-db (key-generation, tests/proxy_unit_tests/…, 0, loadscope, 20)"
+    # which includes every matrix field and gets truncated past the test-path.
+    name: ${{ matrix.test-group }}
     permissions:
       contents: read
       id-token: write
@@ -22,19 +88,145 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # Key generation tests must NOT run in parallel (event loop conflicts with logging worker)
+          # Must run serially — event-loop conflict with the logging worker.
           - test-group: key-generation
             test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py"
             workers: 0
-            timeout: 30
-          - test-group: auth-checks
-            test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py"
-            workers: 8
+            dist: loadscope
             timeout: 20
-          - test-group: remaining
-            test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py"
-            workers: 8
-            timeout: 30
+
+          # ---- auth: split into 2 shards ----
+          - test-group: auth-checks
+            test-path: >-
+              tests/proxy_unit_tests/test_auth_checks.py
+              tests/proxy_unit_tests/test_user_api_key_auth.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: jwt-and-keys
+            test-path: >-
+              tests/proxy_unit_tests/test_jwt.py
+              tests/proxy_unit_tests/test_jwt_key_mapping.py
+              tests/proxy_unit_tests/test_proxy_custom_auth.py
+              tests/proxy_unit_tests/test_key_generate_dynamodb.py
+              tests/proxy_unit_tests/test_deployed_proxy_keygen.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- test_proxy_utils.py, single shard, worksteal distribution ----
+          - test-group: proxy-utils
+            test-path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            workers: 4
+            dist: worksteal
+            timeout: 15
+
+          # ---- proxy server: split into 2 shards ----
+          - test-group: proxy-server-core
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_server.py
+              tests/proxy_unit_tests/test_proxy_server_keys.py
+              tests/proxy_unit_tests/test_proxy_server_caching.py
+              tests/proxy_unit_tests/test_proxy_server_langfuse.py
+              tests/proxy_unit_tests/test_proxy_server_spend.py
+              tests/proxy_unit_tests/test_aproxy_startup.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: proxy-runtime
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_config_unit_test.py
+              tests/proxy_unit_tests/test_proxy_routes.py
+              tests/proxy_unit_tests/test_proxy_gunicorn.py
+              tests/proxy_unit_tests/test_server_root_path.py
+              tests/proxy_unit_tests/test_proxy_pass_user_config.py
+              tests/proxy_unit_tests/test_proxy_token_counter.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- logging: split into 2 shards ----
+          - test-group: custom-logging
+            test-path: >-
+              tests/proxy_unit_tests/test_custom_callback_input.py
+              tests/proxy_unit_tests/test_custom_logger_s3_gcs.py
+              tests/proxy_unit_tests/test_proxy_custom_logger.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: logging-misc
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_reject_logging.py
+              tests/proxy_unit_tests/test_audit_logs_proxy.py
+              tests/proxy_unit_tests/test_search_api_logging.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- db-and-spend: isolate the 170s schema-migration test ----
+          # test_db_schema_migration.py has exactly one test, and that test
+          # is mostly waiting on `prisma migrate deploy` / `prisma migrate
+          # diff` subprocesses (~170s). It does no CPU-bound Python work
+          # inside the test. Running with workers=0 (serial, no xdist)
+          # skips the 4-worker cold-start cost we'd otherwise pay for a
+          # single test, saving ~4 minutes of wall-clock.
+          - test-group: schema-migration
+            test-path: "tests/proxy_unit_tests/test_db_schema_migration.py"
+            workers: 0
+            dist: loadscope
+            timeout: 15
+          - test-group: db-and-spend
+            test-path: >-
+              tests/proxy_unit_tests/test_prisma_client_backoff_retry.py
+              tests/proxy_unit_tests/test_db_schema_changes.py
+              tests/proxy_unit_tests/test_e2e_pod_lock_manager.py
+              tests/proxy_unit_tests/test_skills_db.py
+              tests/proxy_unit_tests/test_update_daily_tag_spend.py
+              tests/proxy_unit_tests/test_update_spend.py
+              tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- guardrails + budget + hooks: split into 2 ----
+          - test-group: guardrails-hooks
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_setting_guardrails.py
+              tests/proxy_unit_tests/test_banned_keyword_list.py
+              tests/proxy_unit_tests/test_unit_test_proxy_hooks.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: budgets
+            test-path: >-
+              tests/proxy_unit_tests/test_default_end_user_budget_simple.py
+              tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py
+              tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          - test-group: endpoints-and-responses
+            test-path: >-
+              tests/proxy_unit_tests/test_blog_posts_endpoint.py
+              tests/proxy_unit_tests/test_models_fallback_endpoint.py
+              tests/proxy_unit_tests/test_google_endpoint_routing.py
+              tests/proxy_unit_tests/test_google_gemini_proxy_request.py
+              tests/proxy_unit_tests/test_get_favicon.py
+              tests/proxy_unit_tests/test_get_image.py
+              tests/proxy_unit_tests/test_ui_path_detection.py
+              tests/proxy_unit_tests/test_prompt_test_endpoint.py
+              tests/proxy_unit_tests/test_check_batch_cost.py
+              tests/proxy_unit_tests/test_check_responses_cost.py
+              tests/proxy_unit_tests/test_response_polling_handler.py
+              tests/proxy_unit_tests/test_response_polling_pre_call_checks.py
+              tests/proxy_unit_tests/test_realtime_cache.py
+              tests/proxy_unit_tests/test_proxy_exception_mapping.py
+              tests/proxy_unit_tests/test_custom_tokenizer_bug.py
+              tests/proxy_unit_tests/test_model_response_typing
+            workers: 4
+            dist: loadscope
+            timeout: 15
     uses: ./.github/workflows/_test-unit-services-base.yml
     with:
       test-path: ${{ matrix.test-path }}
@@ -42,8 +234,5 @@ jobs:
       reruns: 2
       timeout-minutes: ${{ matrix.timeout }}
       enable-postgres: true
+      dist: ${{ matrix.dist }}
       artifact-name: proxy-db-${{ matrix.test-group }}
-    secrets:
-      DATABASE_URL: ${{ secrets.DATABASE_URL }}
-      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test-unit-proxy-endpoints.yml

@@ -36,6 +36,8 @@ jobs:
         tests/test_litellm/proxy/health_endpoints
         tests/test_litellm/proxy/public_endpoints
         tests/test_litellm/proxy/prompts
+        tests/test_litellm/proxy/rag_endpoints
+        tests/test_litellm/proxy/realtime_endpoints
         tests/test_litellm/proxy/ui_crud_endpoints
       workers: 2
       reruns: 2

.github/workflows/test-unit-proxy-legacy.yml

@@ -28,7 +28,7 @@ jobs:
           - name: "key-generation"
             path: "tests/proxy_unit_tests/test_[k-o]*.py"
           - name: "proxy-config"
-            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
           - name: "proxy-server"
             path: "tests/proxy_unit_tests/test_proxy_server.py"
           - name: "proxy-server-extras"

.github/workflows/test-unit-security.yml

@@ -1,6 +1,8 @@
 name: "Unit Tests: Security"
 
-# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
+# Kept push-only (was previously required by DATABASE_URL secret scoping;
+# now the postgres credentials are ephemeral localhost values but the
+# push-trigger stays to match the proxy-db workflow cadence).
 on:
   push:
     branches: [main, "litellm_**"]
@@ -24,7 +26,3 @@ jobs:
       timeout-minutes: 20
       enable-postgres: true
       artifact-name: security
-    secrets:
-      DATABASE_URL: ${{ secrets.DATABASE_URL }}
-      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.npmrc

@@ -2,4 +2,4 @@
 # Packages needing lifecycle scripts: npm rebuild <pkg>
 ignore-scripts=true
 # Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d
+min-release-age=3

AGENTS.md

@@ -23,10 +23,11 @@ LiteLLM is a unified interface for 100+ LLMs that:
 
 ### Key Directories
 - `tests/` - Comprehensive test suites
-- `docs/my-website/` - Documentation website
 - `ui/litellm-dashboard/` - Admin dashboard UI
 - `enterprise/` - Enterprise-specific features
 
+Documentation lives in the separate [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs) repository and is served at [docs.litellm.ai](https://docs.litellm.ai).
+
 ## DEVELOPMENT GUIDELINES
 
 ### MAKING CODE CHANGES
@@ -218,8 +219,8 @@ When opening issues or pull requests, follow these templates:
 
 ## HELPFUL RESOURCES
 
-- Main documentation: https://docs.litellm.ai/
-- Provider-specific docs in `docs/my-website/docs/providers/`
+- Main documentation: https://docs.litellm.ai/ (source: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs))
+- Provider-specific docs: https://docs.litellm.ai/docs/providers/
 - Admin UI for testing proxy features
 
 ## WHEN IN DOUBT

CLAUDE.md

@@ -2,6 +2,10 @@
 
 This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
 
+## Documentation
+
+Documentation lives in a separate repository: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs). It is served at [docs.litellm.ai](https://docs.litellm.ai). Do not create or edit documentation files in this repository — open doc PRs against `BerriAI/litellm-docs` instead.
+
 ## Development Commands
 
 ### Installation
@@ -110,7 +114,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
 
 ### UI Component Library
-- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, plain `<span>`/`<div>` with Tailwind classes (or `Typography.Text`) for text, `Card` from `antd`, etc. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
+- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only `<span>`, `<p>`, `<h*>` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
 
 ### MCP OAuth / OpenAPI Transport Mapping
 - `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).

Dockerfile

@@ -1,9 +1,9 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
 
 FROM $UV_IMAGE AS uvbin
 
@@ -27,10 +27,8 @@ RUN apk add --no-cache \
     npm \
     libsndfile
 
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    UV_PROJECT_ENVIRONMENT=/app/.venv \
+ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
-    XDG_CACHE_HOME=/app/.cache \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
@@ -71,34 +69,26 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
-    npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
+    npm install -g npm@11.12.1 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
     GLOBAL="$(npm root -g)" && \
-    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    for pkg in tar glob @isaacs/brace-expansion brace-expansion minimatch diff picomatch; do \
+        name="${pkg##*/}"; \
+        find "$GLOBAL/npm" -type d -name "$name" -path "*/node_modules/$pkg" | while read d; do \
+            rm -rf "$d" && cp -rL "$GLOBAL/$pkg" "$d"; \
+        done; \
     done && \
-    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-    done && \
-    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
     npm cache clean --force && \
     { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
+ENV PATH="/app/.venv/bin:${PATH}"
 
 COPY --from=builder /app /app
+# Prisma binaries live in $HOME/.cache (default prisma-python location),
+# which is /root/.cache here. Copy them from the builder so they survive
+# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
+# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
+COPY --from=builder /root/.cache /root/.cache
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete

README.md

@@ -484,6 +484,8 @@ make format-check   # Check formatting only
 
 For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
 
+> **📖 Contributing to documentation?** The LiteLLM docs have moved to a separate repository: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs). Please open doc PRs there. Docs are served at [docs.litellm.ai](https://docs.litellm.ai).
+
 ## Code Quality / Linting
 
 LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).

ci_cd/run_migration.py

@@ -1,22 +1,231 @@
+import argparse
 import os
-import subprocess
-from pathlib import Path
-from datetime import datetime
-import testing.postgresql
+import re
 import shutil
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+
+import testing.postgresql
 
 
-def create_migration(migration_name: str = None):
+DESTRUCTIVE_PATTERN = re.compile(r"\bDROP\s+(COLUMN|TABLE|INDEX)\b", re.IGNORECASE)
+DEFAULT_BASE_BRANCH = "litellm_internal_staging"
+
+
+def _find_destructive_statements(sql: str) -> list:
+    """Return SQL lines containing DROP COLUMN, DROP TABLE, or DROP INDEX."""
+    return [
+        line.strip() for line in sql.splitlines() if DESTRUCTIVE_PATTERN.search(line)
+    ]
+
+
+def _print_freshness_failure(
+    base_branch: str, reason: str, stderr_text: str = ""
+) -> None:
+    """Loudly refuse to run when the freshness check can't be completed."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(f"  FRESHNESS CHECK FAILED — COULD NOT VERIFY origin/{base_branch}", file=out)
+    print(banner, file=out)
+    print("", file=out)
+    print(f"Reason: {reason}", file=out)
+    if stderr_text:
+        print("", file=out)
+        print("git stderr:", file=out)
+        for line in stderr_text.rstrip().splitlines():
+            print(f"    {line}", file=out)
+    print("", file=out)
+    print("Common causes:", file=out)
+    print("  - No network access (offline)", file=out)
+    print("  - 'origin' remote not configured, or base branch name is wrong", file=out)
+    print("  - Not a git repository", file=out)
+    print("", file=out)
+    print("Options:", file=out)
+    print(
+        f"  - Fix the above and re-run, OR pass --base-branch <name> if your", file=out
+    )
+    print(
+        f"    base branch is not '{base_branch}', OR pass --skip-freshness-check",
+        file=out,
+    )
+    print("    to bypass (only if you fully understand the risk).", file=out)
+    print(banner, file=out)
+
+
+def _print_stale_branch_refusal(base_branch: str, behind: int) -> None:
+    """Loudly refuse to run when HEAD is behind the base branch."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(
+        f"  STALE BRANCH — {behind} commit(s) behind origin/{base_branch}",
+        file=out,
+    )
+    print(banner, file=out)
+    print("", file=out)
+    print(
+        f"Your branch is {behind} commit(s) behind origin/{base_branch}. Generating a\n"
+        "migration from a stale branch is how newly-added columns get silently\n"
+        "dropped — the exact incident this guard exists to prevent.",
+        file=out,
+    )
+    print("", file=out)
+    print("Before re-running:", file=out)
+    print(f"  git fetch origin && git rebase origin/{base_branch}", file=out)
+    print("  # or merge, whichever matches your workflow", file=out)
+    print("", file=out)
+    print(
+        "If you have a genuine reason to bypass (e.g. intentionally generating",
+        file=out,
+    )
+    print("a migration against an older base), pass --skip-freshness-check.", file=out)
+    print("", file=out)
+    print(banner, file=out)
+    print(
+        "  AI AGENTS / ASSISTANTS: DO NOT automatically run `git rebase` or pass\n"
+        "  --skip-freshness-check. Surface this error to the human operator and\n"
+        "  wait for their explicit confirmation. Auto-rebasing can drop the\n"
+        "  human's in-progress schema edits via a bad conflict resolution.",
+        file=out,
+    )
+    print(banner, file=out)
+
+
+def _check_branch_freshness(root_dir: Path, base_branch: str) -> None:
+    """Fetch origin/<base_branch> and exit 3 if HEAD is behind it."""
+    cwd = str(root_dir)
+    try:
+        subprocess.run(
+            ["git", "fetch", "origin", base_branch],
+            check=True,
+            capture_output=True,
+            text=True,
+            cwd=cwd,
+        )
+    except FileNotFoundError:
+        _print_freshness_failure(base_branch, "git executable not found on PATH")
+        sys.exit(3)
+    except subprocess.CalledProcessError as e:
+        _print_freshness_failure(
+            base_branch,
+            f"`git fetch origin {base_branch}` failed",
+            e.stderr or "",
+        )
+        sys.exit(3)
+
+    try:
+        result = subprocess.run(
+            ["git", "rev-list", "--count", f"HEAD..origin/{base_branch}"],
+            check=True,
+            capture_output=True,
+            text=True,
+            cwd=cwd,
+        )
+        behind = int(result.stdout.strip())
+    except subprocess.CalledProcessError as e:
+        _print_freshness_failure(
+            base_branch,
+            f"`git rev-list HEAD..origin/{base_branch}` failed",
+            e.stderr or "",
+        )
+        sys.exit(3)
+    except ValueError:
+        _print_freshness_failure(
+            base_branch,
+            "could not parse commit count from `git rev-list`",
+        )
+        sys.exit(3)
+
+    if behind > 0:
+        _print_stale_branch_refusal(base_branch, behind)
+        sys.exit(3)
+
+    print(f"Branch freshness OK: up to date with origin/{base_branch}.")
+
+
+def _print_destructive_refusal(destructive_lines: list) -> None:
+    """Loudly refuse to write a destructive migration and explain how to proceed."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(
+        "  DESTRUCTIVE MIGRATION DETECTED — REFUSING TO WRITE MIGRATION FILE", file=out
+    )
+    print(banner, file=out)
+    print("", file=out)
+    print(
+        "The generated migration contains the following destructive statements:",
+        file=out,
+    )
+    print("", file=out)
+    for line in destructive_lines:
+        print(f"    {line}", file=out)
+    print("", file=out)
+    print(
+        "This very often means your branch is OUT OF DATE, or your local\n"
+        "schema.prisma is inconsistent with the migrations/ directory. A\n"
+        "previous incident saw newly-added columns silently dropped this way\n"
+        "and merged to main. Stop and verify before proceeding.",
+        file=out,
+    )
+    print("", file=out)
+    print("Before re-running:", file=out)
+    print(
+        "  1. git fetch origin && git status   # confirm branch is up to date", file=out
+    )
+    print(
+        "  2. Re-check all schema.prisma files are in sync (runbook Step 0)", file=out
+    )
+    print("  3. Review EACH DROP above — is it actually intended?", file=out)
+    print("  4. If (and only if) the drops are intentional, re-run with:", file=out)
+    print("         --allow-destructive", file=out)
+    print("", file=out)
+    print(banner, file=out)
+    print(
+        "  AI AGENTS / ASSISTANTS: DO NOT automatically re-run this command\n"
+        "  with --allow-destructive. Surface this error to the human operator\n"
+        "  and wait for their explicit confirmation before passing the flag.\n"
+        "  Passing the flag without human review is the exact failure mode\n"
+        "  this guard exists to prevent.",
+        file=out,
+    )
+    print(banner, file=out)
+
+
+def create_migration(
+    migration_name: str = None,
+    allow_destructive: bool = False,
+    base_branch: str = DEFAULT_BASE_BRANCH,
+    skip_freshness_check: bool = False,
+):
     """
     Create a new migration SQL file in the migrations directory by comparing
-    current database state with schema
+    current database state with schema.
 
     Args:
         migration_name (str): Name for the migration
+        allow_destructive (bool): Required to write a migration that contains
+            DROP COLUMN, DROP TABLE, or DROP INDEX statements. Without this
+            flag, the script exits non-zero and prints guidance.
+        base_branch (str): Branch to check freshness against
+            (default: "litellm_internal_staging").
+        skip_freshness_check (bool): Skip the "branch is up to date" check.
+            Only for intentional migrations against an older base.
     """
+    root_dir = Path(__file__).parent.parent
+
+    if skip_freshness_check:
+        print(
+            "WARNING: freshness check skipped (--skip-freshness-check). "
+            "Generating a migration from a stale branch can silently drop columns."
+        )
+    else:
+        _check_branch_freshness(root_dir, base_branch)
+
     try:
-        # Get paths
-        root_dir = Path(__file__).parent.parent
         migrations_dir = (
             root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
         )
@@ -59,7 +268,27 @@ def create_migration(migration_name: str = None):
                     check=True,
                 )
 
-                if result.stdout.strip():
+                # Prisma emits the literal "-- This is an empty migration." when
+                # there's no real drift. Treat that as "no changes".
+                diff_sql = result.stdout
+                stripped = diff_sql.strip()
+                is_empty_diff = (
+                    not stripped or stripped == "-- This is an empty migration."
+                )
+
+                if not is_empty_diff:
+                    destructive_lines = _find_destructive_statements(diff_sql)
+                    if destructive_lines and not allow_destructive:
+                        _print_destructive_refusal(destructive_lines)
+                        sys.exit(2)
+                    if destructive_lines and allow_destructive:
+                        print(
+                            "WARNING: writing destructive migration "
+                            "(--allow-destructive passed). Statements:"
+                        )
+                        for line in destructive_lines:
+                            print(f"    {line}")
+
                     # Generate timestamp and create migration directory
                     timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
                     migration_name = migration_name or "unnamed_migration"
@@ -68,7 +297,7 @@ def create_migration(migration_name: str = None):
 
                     # Write the SQL to migration.sql
                     migration_file = migration_dir / "migration.sql"
-                    migration_file.write_text(result.stdout)
+                    migration_file.write_text(diff_sql)
 
                     print(f"Created migration in {migration_dir}")
                     return True
@@ -90,8 +319,48 @@ def create_migration(migration_name: str = None):
 
 
 if __name__ == "__main__":
-    # If running directly, can optionally pass migration name as argument
-    import sys
-
-    migration_name = sys.argv[1] if len(sys.argv) > 1 else None
-    create_migration(migration_name)
+    parser = argparse.ArgumentParser(
+        description=(
+            "Generate a Prisma migration by diffing the temp DB "
+            "(existing migrations applied) against schema.prisma."
+        )
+    )
+    parser.add_argument(
+        "migration_name",
+        nargs="?",
+        default=None,
+        help="Name for the migration (used in the generated directory name).",
+    )
+    parser.add_argument(
+        "--allow-destructive",
+        action="store_true",
+        help=(
+            "Required to write a migration that contains DROP COLUMN, "
+            "DROP TABLE, or DROP INDEX. Without this flag, destructive "
+            "diffs are refused."
+        ),
+    )
+    parser.add_argument(
+        "--base-branch",
+        default=DEFAULT_BASE_BRANCH,
+        help=(
+            f"Branch to check freshness against (default: {DEFAULT_BASE_BRANCH}). "
+            "The script fetches origin/<base-branch> and refuses to run if HEAD "
+            "is behind it."
+        ),
+    )
+    parser.add_argument(
+        "--skip-freshness-check",
+        action="store_true",
+        help=(
+            "Bypass the 'branch is up to date' check. Only for intentional "
+            "migrations against an older base. Pairs poorly with automation."
+        ),
+    )
+    args = parser.parse_args()
+    create_migration(
+        args.migration_name,
+        allow_destructive=args.allow_destructive,
+        base_branch=args.base_branch,
+        skip_freshness_check=args.skip_freshness_check,
+    )

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -47,7 +47,7 @@ spec:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       {{- with .Values.extraInitContainers }}
       initContainers:
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
       containers:
         - name: {{ include "litellm.name" . }}
@@ -212,7 +212,7 @@ spec:
             {{- toYaml . | nindent 12 }}
           {{- end }}
       {{- with .Values.extraContainers }}
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
       volumes:
         {{ if .Values.securityContext.readOnlyRootFilesystem }}

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -37,7 +37,7 @@ spec:
       serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
       {{- with .Values.migrationJob.extraInitContainers }}
       initContainers:
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }} 
       containers:
         - name: prisma-migrations
@@ -96,7 +96,7 @@ spec:
             {{- toYaml . | nindent 12 }}
           {{- end }}
       {{- with .Values.migrationJob.extraContainers }}
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
       {{- with .Values.volumes }}
       volumes:

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -319,3 +319,61 @@ tests:
     asserts:
       - notExists:
           path: spec.minReadySeconds
+  - it: should work with extraInitContainers
+    template: deployment.yaml
+    set:
+      extraInitContainers:
+        - name: init-test
+          image: busybox:latest
+          command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-test
+            image: busybox:latest
+            command: ["echo", "hello"]
+  - it: should support tpl in extraInitContainers
+    template: deployment.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      extraInitContainers:
+        - name: init-tpl
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-tpl
+            image: "ghcr.io/berriai/litellm-database:test"
+            command: ["echo", "hello"]
+  - it: should work with extraContainers
+    template: deployment.yaml
+    set:
+      extraContainers:
+        - name: sidecar
+          image: busybox:latest
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar
+            image: busybox:latest
+  - it: should support tpl in extraContainers
+    template: deployment.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      extraContainers:
+        - name: sidecar-tpl
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar-tpl
+            image: "ghcr.io/berriai/litellm-database:test"

deploy/charts/litellm-helm/tests/migrations-job_tests.yaml

@@ -188,3 +188,69 @@ tests:
       - equal:
           path: spec.template.spec.serviceAccountName
           value: pre-existing-sa
+  - it: should work with extraInitContainers
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        extraInitContainers:
+          - name: init-test
+            image: busybox:latest
+            command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-test
+            image: busybox:latest
+            command: ["echo", "hello"]
+  - it: should support tpl in extraInitContainers
+    template: migrations-job.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      migrationJob:
+        enabled: true
+        extraInitContainers:
+          - name: init-tpl
+            image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+            command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-tpl
+            image: "ghcr.io/berriai/litellm-database:test"
+            command: ["echo", "hello"]
+  - it: should work with extraContainers
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        extraContainers:
+          - name: sidecar
+            image: busybox:latest
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar
+            image: busybox:latest
+  - it: should support tpl in extraContainers
+    template: migrations-job.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      migrationJob:
+        enabled: true
+        extraContainers:
+          - name: sidecar-tpl
+            image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar-tpl
+            image: "ghcr.io/berriai/litellm-database:test"

docker/Dockerfile.alpine

@@ -3,7 +3,7 @@ ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b
 
 # Runtime image
 ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
 
 FROM $UV_IMAGE AS uvbin
 

docker/Dockerfile.database

@@ -1,9 +1,9 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
 
 FROM $UV_IMAGE AS uvbin
 
@@ -26,10 +26,8 @@ RUN apk add --no-cache \
     npm \
     libsndfile
 
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    UV_PROJECT_ENVIRONMENT=/app/.venv \
+ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
-    XDG_CACHE_HOME=/app/.cache \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
@@ -86,17 +84,18 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervi
     find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done && \
-    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
     npm cache clean --force && \
     { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
+ENV PATH="/app/.venv/bin:${PATH}"
 
 COPY --from=builder /app /app
+# Prisma binaries live in $HOME/.cache (default prisma-python location),
+# which is /root/.cache here. Copy them from the builder so they survive
+# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
+# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
+COPY --from=builder /root/.cache /root/.cache
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete

docker/Dockerfile.dev

@@ -3,7 +3,7 @@ ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973a
 
 # Runtime image
 ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
 
 FROM $UV_IMAGE AS uvbin
 

docker/Dockerfile.health_check

@@ -1,4 +1,4 @@
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
 FROM $UV_IMAGE AS uvbin
 
 FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d

docker/Dockerfile.non_root

@@ -1,8 +1,8 @@
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
 ARG PROXY_EXTRAS_SOURCE=published
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
 
 FROM $UV_IMAGE AS uvbin
 
@@ -15,29 +15,21 @@ COPY --from=uvbin /uv /usr/local/bin/uv
 COPY --from=uvbin /uvx /usr/local/bin/uvx
 
 RUN for i in 1 2 3; do \
-    apk add --no-cache \
-      python3 \
-      python3-dev \
-      clang \
-      llvm \
-      lld \
-      gcc \
-      linux-headers \
-      build-base \
-      bash \
-      coreutils \
-      curl \
-      openssl \
-      openssl-dev \
-      nodejs \
-      npm \
-      libsndfile && break || sleep 5; \
+      apk add --no-cache \
+        python3 \
+        python3-dev \
+        gcc \
+        bash \
+        coreutils \
+        curl \
+        openssl \
+        libsndfile \
+        nodejs && break || sleep 5; \
     done
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
-    NVM_DIR=/root/.nvm \
-    PATH="/root/.nvm/versions/node/v20.20.2/bin:/app/.venv/bin:${PATH}" \
+    PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
     PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
     PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
@@ -49,7 +41,8 @@ COPY enterprise/pyproject.toml enterprise/
 COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
 
 # Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
-RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
+RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
+    uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
     --extra proxy \
     --extra proxy-runtime \
     --extra extra_proxy \
@@ -62,38 +55,12 @@ COPY . .
 # Set non-root flag for build time consistency
 ENV LITELLM_NON_ROOT=true
 
-# Build Admin UI once and stage the static output for the runtime image.
-# NOTE: .npmrc files (which may set ignore-scripts=true and min-release-age=3d)
-# are temporarily renamed during npm install/ci so they don't block lifecycle
-# scripts needed by the build. This is safe because npm ci installs from
-# package-lock.json with pinned versions + integrity hashes.
+# Stage the pre-built Admin UI from the checked-in Next.js static export.
+# _experimental/out/ is regenerated as part of the release runbook.
+# Restructure extensionless routes (foo.html -> foo/index.html) to match the layout
+# proxy_server.py expects, and drop a readiness marker.
 RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
-    ([ -f /app/.npmrc ] && mv /app/.npmrc /app/.npmrc.bak || true) && \
-    NVM_VERSION="v0.40.4" && \
-    NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" && \
-    NODE_VERSION="v20.20.2" && \
-    NVM_SCRIPT="/tmp/install-nvm.sh" && \
-    curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" && \
-    echo "${NVM_CHECKSUM}  ${NVM_SCRIPT}" | sha256sum -c - && \
-    bash "$NVM_SCRIPT" && \
-    export NVM_DIR="$HOME/.nvm" && \
-    . "$NVM_DIR/nvm.sh" && \
-    nvm install "${NODE_VERSION}" && \
-    nvm use "${NODE_VERSION}" && \
-    npm install -g npm@11.12.1 && \
-    npm install -g node-gyp@12.2.0 && \
-    ln -sf "$(npm root -g)/node-gyp" "$(npm root -g)/npm/node_modules/node-gyp" && \
-    npm cache clean --force && \
-    cd /app/ui/litellm-dashboard && \
-    if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
-      cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
-    fi && \
-    ([ -f .npmrc ] && mv .npmrc .npmrc.bak || true) && \
-    npm ci --no-audit --no-fund && \
-    ([ -f .npmrc.bak ] && mv .npmrc.bak .npmrc || true) && \
-    ([ -f /app/.npmrc.bak ] && mv /app/.npmrc.bak /app/.npmrc || true) && \
-    npm run build && \
-    cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
+    cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \
     cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
     ( cd /var/lib/litellm/ui && \
       for html_file in *.html; do \
@@ -103,10 +70,10 @@ RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
           mv "$html_file" "$folder_name/index.html"; \
         fi; \
       done && \
-      touch .litellm_ui_ready ) && \
-    cd /app/ui/litellm-dashboard && rm -rf ./out
+      touch .litellm_ui_ready )
 
-RUN if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
+RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
+    if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
       uv sync --frozen --no-default-groups --no-editable \
         --extra proxy \
         --extra proxy-runtime \
@@ -123,10 +90,7 @@ RUN if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
         --python python3; \
     fi
 
-RUN mkdir -p /app/.cache/npm && \
-    prisma generate --schema=./schema.prisma && \
-    prisma --version && \
-    prisma migrate diff --from-empty --to-schema-datamodel ./schema.prisma --script > /dev/null 2>&1 || true
+RUN prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -137,33 +101,11 @@ WORKDIR /app
 USER root
 
 RUN for i in 1 2 3; do \
-    apk upgrade --no-cache && break || sleep 5; \
+      apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python3 bash openssl tzdata nodejs npm supervisor libsndfile && break || sleep 5; \
-    done && \
-    apk upgrade --no-cache nodejs && \
-    npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
-    GLOBAL="$(npm root -g)" && \
-    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-    done && \
-    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-      sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
-    npm cache clean --force && \
-    { apk del --no-cache npm 2>/dev/null || true; }
+      apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \
+    done
 
 COPY --from=builder /app /app
 COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
@@ -179,15 +121,10 @@ ENV PATH="/app/.venv/bin:${PATH}" \
     PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
     PRISMA_HIDE_UPDATE_MESSAGE=1 \
     PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
-    NPM_CONFIG_CACHE=/app/.cache/npm \
-    NPM_CONFIG_PREFER_OFFLINE=true \
     PRISMA_OFFLINE_MODE=true
 
-RUN sed -i 's/\r$//' docker/entrypoint.sh && \
-    sed -i 's/\r$//' docker/prod_entrypoint.sh && \
-    chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
-    mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui /tmp/.npm && \
-    chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm /tmp/.npm && \
+RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \
+    chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \
     PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
     chown -R nobody:nogroup "$PRISMA_PATH" && \
     LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
@@ -201,7 +138,7 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && \
     [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w "$LITELLM_PROXY_EXTRAS_PATH" || true && \
     chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets /app/.cache
 
-USER nobody
+USER 65534
 
 RUN prisma generate --schema=./schema.prisma
 

docker/tests/nonroot.yaml

@@ -2,7 +2,7 @@ schemaVersion: 2.0.0
 
 metadataTest:
   entrypoint: ["docker/prod_entrypoint.sh"]
-  user: "nobody"
+  user: "65534"
   workdir: "/app"
 
 fileExistenceTests:

docs/my-website/.gitignore

@@ -1,23 +0,0 @@
-# Dependencies
-/node_modules
-
-# Production
-/build
-
-# Generated files
-.docusaurus
-.cache-loader
-
-# Misc
-.DS_Store
-.env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
-
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-yarn.lock
-pnpm-lock.yaml

docs/my-website/Dockerfile

@@ -1,32 +0,0 @@
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9
-
-FROM $UV_IMAGE AS uvbin
-
-FROM python:3.14.0a3-slim
-
-COPY --from=uvbin /uv /usr/local/bin/uv
-COPY --from=uvbin /uvx /usr/local/bin/uvx
-COPY . /app
-WORKDIR /app
-
-ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
-    UV_LINK_MODE=copy \
-    PATH="/app/.venv/bin:${PATH}"
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    gcc \
-    python3-dev \
-    libssl-dev \
-    pkg-config \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN uv sync --frozen --no-default-groups --no-editable \
-    --extra proxy \
-    --extra proxy-runtime \
-    --extra extra_proxy \
-    --extra semantic-router \
-    --python python
-
-EXPOSE $PORT 
-
-CMD ["sh", "-c", "litellm --host 0.0.0.0 --port $PORT --workers 10 --config config.yaml"]

docs/my-website/README.md

@@ -1,41 +0,0 @@
-# Website
-
-This website is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator.
-
-### Installation
-
-```
-$ yarn
-```
-
-### Local Development
-
-```
-$ yarn start
-```
-
-This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server.
-
-### Build
-
-```
-$ yarn build
-```
-
-This command generates static content into the `build` directory and can be served using any static contents hosting service.
-
-### Deployment
-
-Using SSH:
-
-```
-$ USE_SSH=true yarn deploy
-```
-
-Not using SSH:
-
-```
-$ GIT_USER=<Your GitHub username> yarn deploy
-```
-
-If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch.

docs/my-website/babel.config.js

@@ -1,3 +0,0 @@
-module.exports = {
-  presets: [require.resolve('@docusaurus/core/lib/babel/preset')],
-};

docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md

@@ -1,138 +0,0 @@
----
-slug: anthropic-wildcard-model-access-incident
-title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload"
-date: 2026-02-23T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-tags: [incident-report, proxy, auth, model-access]
-hide_table_of_contents: false
----
-
-**Date:** Feb 23, 2026  
-**Duration:** ~3 hours  
-**Severity:** High (for users with provider wildcard access rules)  
-**Status:** Resolved
-
-## Summary
-
-When a new Anthropic model (e.g. `claude-sonnet-4-6`) was added to the LiteLLM model cost map and a cost map reload was triggered, requests to the new model were rejected with:
-
-```
-key not allowed to access model. This key can only access models=['anthropic/*']. Tried to access claude-sonnet-4-6.
-```
-
-The reload updated `litellm.model_cost` correctly but never re-ran `add_known_models()`, so `litellm.anthropic_models` (the in-memory set used by the wildcard resolver) remained stale. The new model was invisible to the `anthropic/*` wildcard even though the cost map knew about it.
-
-- **LLM calls:** All requests to newly-added Anthropic models were blocked with a 401.
-- **Existing models:** Unaffected — only models missing from the stale provider set were impacted.
-- **Other providers:** Same bug class existed for any provider wildcard (e.g. `openai/*`, `gemini/*`).
-
-{/* truncate */}
-
----
-
-## Background
-
-LiteLLM supports provider-level wildcard access rules. When an admin configures a key or team with `models=['anthropic/*']`, any model whose provider resolves to `anthropic` should be allowed. The resolution happens in `_model_custom_llm_provider_matches_wildcard_pattern`:
-
-```mermaid
-flowchart TD
-    A["1. Request arrives for claude-sonnet-4-6"] --> B["2. Auth check: can this key call this model?
-    proxy/auth/auth_checks.py"]
-    B --> C["3. Key has models=['anthropic/*']
-    → wildcard match attempted"]
-    C --> D["4. get_llm_provider('claude-sonnet-4-6')
-    checks litellm.anthropic_models set"]
-    D -->|"model IN set"| E["5a. ✅ Provider = 'anthropic'
-    → 'anthropic/claude-sonnet-4-6' matches 'anthropic/*'"]
-    D -->|"model NOT IN set"| F["5b. ❌ Provider unknown
-    → exception raised → wildcard returns False"]
-    E --> G["6. Request allowed"]
-    F --> H["6. 401: key not allowed to access model"]
-
-    style E fill:#d4edda,stroke:#28a745
-    style F fill:#f8d7da,stroke:#dc3545
-    style H fill:#f8d7da,stroke:#dc3545
-    style D fill:#fff3cd,stroke:#ffc107
-```
-
-`litellm.anthropic_models` is a Python `set` populated at import time by `add_known_models()`. It is the source `get_llm_provider()` consults to map a bare model name like `claude-sonnet-4-6` to the provider string `"anthropic"`.
-
----
-
-## Root Cause
-
-`add_known_models()` is called **once** at module import time. Both reload paths in `proxy_server.py` updated `litellm.model_cost` with the fresh map but never called `add_known_models()` again:
-
-```python
-# Before the fix — both reload paths looked like this:
-new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
-litellm.model_cost = new_model_cost_map          # ✅ cost map updated
-_invalidate_model_cost_lowercase_map()           # ✅ cache cleared
-# ❌ add_known_models() never called
-#    → litellm.anthropic_models still has the old set
-#    → new model not in the set
-#    → get_llm_provider() raises for the new model
-#    → wildcard match returns False
-#    → 401 for every request to the new model
-```
-
-The gap existed in two places:
-1. `_check_and_reload_model_cost_map` — the periodic automatic reload (every 10 s)
-2. The `/reload/model_cost_map` admin endpoint — the manual reload
-
-**Timeline:**
-
-1. New model (`claude-sonnet-4-6`) added to `model_prices_and_context_window.json`
-2. Admin triggers cost map reload via UI → `litellm.model_cost` updated
-3. Users with `anthropic/*` wildcard keys attempt requests to `claude-sonnet-4-6`
-4. `get_llm_provider('claude-sonnet-4-6')` raises → wildcard returns False → 401
-5. Admin reloads cost map again — same result (root cause not addressed)
-6. ~3 hours of investigation → root cause identified → fix deployed
-
----
-
-## The Fix
-
-After each reload, `add_known_models()` is called with the freshly fetched map passed explicitly. Passing the map directly (rather than relying on the module-level reference) removes any ambiguity about which dict is iterated:
-
-```python
-# After the fix — both reload paths now do:
-new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
-litellm.model_cost = new_model_cost_map
-_invalidate_model_cost_lowercase_map()
-litellm.add_known_models(model_cost_map=new_model_cost_map)  # ✅ sets repopulated
-```
-
-`add_known_models()` was also updated to accept an optional explicit map so callers cannot accidentally iterate a stale module-level reference:
-
-```python
-# Before
-def add_known_models():
-    for key, value in model_cost.items():   # reads module global — ambiguous after reload
-        ...
-
-# After
-def add_known_models(model_cost_map: Optional[Dict] = None):
-    _map = model_cost_map if model_cost_map is not None else model_cost
-    for key, value in _map.items():         # always iterates the map you just fetched
-        ...
-```
-
-After the fix, the provider sets (`anthropic_models`, `open_ai_chat_completion_models`, etc.) are always consistent with `litellm.model_cost` immediately after every reload. New models become accessible via wildcard rules without any proxy restart.
-
----
-
-## Remediation
-
-| # | Action | Status | Code |
-|---|---|---|---|
-| 1 | Call `add_known_models(model_cost_map=...)` in the periodic reload path | ✅ Done | [`proxy_server.py#L4393`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L4393) |
-| 2 | Call `add_known_models(model_cost_map=...)` in the `/reload/model_cost_map` endpoint | ✅ Done | [`proxy_server.py#L11904`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L11904) |
-| 3 | Update `add_known_models()` to accept an explicit map parameter | ✅ Done | [`__init__.py#L617`](https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py#L617) |
-| 4 | Regression test: `add_known_models(model_cost_map=...)` populates provider sets | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
-| 5 | Regression test: `anthropic/*` wildcard grants/denies access correctly after reload | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
-
----

docs/my-website/blog/april_townhall_announcement/index.md

@@ -1,40 +0,0 @@
----
-slug: april-townhall-announcement
-title: "April Townhall: Security + Product Roadmap"
-date: 2026-04-02T07:30:00
-authors:
-  - krrish
-  - ishaan-alt
-description: "Join the LiteLLM April townhall on Friday, 10 April at 7:30 AM to learn about LiteLLM's security and product roadmap."
-tags: [announcement, townhall]
-hide_table_of_contents: true
----
-
-import Image from '@theme/IdealImage';
-
-We are hosting our April townhall on **Friday, 10 April at 7:30 AM PST**.
-
-<Image
-  img={require('../../img/april_townhall_banner.png')}
-  style={{width: '900px', height: 'auto', display: 'block'}}
-/>
-
-{/* truncate */}
-
-## Agenda
-
-- Product updates and roadmap progress
-- Reliability and security updates
-- Open Q&A with the team
-
-## How to contribute
-
-Add your thoughts to this [ticket](https://github.com/BerriAI/litellm/issues/24825) to help us shape the agenda.
-
-## Register
-
-Register here: [LiteLLM April Townhall Form](https://forms.gle/hvyVXwbFjzJQE7dEA)
-
-We will hold the townhall from **7:30 AM to 8:30 AM PST on Zoom**.
-
-For security, attendance is restricted to corporate emails. If you register with a non-corporate email, we will share the townhall slides and accompanying blog post after the event.

docs/my-website/blog/april_townhall_updates/index.md

@@ -1,162 +0,0 @@
----
-slug: april-townhall-updates
-title: "April Townhall Updates: CI/CD v2, Stability, and Product Roadmap"
-date: 2026-04-10T12:00:00
-authors:
-  - krrish
-  - ishaan-alt
-description: "A recap of the April LiteLLM town hall covering CI/CD v2, product stability work, and the near-term roadmap."
-tags: [townhall, security, reliability, product]
-hide_table_of_contents: false
----
-
-import Image from '@theme/IdealImage';
-
-Thank you to everyone who joined our April town hall.
-
-We used the session to share our CI/CD v2 improvements, product stability work, and what we are prioritizing next across reliability and product roadmap.
-
-{/* truncate */}
-
-## CI/CD v2 improvements
-
-Our CI/CD v2 work is centered around four goals:
-
-1. **Limit** what each package can access
-2. **Reduce** the number of sensitive environment variables
-3. **Avoid** compromised packages
-4. **Reduce the risk of** release tampering
-
-#### New architecture: isolated environments
-
-We have begun moving to isolated environments for distinct CI/CD stages to reduce the chance that a single compromised step can inherit broad access across the entire pipeline.
-
-<Image
-  img={require('../../img/april_townhall_isolated_environments.png')}
-  style={{width: '900px', height: 'auto', display: 'block'}}
-/>
-
-#### Current rollout status
-
-These changes are deployed in our current release workflow. [See here](https://github.com/BerriAI/litellm/tags)
-
-#### Independently verify releases
-
-A key part of CI/CD v2 is supporting independent verification of release artifacts using our published verification process, while reducing reliance on any single credential or release path.
-
-[**Learn more about how to verify releases**](https://docs.litellm.ai/docs/proxy/docker_image_security)
-
-<Image
-  img={require('../../img/verify_releases.png')}
-  style={{width: '900px', height: 'auto', display: 'block'}}
-/>
-
-## Stability improvements
-
-### SDLC improvements
-
-This month, we're focusing on process stability improvements around:
-- Improving main-branch stability
-- Mapping UI QA to built Docker images for 1:1 environment parity
-- Consistent release tags across PyPI and Docker
-- Fixing release notes publication
-
-#### Improving main-branch stability
-
-We're introducing a staging-gated flow:
-
-<Image
-  img={require('../../img/stable_main.png')}
-  style={{width: '900px', height: 'auto', display: 'block'}}
-/>
-
-- Only an internal staging branch can push to `main`.
-- PRs to that staging branch must pass CircleCI LLM API testing.
-- Collision handling happens on staging, which is designed to reduce unstable changes reaching `main`.
-
-#### UI QA in Docker environment
-
-Moving forward, all UI QA will be performed in the built Docker image that users run.
-
-Previously, some UI QA paths were run in local environments that did not fully replicate Docker runtime conditions.
-
-That contributed to release-specific issues, including MCP registration problems in `v1.82.3`.
-
-#### Consistent release tags
-
-Today we publish releases for multiple scenarios:
-- Dev (Built of a PR for a customer-specific scenario)
-- Nightly (Passes all CI/CD checks)
-- Release Candidate (Passes all CI/CD checks + manual UI QA)
-- Stable (intended to pass all CI/CD checks + manual UI QA + 7 days of production testing)
-
-We are targeting a consistent naming convention across PyPI and Docker by the end of April.
-
-#### Release notes
-
-CI/CD v2 changes moved release notes to a manual path. This is a temporary solution while we investigate a better automated workflow. We are targeting a more consistent process by the end of April.
-
-### Product stability improvements
-
-#### Stable Prisma migrations
-
-Today, we have observed several migration failure classes:
-- Migration not applied
-- Migration marked applied but incomplete
-- Migration not applied due to non-root image issues
-
-We're prioritizing this work this month and have assigned an engineering owner to the effort. Our target is to resolve these error classes by the end of April.
-
-#### UI type safety
-
-Another area of focus is improving the stability of the UI. Today, one cause of errors is that the UI maintains its own assumptions about backend API types. This can lead to issues when backend responses differ from UI assumptions.
-
-We aim to move to having the UI and Backend be in sync with each other, and are exploring OpenAPI-driven mapping to achieve this.
-
-## Product roadmap
-
-### Our Assumptions
-
-Over the next few years, we expect:
-- Companies will give employees more AI tools.
-- More AI agents will move into production workflows across HR, finance, support, and operations.
-
-### Our Inferences
-#### Near-term
-
-- AI spend will increase.
-- Uptime and latency will become even more important.
-- More AI resources (skills, CLIs, and related assets) will require governance.
-- Agent and MCP usage patterns will require deeper controls.
-- Broader developer adoption will increase the need for simpler, more discoverable tooling.
-
-#### Long-term 
-
-- We expect many organizations to treat agent auditability (how decisions were made across LLM + MCP + sub-agent inputs/outputs) as a compliance expectation.
-- Permission management will get more complex as user-agent interaction chains deepen.
-
-Roadmap timelines in this post are targets and may evolve based on validation and user feedback.
-
-## April investments
-
-### Reliability
-
-- Increase uptime for 10k+ RPS scenarios.
-- Investigate latency overhead for long-running Claude Code requests.
-
-### Feature reliability
-
-- Polish MCP authentication.
-- Better understand how teams are using agents through LiteLLM.
-
-### Governance
-
-- Launch Skills as a first-class citizen in LiteLLM.
-
-## Q&A
-
-Thank you again for all the questions and direct feedback. We will keep sharing concrete progress updates as these efforts ship.
-
-## Hiring
-
-We are actively hiring across several roles, please apply [here](https://jobs.ashbyhq.com/litellm) if you're interested!

docs/my-website/blog/authors.yml

@@ -1,48 +0,0 @@
-litellm:
-  name: LiteLLM Team
-  title: LiteLLM Core Team
-  url: https://github.com/BerriAI/litellm
-  image_url: https://github.com/BerriAI.png
-
-sameer:
-  name: Sameer Kankute
-  title: SWE @ LiteLLM (LLM Translation)
-  url: https://www.linkedin.com/in/sameer-kankute/
-  image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-
-krrish:
-  name: Krrish Dholakia
-  title: CEO, LiteLLM
-  url: https://www.linkedin.com/in/krish-d/
-  image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-
-ishaan:
-  name: Ishaan Jaffer
-  title: CTO, LiteLLM
-  url: https://www.linkedin.com/in/reffajnaahsi/
-  image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-
-# Alias for typo in name
-ishaan-alt:
-  name: Ishaan Jaffer
-  title: CTO, LiteLLM
-  url: https://www.linkedin.com/in/reffajnaahsi/
-  image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-
-ryan:
-  name: Ryan Crabbe
-  title: Performance Engineer, LiteLLM
-  url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
-  image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
-
-alexsander:
-  name: Alexsander Hamir
-  title: Performance Engineer, LiteLLM
-  url: https://www.linkedin.com/in/alexsander-baptista/
-  image_url: https://github.com/AlexsanderHamir.png
-
-yuneng:
-  name: Yuneng Jiang
-  title: SWE @ LiteLLM (Full Stack)
-  url: https://www.linkedin.com/in/yuneng-david-jiang-455676139/
-  image_url: https://avatars.githubusercontent.com/u/171294688?v=4

docs/my-website/blog/ci_cd_v2_improvements/index.md

@@ -1,90 +0,0 @@
----
-slug: ci-cd-v2-improvements
-title: "Announcing CI/CD v2 for LiteLLM"
-date: 2026-03-30T21:30:00
-authors:
-  - krrish
-description: "CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM."
-tags: [engineering, ci-cd, security]
-hide_table_of_contents: false
----
-
-import Image from '@theme/IdealImage';
-
-The CI/CD v2 is now live for LiteLLM.
-
-<Image
-  img={require('../../img/ci_cd_architecture.png')}
-  style={{width: '700px', height: 'auto', display: 'block'}}
-/>
-
-<br/>
-Building on the roadmap from our [security incident](https://docs.litellm.ai/blog/security-townhall-updates#roadmap), CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM.
-
-## What changed
-
-- Security scans and unit tests run in isolated environments.
-- Validation and release are separated into different repositories, making it harder for an attacker to reach release credentials.
-- Trusted Publishing for PyPI releases - this means no long-lived credentials are used to publish releases.
-- Immutable Docker release tags - this means no tampering of Docker release tags after they are published [Learn more](https://docs.docker.com/docker-hub/repos/manage/hub-images/immutable-tags/). Note: work for GHCR docker releases is planned as well.
-- Docker image signing with [Cosign](https://github.com/sigstore/cosign) - all release images are signed so users can independently verify they came from us.
-
-## Verify Docker image signatures
-
-Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
-
-**Verify using the pinned commit hash (recommended):**
-
-A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-**Verify using a release tag (convenience):**
-
-Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
-
-Expected output:
-
-```
-The following checks were performed on each of these signatures:
-  - The cosign claims were validated
-  - The signatures were verified against the specified public key
-```
-
-## What's next
-
-Moving forward, we plan on:
-- Adopting OpenSSF (this is a set of security criteria that projects should meet to demonstrate a strong security posture - [Learn more](https://baseline.openssf.org/versions/2026-02-19.html))
-  - We've added Scorecard and Allstar to our Github
-
-- Adding SLSA Build Provenance to our CI/CD pipeline - this means we allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published.
-
-
-We hope that this will mean you can be confident that the releases you are using are safe and from us.
-
-
-## The principle
-
-The new CI/CD pipeline reflects the principles, outlined below, and is designed to be more secure and reliable:
-
-- **Limit** what each package can access
-- **Reduce** the number of sensitive environment variables
-- **Avoid** compromised packages
-- **Prevent** release tampering
-
-
-## How to help: 
-
-Help us plan April's stability sprint - https://github.com/BerriAI/litellm/issues/24825

docs/my-website/blog/claude_code_beta_headers/index.md

@@ -1,168 +0,0 @@
----
-slug: claude-code-beta-headers-incident
-title: "Incident Report: Invalid beta headers with Claude Code"
-date: 2026-02-16T10:00:00
-authors:
-  - sameer
-  - ishaan-alt
-  - krrish
-tags: [incident-report, anthropic, stability]
-hide_table_of_contents: false
----
-
-**Date:** February 13, 2026
-**Duration:** ~3 hours
-**Severity:** High
-**Status:** Resolved
-
-> **Note:** This fix will be available starting from `v1.81.13-nightly` or higher of LiteLLM.
-
-## Summary
-
-Claude Code began sending unsupported Anthropic beta headers to non-Anthropic providers (Bedrock, Azure AI, Vertex AI), causing `invalid beta flag` errors. LiteLLM was forwarding all beta headers without provider-specific validation. Users experienced request failures when routing Claude Code requests through LiteLLM to these providers.
-
-- **LLM calls to Anthropic:** No impact.
-- **LLM calls to Bedrock/Azure/Vertex:** Failed with `invalid beta flag` errors when unsupported headers were present.
-- **Cost tracking and routing:** No impact.
-
-{/* truncate */}
-
----
-
-## Background
-
-Anthropic uses beta headers to enable experimental features in Claude. When Claude Code makes API requests, it includes headers like `anthropic-beta: prompt-caching-scope-2026-01-05,advanced-tool-use-2025-11-20`. However, not all providers support all Anthropic beta features.
-
-Before this incident, LiteLLM forwarded all beta headers to all providers without validation:
-
-```mermaid
-sequenceDiagram
-    participant CC as Claude Code
-    participant LP as LiteLLM (old behavior)
-    participant Provider as Provider (Bedrock/Azure/Vertex)
-
-    CC->>LP: Request with beta headers
-    Note over CC,LP: anthropic-beta: header1,header2,header3
-
-    LP->>Provider: Forward ALL headers (no validation)
-    Note over LP,Provider: anthropic-beta: header1,header2,header3
-
-    Provider-->>LP: ❌ Error: invalid beta flag
-    LP-->>CC: Request fails
-```
-
-Requests succeeded for Anthropic (native support) but failed for other providers when Claude Code sent headers those providers didn't support.
-
----
-
-## Root cause
-
-LiteLLM lacked provider-specific beta header validation. When Claude Code introduced new beta features or sent headers that specific providers didn't support, those headers were blindly forwarded, causing provider API errors.
-
----
-
-## Remediation
-
-| # | Action | Status | Code |
-|---|---|---|---|
-| 1 | Create `anthropic_beta_headers_config.json` with provider-specific mappings | ✅ Done | [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) |
-| 2 | Implement strict validation: headers must be explicitly mapped to be forwarded | ✅ Done | [`litellm_logging.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/litellm_logging.py) |
-| 3 | Add `/reload/anthropic_beta_headers` endpoint for dynamic config updates | ✅ Done | Proxy management endpoints |
-| 4 | Add `/schedule/anthropic_beta_headers_reload` for automatic periodic updates | ✅ Done | Proxy management endpoints |
-| 5 | Support `LITELLM_ANTHROPIC_BETA_HEADERS_URL` for custom config sources | ✅ Done | Environment configuration |
-| 6 | Support `LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS` for air-gapped deployments | ✅ Done | Environment configuration |
-
-Now LiteLLM validates and transforms headers per-provider:
-
-```mermaid
-sequenceDiagram
-    participant CC as Claude Code
-    participant LP as LiteLLM (new behavior)
-    participant Config as Beta Headers Config
-    participant Provider as Provider (Bedrock/Azure/Vertex)
-
-    CC->>LP: Request with beta headers
-    Note over CC,LP: anthropic-beta: header1,header2,header3
-
-    LP->>Config: Load header mapping for provider
-    Config-->>LP: Returns mapping (header→value or null)
-
-    Note over LP: Validate & Transform:<br/>1. Check if header exists in mapping<br/>2. Filter out null values<br/>3. Map to provider-specific names
-
-    LP->>Provider: Request with filtered & mapped headers
-    Note over LP,Provider: anthropic-beta: mapped-header2<br/>(header1, header3 filtered out)
-
-    Provider-->>LP: ✅ Success response
-    LP-->>CC: Response
-```
-
----
-
-## Dynamic configuration updates
-
-A key improvement is zero-downtime configuration updates. When Anthropic releases new beta features, users can update their configuration without restarting:
-
-```bash
-# Manually trigger reload (no restart needed)
-curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-  -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
-
-# Or schedule automatic reloads every 24 hours
-curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
-  -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
-```
-
-This prevents future incidents where Claude Code introduces new headers before LiteLLM configuration is updated.
-
----
-
-## Configuration format
-
-The `anthropic_beta_headers_config.json` file maps input headers to provider-specific output headers:
-
-```json
-{
-  "description": "Mapping of Anthropic beta headers for each provider.",
-  "anthropic": {
-    "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
-    "computer-use-2025-01-24": "computer-use-2025-01-24"
-  },
-  "bedrock_converse": {
-    "advanced-tool-use-2025-11-20": null,
-    "computer-use-2025-01-24": "computer-use-2025-01-24"
-  },
-  "azure_ai": {
-    "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
-    "computer-use-2025-01-24": "computer-use-2025-01-24"
-  }
-}
-```
-
-**Validation rules:**
-1. Headers must exist in the mapping for the target provider
-2. Headers with `null` values are filtered out (unsupported)
-3. Header names can be transformed per-provider (e.g., Bedrock uses different names for some features)
-
----
-
-## Resolution steps for users
-
-For users still experiencing issues, update to the latest LiteLLM version if < v1.81.11-nightly:
-
-```bash
-pip install --upgrade litellm
-```
-
-Or manually reload the configuration without restarting:
-
-```bash
-curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-  -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
-```
-
----
-
-## Related documentation
-
-- [Managing Anthropic Beta Headers](../../docs/proxy/sync_anthropic_beta_headers) - Complete configuration guide
-- [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) - Current configuration file

docs/my-website/blog/claude_opus_4_6/index.md

@@ -1,723 +0,0 @@
----
-slug: claude_opus_4_6
-title: "Day 0 Support: Claude Opus 4.6"
-date: 2026-02-05T10:00:00
-authors:
-  - sameer
-  - ishaan-alt
-  - krrish
-description: "Day 0 support for Claude Opus 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
-tags: [anthropic, claude, opus 4.6]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports Claude Opus 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
-
-{/* truncate */}
-
-## Docker Image
-
-```bash
-docker pull ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6
-```
-
-## Usage - Anthropic
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-6
-    litellm_params:
-      model: anthropic/claude-opus-4-6
-      api_key: os.environ/ANTHROPIC_API_KEY
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Azure
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-6
-    litellm_params:
-      model: azure_ai/claude-opus-4-6
-      api_key: os.environ/AZURE_AI_API_KEY
-      api_base: os.environ/AZURE_AI_API_BASE  # https://<resource>.services.ai.azure.com
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
-  -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Vertex AI
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-6
-    litellm_params:
-      model: vertex_ai/claude-opus-4-6
-      vertex_project: os.environ/VERTEX_PROJECT
-      vertex_location: us-east5
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e VERTEX_PROJECT=$VERTEX_PROJECT \
-  -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  -v $(pwd)/credentials.json:/app/credentials.json \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Bedrock
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-6
-    litellm_params:
-      model: bedrock/anthropic.claude-opus-4-6-v1
-      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
-      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
-      aws_region_name: us-east-1
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-  -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Advanced Features
-
-### Compaction
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-Litellm supports enabling compaction for the new claude-opus-4-6.
-
-**Enabling Compaction**
-
-To enable compaction, add the `context_management` parameter with the `compact_20260112` edit type:
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "What is the weather in San Francisco?"
-    }
-  ],
-  "context_management": {
-    "edits": [
-      {
-        "type": "compact_20260112"
-      }
-    ]
-  },
-  "max_tokens": 100
-}'
-```
-All the parameters supported for context_management by anthropic are supported and can be directly added. Litellm automatically adds the `compact-2026-01-12` beta header in the request.
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-Enable compaction to reduce context size while preserving key information. LiteLLM automatically adds the `compact-2026-01-12` beta header when compaction is enabled.
-
-:::info
-**Provider Support:** Compaction is supported on Anthropic, Azure AI, and Vertex AI. It is **not supported** on Bedrock (Invoke or Converse APIs).
-:::
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 4096,
-    "messages": [
-        {
-            "role": "user",
-            "content": "Hi"
-        }
-    ],
-    "context_management": {
-        "edits": [
-            {
-                "type": "compact_20260112"
-            }
-        ]
-    }
-}'
-```
-
-</TabItem>
-</Tabs>
-
-
-**Response with Compaction Block**
-
-The response will include the compaction summary in `provider_specific_fields.compaction_blocks`:
-
-```json
-{
-  "id": "chatcmpl-a6c105a3-4b25-419e-9551-c800633b6cb2",
-  "created": 1770357619,
-  "model": "claude-opus-4-6",
-  "object": "chat.completion",
-  "choices": [
-    {
-      "finish_reason": "length",
-      "index": 0,
-      "message": {
-        "content": "I don't have access to real-time data, so I can't provide the current weather in San Francisco. To get up-to-date weather information, I'd recommend checking:\n\n- **Weather websites** like weather.com, accuweather.com, or wunderground.com\n- **Search engines** – just Google \"San Francisco weather\"\n- **Weather apps** on your phone (e.g., Apple Weather, Google Weather)\n- **National",
-        "role": "assistant",
-        "provider_specific_fields": {
-          "compaction_blocks": [
-            {
-              "type": "compaction",
-              "content": "Summary of the conversation: The user requested help building a web scraper..."
-            }
-          ]
-        }
-      }
-    }
-  ],
-  "usage": {
-    "completion_tokens": 100,
-    "prompt_tokens": 86,
-    "total_tokens": 186
-  }
-}
-```
-
-**Using Compaction Blocks in Follow-up Requests**
-
-To continue the conversation with compaction, include the compaction block in the assistant message's `provider_specific_fields`:
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "How can I build a web scraper?"
-    },
-    {
-      "role": "assistant",
-      "content": [
-        {
-          "type": "text",
-          "text": "Certainly! To build a basic web scraper, you'll typically use a programming language like Python along with libraries such as `requests` (for fetching web pages) and `BeautifulSoup` (for parsing HTML). Here's a basic example:\n\n```python\nimport requests\nfrom bs4 import BeautifulSoup\n\nurl = 'https://example.com'\nresponse = requests.get(url)\nsoup = BeautifulSoup(response.text, 'html.parser')\n\n# Extract and print all text\ntext = soup.get_text()\nprint(text)\n```\n\nLet me know what you're interested in scraping or if you need help with a specific website!"
-        }
-      ],
-      "provider_specific_fields": {
-        "compaction_blocks": [
-          {
-            "type": "compaction",
-            "content": "Summary of the conversation: The user asked how to build a web scraper, and the assistant gave an overview using Python with requests and BeautifulSoup."
-          }
-        ]
-      }
-    },
-    {
-      "role": "user",
-      "content": "How do I use it to scrape product prices?"
-    }
-  ],
-  "context_management": {
-    "edits": [
-      {
-        "type": "compact_20260112"
-      }
-    ]
-  },
-  "max_tokens": 100
-}'
-```
-
-**Streaming Support**
-
-Compaction blocks are also supported in streaming mode. You'll receive:
-- `compaction_start` event when a compaction block begins
-- `compaction_delta` events with the compaction content
-- The accumulated `compaction_blocks` in `provider_specific_fields`
-
-### Adaptive Thinking
-
-:::note
-When using `reasoning_effort` with Claude Opus 4.6, all values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets with `type: "enabled"`, pass the native `thinking` parameter directly (see "Native thinking param" tab below).
-:::
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-LiteLLM supports adaptive thinking through the `reasoning_effort` parameter:
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Solve this complex problem: What is the optimal strategy for..."
-    }
-  ],
-  "reasoning_effort": "high"
-}'
-```
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-Use the `thinking` parameter with `type: "adaptive"` to enable adaptive thinking mode:
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 16000,
-    "thinking": {
-        "type": "adaptive"
-    },
-    "messages": [
-        {
-            "role": "user",
-            "content": "Explain why the sum of two even numbers is always even."
-        }
-    ]
-}'
-```
-
-</TabItem>
-<TabItem value="native" label="Native thinking param">
-
-Use the `thinking` parameter directly for adaptive thinking via the SDK:
-
-```python
-import litellm
-
-response = litellm.completion(
-  model="anthropic/claude-opus-4-6",
-  messages=[{"role": "user", "content": "Solve this complex problem: What is the optimal strategy for..."}],
-  thinking={"type": "adaptive"},
-)
-```
-
-</TabItem>
-</Tabs>
-
-### Effort Levels
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter:
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Explain quantum computing"
-    }
-  ],
-  "output_config": {
-        "effort": "medium"
-    }
-}'
-```
-
-You can use reasoning effort plus output_config to have more control on the model.
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter:
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 4096,
-    "messages": [
-        {
-            "role": "user",
-            "content": "Explain quantum computing"
-        }
-    ],
-    "output_config": {
-        "effort": "medium"
-    }
-}'
-```
-
-</TabItem>
-</Tabs>
-
-### 1M Token Context (Beta)
-
-Opus 4.6 supports 1M token context. Premium pricing applies for prompts exceeding 200k tokens ($10/$37.50 per million input/output tokens). LiteLLM supports cost calculations for 1M token contexts.
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider.
-
-**Step 1: Enable header forwarding in your config**
-
-```yaml
-general_settings:
-  forward_client_headers_to_llm_api: true
-```
-
-**Step 2: Send requests with the beta header**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---header 'anthropic-beta: context-1m-2025-08-07' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Analyze this large document..."
-    }
-  ]
-}'
-```
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider.
-
-**Step 1: Enable header forwarding in your config**
-
-```yaml
-general_settings:
-  forward_client_headers_to_llm_api: true
-```
-
-**Step 2: Send requests with the beta header**
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'anthropic-beta: context-1m-2025-08-07' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 16000,
-    "messages": [
-        {
-            "role": "user",
-            "content": "Analyze this large document..."
-        }
-    ]
-}'
-```
-
-:::tip
-You can combine multiple beta headers by separating them with commas:
-```bash
---header 'anthropic-beta: context-1m-2025-08-07,compact-2026-01-12'
-```
-:::
-
-</TabItem>
-</Tabs>
-
-### US-Only Inference
-
-Available at 1.1× token pricing. LiteLLM automatically tracks costs for US-only inference.
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-Use the `inference_geo` parameter to specify US-only inference:
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "What is the capital of France?"
-    }
-  ],
-  "inference_geo": "us"
-}'
-```
-
-LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking.
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-Use the `inference_geo` parameter to specify US-only inference:
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 4096,
-    "messages": [
-        {
-            "role": "user",
-            "content": "What is the capital of France?"
-        }
-    ],
-    "inference_geo": "us"
-}'
-```
-
-LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking.
-
-</TabItem>
-</Tabs>
-
-### Fast Mode
-
-:::info
-Fast mode is **only supported on the Anthropic provider** (`anthropic/claude-opus-4-6`). It is not available on Azure AI, Vertex AI, or Bedrock.
-:::
-
-**Pricing:**
-- Standard: $5 input / $25 output per MTok
-- Fast: $30 input / $150 output per MTok (6× premium)
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Refactor this module..."
-    }
-  ],
-  "max_tokens": 4096,
-  "speed": "fast"
-}'
-```
-
-**Using OpenAI SDK:**
-
-```python
-import openai
-
-client = openai.OpenAI(
-    api_key="your-litellm-key",
-    base_url="http://0.0.0.0:4000"
-)
-
-response = client.chat.completions.create(
-    model="claude-opus-4-6",
-    messages=[{"role": "user", "content": "Refactor this module..."}],
-    max_tokens=4096,
-    extra_body={"speed": "fast"}
-)
-```
-
-**Using LiteLLM SDK:**
-
-```python
-from litellm import completion
-
-response = completion(
-    model="anthropic/claude-opus-4-6",
-    messages=[{"role": "user", "content": "Refactor this module..."}],
-    max_tokens=4096,
-    speed="fast"
-)
-```
-
-LiteLLM automatically tracks the higher costs for fast mode in usage and cost calculations.
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 4096,
-    "speed": "fast",
-    "messages": [
-        {
-            "role": "user",
-            "content": "Refactor this module..."
-        }
-    ]
-}'
-```
-
-LiteLLM automatically:
-- Adds the `fast-mode-2026-02-01` beta header
-- Tracks the 6× premium pricing in cost calculations
-
-</TabItem>
-</Tabs>

docs/my-website/blog/claude_opus_4_7/index.md

@@ -1,366 +0,0 @@
----
-slug: claude_opus_4_7
-title: "Day 0 Support: Claude Opus 4.7"
-date: 2026-04-16T10:00:00
-authors:
-  - sameer
-  - ishaan-alt
-  - krrish
-description: "Day 0 support for Claude Opus 4.7 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
-tags: [anthropic, claude, opus 4.7]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports [Claude Opus 4.7](https://www.anthropic.com/news/claude-opus-4-7) on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
-
-{/* truncate */}
-
-## Docker Image
-
-```bash
-docker pull ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7
-```
-
-## Usage - Anthropic
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-7
-    litellm_params:
-      model: anthropic/claude-opus-4-7
-      api_key: os.environ/ANTHROPIC_API_KEY
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-7",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Azure
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-7
-    litellm_params:
-      model: azure_ai/claude-opus-4-7
-      api_key: os.environ/AZURE_AI_API_KEY
-      api_base: os.environ/AZURE_AI_API_BASE  # https://<resource>.services.ai.azure.com
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
-  -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-7",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Vertex AI
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-7
-    litellm_params:
-      model: vertex_ai/claude-opus-4-7
-      vertex_project: os.environ/VERTEX_PROJECT
-      vertex_location: us-east5
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e VERTEX_PROJECT=$VERTEX_PROJECT \
-  -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  -v $(pwd)/credentials.json:/app/credentials.json \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-7",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Bedrock
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-7
-    litellm_params:
-      model: bedrock/anthropic.claude-opus-4-7
-      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
-      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
-      aws_region_name: us-east-1
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-  -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-7",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Advanced Features
-
-### Adaptive Thinking
-
-:::note
-When using `reasoning_effort` with Claude Opus 4.7, all values (`low`, `medium`, `high`, `xhigh`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets with `type: "enabled"`, pass the native `thinking` parameter directly.
-:::
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-LiteLLM supports adaptive thinking through the `reasoning_effort` parameter:
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-7",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Solve this complex problem: What is the optimal strategy for..."
-    }
-  ],
-  "reasoning_effort": "high"
-}'
-```
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-Use the `thinking` parameter with `type: "adaptive"` to enable adaptive thinking mode:
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-7",
-    "max_tokens": 16000,
-    "thinking": {
-        "type": "adaptive"
-    },
-    "messages": [
-        {
-            "role": "user",
-            "content": "Explain why the sum of two even numbers is always even."
-        }
-    ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-### Effort Levels
-
-Claude Opus 4.7 supports four effort levels: `low`, `medium`, `high` (default), and `xhigh`. These give you finer-grained control over how much reasoning the model applies to a task. Pass the effort level via the `output_config` parameter.
-
-`xhigh` is a new effort level introduced with Opus 4.7 that sits above `high`. The `max` effort level is Claude Opus 4.6 only and is not available on 4.7.
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-7",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Explain quantum computing"
-    }
-  ],
-  "output_config": {
-    "effort": "xhigh"
-  }
-}'
-```
-
-**Using OpenAI SDK:**
-
-```python
-import openai
-
-client = openai.OpenAI(
-    api_key="your-litellm-key",
-    base_url="http://0.0.0.0:4000"
-)
-
-response = client.chat.completions.create(
-    model="claude-opus-4-7",
-    messages=[{"role": "user", "content": "Explain quantum computing"}],
-    extra_body={"output_config": {"effort": "xhigh"}}
-)
-```
-
-**Using LiteLLM SDK:**
-
-```python
-from litellm import completion
-
-response = completion(
-    model="anthropic/claude-opus-4-7",
-    messages=[{"role": "user", "content": "Explain quantum computing"}],
-    output_config={"effort": "xhigh"},
-)
-```
-
-You can combine `reasoning_effort` with `output_config` for even more fine-grained control over the model's behavior.
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-7",
-    "max_tokens": 4096,
-    "messages": [
-        {
-            "role": "user",
-            "content": "Explain quantum computing"
-        }
-    ],
-    "output_config": {
-        "effort": "xhigh"
-    }
-}'
-```
-
-</TabItem>
-</Tabs>
-
-**Effort level guide:**
-
-| Effort | When to use |
-|--------|-------------|
-| `low` | Short, fast responses — simple lookups, formatting, classification |
-| `medium` | Balanced tradeoff for everyday Q&A and light reasoning |
-| `high` (default) | Complex reasoning, code generation, analysis |
-| `xhigh` | Hardest problems — multi-step math, deep research, agentic planning |
-

docs/my-website/blog/claude_sonnet_4_6/index.md

@@ -1,279 +0,0 @@
----
-slug: claude_sonnet_4_6
-title: "Day 0 Support: Claude Sonnet 4.6"
-date: 2026-02-17T10:00:00
-authors:
-  - ishaan-alt
-  - krrish
-description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
-tags: [anthropic, claude, sonnet 4.6]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
-
-{/* truncate */}
-
-## Docker Image
-
-```bash
-docker pull ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6
-```
-
-## Usage - Anthropic
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-sonnet-4-6
-    litellm_params:
-      model: anthropic/claude-sonnet-4-6
-      api_key: os.environ/ANTHROPIC_API_KEY
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-sonnet-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-response = completion(
-    model="anthropic/claude-sonnet-4-6",
-    messages=[{"role": "user", "content": "what llm are you"}]
-)
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Azure
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-sonnet-4-6
-    litellm_params:
-      model: azure_ai/claude-sonnet-4-6
-      api_key: os.environ/AZURE_AI_API_KEY
-      api_base: os.environ/AZURE_AI_API_BASE  # https://<resource>.services.ai.azure.com
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
-  -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-sonnet-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-response = completion(
-    model="azure_ai/claude-sonnet-4-6",
-    api_key="your-azure-api-key",
-    api_base="https://<resource>.services.ai.azure.com",
-    messages=[{"role": "user", "content": "what llm are you"}]
-)
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Vertex AI
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-sonnet-4-6
-    litellm_params:
-      model: vertex_ai/claude-sonnet-4-6
-      vertex_project: os.environ/VERTEX_PROJECT
-      vertex_location: us-east5
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e VERTEX_PROJECT=$VERTEX_PROJECT \
-  -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  -v $(pwd)/credentials.json:/app/credentials.json \
-  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-sonnet-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-response = completion(
-    model="vertex_ai/claude-sonnet-4-6",
-    vertex_project="your-project-id",
-    vertex_location="us-east5",
-    messages=[{"role": "user", "content": "what llm are you"}]
-)
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Bedrock
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-sonnet-4-6
-    litellm_params:
-      model: bedrock/anthropic.claude-sonnet-4-6-v1
-      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
-      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
-      aws_region_name: us-east-1
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-  -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-sonnet-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-response = completion(
-    model="bedrock/anthropic.claude-sonnet-4-6-v1",
-    aws_access_key_id="your-access-key",
-    aws_secret_access_key="your-secret-key",
-    aws_region_name="us-east-1",
-    messages=[{"role": "user", "content": "what llm are you"}]
-)
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>

docs/my-website/blog/fastapi_middleware_performance/index.mdx

@@ -1,211 +0,0 @@
----
-slug: fastapi-middleware-performance
-title: "Your Middleware Could Be a Bottleneck"
-date: 2026-02-07T10:00:00
-authors:
-  - krrish
-  - ishaan-alt
-  - ryan
-description: "How we improved LiteLLM proxy latency and throughput by replacing a single middleware base class"
-tags: [performance, fastapi, middleware]
-hide_table_of_contents: false
----
-
-import { BaseHTTPMiddlewareAnimation, PureASGIAnimation, BenchmarkVisualization } from '@site/src/components/MiddlewareDiagrams';
-
-> How we improved LiteLLM proxy latency and throughput by replacing a single, simple middleware base class
-
----
-
-## Our Setup
-
-The LiteLLM proxy server has two middleware layers. The first is Starlette's `CORSMiddleware` (re-exported by FastAPI), which is a pure ASGI middleware. Then we have a simple BaseHTTPMiddleware called PrometheusAuthMiddleware.
-
-The job of `PrometheusAuthMiddleware` is to authenticate requests to the `/metrics` endpoint. It's not on by default, you enable it with a flag in your proxy config:
-
-<details>
-<summary>Proxy config flag</summary>
-
-```yaml
-litellm_settings:
-    require_auth_for_metrics_endpoint: true
-```
-
-</details>
-
-The middleware checks two things: is the request hitting `/metrics`, and is auth even enabled? If both checks fail, which they do for the vast majority of requests, it just passes the request through unchanged.
-
-<details>
-<summary>PrometheusAuthMiddleware source</summary>
-
-```python
-class PrometheusAuthMiddleware(BaseHTTPMiddleware):
-    async def dispatch(self, request: Request, call_next):
-        if self._is_prometheus_metrics_endpoint(request):
-            if self._should_run_auth_on_metrics_endpoint() is True:
-                try:
-                    await user_api_key_auth(request=request, api_key=...)
-                except Exception as e:
-                    return JSONResponse(status_code=401, content=...)
-        response = await call_next(request)
-        return response
-
-    @staticmethod
-    def _is_prometheus_metrics_endpoint(request: Request):
-        if "/metrics" in request.url.path:
-            return True
-        return False
-```
-
-</details>
-
-Looks harmless. Subclass `BaseHTTPMiddleware`, implement `dispatch()`, done. This is what you will see in Starlette's documentation<sup>[1](#footnote-1)</sup>.
-
-{/* truncate */}
-
----
-
-## What BaseHTTPMiddleware Actually Does
-
-When you write a `dispatch()` method, you'd expect the request to flow straight through your function and out the other side. What actually happens is much more involved. 
-
-On every request, even a pure passthrough (meaning nothing happens), `BaseHTTPMiddleware` creates **7 intermediate objects and tasks**:
-
-<BaseHTTPMiddlewareAnimation />
-
-It wraps the request in a new object to track body state, creates a synchronization event, allocates an in-memory channel to pass messages between your middleware and the inner app, sets up a task group to manage the lifecycle, and then runs your actual route handler in a *separate background task* when you call `call_next()`. The response body then flows back through that in-memory channel, gets re-wrapped in a streaming response object, and finally reaches the caller. That's a lot.
-
-For a middleware that for us, does nothing on 99.9% of requests, paying this cost doesn't make sense.
-
-Compare that to a pure ASGI middleware, which we can have just check the request path and continue along. 
-
-<PureASGIAnimation />
-
-Our middleware is doing something really simple. For the vast majority of requests it doesn't need to do anything at all but just let the request pass through. It doesn't need task groups, memory streams, or cancel scopes. It needs a function call.
-
----
-
-## Comparing Both
-
-We replaced the `BaseHTTPMiddleware` subclass with a pure ASGI middleware. To benchmark the difference, we used Apache Bench<sup>[2](#footnote-2)</sup> to compare both configurations of LiteLLM's middleware stack: the old setup (1 pure ASGI + 1 `BaseHTTPMiddleware`) against the new setup (2 pure ASGI).
-
-A minimal FastAPI app serves `GET /health` → `PlainTextResponse("ok")`. The endpoint does zero work to isolate the middleware overhead: any difference between configs is purely the cost of the middleware plumbing itself. Both middlewares are just calling the next layer. Same work, different base class.
-
-Apache Bench (`ab`) fires requests at the server with 1,000 concurrent connections and a single uvicorn worker. One worker means one event loop, so the benchmark directly measures how each middleware design handles concurrent load on a single thread.
-
-<BenchmarkVisualization />
-
-<details>
-<summary>Try it yourself</summary>
-
-Save the script below as `benchmark_middleware.py`, then run:
-
-```bash
-# Terminal 1 — start the "before" server (1 ASGI + 1 BaseHTTPMiddleware)
-python benchmark_middleware.py --middleware mixed
-
-# Terminal 2 — benchmark it
-ab -n 50000 -c 1000 http://localhost:8000/health
-
-# Stop the server, then start the "after" server (2x pure ASGI)
-python benchmark_middleware.py --middleware asgi
-
-# Terminal 2 — benchmark again
-ab -n 50000 -c 1000 http://localhost:8000/health
-```
-
-```python
-import argparse
-import uvicorn
-from fastapi import FastAPI
-from fastapi.responses import PlainTextResponse
-from starlette.middleware.base import BaseHTTPMiddleware
-from starlette.requests import Request
-from starlette.types import ASGIApp, Receive, Scope, Send
-
-
-class NoOpBaseHTTPMiddleware(BaseHTTPMiddleware):
-    async def dispatch(self, request: Request, call_next):
-        return await call_next(request)
-
-
-class NoOpPureASGIMiddleware:
-    def __init__(self, app: ASGIApp) -> None:
-        self.app = app
-
-    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
-        await self.app(scope, receive, send)
-
-
-def create_app(middleware_type: str | None = None, layers: int = 2) -> FastAPI:
-    app = FastAPI()
-
-    @app.get("/health")
-    async def health():
-        return PlainTextResponse("ok")
-
-    if middleware_type == "mixed":
-        app.add_middleware(NoOpBaseHTTPMiddleware)
-        app.add_middleware(NoOpPureASGIMiddleware)
-    elif middleware_type == "asgi":
-        for _ in range(layers):
-            app.add_middleware(NoOpPureASGIMiddleware)
-
-    return app
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--middleware", choices=["asgi", "mixed"], default=None)
-    parser.add_argument("--layers", type=int, default=2)
-    parser.add_argument("--port", type=int, default=8000)
-    args = parser.parse_args()
-
-    app = create_app(middleware_type=args.middleware, layers=args.layers)
-    uvicorn.run(app, host="0.0.0.0", port=args.port, workers=1, log_level="warning")
-```
-
-</details>
-
----
-
-## Our Change
-
-Here's what we replaced it with:
-
-```python
-class PrometheusAuthMiddleware:
-    def __init__(self, app: ASGIApp) -> None:
-        self.app = app
-
-    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
-        if scope["type"] != "http" or "/metrics" not in scope.get("path", ""):
-            await self.app(scope, receive, send)
-            return
-
-        if litellm.require_auth_for_metrics_endpoint is True:
-            request = Request(scope, receive)
-            api_key = request.headers.get("Authorization") or ""
-            try:
-                await user_api_key_auth(request=request, api_key=api_key)
-            except Exception as e:
-                # send 401 directly via ASGI protocol
-                ...
-                return
-
-        await self.app(scope, receive, send)
-```
-
-For the 99.9% of requests that aren't hitting `/metrics`, the middleware is now one dict lookup, one string check, and one function call. No objects allocated, no tasks spawned. 
-
-It's important to evaluate if the tools you're using are the right fit for the job as your software grows and handles more responsiblity. We're now putting in a static analysis check to prevent this from happening again with any newly introduced middlewares. If we find the use case is necessary then that's okay and we'll reevalute but for everything LiteLLM needs to do at the moment it's not. 
-
-This middleware change was one part of a broader optimization effort on the LiteLLM proxy. Across all optimizations combined, we've measured about a **30% reduction in proxy overhead** over the past two weeks. 
-
----
-
-<a id="footnote-1"></a>
-<sup>1</sup> [Starlette Middleware — BaseHTTPMiddleware](https://starlette.dev/middleware/#basehttpmiddleware)
-
-<a id="footnote-2"></a>
-<sup>2</sup> [Apache HTTP server benchmarking tool (`ab`)](https://httpd.apache.org/docs/2.4/programs/ab.html)

docs/my-website/blog/gemin_3.1/index.md

@@ -1,142 +0,0 @@
----
-slug: gemini_3_1_pro
-title: "DAY 0 Support: Gemini 3.1 Pro on LiteLLM"
-date: 2026-02-19T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Guide to using Gemini 3.1 Pro on LiteLLM Proxy and SDK with day 0 support."
-tags: [gemini, day 0 support, llms]
-hide_table_of_contents: false
----
-
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Gemini 3.1 Pro Day 0 Support 
-
-LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along with it.
-
-{/* truncate */}
-
-## Deploy this version
-
-<Tabs>
-<TabItem value="docker" label="Docker">
-
-``` showLineNumbers title="docker run litellm"
-docker run \
--e STORE_MODEL_IN_DB=True \
--p 4000:4000 \
-ghcr.io/berriai/litellm:main-v1.81.9-stable.gemini.3.1-pro
-```
-
-</TabItem>
-
-<TabItem value="pip" label="Pip">
-
-``` showLineNumbers title="pip install litellm"
-pip install litellm==v1.81.9-stable.gemini.3.1-pro
-```
-
-</TabItem>
-</Tabs>
-
-## What's New
-
-### 1. New Thinking Levels: `thinkingLevel` with MINIMAL & MEDIUM
-
-Gemini 3.1 Pro introduces support for **medium** thinking level
-
-LiteLLM automatically maps the OpenAI `reasoning_effort` parameter to Gemini's `thinkingLevel`, so you can use familiar `reasoning_effort` values (`minimal`, `low`, `medium`, `high`) without changing your code!
-
----
-## Supported Endpoints
-
-LiteLLM provides **full end-to-end support** for Gemini 3.1 Pro on:
-
-- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
-- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
-- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
-- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
-
-All endpoints support:
-- Streaming and non-streaming responses
-- Function calling with thought signatures
-- Multi-turn conversations
-- All Gemini 3-specific features
-- Conversion of provider specific thinking related param to thinkingLevel
-
-## Quick Start
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-**Basic Usage with MEDIUM thinking (NEW)**
-
-```python
-from litellm import completion
-
-# No need to make any changes to your code as we map openai reasoning param to thinkingLevel
-response = completion(
-    model="gemini/gemini-3.1-pro-preview",
-    messages=[{"role": "user", "content": "Solve this complex math problem: 25 * 4 + 10"}],
-    reasoning_effort="medium",  # NEW: MEDIUM thinking level
-)
-
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="PROXY">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gemini-3.1-pro-preview
-    litellm_params:
-      model: gemini/gemini-3.1-pro-preview
-      api_key: os.environ/GEMINI_API_KEY
-  - model_name: vertex-gemini-3.1-pro-preview
-    litellm_params:
-      model: vertex_ai/gemini-3.1-pro-preview
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-**3. Call with MEDIUM thinking**
-
-```bash
-curl -X POST http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-  -d '{
-    "model": "gemini-3.1-pro-preview",
-    "messages": [{"role": "user", "content": "Complex reasoning task"}],
-    "reasoning_effort": "medium"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
----
-
-## `reasoning_effort` Mapping for Gemini 3+
-
-| reasoning_effort | thinking_level | 
-|------------------|----------------|
-| `minimal` | `minimal` |
-| `low` | `low` |
-| `medium` | `medium` |
-| `high` | `high` |
-| `disable` | `minimal` |
-| `none` | `minimal` |

docs/my-website/blog/gemini_3/index.md

@@ -1,975 +0,0 @@
----
-slug: gemini_3
-title: "DAY 0 Support: Gemini 3 on LiteLLM"
-date: 2025-11-19T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK."
-tags: [gemini, day 0 support, llms]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-:::info
-
-This guide covers common questions and best practices for using `gemini-3-pro-preview` with LiteLLM Proxy and SDK.
-
-:::
-
-{/* truncate */}
-
-## Quick Start
-
-<Tabs>
-<TabItem value="sdk" label="Python SDK">
-
-```python
-from litellm import completion
-import os
-
-os.environ["GEMINI_API_KEY"] = "your-api-key"
-
-response = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "Hello!"}],
-    reasoning_effort="low"
-)
-
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Add to config.yaml:**
-
-```yaml
-model_list:
-  - model_name: gemini-3-pro-preview
-    litellm_params:
-      model: gemini/gemini-3-pro-preview
-      api_key: os.environ/GEMINI_API_KEY
-```
-
-**2. Start proxy:**
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-**3. Make request:**
-
-```bash
-curl http://0.0.0.0:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",
-    "messages": [{"role": "user", "content": "Hello!"}],
-    "reasoning_effort": "low"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-## Supported Endpoints
-
-LiteLLM provides **full end-to-end support** for Gemini 3 Pro Preview on:
-
-- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
-- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
-- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
-- ✅ `/v1/generateContent` – [Google Gemini API](https://cloud.google.com/vertex-ai/docs/generative-ai/model-reference/gemini#rest) compatible endpoint (for code, see: `client.models.generate_content(...)`)
-
-All endpoints support:
-- Streaming and non-streaming responses
-- Function calling with thought signatures
-- Multi-turn conversations
-- All Gemini 3-specific features
-
-## Thought Signatures
-
-#### What are Thought Signatures?
-
-Thought signatures are encrypted representations of the model's internal reasoning process. They're essential for maintaining context across multi-turn conversations, especially with function calling.
-
-#### How Thought Signatures Work
-
-1. **Automatic Extraction**: When Gemini 3 returns a function call, LiteLLM automatically extracts the `thought_signature` from the response
-2. **Storage**: Thought signatures are stored in `provider_specific_fields.thought_signature` of tool calls
-3. **Automatic Preservation**: When you include the assistant's message in conversation history, LiteLLM automatically preserves and returns thought signatures to Gemini
-
-## Example: Multi-Turn Function Calling
-
-#### Streaming with Thought Signatures
-
-When using streaming mode with `stream_chunk_builder()`, thought signatures are now automatically preserved:
-
-<Tabs>
-<TabItem value="streaming" label="Streaming SDK">
-
-```python
-import os
-import litellm
-from litellm import completion
-
-os.environ["GEMINI_API_KEY"] = "your-api-key"
-
-MODEL = "gemini/gemini-3-pro-preview"
-
-messages = [
-    {"role": "system", "content": "You are a helpful assistant. Use the calculate tool."},
-    {"role": "user", "content": "What is 2+2?"},
-]
-
-tools = [{
-    "type": "function",
-    "function": {
-        "name": "calculate",
-        "description": "Calculate a mathematical expression",
-        "parameters": {
-            "type": "object",
-            "properties": {"expression": {"type": "string"}},
-            "required": ["expression"],
-        },
-    },
-}]
-
-print("Step 1: Sending request with stream=True...")
-response = completion(
-    model=MODEL,
-    messages=messages,
-    stream=True,
-    tools=tools,
-    reasoning_effort="low"
-)
-
-# Collect all chunks
-chunks = []
-for part in response:
-    chunks.append(part)
-
-# Reconstruct message using stream_chunk_builder
-# Thought signatures are now preserved automatically!
-full_response = litellm.stream_chunk_builder(chunks, messages=messages)
-print(f"Full response: {full_response}")
-
-assistant_msg = full_response.choices[0].message
-
-# ✅ Thought signature is now preserved in provider_specific_fields
-if assistant_msg.tool_calls and assistant_msg.tool_calls[0].provider_specific_fields:
-    thought_sig = assistant_msg.tool_calls[0].provider_specific_fields.get("thought_signature")
-    print(f"Thought signature preserved: {thought_sig is not None}")
-
-# Append assistant message (includes thought signatures automatically)
-messages.append(assistant_msg)
-
-# Mock tool execution
-messages.append({
-    "role": "tool",
-    "content": "4",
-    "tool_call_id": assistant_msg.tool_calls[0].id
-})
-
-print("\nStep 2: Sending tool result back to model...")
-response_2 = completion(
-    model=MODEL,
-    messages=messages,
-    stream=True,
-    tools=tools,
-    reasoning_effort="low"
-)
-
-for part in response_2:
-    if part.choices[0].delta.content:
-        print(part.choices[0].delta.content, end="")
-print()  # New line
-```
-
-**Key Points:**
-- ✅ `stream_chunk_builder()` now preserves `provider_specific_fields` including thought signatures
-- ✅ Thought signatures are automatically included when appending `assistant_msg` to conversation history
-- ✅ Multi-turn conversations work seamlessly with streaming
-
-</TabItem>
-<TabItem value="sdk" label="Non-Streaming SDK">
-
-```python
-from openai import OpenAI
-import json
-
-client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
-
-# Define tools
-tools = [
-    {
-        "type": "function",
-        "function": {
-            "name": "get_weather",
-            "description": "Get the current weather",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "location": {"type": "string"}
-                },
-                "required": ["location"]
-            }
-        }
-    }
-]
-
-# Step 1: Initial request
-messages = [{"role": "user", "content": "What's the weather in Tokyo?"}]
-
-response = client.chat.completions.create(
-    model="gemini-3-pro-preview",
-    messages=messages,
-    tools=tools,
-    reasoning_effort="low"
-)
-
-# Step 2: Append assistant message (thought signatures automatically preserved)
-messages.append(response.choices[0].message)
-
-# Step 3: Execute tool and append result
-for tool_call in response.choices[0].message.tool_calls:
-    if tool_call.function.name == "get_weather":
-        result = {"temperature": 30, "unit": "celsius"}
-        messages.append({
-            "role": "tool",
-            "content": json.dumps(result),
-            "tool_call_id": tool_call.id
-        })
-
-# Step 4: Follow-up request (thought signatures automatically included)
-response2 = client.chat.completions.create(
-    model="gemini-3-pro-preview",
-    messages=messages,
-    tools=tools,
-    reasoning_effort="low"
-)
-
-print(response2.choices[0].message.content)
-```
-
-**Key Points:**
-- ✅ Thought signatures are automatically extracted from `response.choices[0].message.tool_calls[].provider_specific_fields.thought_signature`
-- ✅ When you append `response.choices[0].message` to your conversation history, thought signatures are automatically preserved
-- ✅ You don't need to manually extract or manage thought signatures
-
-</TabItem>
-<TabItem value="proxy" label="cURL">
-
-```bash
-# Step 1: Initial request
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",
-    "messages": [
-      {"role": "user", "content": "What'\''s the weather in Tokyo?"}
-    ],
-    "tools": [
-      {
-        "type": "function",
-        "function": {
-          "name": "get_weather",
-          "description": "Get the current weather",
-          "parameters": {
-            "type": "object",
-            "properties": {
-              "location": {"type": "string"}
-            },
-            "required": ["location"]
-          }
-        }
-      }
-    ],
-    "reasoning_effort": "low"
-  }'
-```
-
-**Response includes thought signature:**
-
-```json
-{
-  "choices": [{
-    "message": {
-      "role": "assistant",
-      "tool_calls": [{
-        "id": "call_abc123",
-        "type": "function",
-        "function": {
-          "name": "get_weather",
-          "arguments": "{\"location\": \"Tokyo\"}"
-        },
-        "provider_specific_fields": {
-          "thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..."
-        }
-      }]
-    }
-  }]
-}
-```
-
-```bash
-# Step 2: Follow-up request (include assistant message with thought signature)
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",
-    "messages": [
-      {"role": "user", "content": "What'\''s the weather in Tokyo?"},
-      {
-        "role": "assistant",
-        "content": null,
-        "tool_calls": [{
-          "id": "call_abc123",
-          "type": "function",
-          "function": {
-            "name": "get_weather",
-            "arguments": "{\"location\": \"Tokyo\"}"
-          },
-          "provider_specific_fields": {
-            "thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..."
-          }
-        }]
-      },
-      {
-        "role": "tool",
-        "content": "{\"temperature\": 30, \"unit\": \"celsius\"}",
-        "tool_call_id": "call_abc123"
-      }
-    ],
-    "tools": [...],
-    "reasoning_effort": "low"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-#### Important Notes on Thought Signatures
-
-1. **Automatic Handling**: LiteLLM automatically extracts and preserves thought signatures. You don't need to manually manage them.
-
-2. **Parallel Function Calls**: When the model makes parallel function calls, only the **first function call** has a thought signature.
-
-3. **Sequential Function Calls**: In multi-step function calling, each step's first function call has its own thought signature that must be preserved.
-
-4. **Required for Context**: Thought signatures are essential for maintaining reasoning context. Without them, the model may lose context of its previous reasoning.
-
-## Conversation History: Switching from Non-Gemini-3 Models
-
-#### Common Question: Will switching from a non-Gemini-3 model to Gemini-3 break conversation history?
-
-**Answer: No!** LiteLLM automatically handles this by adding dummy thought signatures when needed.
-
-#### How It Works
-
-When you switch from a model that doesn't use thought signatures (e.g., `gemini-2.5-flash`) to Gemini 3, LiteLLM:
-
-1. **Detects missing signatures**: Identifies assistant messages with tool calls that lack thought signatures
-2. **Adds dummy signature**: Automatically injects a dummy thought signature (`skip_thought_signature_validator`) for compatibility
-3. **Maintains conversation flow**: Your conversation history continues to work seamlessly
-
-#### Example: Switching Models Mid-Conversation
-
-<Tabs>
-<TabItem value="sdk" label="Python SDK">
-
-```python
-from openai import OpenAI
-
-client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
-
-# Step 1: Start with gemini-2.5-flash (no thought signatures)
-messages = [{"role": "user", "content": "What's the weather?"}]
-
-response1 = client.chat.completions.create(
-    model="gemini-2.5-flash",
-    messages=messages,
-    tools=[...],
-    reasoning_effort="low"
-)
-
-# Append assistant message (no tool call thought signature from gemini-2.5-flash)
-messages.append(response1.choices[0].message)
-
-# Step 2: Switch to gemini-3-pro-preview
-# LiteLLM automatically adds dummy thought signature to the previous assistant message
-response2 = client.chat.completions.create(
-    model="gemini-3-pro-preview",  # 👈 Switched model
-    messages=messages,  # 👈 Same conversation history
-    tools=[...],
-    reasoning_effort="low"
-)
-
-# ✅ Works seamlessly! No errors, no breaking changes
-print(response2.choices[0].message.content)
-```
-
-</TabItem>
-<TabItem value="proxy" label="cURL">
-
-```bash
-# Step 1: Start with gemini-2.5-flash
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-2.5-flash",
-    "messages": [{"role": "user", "content": "What'\''s the weather?"}],
-    "tools": [...],
-    "reasoning_effort": "low"
-  }'
-
-# Step 2: Switch to gemini-3-pro-preview with same conversation history
-# LiteLLM automatically handles the missing thought signature
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",  # 👈 Switched model
-    "messages": [
-      {"role": "user", "content": "What'\''s the weather?"},
-      {
-        "role": "assistant",
-        "tool_calls": [...]  # 👈 No thought_signature from gemini-2.5-flash
-      }
-    ],
-    "tools": [...],
-    "reasoning_effort": "low"
-  }'
-# ✅ Works! LiteLLM adds dummy signature automatically
-```
-
-</TabItem>
-</Tabs>
-
-#### Dummy Signature Details
-
-The dummy signature used is: `base64("skip_thought_signature_validator")`
-
-This is the recommended approach by Google for handling conversation history from models that don't support thought signatures. It allows Gemini 3 to:
-- Accept the conversation history without validation errors
-- Continue the conversation seamlessly
-- Maintain context across model switches
-
-## Thinking Level Parameter
-
-#### How `reasoning_effort` Maps to `thinking_level`
-
-For Gemini 3 Pro Preview, LiteLLM automatically maps `reasoning_effort` to the new `thinking_level` parameter:
-
-| `reasoning_effort` | `thinking_level` | Notes |
-|-------------------|------------------|-------|
-| `"minimal"` | `"low"` | Maps to low thinking level |
-| `"low"` | `"low"` | Default for most use cases |
-| `"medium"` | `"high"` | Medium not available yet, maps to high |
-| `"high"` | `"high"` | Maximum reasoning depth |
-| `"disable"` | `"low"` | Gemini 3 cannot fully disable thinking |
-| `"none"` | `"low"` | Gemini 3 cannot fully disable thinking |
-
-#### Default Behavior
-
-If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for Gemini 3 models, to avoid high costs. 
-
-### Example Usage
-
-<Tabs>
-<TabItem value="sdk" label="Python SDK">
-
-```python
-from litellm import completion
-
-# Low thinking level (faster, lower cost)
-response = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "What's the weather?"}],
-    reasoning_effort="low"  # Maps to thinking_level="low"
-)
-
-# High thinking level (deeper reasoning, higher cost)
-response = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "Solve this complex math problem step by step."}],
-    reasoning_effort="high"  # Maps to thinking_level="high"
-)
-```
-
-</TabItem>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-```bash
-# Low thinking level
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",
-    "messages": [{"role": "user", "content": "What'\''s the weather?"}],
-    "reasoning_effort": "low"
-  }'
-
-# High thinking level
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",
-    "messages": [{"role": "user", "content": "Solve this complex problem."}],
-    "reasoning_effort": "high"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-## Important Notes
-
-1. **Gemini 3 Cannot Disable Thinking**: Unlike Gemini 2.5 models, Gemini 3 cannot fully disable thinking. Even when you set `reasoning_effort="none"` or `"disable"`, it maps to `thinking_level="low"`.
-
-2. **Temperature Recommendation**: For Gemini 3 models, LiteLLM defaults `temperature` to `1.0` and strongly recommends keeping it at this default. Setting `temperature < 1.0` can cause:
-   - Infinite loops
-   - Degraded reasoning performance
-   - Failure on complex tasks
-
-3. **Automatic Defaults**: If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for optimal performance.
-
-## Cost Tracking: Prompt Caching & Context Window
-
-LiteLLM provides comprehensive cost tracking for Gemini 3 Pro Preview, including support for prompt caching and tiered pricing based on context window size.
-
-### Prompt Caching Cost Tracking
-
-Gemini 3 supports prompt caching, which allows you to cache frequently used prompt prefixes to reduce costs. LiteLLM automatically tracks and calculates costs for:
-
-- **Cache Hit Tokens**: Tokens that are read from cache (charged at a lower rate)
-- **Cache Creation Tokens**: Tokens that are written to cache (one-time cost)
-- **Text Tokens**: Regular prompt tokens that are processed normally
-
-#### How It Works
-
-LiteLLM extracts caching information from the `prompt_tokens_details` field in the usage object:
-
-```python
-{
-  "usage": {
-    "prompt_tokens": 50000,
-    "completion_tokens": 1000,
-    "total_tokens": 51000,
-    "prompt_tokens_details": {
-      "cached_tokens": 30000,  # Cache hit tokens
-      "cache_creation_tokens": 5000,  # Tokens written to cache
-      "text_tokens": 15000  # Regular processed tokens
-    }
-  }
-}
-```
-
-### Context Window Tiered Pricing
-
-Gemini 3 Pro Preview supports up to 1M tokens of context, with tiered pricing that automatically applies when your prompt exceeds 200k tokens.
-
-#### Automatic Tier Detection
-
-LiteLLM automatically detects when your prompt exceeds the 200k token threshold and applies the appropriate tiered pricing:
-
-```python
-from litellm import completion_cost
-
-# Example: Small prompt (< 200k tokens)
-response_small = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "Hello!"}]
-)
-# Uses base pricing: $0.000002/input token, $0.000012/output token
-
-# Example: Large prompt (> 200k tokens)
-response_large = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "..." * 250000}]  # 250k tokens
-)
-# Automatically uses tiered pricing: $0.000004/input token, $0.000018/output token
-```
-
-#### Cost Breakdown
-
-The cost calculation includes:
-
-1. **Text Processing Cost**: Regular tokens processed at base or tiered rate
-2. **Cache Read Cost**: Cached tokens read at discounted rate
-3. **Cache Creation Cost**: One-time cost for writing tokens to cache (applies tiered rate if above 200k)
-4. **Output Cost**: Generated tokens at base or tiered rate
-
-### Example: Viewing Cost Breakdown
-
-You can view the detailed cost breakdown using LiteLLM's cost tracking:
-
-```python
-from litellm import completion, completion_cost
-
-response = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "Explain prompt caching"}],
-    caching=True  # Enable prompt caching
-)
-
-# Get total cost
-total_cost = completion_cost(completion_response=response)
-print(f"Total cost: ${total_cost:.6f}")
-
-# Access usage details
-usage = response.usage
-print(f"Prompt tokens: {usage.prompt_tokens}")
-print(f"Completion tokens: {usage.completion_tokens}")
-
-# Access caching details
-if usage.prompt_tokens_details:
-    print(f"Cache hit tokens: {usage.prompt_tokens_details.cached_tokens}")
-    print(f"Cache creation tokens: {usage.prompt_tokens_details.cache_creation_tokens}")
-    print(f"Text tokens: {usage.prompt_tokens_details.text_tokens}")
-```
-
-### Cost Optimization Tips
-
-1. **Use Prompt Caching**: For repeated prompt prefixes, enable caching to reduce costs by up to 90% for cached portions
-2. **Monitor Context Size**: Be aware that prompts above 200k tokens use tiered pricing (2x for input, 1.5x for output)
-3. **Cache Management**: Cache creation tokens are charged once when writing to cache, then subsequent reads are much cheaper
-4. **Track Usage**: Use LiteLLM's built-in cost tracking to monitor spending across different token types
-
-### Integration with LiteLLM Proxy
-
-When using LiteLLM Proxy, all cost tracking is automatically logged and available through:
-
-- **Usage Logs**: Detailed token and cost breakdowns in proxy logs
-- **Budget Management**: Set budgets and alerts based on actual usage
-- **Analytics Dashboard**: View cost trends and breakdowns by token type
-
-```yaml
-# config.yaml
-model_list:
-  - model_name: gemini-3-pro-preview
-    litellm_params:
-      model: gemini/gemini-3-pro-preview
-      api_key: os.environ/GEMINI_API_KEY
-
-litellm_settings:
-  # Enable detailed cost tracking
-  success_callback: ["langfuse"]  # or your preferred logging service
-```
-
-## Using with Claude Code CLI
-
-You can use `gemini-3-pro-preview` with **Claude Code CLI** - Anthropic's command-line interface. This allows you to use Gemini 3 Pro Preview with Claude Code's native syntax and workflows.
-
-### Setup
-
-**1. Add Gemini 3 Pro Preview to your `config.yaml`:**
-
-```yaml
-model_list:
-  - model_name: gemini-3-pro-preview
-    litellm_params:
-      model: gemini/gemini-3-pro-preview
-      api_key: os.environ/GEMINI_API_KEY
-
-litellm_settings:
-  master_key: os.environ/LITELLM_MASTER_KEY
-```
-
-**2. Set environment variables:**
-
-```bash
-export GEMINI_API_KEY="your-gemini-api-key"
-export LITELLM_MASTER_KEY="sk-1234567890"  # Generate a secure key
-```
-
-**3. Start LiteLLM Proxy:**
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING on http://0.0.0.0:4000
-```
-
-**4. Configure Claude Code to use LiteLLM Proxy:**
-
-```bash
-export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
-export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
-```
-
-**5. Use Gemini 3 Pro Preview with Claude Code:**
-
-```bash
-# Claude Code will use gemini-3-pro-preview from your LiteLLM proxy
-claude --model gemini-3-pro-preview
-
-```
-
-### Example Usage
-
-Once configured, you can interact with Gemini 3 Pro Preview using Claude Code's native interface:
-
-```bash
-$ claude --model gemini-3-pro-preview
-> Explain how thought signatures work in multi-turn conversations.
-
-# Gemini 3 Pro Preview responds through Claude Code interface
-```
-
-### Benefits
-
-- ✅ **Native Claude Code Experience**: Use Gemini 3 Pro Preview with Claude Code's familiar CLI interface
-- ✅ **Unified Authentication**: Single API key for all models through LiteLLM proxy
-- ✅ **Cost Tracking**: All usage tracked through LiteLLM's centralized logging
-- ✅ **Seamless Model Switching**: Easily switch between Claude and Gemini models
-- ✅ **Full Feature Support**: All Gemini 3 features (thought signatures, function calling, etc.) work through Claude Code
-
-### Troubleshooting
-
-**Claude Code not finding the model:**
-- Ensure the model name in Claude Code matches exactly: `gemini-3-pro-preview`
-- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
-- Check that `ANTHROPIC_BASE_URL` points to your LiteLLM proxy
-
-**Authentication errors:**
-- Verify `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
-- Ensure `GEMINI_API_KEY` is set correctly
-- Check LiteLLM proxy logs for detailed error messages
-
-## Responses API Support
-
-LiteLLM fully supports the OpenAI Responses API for Gemini 3 Pro Preview, including both streaming and non-streaming modes. The Responses API provides a structured way to handle multi-turn conversations with function calling, and LiteLLM automatically preserves thought signatures throughout the conversation.
-
-### Example: Using Responses API with Gemini 3
-
-<Tabs>
-<TabItem value="sdk" label="Non-Streaming">
-
-```python
-from openai import OpenAI
-import json
-
-client = OpenAI()
-
-# 1. Define a list of callable tools for the model
-tools = [
-    {
-        "type": "function",
-        "name": "get_horoscope",
-        "description": "Get today's horoscope for an astrological sign.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "sign": {
-                    "type": "string",
-                    "description": "An astrological sign like Taurus or Aquarius",
-                },
-            },
-            "required": ["sign"],
-        },
-    },
-]
-
-def get_horoscope(sign):
-    return f"{sign}: Next Tuesday you will befriend a baby otter."
-
-# Create a running input list we will add to over time
-input_list = [
-    {"role": "user", "content": "What is my horoscope? I am an Aquarius."}
-]
-
-# 2. Prompt the model with tools defined
-response = client.responses.create(
-    model="gemini-3-pro-preview",
-    tools=tools,
-    input=input_list,
-)
-
-# Save function call outputs for subsequent requests
-input_list += response.output
-
-for item in response.output:
-    if item.type == "function_call":
-        if item.name == "get_horoscope":
-            # 3. Execute the function logic for get_horoscope
-            horoscope = get_horoscope(json.loads(item.arguments))
-            
-            # 4. Provide function call results to the model
-            input_list.append({
-                "type": "function_call_output",
-                "call_id": item.call_id,
-                "output": json.dumps({
-                  "horoscope": horoscope
-                })
-            })
-
-print("Final input:")
-print(input_list)
-
-response = client.responses.create(
-    model="gemini-3-pro-preview",
-    instructions="Respond only with a horoscope generated by a tool.",
-    tools=tools,
-    input=input_list,
-)
-
-# 5. The model should be able to give a response!
-print("Final output:")
-print(response.model_dump_json(indent=2))
-print("\n" + response.output_text)
-```
-
-**Key Points:**
-- ✅ Thought signatures are automatically preserved in function calls
-- ✅ Works seamlessly with multi-turn conversations
-- ✅ All Gemini 3-specific features are fully supported
-
-</TabItem>
-<TabItem value="streaming" label="Streaming">
-
-```python
-from openai import OpenAI
-import json
-
-client = OpenAI()
-
-tools = [
-    {
-        "type": "function",
-        "name": "get_horoscope",
-        "description": "Get today's horoscope for an astrological sign.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "sign": {
-                    "type": "string",
-                    "description": "An astrological sign like Taurus or Aquarius",
-                },
-            },
-            "required": ["sign"],
-        },
-    },
-]
-
-def get_horoscope(sign):
-    return f"{sign}: Next Tuesday you will befriend a baby otter."
-
-input_list = [
-    {"role": "user", "content": "What is my horoscope? I am an Aquarius."}
-]
-
-# Streaming mode
-response = client.responses.create(
-    model="gemini-3-pro-preview",
-    tools=tools,
-    input=input_list,
-    stream=True,
-)
-
-# Collect all chunks
-chunks = []
-for chunk in response:
-    chunks.append(chunk)
-    # Process streaming chunks as they arrive
-    print(chunk)
-
-# Thought signatures are automatically preserved in streaming mode
-```
-
-**Key Points:**
-- ✅ Streaming mode fully supported
-- ✅ Thought signatures preserved across streaming chunks
-- ✅ Real-time processing of function calls and responses
-
-</TabItem>
-</Tabs>
-
-### Responses API Benefits
-
-- ✅ **Structured Output**: Responses API provides a clear structure for handling function calls and multi-turn conversations
-- ✅ **Thought Signature Preservation**: LiteLLM automatically preserves thought signatures in both streaming and non-streaming modes
-- ✅ **Seamless Integration**: Works with existing OpenAI SDK patterns
-- ✅ **Full Feature Support**: All Gemini 3 features (thought signatures, function calling, reasoning) are fully supported
-
-
-## Best Practices
-
-#### 1. Always Include Thought Signatures in Conversation History
-
-When building multi-turn conversations with function calling:
-
-✅ **Do:**
-```python
-# Append the full assistant message (includes thought signatures)
-messages.append(response.choices[0].message)
-```
-
-❌ **Don't:**
-```python
-# Don't manually construct assistant messages without thought signatures
-messages.append({
-    "role": "assistant",
-    "tool_calls": [...]  # Missing thought signatures!
-})
-```
-
-#### 2. Use Appropriate Thinking Levels
-
-- **`reasoning_effort="low"`**: For simple queries, quick responses, cost optimization
-- **`reasoning_effort="high"`**: For complex problems requiring deep reasoning
-
-#### 3. Keep Temperature at Default
-
-For Gemini 3 models, always use `temperature=1.0` (default). Lower temperatures can cause issues.
-
-#### 4. Handle Model Switches Gracefully
-
-When switching from non-Gemini-3 to Gemini-3:
-- ✅ LiteLLM automatically handles missing thought signatures
-- ✅ No manual intervention needed
-- ✅ Conversation history continues seamlessly
-
-
-## Troubleshooting
-
-#### Issue: Missing Thought Signatures
-
-**Symptom**: Error when including assistant messages in conversation history
-
-**Solution**: Ensure you're appending the full assistant message from the response:
-```python
-messages.append(response.choices[0].message)  # ✅ Includes thought signatures
-```
-
-#### Issue: Conversation Breaks When Switching Models
-
-**Symptom**: Errors when switching from gemini-2.5-flash to gemini-3-pro-preview
-
-**Solution**: This should work automatically! LiteLLM adds dummy signatures. If you see errors, ensure you're using the latest LiteLLM version.
-
-#### Issue: Infinite Loops or Poor Performance
-
-**Symptom**: Model gets stuck or produces poor results
-
-**Solution**: 
-- Ensure `temperature=1.0` (default for Gemini 3)
-- Check that `reasoning_effort` is set appropriately
-- Verify you're using the correct model name: `gemini/gemini-3-pro-preview`
-
-## Additional Resources
-
-- [Gemini Provider Documentation](../../docs/providers/gemini)
-- [Thought Signatures Guide](../../docs/providers/gemini#thought-signatures)
-- [Reasoning Content Documentation](../../docs/reasoning_content)
-- [Function Calling Guide](../../docs/completion/function_call)

docs/my-website/blog/gemini_3_1_flash_lite/index.md

@@ -1,168 +0,0 @@
----
-slug: gemini_3_1_flash_lite_preview
-title: "DAY 0 Support: Gemini 3.1 Flash Lite Preview on LiteLLM"
-date: 2026-03-03T08:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Guide to using Gemini 3.1 Flash Lite Preview on LiteLLM Proxy and SDK with day 0 support."
-tags: [gemini, day 0 support, llms, supernova]
-hide_table_of_contents: false
----
-
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Gemini 3.1 Flash Lite Preview Day 0 Support 
-
-LiteLLM now supports `gemini-3.1-flash-lite-preview` with full day 0 support!
-
-:::note
-If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
-:::
-
-{/* truncate */}
-
-## Deploy this version
-
-<Tabs>
-<TabItem value="docker" label="Docker">
-
-``` showLineNumbers title="docker run litellm"
-docker run \
--e STORE_MODEL_IN_DB=True \
--p 4000:4000 \
-ghcr.io/berriai/litellm:main-v1.80.8-stable.1
-```
-
-</TabItem>
-
-<TabItem value="pip" label="Pip">
-
-``` showLineNumbers title="pip install litellm"
-pip install litellm==v1.80.8-stable.1
-```
-
-</TabItem>
-</Tabs>
-
-## What's New
-
-Supports all four thinking levels:
-- **MINIMAL**: Ultra-fast responses with minimal reasoning
-- **LOW**: Simple instruction following
-- **MEDIUM**: Balanced reasoning for complex tasks
-- **HIGH**: Maximum reasoning depth (dynamic)
-
----
-
-## Quick Start
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-**Basic Usage**
-
-```python
-from litellm import completion
-
-response = completion(
-    model="gemini/gemini-3.1-flash-lite-preview",
-    messages=[{"role": "user", "content": "Extract key entities from this text: ..."}],
-)
-
-print(response.choices[0].message.content)
-```
-
-**With Thinking Levels**
-
-```python
-from litellm import completion
-
-# Use MEDIUM thinking for complex reasoning tasks
-response = completion(
-    model="gemini/gemini-3.1-flash-lite-preview",
-    messages=[{"role": "user", "content": "Analyze this dataset and identify patterns"}],
-    reasoning_effort="medium",  # low, medium , high
-)
-
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="PROXY">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gemini-3.1-flash-lite
-    litellm_params:
-      model: gemini/gemini-3.1-flash-lite-preview
-      api_key: os.environ/GEMINI_API_KEY
-  
-  # Or use Vertex AI
-  - model_name: vertex-gemini-3.1-flash-lite
-    litellm_params:
-      model: vertex_ai/gemini-3.1-flash-lite-preview
-      vertex_project: your-project-id
-      vertex_location: us-central1
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-**3. Make requests**
-
-```bash
-curl -X POST http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-  -d '{
-    "model": "gemini-3.1-flash-lite",
-    "messages": [{"role": "user", "content": "Extract structured data from this text"}],
-    "reasoning_effort": "low"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
----
-
-## Supported Endpoints
-
-LiteLLM provides **full end-to-end support** for Gemini 3.1 Flash Lite Preview on:
-
-- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
-- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
-- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
-- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
-
-All endpoints support:
-- Streaming and non-streaming responses
-- Function calling with thought signatures
-- Multi-turn conversations
-- All Gemini 3-specific features (thinking levels, thought signatures)
-- Full multimodal support (text, image, audio, video)
-
----
-
-## `reasoning_effort` Mapping for Gemini 3.1
-
-LiteLLM automatically maps OpenAI's `reasoning_effort` parameter to Gemini's `thinkingLevel`:
-
-| reasoning_effort | thinking_level | Use Case |
-|------------------|----------------|----------|
-| `minimal` | `minimal` | Ultra-fast responses, simple queries |
-| `low` | `low` | Basic instruction following |
-| `medium` | `medium` | Balanced reasoning for moderate complexity |
-| `high` | `high` | Maximum reasoning depth, complex problems |
-| `disable` | `minimal` | Disable extended reasoning |
-| `none` | `minimal` | No extended reasoning |

docs/my-website/blog/gemini_3_flash/index.md

@@ -1,247 +0,0 @@
----
-slug: gemini_3_flash
-title: "DAY 0 Support: Gemini 3 Flash on LiteLLM"
-date: 2025-12-17T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support."
-tags: [gemini, day 0 support, llms]
-hide_table_of_contents: false
----
-
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Gemini 3 Flash Day 0 Support 
-
-LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along with it.
-
-:::note
-If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
-:::
-
-{/* truncate */}
-
-## Deploy this version
-
-<Tabs>
-<TabItem value="docker" label="Docker">
-
-``` showLineNumbers title="docker run litellm"
-docker run \
--e STORE_MODEL_IN_DB=True \
--p 4000:4000 \
-ghcr.io/berriai/litellm:main-v1.80.8-stable.1
-```
-
-</TabItem>
-
-<TabItem value="pip" label="Pip">
-
-``` showLineNumbers title="pip install litellm"
-pip install litellm==1.80.8.post1
-```
-
-</TabItem>
-</Tabs>
-
-## What's New
-
-### 1. New Thinking Levels: `thinkingLevel` with MINIMAL & MEDIUM
-
-Gemini 3 Flash introduces granular thinking control with `thinkingLevel` instead of `thinkingBudget`.
-- **MINIMAL**: Ultra-lightweight thinking for fast responses
-- **MEDIUM**: Balanced thinking for complex reasoning  
-- **HIGH**: Maximum reasoning depth
-
-LiteLLM automatically maps the OpenAI `reasoning_effort` parameter to Gemini's `thinkingLevel`, so you can use familiar `reasoning_effort` values (`minimal`, `low`, `medium`, `high`) without changing your code!
-
-### 2. Thought Signatures
-
-Like `gemini-3-pro`, this model also includes thought signatures for tool calls. LiteLLM handles signature extraction and embedding internally. [Learn more about thought signatures](../gemini_3/index.md#thought-signatures).
-
-**Edge Case Handling**: If thought signatures are missing in the request, LiteLLM adds a dummy signature ensuring the API call doesn't break
-
----
-## Supported Endpoints
-
-LiteLLM provides **full end-to-end support** for Gemini 3 Flash on:
-
-- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
-- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
-- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
-- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
-All endpoints support:
-- Streaming and non-streaming responses
-- Function calling with thought signatures
-- Multi-turn conversations
-- All Gemini 3-specific features
-- Converstion of provider specific thinking related param to thinkingLevel
-
-## Quick Start
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-**Basic Usage with MEDIUM thinking (NEW)**
-
-```python
-from litellm import completion
-
-# No need to make any changes to your code as we map openai reasoning param to thinkingLevel
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "Solve this complex math problem: 25 * 4 + 10"}],
-    reasoning_effort="medium",  # NEW: MEDIUM thinking level
-)
-
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="PROXY">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gemini-3-flash
-    litellm_params:
-      model: gemini/gemini-3-flash-preview
-      api_key: os.environ/GEMINI_API_KEY
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-**3. Call with MEDIUM thinking**
-
-```bash
-curl -X POST http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-  -d '{
-    "model": "gemini-3-flash",
-    "messages": [{"role": "user", "content": "Complex reasoning task"}],
-    "reasoning_effort": "medium"
-  }'
-``'
-
-</TabItem>
-</Tabs>
-
----
-
-## All `reasoning_effort` Levels
-
-<Tabs>
-<TabItem value="minimal" label="MINIMAL">
-
-**Ultra-fast, minimal reasoning**
-
-```python
-from litellm import completion
-
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "What's 2+2?"}],
-    reasoning_effort="minimal",
-)
-```
-
-</TabItem>
-
-<TabItem value="low" label="LOW">
-
-**Simple instruction following**
-
-```python
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "Write a haiku about coding"}],
-    reasoning_effort="low",
-)
-```
-
-</TabItem>
-
-<TabItem value="medium" label="MEDIUM (NEW)">
-
-**Balanced reasoning for complex tasks** ✨
-
-```python
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "Analyze this dataset and find patterns"}],
-    reasoning_effort="medium",  # NEW!
-)
-```
-
-</TabItem>
-
-<TabItem value="high" label="HIGH">
-
-**Maximum reasoning depth**
-
-```python
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "Prove this mathematical theorem"}],
-    reasoning_effort="high",
-)
-```
-
-</TabItem>
-</Tabs>
-
----
-
-## Key Features
-
-✅ **Thinking Levels**: MINIMAL, LOW, MEDIUM, HIGH  
-✅ **Thought Signatures**: Track reasoning with unique identifiers  
-✅ **Seamless Integration**: Works with existing OpenAI-compatible client  
-✅ **Backward Compatible**: Gemini 2.5 models continue using `thinkingBudget`  
-
----
-
-## Installation
-
-```bash
-pip install litellm --upgrade
-```
-
-```python
-import litellm
-from litellm import completion
-
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "Your question here"}],
-    reasoning_effort="medium",  # Use MEDIUM thinking
-)
-print(response)
-```
-
-:::note
-If using this model via vertex_ai, keep the location as global as this is the only supported location as of now.
-:::
-
-
-## `reasoning_effort` Mapping for Gemini 3+
-
-| reasoning_effort | thinking_level | 
-|------------------|----------------|
-| `minimal` | `minimal` |
-| `low` | `low` |
-| `medium` | `medium` |
-| `high` | `high` |
-| `disable` | `minimal` |
-| `none` | `minimal` |

docs/my-website/blog/gemini_embedding_2_multimodal/index.md

@@ -1,168 +0,0 @@
----
-slug: gemini_embedding_2_multimodal
-title: "Gemini Embedding 2 Preview: Multimodal Embeddings on LiteLLM"
-date: 2025-03-11T10:00:00
-authors:
-  - sameer
-description: "Generate embeddings from text, images, audio, video, and PDFs with gemini-embedding-2-preview on LiteLLM via Gemini API and Vertex AI."
-tags: [gemini, embeddings, multimodal, vertex ai]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Gemini Embedding 2 Preview: Multimodal Embeddings
-
-LiteLLM now supports **multimodal embeddings** with `gemini-embedding-2-preview`—generating a single embedding from a mix of text, images, audio, video, and PDF content. Available via both the **Gemini API** (API key) and **Vertex AI** (GCP credentials).
-
-{/* truncate */}
-
-## Supported Input Types
-
-| Modality | Supported Formats | 
-|----------|-------------------|
-| **Text** | Plain text |
-| **Image** | PNG, JPEG | 
-| **Audio** | MP3, WAV | 
-| **Video** | MP4, MOV | 
-| **Documents** | PDF | 
-
-## Input Formats
-
-LiteLLM accepts three input formats for multimodal content:
-
-1. **Data URIs** – Base64-encoded inline: `data:image/png;base64,<encoded_data>`
-2. **GCS URLs** – Cloud Storage paths (Vertex AI): `gs://bucket/path/to/file.png`
-3. **Gemini File References** – Pre-uploaded files (Gemini API): `files/abc123`
-
-## Quick Start
-
-<Tabs>
-<TabItem value="gemini" label="Gemini API">
-
-```python
-from litellm import embedding
-import os
-
-os.environ["GEMINI_API_KEY"] = "your-api-key"
-
-# Text + Image (base64)
-response = embedding(
-    model="gemini/gemini-embedding-2-preview",
-    input=[
-        "The food was delicious and the waiter...",
-        "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
-    ],
-)
-print(response)
-```
-
-</TabItem>
-
-<TabItem value="vertex" label="Vertex AI">
-
-```python
-import litellm
-from litellm import embedding
-
-litellm.vertex_project = "your-project-id"
-litellm.vertex_location = "us-central1"
-
-# Text + Image (GCS URL)
-response = embedding(
-    model="vertex_ai/gemini-embedding-2-preview",
-    input=[
-        "Describe this image",
-        "gs://my-bucket/images/photo.png"
-    ],
-)
-print(response)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Config (config.yaml)**
-
-```yaml
-model_list:
-  - model_name: gemini-embedding-2-preview
-    litellm_params:
-      model: gemini/gemini-embedding-2-preview
-      api_key: os.environ/GEMINI_API_KEY
-  - model_name: vertex-gemini-embedding-2-preview
-    litellm_params:
-      model: vertex_ai/gemini-embedding-2-preview
-      vertex_project: os.environ/VERTEXAI_PROJECT
-      vertex_location: os.environ/VERTEXAI_LOCATION
-
-general_settings:
-  master_key: sk-1234
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config config.yaml
-```
-
-**3. Call embeddings**
-
-```bash
-curl -X POST http://localhost:4000/embeddings \
-  -H "Authorization: Bearer sk-1234" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "gemini-embedding-2-preview",
-    "input": [
-      "The food was delicious and the waiter...",
-      "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
-    ]
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-## Input Format Examples
-
-| Format | Example | Provider |
-|--------|---------|----------|
-| **Data URI** | `data:image/png;base64,...` | Gemini, Vertex AI |
-| **GCS URL** | `gs://bucket/path/image.png` | Vertex AI |
-| **File reference** | `files/abc123` | Gemini API only |
-
-### Supported MIME Types for Data URIs
-
-- **Images:** `image/png`, `image/jpeg`
-- **Audio:** `audio/mpeg`, `audio/wav`
-- **Video:** `video/mp4`, `video/quicktime`
-- **Documents:** `application/pdf`
-
-### GCS URL MIME Inference
-
-For Vertex AI, MIME types are inferred from file extensions:
-
-- `.png` → `image/png`
-- `.jpg` / `.jpeg` → `image/jpeg`
-- `.mp3` → `audio/mpeg`
-- `.wav` → `audio/wav`
-- `.mp4` → `video/mp4`
-- `.mov` → `video/quicktime`
-- `.pdf` → `application/pdf`
-
-## Optional Parameters
-
-| Parameter | Description | Maps to |
-|-----------|-------------|---------|
-| `dimensions` | Output embedding size | `outputDimensionality` |
-
-```python
-response = embedding(
-    model="gemini/gemini-embedding-2-preview",
-    input=["text to embed"],
-    dimensions=768,  # Optional: control output vector size
-)
-```

docs/my-website/blog/gpt_5_3_codex/index.md

@@ -1,138 +0,0 @@
----
-slug: gpt_5_3_codex
-title: "Day 0 Support: GPT-5.3-Codex"
-date: 2026-02-24T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API."
-tags: [openai, gpt-5.3-codex, codex, day 0 support]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items.
-
-{/* truncate */}
-
-## Why `phase` matters for GPT-5.3-Codex
-
-`phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses.
-
-Reference: [Phase parameter docs](https://developers.openai.com/api/reference/overview)
-
-Supported values:
-- `null`
-- `"commentary"`
-- `"final_answer"`
-
-Important:
-- Persist assistant output items with `phase` exactly as returned.
-- Send those assistant items back on the next turn.
-- Do **not** add `phase` to user messages.
-
-## Docker Image
-
-```bash
-docker pull ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3
-```
-
-## Usage 
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gpt-5.3-codex
-    litellm_params:
-      model: openai/gpt-5.3-codex
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e ANTHROPIC_API_KEY=$OPENAI_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3 \
-  --config /app/config.yaml
-```
-
-
-**3. Test it**
-
-```bash
-curl -X POST "http://0.0.0.0:4000/v1/responses" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_KEY" \
-  -d '{
-    "model": "gpt-5.3-codex",
-    "input": "Write a Python script that checks if a number is prime."
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-## Python Example: Persist `phase` with OpenAI Client + LiteLLM Base URL
-
-```python
-from openai import OpenAI
-
-client = OpenAI(
-    base_url="http://0.0.0.0:4000/v1",  # LiteLLM Proxy
-    api_key="your-litellm-api-key",
-)
-
-items = []  # Persist this per conversation/thread
-
-
-def _item_get(item, key, default=None):
-    if isinstance(item, dict):
-        return item.get(key, default)
-    return getattr(item, key, default)
-
-
-def run_turn(user_text: str):
-    global items
-
-    # User message: no phase field
-    items.append(
-        {
-            "type": "message",
-            "role": "user",
-            "content": [{"type": "input_text", "text": user_text}],
-        }
-    )
-
-    resp = client.responses.create(
-        model="gpt-5.3-codex",
-        input=items,
-    )
-
-    # Persist assistant output items verbatim, including phase
-    for out_item in (resp.output or []):
-        items.append(out_item)
-
-    # Optional: inspect latest phase for UI/telemetry routing
-    latest_phase = None
-    for out_item in reversed(resp.output or []):
-        if _item_get(out_item, "type") == "output_item.done" and _item_get(out_item, "phase") is not None:
-            latest_phase = _item_get(out_item, "phase")
-            break
-
-    return resp, latest_phase
-```
-
-## Notes
-
-- Use `/v1/responses` for GPT Codex models.
-- Preserve full assistant output history for best multi-turn behavior.
-- If `phase` metadata is dropped during history reconstruction, output quality can degrade on long-running tasks.

docs/my-website/blog/gpt_5_4/index.md

@@ -1,90 +0,0 @@
----
-slug: gpt_5_4
-title: "Day 0 Support: GPT-5.4"
-date: 2026-03-05T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "GPT-5.4 model support in LiteLLM"
-tags: [openai, gpt-5.4, completion]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports fully GPT-5.4!
-
-{/* truncate */}
-
-## Docker Image
-
-```bash
-docker pull ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch
-```
-
-## Usage
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gpt-5.4
-    litellm_params:
-      model: openai/gpt-5.4
-      api_key: os.environ/OPENAI_API_KEY
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e OPENAI_API_KEY=$OPENAI_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch \
-  --config /app/config.yaml
-```
-
-**3. Test it**
-
-```bash
-curl -X POST "http://0.0.0.0:4000/chat/completions" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_KEY" \
-  -d '{
-    "model": "gpt-5.4",
-    "messages": [
-      {"role": "user", "content": "Write a Python function to check if a number is prime."}
-    ]
-  }'
-```
-
-</TabItem>
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-response = completion(
-    model="openai/gpt-5.4",
-    messages=[
-        {"role": "user", "content": "Write a Python function to check if a number is prime."}
-    ],
-)
-
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>
-
-## Notes
-
-- Restart your container to get the cost tracking for this model.
-- Use `/responses` for better model performance.
-- GPT-5.4 supports reasoning, function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.

docs/my-website/blog/gpt_5_4_mini_nano/index.md

@@ -1,106 +0,0 @@
----
-slug: gpt_5_4_mini_nano
-title: "Day 0 Support: GPT-5.4-mini and GPT-5.4-nano"
-date: 2026-03-17T10:00:00
-authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-description: "GPT-5.4-mini and GPT-5.4-nano model support in LiteLLM"
-tags: [openai, gpt-5.4-mini, gpt-5.4-nano, completion]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports GPT-5.4-mini and GPT-5.4-nano — cost-effective models for simple completions and high-throughput workloads.
-
-:::note
-If you're on **v1.82.3-stable** or above, you don't need any update to use these models.
-:::
-
-## Usage
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gpt-5.4-mini
-    litellm_params:
-      model: openai/gpt-5.4-mini
-      api_key: os.environ/OPENAI_API_KEY
-  - model_name: gpt-5.4-nano
-    litellm_params:
-      model: openai/gpt-5.4-nano
-      api_key: os.environ/OPENAI_API_KEY
-```
-
-**2. Start the proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-**3. Test it**
-
-```bash
-# GPT-5.4-mini
-curl -X POST "http://localhost:4000/v1/chat/completions" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_KEY" \
-  -d '{
-    "model": "gpt-5.4-mini",
-    "messages": [{"role": "user", "content": "What is the capital of France?"}]
-  }'
-
-# GPT-5.4-nano
-curl -X POST "http://localhost:4000/v1/chat/completions" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_KEY" \
-  -d '{
-    "model": "gpt-5.4-nano",
-    "messages": [{"role": "user", "content": "What is 2 + 2?"}]
-  }'
-```
-
-</TabItem>
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-# GPT-5.4-mini
-response = completion(
-    model="openai/gpt-5.4-mini",
-    messages=[{"role": "user", "content": "What is the capital of France?"}],
-)
-print(response.choices[0].message.content)
-
-# GPT-5.4-nano
-response = completion(
-    model="openai/gpt-5.4-nano",
-    messages=[{"role": "user", "content": "What is 2 + 2?"}],
-)
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>
-
-## Notes
-
-- Both models support function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.
-- GPT-5.4-nano is the most cost-effective option for simple tasks; GPT-5.4-mini offers a balance of speed and capability.

docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md

@@ -1,78 +0,0 @@
----
-slug: guardrail-logging-secret-exposure-incident
-title: "Incident Report: Guardrail logging exposed secret headers in spend logs and traces"
-date: 2026-03-18T10:00:00
-authors:
-  - litellm
-tags: [incident-report, security, guardrails]
-hide_table_of_contents: false
----
-
-**Date:** March 18, 2026
-**Duration:** Unknown
-**Severity:** High
-**Status:** Resolved
-
-## Summary
-
-When a custom guardrail returned the full LiteLLM request/data dictionary, the guardrail response logged by LiteLLM could include `secret_fields.raw_headers`, including plaintext `Authorization` headers containing API keys or other credentials.
-
-This information could then propagate to logging and observability surfaces that consume guardrail metadata, including:
-
-- **Spend logs in the LiteLLM UI:** visible to admins with access to spend-log data
-- **OpenTelemetry traces:** visible to anyone with access to the relevant telemetry backend
-
-LLM calls, proxy routing, and provider execution were not blocked by this bug. The impact was exposure of sensitive request headers in observability and logging paths.
-
-{/* truncate */}
-
----
-
-## Background
-
-LiteLLM keeps internal request data (including request headers) for use during the call. That data is not meant to be written to logs or telemetry.
-
-When custom guardrails run, their outcomes are logged so they can appear in spend logs, OpenTelemetry traces, and other observability backends. If a guardrail returned the full request payload instead of a minimal result, that internal request data could be included in what was logged. Before the fix, the guardrail logging path did not strip that data before sending it to those systems.
-
-```mermaid
-flowchart TD
-    inboundRequest["1. Incoming proxy request"] --> storeSecrets["2. Store internal request data"]
-    storeSecrets --> guardrailRuns["3. Custom guardrail runs"]
-    guardrailRuns --> fullDataReturn["4. Guardrail returns full request payload"]
-    fullDataReturn --> loggingBuild["5. Build guardrail log payload"]
-    loggingBuild --> spendLogs["6a. Persist to spend logs / UI"]
-    loggingBuild --> otelTraces["6b. Attach to OTEL guardrail spans"]
-```
-
----
-
-## Root Cause
-
-The root cause was incomplete sanitization in the guardrail logging path. When building the payload that gets sent to spend logs and traces, LiteLLM prepared guardrail responses for logging but did not strip internal request data (such as headers) from them. If a guardrail returned a response that included that data, it was passed through to the logging and observability systems unchanged.
-
----
-
-## Impact
-
-This issue required all of the following:
-
-1. A custom guardrail returned the full LiteLLM request/data dictionary, or another response object containing `secret_fields`.
-2. LiteLLM logged that guardrail response through the standard guardrail logging path.
-3. An operator, admin, or telemetry consumer had access to the resulting logs or traces.
-
-When those conditions were met, sensitive values could become visible through:
-
-- **Spend logs / UI responses:** guardrail metadata could be included in spend-log payloads rendered in the admin UI.
-- **OpenTelemetry traces:** `guardrail_response` could be written as a span attribute on guardrail spans.
-- **Other downstream observability backends:** any integration consuming the same guardrail metadata could receive the leaked values.
-
-This was a logging and telemetry exposure bug. It did not let callers bypass auth, access other tenants directly, or change model behavior, but it could expose plaintext credentials to people with access to those observability systems.
-
----
-
-## Guidance For Users
-
-- Upgrade to LiteLLM 1.82.3+.
-- If you operated custom guardrails that return the full request/data dict, review whether spend logs or telemetry traces were retained during the affected period.
-- Rotate any credentials that may have appeared in `Authorization` or other forwarded request headers in those systems.
-- Apply least-privilege access controls to spend-log views and telemetry backends that may contain request-derived metadata.

docs/my-website/blog/httpx_cache_eviction_incident/index.md

@@ -1,126 +0,0 @@
----
-slug: httpx-cache-eviction-incident
-title: "Incident Report: Cache Eviction Closes In-Use httpx Clients"
-date: 2026-02-27T10:00:00
-authors:
-  - ryan
-  - ishaan-alt
-  - krrish
-tags: [incident-report, caching, stability]
-hide_table_of_contents: false
----
-
-**Date:** February 27, 2026
-**Duration:** ~6 days (Feb 21 merge -> Feb 27 fix)
-**Severity:** High
-**Status:** Resolved
-
-> **Note:** This fix is available starting from LiteLLM `v1.81.14.rc.2` or higher.
-
-## Summary
-
-A change to improve Redis connection pool cleanup introduced a regression that closed **httpx clients** that were still actively being used by the proxy. The `LLMClientCache` (an in-memory TTL cache) stores both Redis clients *and* httpx clients under the same eviction policy. When a cache entry expired or was evicted, the new cleanup code called `aclose()`/`close()` on the evicted value which worked correctly for Redis clients, but destroyed httpx clients that other parts of the system still held references to and were actively using for LLM API calls.
-
-**Impact:** Any proxy instance that hit the cache TTL (default 10 minutes) or capacity limit (200 entries) would have its httpx clients closed out from under it, causing requests to LLM providers to fail with connection errors.
-
-{/* truncate */}
-
----
-
-## Background
-
-`LLMClientCache` extends `InMemoryCache` and is used to cache SDK clients (OpenAI, Anthropic, etc.) to avoid re-creating them on every request. These clients are keyed by configuration + event loop ID. The cache has:
-
-- **Max size:** 200 entries
-- **Default TTL:** 10 minutes
-
-When the cache is full or entries expire, `InMemoryCache.evict_cache()` calls `_remove_key()` to drop entries.
-
-The cached values are a mix of:
-- **Redis/async Redis clients** — owned exclusively by the cache, safe to close on eviction
-- **httpx-backed SDK clients** (OpenAI, Anthropic, etc.) — shared references, still in use by router/model instances
-
----
-
-## Root Cause
-
-[PR #21717](https://github.com/BerriAI/litellm/pull/21717) overrode `_remove_key()` in `LLMClientCache` to close async clients on eviction:
-
-<details>
-<summary>Problematic code added in PR #21717</summary>
-
-```python
-class LLMClientCache(InMemoryCache):
-    def _remove_key(self, key: str) -> None:
-        value = self.cache_dict.get(key)
-        super()._remove_key(key)
-        if value is not None:
-            close_fn = getattr(value, "aclose", None) or getattr(value, "close", None)
-            if close_fn and asyncio.iscoroutinefunction(close_fn):
-                try:
-                    asyncio.get_running_loop().create_task(close_fn())
-                except RuntimeError:
-                    pass
-            elif close_fn and callable(close_fn):
-                try:
-                    close_fn()
-                except Exception:
-                    pass
-```
-
-</details>
-
-The intent was correct for Redis clients — prevent connection pool leaks when cached Redis clients expire. But `LLMClientCache` also stores httpx-backed SDK clients (e.g., `AsyncOpenAI`, `AsyncAnthropic`). These clients:
-
-1. Have an `aclose()` method (inherited from httpx)
-2. Are still held by references elsewhere in the codebase (router, model instances)
-3. Were being closed without any check on whether they were still in use
-
-So when the cache evicted an entry, it would call `aclose()` on an httpx client that was still being used for active LLM requests → closed transport → connection errors.
-
----
-
-## The Fix
-
-[PR #22247](https://github.com/BerriAI/litellm/pull/22247) removed the `_remove_key` override entirely:
-
-<details>
-<summary>The fix (PR #22247)</summary>
-
-```diff
- class LLMClientCache(InMemoryCache):
--    def _remove_key(self, key: str) -> None:
--        """Close async clients before evicting them to prevent connection pool leaks."""
--        value = self.cache_dict.get(key)
--        super()._remove_key(key)
--        if value is not None:
--            close_fn = getattr(value, "aclose", None) or getattr(
--                value, "close", None
--            )
--            ...
--
-     def update_cache_key_with_event_loop(self, key):
-```
-
-</details>
-
-The eviction now simply drops the reference and lets Python's GC handle cleanup, which is safe because:
-- httpx clients that are still referenced elsewhere stay alive
-- Unreferenced clients get cleaned up by GC naturally
-
-The other improvements from PR #21717 were kept:
-- **`max_connections` respected for URL-based Redis configs**, previously silently dropped
-- **`disconnect()` now closes both sync and async Redis clients**, sync client was previously leaked
-- **Connection pool passthrough**, when a pool is provided with a URL config, it's used directly instead of creating a duplicate
-
----
-
-## Remediation
-
-| Action | Status | Code |
-|--------|--------|------|
-| Remove `_remove_key` override that closes shared clients on eviction | ✅ Done | [PR #22247](https://github.com/BerriAI/litellm/pull/22247) |
-| Add e2e test: evicted client still usable (capacity) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
-| Add e2e test: expired client still usable (TTL) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
-
-The e2e tests go through `get_async_httpx_client()` the same code path the proxy uses in production and assert the client is still functional after eviction. These run in CI on every PR against `main`. If anyone modifies `LLMClientCache` eviction behavior, overrides `_remove_key`, or adds any form of client cleanup on eviction, these tests will fail regardless of the implementation approach.

docs/my-website/blog/litellm_observatory/index.md

@@ -1,128 +0,0 @@
----
-slug: litellm-observatory
-title: "Improve release stability with 24 hour load tests"
-date: 2026-02-06T10:00:00
-authors:
-  - alexsander
-  - krrish
-  - ishaan-alt
-description: "How we built a long-running, release-validation system to catch regressions before they reach users."
-tags: [testing, observability, reliability, releases]
-hide_table_of_contents: false
----
-
-![LiteLLM Observatory](https://raw.githubusercontent.com/AlexsanderHamir/assets/main/Screenshot%202026-01-31%20175355.png)
-
-# Improve release stability with 24 hour load tests
-
-As LiteLLM adoption has grown, so have expectations around reliability, performance, and operational safety. Meeting those expectations requires more than correctness-focused tests, it requires validating how the system behaves over time, under real-world conditions.
-
-This post introduces **LiteLLM Observatory**, a long-running release-validation system we built to catch regressions before they reach users.
-
-{/* truncate */}
-
----
-
-## Why We Built the Observatory
-
-LiteLLM operates at the intersection of external providers, long-lived network connections, and high-throughput workloads. While our unit and integration tests do an excellent job validating correctness, they are not designed to surface issues that only appear after extended operation.
-
-A subtle lifecycle edge case discovered in v1.81.3 reinforced the need for stronger release validation in this area.
-
----
-
-## A Real-World Lifecycle Edge Case
-
-In v1.81.3, we shipped a fix for an HTTP client memory leak. The change passed unit and integration tests and behaved correctly in short-lived runs.
-
-The issue that surfaced was not caused by a single incorrect line of logic, but by how multiple components interacted over time:
-
-- A cached `httpx` client was configured with a 1-hour TTL
-- When the cache expired, the underlying HTTP connection was closed as expected
-- A higher-level client continued to hold a reference to that connection
-- Subsequent requests failed with:
-
-```
-Cannot send a request, as the client has been closed
-```
-
-**Before (with bug):**
-
-| Provider | Requests | Success | Failures | Fail % |
-|----------|----------|---------|----------|--------|
-| OpenAI   | 720,000  | 432,000 | 288,000  | 40%    |
-| Azure    | 692,000  | 415,200 | 276,800  | 40%    |
-
-**After (fixed):**
-
-| Provider | Requests   | Success   | Failures | Fail %  |
-|----------|------------|-----------|----------|---------|
-| OpenAI   | 1,200,000  | 1,199,988 | 12       | 0.001%  |
-| Azure    | 1,150,000  | 1,149,982 | 18       | 0.002%  |
-
-Our focus moving forward is on being the first to detect issues, even when they aren’t covered by unit tests. LiteLLM Observatory is designed to surface latency regressions, OOMs, and failure modes that only appear under real traffic patterns in **our own production deployments** during release validation.
-
-
----
-
-### How the Observatory Works
-
-[LiteLLM Observatory](https://github.com/BerriAI/litellm-observatory) is a testing service that runs long-running tests against our LiteLLM deployments. We trigger tests by sending API requests, and results are automatically sent to Slack when tests complete.
-
-#### How Tests Run
-
-1. **Start a Test**: We send a request to the Observatory API with:
-   - Which LiteLLM deployment to test (URL and API key)
-   - Which test to run (e.g., `TestOAIAzureRelease`)
-   - Test settings (which models to test, how long to run, failure thresholds)
-
-2. **Smart Queueing**:
-   - The system checks whether we are attempting to run the exact same test more than once
-   - If a duplicate test is already running or queued, we receive an error to avoid wasting resources
-   - Otherwise, the test is added to a queue and runs when capacity is available (up to 5 tests can run concurrently by default)
-
-3. **Instant Response**: The API responds immediately—we do not wait for the test to finish. Tests may run for hours, but the request itself completes in milliseconds.
-
-4. **Background Execution**:
-   - The test runs in the background, issuing requests against our LiteLLM deployment
-   - It tracks request success and failure rates over time
-   - When the test completes, results are automatically posted to our Slack channel
-
-#### Example: The OpenAI / Azure Reliability Test
-
-The `TestOAIAzureRelease` test is designed to catch a class of bugs that only surface after sustained runtime:
-
-- **Duration**: Runs continuously for 3 hours
-- **Behavior**: Cycles through specified models (such as `gpt-4` and `gpt-3.5-turbo`), issuing requests continuously
-- **Why 3 Hours**: This helps catch issues where HTTP clients degrade or fail after extended use (for example, a bug observed in LiteLLM v1.81.3)
-- **Pass / Fail Criteria**: The test passes if fewer than 1% of requests fail. If the failure rate exceeds 1%, the test fails and we are notified in Slack
-- **Key Detail**: The same HTTP client is reused for the entire run, allowing us to detect lifecycle-related bugs that only appear under prolonged reuse
-
-#### When We Use It
-
-- **Before Deployments**: Run tests before promoting a new LiteLLM version to production
-- **Routine Validation**: Schedule regular runs (daily or weekly) to catch regressions early
-- **Issue Investigation**: Run tests on demand when we suspect a deployment issue
-- **Long-Running Failure Detection**: Identify bugs that only appear under sustained load, beyond what short smoke tests can reveal
-
-
-### Complementing Unit Tests
-
-Unit tests remain a foundational part of our development process. They are fast and precise, but they don’t cover:
-
-- Real provider behavior
-- Long-lived network interactions
-- Resource lifecycle edge cases
-- Time-dependent regressions
-
-LiteLLM Observatory complements unit tests by validating the system as it actually runs in production-like environments.
-
----
-
-### Looking Ahead
-
-Reliability is an ongoing investment.
-
-LiteLLM Observatory is one of several systems we’re building to continuously raise the bar on release quality and operational safety. As LiteLLM evolves, so will our validation tooling, informed by real-world usage and lessons learned.
-
-We’ll continue to share those improvements openly as we go.

docs/my-website/blog/minimax_m2_5/index.md

@@ -1,387 +0,0 @@
----
-slug: minimax_m2_5
-title: "Day 0 Support: MiniMax-M2.5"
-date: 2026-02-12T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Day 0 support for MiniMax-M2.5 on LiteLLM"
-tags: [minimax, M2.5, llm]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway.
-
-{/* truncate */}
-
-## Supported Models
-
-LiteLLM supports the following MiniMax models:
-
-| Model | Description | Input Cost | Output Cost | Context Window |
-|-------|-------------|------------|-------------|----------------|
-| **MiniMax-M2.5** | Advanced reasoning, Agentic capabilities | $0.3/M tokens | $1.2/M tokens | 1M tokens |
-| **MiniMax-M2.5-lightning** | Faster and More Agile (~100 tps) | $0.3/M tokens | $2.4/M tokens | 1M tokens |
-
-## Features Supported
-
-- **Prompt Caching**: Reduce costs with cached prompts ($0.03/M tokens for cache read, $0.375/M tokens for cache write)
-- **Function Calling**: Built-in tool calling support
-- **Reasoning**: Advanced reasoning capabilities with thinking support
-- **System Messages**: Full system message support
-- **Cost Tracking**: Automatic cost calculation for all requests
-
-## Docker Image
-
-```bash
-docker pull litellm/litellm:v1.81.3-stable
-```
-
-## Usage - OpenAI Compatible API (/v1/chat/completions)
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: minimax-m2-5
-    litellm_params:
-      model: minimax/MiniMax-M2.5
-      api_key: os.environ/MINIMAX_API_KEY
-      api_base: https://api.minimax.io/v1
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e MINIMAX_API_KEY=$MINIMAX_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.3-stable \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "minimax-m2-5",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-### With Reasoning Split
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "minimax-m2-5",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Solve: 2+2=?"
-    }
-  ],
-  "extra_body": {
-    "reasoning_split": true
-  }
-}'
-```
-
-## Usage - Anthropic Compatible API (/v1/messages)
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: minimax-m2-5
-    litellm_params:
-      model: minimax/MiniMax-M2.5
-      api_key: os.environ/MINIMAX_API_KEY
-      api_base: https://api.minimax.io/anthropic/v1/messages
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e MINIMAX_API_KEY=$MINIMAX_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.3-stable \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "minimax-m2-5",
-  "max_tokens": 1000,
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-### With Thinking
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "minimax-m2-5",
-  "max_tokens": 1000,
-  "thinking": {
-    "type": "enabled",
-    "budget_tokens": 1000
-  },
-  "messages": [
-    {
-      "role": "user",
-      "content": "Solve: 2+2=?"
-    }
-  ]
-}'
-```
-
-## Usage - LiteLLM SDK
-
-### OpenAI-compatible API
-
-```python
-import litellm
-
-response = litellm.completion(
-    model="minimax/MiniMax-M2.5",
-    messages=[
-        {"role": "user", "content": "Hello, how are you?"}
-    ],
-    api_key="your-minimax-api-key",
-    api_base="https://api.minimax.io/v1"
-)
-
-print(response.choices[0].message.content)
-```
-
-### Anthropic-compatible API
-
-```python
-import litellm
-
-response = litellm.anthropic.messages.acreate(
-    model="minimax/MiniMax-M2.5",
-    messages=[{"role": "user", "content": "Hello, how are you?"}],
-    api_key="your-minimax-api-key",
-    api_base="https://api.minimax.io/anthropic/v1/messages",
-    max_tokens=1000
-)
-
-print(response.choices[0].message.content)
-```
-
-### With Thinking
-
-```python
-response = litellm.anthropic.messages.acreate(
-    model="minimax/MiniMax-M2.5",
-    messages=[{"role": "user", "content": "Solve: 2+2=?"}],
-    thinking={"type": "enabled", "budget_tokens": 1000},
-    api_key="your-minimax-api-key"
-)
-
-# Access thinking content
-for block in response.choices[0].message.content:
-    if hasattr(block, 'type') and block.type == 'thinking':
-        print(f"Thinking: {block.thinking}")
-```
-
-### With Reasoning Split (OpenAI API)
-
-```python
-response = litellm.completion(
-    model="minimax/MiniMax-M2.5",
-    messages=[
-        {"role": "user", "content": "Solve: 2+2=?"}
-    ],
-    extra_body={"reasoning_split": True},
-    api_key="your-minimax-api-key",
-    api_base="https://api.minimax.io/v1"
-)
-
-# Access thinking and response
-if hasattr(response.choices[0].message, 'reasoning_details'):
-    print(f"Thinking: {response.choices[0].message.reasoning_details}")
-print(f"Response: {response.choices[0].message.content}")
-```
-
-## Cost Tracking
-
-LiteLLM automatically tracks costs for MiniMax-M2.5 requests. The pricing is:
-
-- **Input**: $0.3 per 1M tokens
-- **Output**: $1.2 per 1M tokens
-- **Cache Read**: $0.03 per 1M tokens
-- **Cache Write**: $0.375 per 1M tokens
-
-### Accessing Cost Information
-
-```python
-response = litellm.completion(
-    model="minimax/MiniMax-M2.5",
-    messages=[{"role": "user", "content": "Hello!"}],
-    api_key="your-minimax-api-key"
-)
-
-# Access cost information
-print(f"Cost: ${response._hidden_params.get('response_cost', 0)}")
-```
-
-## Streaming Support
-
-### OpenAI API
-
-```python
-response = litellm.completion(
-    model="minimax/MiniMax-M2.5",
-    messages=[{"role": "user", "content": "Tell me a story"}],
-    stream=True,
-    api_key="your-minimax-api-key",
-    api_base="https://api.minimax.io/v1"
-)
-
-for chunk in response:
-    if chunk.choices[0].delta.content:
-        print(chunk.choices[0].delta.content, end="")
-```
-
-### Streaming with Reasoning Split
-
-```python
-stream = litellm.completion(
-    model="minimax/MiniMax-M2.5",
-    messages=[
-        {"role": "user", "content": "Tell me a story"},
-    ],
-    extra_body={"reasoning_split": True},
-    stream=True,
-    api_key="your-minimax-api-key",
-    api_base="https://api.minimax.io/v1"
-)
-
-reasoning_buffer = ""
-text_buffer = ""
-
-for chunk in stream:
-    if hasattr(chunk.choices[0].delta, "reasoning_details") and chunk.choices[0].delta.reasoning_details:
-        for detail in chunk.choices[0].delta.reasoning_details:
-            if "text" in detail:
-                reasoning_text = detail["text"]
-                new_reasoning = reasoning_text[len(reasoning_buffer):]
-                if new_reasoning:
-                    print(new_reasoning, end="", flush=True)
-                    reasoning_buffer = reasoning_text
-
-    if chunk.choices[0].delta.content:
-        content_text = chunk.choices[0].delta.content
-        new_text = content_text[len(text_buffer):] if text_buffer else content_text
-        if new_text:
-            print(new_text, end="", flush=True)
-            text_buffer = content_text
-```
-
-## Using with Native SDKs
-
-### Anthropic SDK via LiteLLM Proxy
-
-```python
-import os
-os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
-os.environ["ANTHROPIC_API_KEY"] = "sk-1234"  # Your LiteLLM proxy key
-
-import anthropic
-
-client = anthropic.Anthropic()
-
-message = client.messages.create(
-    model="minimax-m2-5",
-    max_tokens=1000,
-    system="You are a helpful assistant.",
-    messages=[
-        {
-            "role": "user",
-            "content": [
-                {
-                    "type": "text",
-                    "text": "Hi, how are you?"
-                }
-            ]
-        }
-    ]
-)
-
-for block in message.content:
-    if block.type == "thinking":
-        print(f"Thinking:\n{block.thinking}\n")
-    elif block.type == "text":
-        print(f"Text:\n{block.text}\n")
-```
-
-### OpenAI SDK via LiteLLM Proxy
-
-```python
-import os
-os.environ["OPENAI_BASE_URL"] = "http://localhost:4000"
-os.environ["OPENAI_API_KEY"] = "sk-1234"  # Your LiteLLM proxy key
-
-from openai import OpenAI
-
-client = OpenAI()
-
-response = client.chat.completions.create(
-    model="minimax-m2-5",
-    messages=[
-        {"role": "system", "content": "You are a helpful assistant."},
-        {"role": "user", "content": "Hi, how are you?"},
-    ],
-    extra_body={"reasoning_split": True},
-)
-
-# Access thinking and response
-if hasattr(response.choices[0].message, 'reasoning_details'):
-    print(f"Thinking:\n{response.choices[0].message.reasoning_details[0]['text']}\n")
-print(f"Text:\n{response.choices[0].message.content}\n")
-```

docs/my-website/blog/model_cost_map_incident/index.md

@@ -1,92 +0,0 @@
----
-slug: model-cost-map-incident
-title: "Incident Report: Invalid model cost map on main"
-date: 2026-02-10T10:00:00
-authors:
-  - ishaan
-tags: [incident-report, stability]
-hide_table_of_contents: false
----
-
-**Date:** January 27, 2026
-**Duration:** ~20 minutes
-**Severity:** Low
-**Status:** Resolved
-
-## Summary
-
-A malformed JSON entry in `model_prices_and_context_window.json` was merged to `main` ([`562f0a0`](https://github.com/BerriAI/litellm/commit/562f0a028251750e3d75386bee0e630d9796d0df)). This caused LiteLLM to silently fall back to a stale local copy of the model cost map. Users on older package versions lost cost tracking for newer models only (e.g. `azure/gpt-5.2`). No LLM calls were blocked.
-
-- **LLM calls and proxy routing:** No impact.
-- **Cost tracking:** Impacted for newer models not present in the local backup. Older models were unaffected. The incident lasted ~20 minutes until the commit was reverted.
-
-{/* truncate */}
-
----
-
-## Background
-
-The model cost map is not in the request path. It is used after the LLM response comes back, inside a try/catch, to calculate spend. A missing entry never blocks a call.
-
-```mermaid
-flowchart TD
-    A["1. litellm.completion() receives request
-    litellm/main.py"] --> B["2. Route to provider
-    litellm/litellm_core_utils/get_llm_provider_logic.py"]
-    B --> C["3. LLM returns response
-    litellm/main.py"]
-    C --> D["4. Post-call: look up model in cost map
-    litellm/cost_calculator.py"]
-    D -->|"found"| E["5a. Attach cost to response"]
-    D -->|"not found (try/catch)"| F["5b. Log warning, set cost=0"]
-    E --> G["6. Return response to caller"]
-    F --> G
-
-    style D fill:#fff3cd,stroke:#ffc107
-    style F fill:#fff3cd,stroke:#ffc107
-    style E fill:#d4edda,stroke:#28a745
-    style G fill:#d4edda,stroke:#28a745
-```
-
-Both paths return a response to the caller. When the cost map lookup fails, the only difference is `cost=0` on that request.
-
----
-
-## Root cause
-
-LiteLLM fetches the model cost map from GitHub `main` at import time. If the fetch fails, it falls back to a local backup bundled with the package. Before this incident, the fallback was completely silent -- no warning was logged.
-
-A contributor PR introduced an extra `{` bracket, producing invalid JSON. The remote fetch failed with `JSONDecodeError`, triggering the silent fallback. Users on older package versions had backup files missing newer models.
-
-**Timeline:**
-
-1. Malformed JSON merged to `main`
-2. LiteLLM installations fall back to local backup on next import
-3. Users report `"This model isn't mapped yet"` for newer models
-4. Bad commit identified and reverted (~20 minutes)
-
----
-
-## Remediation
-
-| # | Action | Status | Code |
-|---|---|---|---|
-| 1 | CI validation on `model_prices_and_context_window.json` | ✅ Done | [`test-model-map.yaml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test-model-map.yaml) |
-| 2 | Warning log on fallback to local backup | ✅ Done | [`get_model_cost_map.py#L57-L68`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L57-L68) |
-| 3 | `GetModelCostMap` class with integrity validation helpers | ✅ Done | [`get_model_cost_map.py#L24-L149`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L24-L149) |
-| 4 | Resilience test suite (bad hosted map, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py#L150-L291`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L150-L291) |
-| 5 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py#L213-L228`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L213-L228) |
-
-Enterprises that require zero external dependencies at import time can set `LITELLM_LOCAL_MODEL_COST_MAP=True` to skip the GitHub fetch entirely.
-
----
-
-## Other dependencies on external resources
-
-| Dependency | Impact if unavailable | Fallback |
-|---|---|---|
-| Model cost map (GitHub) | Cost tracking for newer models | Local backup (now with warning) |
-| JWT public keys (IDP/SSO) | Auth fails | None |
-| OIDC UserInfo (IDP/SSO) | Auth fails | None |
-| HuggingFace model API | HF provider calls fail | None |
-| Ollama tags (localhost) | Ollama model list stale | Static list |

docs/my-website/blog/realtime_webrtc_http_endpoints/index.md

@@ -1,111 +0,0 @@
----
-slug: realtime_webrtc_http_endpoints
-title: "Realtime WebRTC HTTP Endpoints"
-date: 2026-03-12T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange."
-tags: [realtime, webrtc, proxy, openai]
-hide_table_of_contents: false
----
-
-import WebRTCTester from '@site/src/components/WebRTCTester';
-
-Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth and key management.
-
-{/* truncate */}
-
-## How it works
-
-![WebRTC flow: Browser, LiteLLM Proxy, and OpenAI/Azure](../../img/webrtc_flow.png)
-
-**Flow of generating ephemeral token**
-
-![Ephemeral token flow: Browser requests token, LiteLLM gets real token from OpenAI, returns encrypted token](../../img/ephemeral_token.png)
-
-
-## Proxy Setup
-
-```yaml
-model_list:
-  - model_name: gpt-4o-realtime
-    litellm_params:
-      model: openai/gpt-4o-realtime-preview-2024-12-17
-      api_key: os.environ/OPENAI_API_KEY
-    model_info:
-      mode: realtime
-```
-
-**Azure:** use `model: azure/gpt-4o-realtime-preview`, `api_key`, `api_base`.
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-## Try it live
-
-<WebRTCTester />
-
-## Client Usage
-
-**1. Get token** - `POST /v1/realtime/client_secrets` with LiteLLM API key and `{ model }`.
-
-**2. WebRTC handshake** - Create `RTCPeerConnection`, add mic track, create data channel `oai-events`, send SDP offer to `POST /v1/realtime/calls` with `Authorization: Bearer <encrypted_token>` and `Content-Type: application/sdp`.
-
-**3. Events** - Use the data channel for `session.update` and other events.
-
-<details>
-<summary>Full code example</summary>
-
-```javascript
-// 1. Token
-const r = await fetch("http://proxy:4000/v1/realtime/client_secrets", {
-  method: "POST",
-  headers: { "Authorization": "Bearer sk-litellm-key", "Content-Type": "application/json" },
-  body: JSON.stringify({ model: "gpt-4o-realtime" }),
-});
-const { client_secret } = await r.json();
-const token = client_secret.value;
-
-// 2. WebRTC
-const pc = new RTCPeerConnection();
-const audio = document.createElement("audio");
-audio.autoplay = true;
-pc.ontrack = (e) => (audio.srcObject = e.streams[0]);
-const ms = await navigator.mediaDevices.getUserMedia({ audio: true });
-pc.addTrack(ms.getTracks()[0]);
-const dc = pc.createDataChannel("oai-events");
-const offer = await pc.createOffer();
-await pc.setLocalDescription(offer);
-
-const sdpRes = await fetch("http://proxy:4000/v1/realtime/calls", {
-  method: "POST",
-  headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/sdp" },
-  body: offer.sdp,
-});
-await pc.setRemoteDescription({ type: "answer", sdp: await sdpRes.text() });
-
-// 3. Events
-dc.send(JSON.stringify({ type: "session.update", session: { instructions: "..." } }));
-```
-
-</details>
-
-## FAQ
-
-**Q: What do I do if I get a 401 Token expired error?**  
-A: Tokens are short-lived. Get a fresh token right before creating the WebRTC offer.
-
-**Q: Which key should I use for `/v1/realtime/calls`?**  
-A: Use the **encrypted token** from `client_secrets`, not your raw API key.
-
-**Q: Should I pass the `model` parameter when making the call?**  
-A: No, the encrypted token already encodes all routing information including model.
-
-**Q: How do I resolve Azure `api-version` errors?**  
-A: Set the correct `api_version` in `litellm_params` (or via the `AZURE_API_VERSION` environment variable), along with the right `api_base` and deployment values.
-
-**Q: What if I get no audio?**  
-A: Make sure you grant microphone permission, ensure `pc.ontrack` assigns the audio element with `autoplay` enabled, check your network/firewall for WebRTC traffic, and inspect the browser console for ICE or SDP errors.

docs/my-website/blog/redis_circuit_breaker/diagrams.js

@@ -1,159 +0,0 @@
-import React from 'react';
-
-const s = {
-  fig: {margin: '2.5rem 0', fontFamily: 'inherit'},
-  box: {borderRadius: 12, border: '1px solid #e5e7eb', background: '#fff', padding: '2rem 2.5rem'},
-  label: {fontSize: 11, fontWeight: 700, textTransform: 'uppercase', letterSpacing: '0.12em', color: '#9ca3af', textAlign: 'center', marginBottom: '1.5rem'},
-  caption: {textAlign: 'center', fontSize: 12, color: '#9ca3af', marginTop: 12},
-  node: (border='#d1d5db', bg='#f9fafb') => ({
-    border: `1px solid ${border}`, borderRadius: 6, padding: '8px 20px',
-    fontSize: 13, background: bg, display: 'inline-block',
-  }),
-  arrow: {display: 'flex', flexDirection: 'column', alignItems: 'center'},
-};
-
-const SmallArrow = ({color='#9ca3af'}) => (
-  <svg width="2" height="28" style={{display:'block'}}>
-    <line x1="1" y1="0" x2="1" y2="22" stroke={color} strokeWidth="1.5"/>
-    <polygon points="1,28 -2,21 4,21" fill={color}/>
-  </svg>
-);
-
-export function CascadeFailure() {
-  return (
-    <figure style={s.fig}>
-      <div style={s.box}>
-        <p style={s.label}>Without circuit breaker — cascade failure</p>
-        <div style={{display:'flex', flexDirection:'column', alignItems:'center', gap:0}}>
-          <div style={s.node()}>LiteLLM Pod (×100)</div>
-          <SmallArrow />
-          <div style={s.node()}>Rate limit / cache check</div>
-          <div style={{position:'relative', display:'flex', flexDirection:'column', alignItems:'center'}}>
-            <SmallArrow color="#f87171"/>
-            <span style={{position:'absolute', left:8, top:4, fontSize:11, color:'#f87171', whiteSpace:'nowrap'}}>hangs 30s per request</span>
-          </div>
-          <div style={s.node('#fca5a5','#fef2f2')}><span style={{color:'#b91c1c', fontWeight:600}}>Redis — degraded, timing out</span></div>
-          <SmallArrow color="#fb923c"/>
-          <div style={s.node('#fdba74','#fff7ed')}><span style={{color:'#c2410c', fontWeight:600}}>Postgres — 100× normal read load</span></div>
-          <SmallArrow />
-          <div style={{...s.node('#111827','#111827'), color:'#fff', fontWeight:600}}>Total outage — gateway down</div>
-        </div>
-      </div>
-      <figcaption style={s.caption}>Slow Redis → every auth check times out → database overwhelmed → full cascade</figcaption>
-    </figure>
-  );
-}
-
-export function CircuitBreakerStates() {
-  const circle = (border, color, label, sub) => (
-    <div style={{display:'flex', flexDirection:'column', alignItems:'center', width: 140}}>
-      <div style={{width:88, height:88, borderRadius:'50%', border:`2px solid ${border}`, background:'#fff', display:'flex', flexDirection:'column', alignItems:'center', justifyContent:'center'}}>
-        <span style={{fontSize:11, fontWeight:700, color, letterSpacing:'0.06em'}}>{label}</span>
-        <span style={{fontSize:10, color:'#9ca3af', marginTop:2}}>{sub}</span>
-      </div>
-      <p style={{fontSize:11, color:'#6b7280', textAlign:'center', marginTop:10, lineHeight:1.5}}>{'\u00a0'}</p>
-    </div>
-  );
-  const arrow = (label) => (
-    <div style={{display:'flex', flexDirection:'column', alignItems:'center', marginTop:36, marginLeft:4, marginRight:4}}>
-      <span style={{fontSize:10, color:'#6b7280', marginBottom:4}}>{label}</span>
-      <div style={{display:'flex', alignItems:'center'}}>
-        <div style={{height:1, width:48, background:'#9ca3af'}}/>
-        <svg width="8" height="8" style={{marginLeft:-1}}><polygon points="0,0 8,4 0,8" fill="#6b7280"/></svg>
-      </div>
-    </div>
-  );
-  return (
-    <figure style={s.fig}>
-      <div style={s.box}>
-        <p style={s.label}>Circuit breaker state machine</p>
-        <div style={{display:'flex', justifyContent:'center', alignItems:'flex-start'}}>
-          {circle('#1f2937','#111827','CLOSED','normal')}
-          {arrow('5 failures')}
-          {circle('#f87171','#dc2626','OPEN','fast-fail')}
-          {arrow('60s timeout')}
-          {circle('#fbbf24','#b45309','HALF-OPEN','probing')}
-        </div>
-        <div style={{display:'flex', justifyContent:'center', gap:32, marginTop:24}}>
-          <div style={{display:'flex', flexDirection:'column', alignItems:'center', gap:4}}>
-            <div style={{display:'flex', alignItems:'center', gap:4}}>
-              <svg width="8" height="8"><polygon points="8,0 0,4 8,8" fill="#16a34a"/></svg>
-              <div style={{height:1, width:100, background:'#16a34a'}}/>
-            </div>
-            <span style={{fontSize:10, color:'#16a34a'}}>probe success → CLOSED</span>
-          </div>
-          <div style={{display:'flex', flexDirection:'column', alignItems:'center', gap:4}}>
-            <div style={{display:'flex', alignItems:'center', gap:4}}>
-              <svg width="8" height="8"><polygon points="8,0 0,4 8,8" fill="#ef4444"/></svg>
-              <div style={{height:1, width:100, borderTop:'2px dashed #f87171'}}/>
-            </div>
-            <span style={{fontSize:10, color:'#ef4444'}}>probe failure → OPEN again</span>
-          </div>
-        </div>
-      </div>
-    </figure>
-  );
-}
-
-export function CircuitBreakerFlow() {
-  return (
-    <figure style={s.fig}>
-      <div style={s.box}>
-        <p style={s.label}>With circuit breaker — graceful degradation</p>
-        <div style={{display:'flex', flexDirection:'column', alignItems:'center'}}>
-          <div style={s.node()}>Incoming request</div>
-          <SmallArrow />
-          <div style={{...s.node('#111827'), border:'2px solid #111827', fontWeight:600}}>Circuit Breaker</div>
-          <div style={{display:'flex', gap:80, marginTop:20, alignItems:'flex-start'}}>
-            <div style={{display:'flex', flexDirection:'column', alignItems:'center', gap:8}}>
-              <SmallArrow />
-              <span style={{fontSize:10, fontWeight:700, textTransform:'uppercase', letterSpacing:'0.06em', color:'#6b7280', border:'1px solid #e5e7eb', borderRadius:4, padding:'2px 8px'}}>Closed</span>
-              <div style={{...s.node(), textAlign:'center', fontSize:13}}>Redis call<br/><span style={{fontSize:11, color:'#9ca3af'}}>normal latency</span></div>
-            </div>
-            <div style={{display:'flex', flexDirection:'column', alignItems:'center', gap:8}}>
-              <SmallArrow />
-              <span style={{fontSize:10, fontWeight:700, textTransform:'uppercase', letterSpacing:'0.06em', color:'#ef4444', border:'1px solid #fca5a5', borderRadius:4, padding:'2px 8px'}}>Open</span>
-              <div style={{...s.node('#fca5a5'), textAlign:'center', fontSize:13}}>Fast-fail — 0ms<br/><span style={{fontSize:11, color:'#9ca3af'}}>no network call</span></div>
-              <SmallArrow />
-              <div style={{...s.node(), textAlign:'center', fontSize:13}}>DB fallback<br/><span style={{fontSize:11, color:'#9ca3af'}}>bounded load</span></div>
-            </div>
-          </div>
-          <div style={{...s.node('#111827','#111827'), color:'#fff', fontWeight:600, marginTop:24}}>Request completes — gateway stays up</div>
-        </div>
-      </div>
-      <figcaption style={s.caption}>Redis down → circuit opens → 0ms rejection → DB absorbs bounded fallback traffic</figcaption>
-    </figure>
-  );
-}
-
-export function IncidentTimeline() {
-  const row = (color, text) => (
-    <div style={{display:'flex', alignItems:'flex-start', gap:10, marginBottom:12}}>
-      <div style={{marginTop:5, width:6, height:6, borderRadius:'50%', background:color, flexShrink:0}}/>
-      <p style={{fontSize:13, color:'#4b5563', margin:0, lineHeight:1.5}}>{text}</p>
-    </div>
-  );
-  return (
-    <figure style={s.fig}>
-      <div style={s.box}>
-        <p style={s.label}>Redis degrades — before vs. after</p>
-        <div style={{display:'grid', gridTemplateColumns:'1fr 1fr', gap:20}}>
-          <div style={{border:'1px solid #e5e7eb', borderRadius:8, padding:20}}>
-            <p style={{fontSize:10, fontWeight:700, textTransform:'uppercase', letterSpacing:'0.1em', color:'#9ca3af', marginBottom:16}}>Without circuit breaker</p>
-            {row('#f87171','All 100 pods hang for 30s on each auth check')}
-            {row('#f87171','Threadpools fill up, requests queue')}
-            {row('#f87171','100× simultaneous DB fallbacks overwhelm Postgres')}
-            {row('#f87171','Requires manual intervention to recover')}
-          </div>
-          <div style={{border:'1px solid #111827', borderRadius:8, padding:20}}>
-            <p style={{fontSize:10, fontWeight:700, textTransform:'uppercase', letterSpacing:'0.1em', color:'#9ca3af', marginBottom:16}}>With circuit breaker</p>
-            {row('#111827','Circuit opens after 5 failures — 0ms fast-fail')}
-            {row('#111827','Auth falls back to DB — bounded, not 100× load')}
-            {row('#111827','Cache miss rate temporarily elevated — gateway stays up')}
-            {row('#111827','Auto-recovers when Redis comes back — no intervention needed')}
-          </div>
-        </div>
-      </div>
-    </figure>
-  );
-}

docs/my-website/blog/redis_circuit_breaker/index.md

@@ -1,141 +0,0 @@
----
-slug: redis-circuit-breaker
-title: "Making the AI Gateway Resilient to Redis Failures"
-date: 2026-04-11T09:00:00
-authors:
-  - ishaan
-description: "How LiteLLM's production AI Gateway handles Redis degradation at scale without cascading failures — circuit breaker pattern, 0ms fast-fail, automatic recovery."
-tags: [reliability, redis, infrastructure, engineering, ai-gateway]
-hide_table_of_contents: true
----
-
-import { CascadeFailure, CircuitBreakerStates, CircuitBreakerFlow, IncidentTimeline } from './diagrams';
-
-*Last Updated: April 2026*
-
-Enterprise AI Gateway deployments put Redis in the hot path for nearly every request: rate limiting, cache lookups, spend tracking. When Redis is healthy, the latency contribution is single-digit milliseconds — invisible to end users. When it degrades, a production AI Gateway needs to stay up regardless.
-
-Running LiteLLM at scale across 100+ pods means designing for failure modes before they appear. The easy case is Redis going fully down: fail fast, fall through to the database, continue serving requests. The hard case — the one that takes down gateways — is a *slow* Redis: still accepting connections, still responding, but timing out after 20-30 seconds per operation.
-
-{/* truncate */}
-
-## Why slow Redis is harder than a full outage
-
-<CascadeFailure />
-
-With 100 pods each hanging 30 seconds on every auth check, threadpools fill up and requests queue. By the time Redis times out and falls through to Postgres, the database receives 100× its normal load from simultaneous fallbacks. A slow Redis becomes a database outage becomes a full gateway outage. A production-grade AI Gateway cannot allow one degraded dependency to cascade into total failure.
-
-## The fix: circuit breaker pattern
-
-The circuit breaker pattern tracks consecutive failures and cuts off the unhealthy dependency before it cascades. Instead of hanging 30 seconds on each Redis call, the circuit opens after 5 consecutive failures and fast-fails at 0ms — no network call, no wait.
-
-<CircuitBreakerStates />
-
-Three states:
-
-- **CLOSED** — normal. All Redis calls pass through.
-- **OPEN** — Redis is unhealthy. Every call fast-fails instantly. Requests continue with degraded-but-functional behavior: auth and rate limiting fall back to the database.
-- **HALF-OPEN** — after 60 seconds, one probe request tests recovery. Success closes the circuit; failure resets the timer.
-
-This is how a reliable AI Gateway handles infrastructure degradation: stay up, degrade gracefully, recover automatically.
-
-## How requests flow through the AI Gateway
-
-<CircuitBreakerFlow />
-
-When the circuit is open, the gateway does not stall. Auth checks fall back to Postgres — slower, but bounded. The database absorbs the load because it receives *some* requests via DB fallback, not *all* 100 pods simultaneously dumping their queued requests after a 30-second timeout.
-
-The difference between a resilient AI Gateway and a fragile one: controlled degradation vs. uncontrolled cascade.
-
-## The implementation
-
-```python
-class RedisCircuitBreaker:
-    def __init__(self, failure_threshold: int, recovery_timeout: int):
-        self.failure_threshold = failure_threshold  # default: 5
-        self.recovery_timeout = recovery_timeout    # default: 60s
-        self._failure_count = 0
-        self._state = self.CLOSED
-
-    def is_open(self) -> bool:
-        if self._state == self.OPEN:
-            if time.time() - self._opened_at > self.recovery_timeout:
-                self._state = self.HALF_OPEN
-                return False  # this caller is the recovery probe
-            return True       # fast-fail
-        return False
-
-    def record_failure(self):
-        self._failure_count += 1
-        self._opened_at = time.time()
-        if self._failure_count >= self.failure_threshold:
-            self._state = self.OPEN  # open the circuit
-
-    def record_success(self):
-        self._failure_count = 0
-        self._state = self.CLOSED   # Redis recovered
-```
-
-Every async Redis operation goes through a decorator that checks the breaker before touching the network. When open, it raises immediately:
-
-```python
-@_redis_circuit_breaker_guard
-async def async_get_cache(self, key: str):
-    ...
-```
-
-The decorator handles all bookkeeping — success resets nothing, failures increment the counter, exceptions trigger `record_failure()`. The caller sees a clean exception and falls through to its normal non-Redis path. No changes required in calling code.
-
-## AI Gateway resilience in production
-
-<IncidentTimeline />
-
-Redis degradation events no longer cascade in production. The observable symptom during a Redis slowdown is a temporary bump in cache miss rate — the right failure mode for a resilient AI Gateway. Auth still works. Rate limiting still works. Spend tracking still works, at slightly higher DB cost. Recovery is fully automatic when Redis comes back.
-
-```bash
-# configure via environment variables
-REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD=5   # failures before opening
-REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT=60  # seconds before probe
-```
-
-The circuit breaker ships on by default in all LiteLLM versions since `v1.82.0`. No configuration needed for most deployments.
-
-## Key Takeaways
-
-- A slow Redis is more dangerous than a downed one: 30-second timeouts across 100+ pods overwhelm Postgres at 100× normal load
-- LiteLLM's AI Gateway uses a circuit breaker that fast-fails Redis calls at 0ms after 5 consecutive failures
-- Three states: CLOSED (normal), OPEN (fast-fail + DB fallback), HALF-OPEN (probe recovery)
-- Auth, rate limiting, and spend tracking continue working during Redis outages
-- Resilient, production-grade behavior — enabled by default since `v1.82.0`, no configuration required
-
----
-
-### Frequently Asked Questions
-
-### Does the circuit breaker affect normal Redis performance?
-
-No. When Redis is healthy (circuit CLOSED), every call passes through with zero overhead. The breaker only activates after 5 consecutive failures — transparent under normal conditions.
-
-### What happens to rate limiting when the circuit is open?
-
-Rate limiting falls back to Postgres with bounded load. Limits remain enforced at slightly higher DB cost until Redis recovers and the circuit closes automatically.
-
-### How is this different from basic Redis retry logic?
-
-Retry logic still waits for each timeout (30s × retries). The circuit breaker cuts the connection immediately at 0ms after the failure threshold, preventing threadpool exhaustion across all pods simultaneously. Retries make slow-Redis worse; the circuit breaker contains it.
-
-### Is this available in LiteLLM OSS?
-
-Yes. The circuit breaker ships in LiteLLM OSS (Apache 2.0) by default since `v1.82.0`. [LiteLLM Enterprise](https://litellm.ai/enterprise) adds SSO/SCIM, air-gapped deployment, 24/7 SLA support, and advanced guardrails on top of the OSS foundation.
-
----
-
-## Conclusion
-
-Redis resilience is one layer of what makes LiteLLM a production-grade, reliable AI Gateway at scale. The circuit breaker pattern ensures infrastructure degradation stays contained — the right failure mode is a temporary cache miss rate bump, not a full outage. This is how AI Gateway infrastructure should behave under pressure: degrade gracefully, recover automatically, keep serving traffic. For teams with strict uptime and compliance requirements, [LiteLLM Enterprise](https://litellm.ai/enterprise) provides the additional controls needed for regulated production environments.
-
-## Recommended Reading
-
-- [LiteLLM AI Gateway — full feature overview](https://docs.litellm.ai/docs/simple_proxy)
-- [Load balancing and routing across 100+ LLM providers](https://docs.litellm.ai/docs/routing)
-- [Spend tracking and budget controls](https://docs.litellm.ai/docs/proxy/cost_tracking)

docs/my-website/blog/responses_api_encrypted_content_incident/index.md

@@ -1,312 +0,0 @@
----
-slug: responses-api-encrypted-content-incident
-title: "Incident Report: Encrypted Content Failures in Multi-Region Responses API Load Balancing"
-date: 2026-02-24T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-tags: [incident-report, proxy, responses-api, load-balancing]
-hide_table_of_contents: false
----
-
-**Date:** Feb 24, 2026  
-**Duration:** Ongoing (until fix deployed)  
-**Severity:** High (for users load balancing Responses API across different API keys)  
-**Status:** Resolved
-
-## Summary
-
-When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or OpenAI organizations), follow-up requests containing encrypted content items (like `rs_...` reasoning items) would fail with:
-
-```json
-{
-  "error": {
-    "message": "The encrypted content for item rs_0d09d6e56879e76500699d6feee41c8197bd268aae76141f87 could not be verified. Reason: Encrypted content organization_id did not match the target organization.",
-    "type": "invalid_request_error",
-    "code": "invalid_encrypted_content"
-  }
-}
-```
-
-Encrypted content items are cryptographically tied to the API key's organization that created them. When the router load balanced a follow-up request to a deployment with a different API key, decryption failed.
-
-- **Responses API calls with encrypted content:** Complete failure when routed to wrong deployment
-- **Initial requests:** Unaffected — only follow-up requests containing encrypted items failed
-- **Other API endpoints:** No impact — chat completions, embeddings, etc. functioned normally
-
-{/* truncate */}
-
----
-
-## Background
-
-OpenAI's Responses API can return encrypted "reasoning items" (with IDs like `rs_...`) that contain intermediate reasoning steps. These items are encrypted with the organization's key and can only be decrypted by the same organization's API key.
-
-When load balancing across deployments with different API keys, the existing affinity mechanisms were insufficient:
-
-- **`responses_api_deployment_check`**: Requires `previous_response_id` which some clients (like Codex) don't provide
-- **`deployment_affinity`**: Too broad — pins *all* requests from a user to one deployment, reducing effective quota by the number of users
-- **`session_affinity`**: Requires explicit session IDs and still reduces quota
-
-```mermaid
-flowchart TD
-    A["1. Initial request to Responses API
-    router.aresponses()"] --> B["2. Router load balances to Deployment A
-    (API Key 1, Azure East US)"]
-    B --> C["3. Response contains encrypted item
-    rs_abc123 (encrypted with Org 1 key)"]
-    C --> D["4. Follow-up request includes rs_abc123 in input"]
-    D --> E["5. Router load balances to Deployment B
-    (API Key 2, Azure West Europe)"]
-    E -->|"Different API key"| F["6. ❌ Deployment B cannot decrypt rs_abc123
-    Error: invalid_encrypted_content"]
-    
-    D -.->|"With encrypted_content_affinity"| G["5b. Router detects rs_abc123 was created by Deployment A"]
-    G --> H["6b. ✅ Routes to Deployment A (bypasses rate limits)
-    Request succeeds"]
-
-    style F fill:#f8d7da,stroke:#dc3545
-    style H fill:#d4edda,stroke:#28a745
-    style E fill:#fff3cd,stroke:#ffc107
-    style G fill:#d4edda,stroke:#28a745
-```
-
----
-
-## Root Cause
-
-LiteLLM's router had no mechanism to track which deployment created specific encrypted content items and route follow-up requests accordingly. The router treated all deployments as interchangeable, leading to decryption failures when encrypted content crossed organizational boundaries.
-
-**The Problem Flow:**
-
-1. User calls `router.aresponses()` with model `gpt-5.1-codex`
-2. Router load balances to Deployment A (Azure East US, API Key 1)
-3. Response contains encrypted reasoning item `rs_abc123` (encrypted with Org 1's key)
-4. User makes follow-up request with `rs_abc123` in the input
-5. Router load balances to Deployment B (Azure West Europe, API Key 2)
-6. Deployment B tries to decrypt `rs_abc123` with Org 2's key → **fails**
-
-**Why Existing Solutions Didn't Work:**
-
-- **`previous_response_id`**: Not provided by all clients (e.g., Codex)
-- **`deployment_affinity`**: Pins *all* user requests to one deployment → reduces quota to 1/N where N = number of deployments
-- **`session_affinity`**: Requires explicit session management and still reduces quota
-
-**Timeline:**
-
-1. Users configured multi-region Responses API load balancing with different API keys
-2. Initial requests succeeded, but follow-up requests with encrypted content failed intermittently
-3. Error rate correlated with number of deployments (more deployments = higher chance of routing to wrong one)
-4. Investigation revealed encrypted content was organization-bound
-5. Existing affinity mechanisms deemed unsuitable (quota reduction, missing `previous_response_id`)
-6. New solution designed and implemented: `encrypted_content_affinity`
-
----
-
-## The Fix
-
-Implemented a new `encrypted_content_affinity` pre-call check that intelligently tracks encrypted content and routes follow-up requests **only when necessary**.
-
-### Implementation
-
-**1. Encoding `model_id` into output items** ([`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py))
-
-The same approach used for `previous_response_id` affinity — no cache needed. When a response contains output items with `encrypted_content`, LiteLLM encodes the originating deployment's `model_id` in **two places** for redundancy:
-
-1. **Into the item ID** (if present): `rs_abc123` → `encitem_{base64("litellm:model_id:{model_id};item_id:rs_abc123")}`
-2. **Into the encrypted_content itself**: Wraps the content with `litellm_enc:{base64("model_id:{model_id}")};{original_encrypted_content}`
-
-```python
-# Encoding item IDs (when present)
-def _build_encrypted_item_id(model_id: str, item_id: str) -> str:
-    assembled = f"litellm:model_id:{model_id};item_id:{item_id}"
-    encoded = base64.b64encode(assembled.encode("utf-8")).decode("utf-8")
-    return f"encitem_{encoded}"
-
-# Wrapping encrypted_content (always, for redundancy)
-def _wrap_encrypted_content_with_model_id(encrypted_content: str, model_id: str) -> str:
-    metadata = f"model_id:{model_id}"
-    encoded_metadata = base64.b64encode(metadata.encode("utf-8")).decode("utf-8")
-    return f"litellm_enc:{encoded_metadata};{encrypted_content}"
-```
-
-**Why wrap encrypted_content directly?** Some clients (like Codex) don't consistently send item IDs in follow-up requests, but they always send the `encrypted_content` itself. By embedding `model_id` into the content, affinity works even when IDs are missing.
-
-**Streaming responses:** The wrapping logic is applied to both:
-- Final response objects (non-streaming)
-- Individual streaming events (`response.output_item.added`, `response.output_item.done`)
-
-This ensures clients receiving streaming responses get wrapped content they can send back.
-
-Before forwarding to the upstream provider, LiteLLM restores the original item IDs and unwraps encrypted_content so the provider never sees the encoded form:
-
-```python
-# In responses/main.py — before calling the handler
-input = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(input)
-```
-
-**2. `EncryptedContentAffinityCheck` — routing only** ([`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py))
-
-No `async_log_success_event` or cache lookups — the `model_id` is decoded directly from the item ID or encrypted_content:
-
-```python
-class EncryptedContentAffinityCheck(CustomLogger):
-    async def async_filter_deployments(self, model, healthy_deployments, ...):
-        """Extract model_id from input items (ID or encrypted_content) and pin to that deployment."""
-        for item in request_kwargs.get("input", []):
-            # Try to extract model_id from two sources:
-            model_id = self._extract_model_id_from_input(item)
-            
-            if model_id:
-                deployment = self._find_deployment_by_model_id(
-                    healthy_deployments, model_id
-                )
-                if deployment:
-                    request_kwargs["_encrypted_content_affinity_pinned"] = True
-                    return [deployment]
-        return healthy_deployments
-    
-    def _extract_model_id_from_input(self, item: dict) -> Optional[str]:
-        """Extract model_id from either encoded ID or wrapped encrypted_content."""
-        # 1. Try decoding from item ID (if present)
-        item_id = item.get("id", "")
-        if item_id:
-            decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id)
-            if decoded:
-                return decoded["model_id"]
-        
-        # 2. Try unwrapping from encrypted_content (fallback for clients that omit IDs)
-        encrypted_content = item.get("encrypted_content", "")
-        if encrypted_content and encrypted_content.startswith("litellm_enc:"):
-            model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
-                encrypted_content
-            )
-            return model_id
-        
-        return None
-```
-
-**3. Rate Limit Bypass** ([`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py))
-
-When encrypted content requires a specific deployment, RPM/TPM limits are bypassed (the request would fail on any other deployment anyway):
-
-```python
-# In async_get_available_deployment, after filtering healthy deployments:
-if (
-    request_kwargs.get("_encrypted_content_affinity_pinned")
-    and len(healthy_deployments) == 1
-):
-    return healthy_deployments[0]  # Bypass routing strategy (RPM/TPM checks)
-```
-
-**3. Configuration**
-
-```yaml
-router_settings:
-  routing_strategy: usage-based-routing-v2
-  enable_pre_call_checks: true
-  optional_pre_call_checks:
-    - encrypted_content_affinity
-  deployment_affinity_ttl_seconds: 86400  # 24 hours
-```
-
-### Key Benefits
-
-✅ **No quota reduction**: Only pins requests containing encrypted items  
-✅ **Bypasses rate limits**: When encrypted content requires a specific deployment, RPM/TPM limits don't block it  
-✅ **No `previous_response_id` required**: Works by encoding `model_id` directly into the item ID  
-✅ **No cache required**: `model_id` is decoded on-the-fly from the item ID — no Redis, no TTL  
-✅ **Globally safe**: Can be enabled for all models; non-Responses-API calls are unaffected  
-✅ **Surgical precision**: Normal requests continue to load balance freely
-
----
-
-## Remediation
-
-| # | Action | Status | Code |
-|---|---|---|---|
-| 1 | Encode `model_id` into encrypted-content item IDs on response | ✅ Done | [`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py) |
-| 2 | Restore original item IDs before forwarding to upstream provider | ✅ Done | [`responses/main.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/main.py) |
-| 3 | `EncryptedContentAffinityCheck`: decode item IDs to route (no cache) | ✅ Done | [`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py) |
-| 4 | Add `encrypted_content_affinity` to `OptionalPreCallChecks` type | ✅ Done | [`types/router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/types/router.py) |
-| 5 | Implement rate limit bypass for affinity-pinned requests | ✅ Done | [`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py) |
-| 6 | Unit tests: encoding/decoding utilities, routing, RPM bypass | ✅ Done | [`test_encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py) |
-| 7 | Documentation: Responses API guide, load balancing guide, config reference | ✅ Done | [Docs](https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing) |
-| 8 | **[Mar 3]** Fix streaming events to wrap encrypted_content | ✅ Done | [`responses/streaming_iterator.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/streaming_iterator.py) |
-
----
-
-## Follow-up Fix: Streaming Responses (Mar 3, 2026)
-
-### The Issue
-
-After the initial fix was deployed, users reported that the `invalid_encrypted_content` error **still occurred** when using streaming responses with clients like Codex. Investigation revealed:
-
-- ✅ Non-streaming responses: `encrypted_content` was correctly wrapped with `litellm_enc:` prefix
-- ❌ Streaming responses: Individual `response.output_item.added` and `response.output_item.done` events contained **raw, unwrapped** `encrypted_content`
-
-Since Codex and other clients consume responses as streams, they received unwrapped content in these events and sent it back in follow-up requests, causing the affinity check to fail.
-
-### The Root Cause
-
-The `_update_encrypted_content_item_ids_in_response` function only modified the **final** response object, which is used for non-streaming responses. For streaming responses, individual chunks are processed by `ResponsesAPIStreamingIterator._process_chunk`, which was **not** applying the wrapping logic to streaming events.
-
-### The Fix
-
-Modified `litellm/litellm/responses/streaming_iterator.py` to wrap `encrypted_content` in streaming events:
-
-```python
-# In ResponsesAPIStreamingIterator._process_chunk
-if (
-    self.litellm_metadata
-    and self.litellm_metadata.get("encrypted_content_affinity_enabled")
-):
-    event_type = getattr(openai_responses_api_chunk, "type", None)
-    if event_type in (
-        ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
-        ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
-    ):
-        item = getattr(openai_responses_api_chunk, "item", None)
-        if item:
-            encrypted_content = getattr(item, "encrypted_content", None)
-            if encrypted_content and isinstance(encrypted_content, str):
-                model_id = (
-                    self.litellm_metadata.get("model_info", {}).get("id")
-                    if self.litellm_metadata
-                    else None
-                )
-                if model_id:
-                    wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
-                        encrypted_content, model_id
-                    )
-                    setattr(item, "encrypted_content", wrapped_content)
-```
-
-This ensures that **all** `encrypted_content` sent to clients (streaming or non-streaming) is wrapped with `model_id` metadata, enabling consistent affinity routing.
-
----
-
-## Migration Guide
-
-### Before (Using `deployment_affinity`)
-
-```yaml
-router_settings:
-  optional_pre_call_checks:
-    - deployment_affinity  # ❌ Reduces quota by number of users
-```
-
-**Problem:** All requests from a user pin to one deployment, reducing effective quota to 1/N.
-
-### After (Using `encrypted_content_affinity`)
-
-```yaml
-router_settings:
-  optional_pre_call_checks:
-    - encrypted_content_affinity  # ✅ Only pins requests with encrypted content
-```
-
-**Benefit:** Normal requests load balance freely, only encrypted content requests pin when necessary.
-
----

docs/my-website/blog/security_hardening_april_2026/index.md

@@ -1,66 +0,0 @@
----
-slug: security-hardening-april-2026
-title: "Security Update: Vulnerability Disclosures and Ongoing Hardening"
-date: 2026-04-03T12:00:00
-authors:
-  - krrish
-  - ishaan-alt
-description: "Disclosure of security vulnerabilities fixed in LiteLLM v1.83.0, and the launch of our bug bounty program."
-tags: [security]
-hide_table_of_contents: false
----
-
-After the [supply chain incident](https://docs.litellm.ai/blog/security-update-march-2026) in March, we brought in [Veria Labs](https://verialabs.com/) to audit the LiteLLM proxy and fixed a number of vulnerability reports from independent researchers. All issues below are fixed in v1.83.0. If you are affected, particularly if you have JWT auth enabled, we recommend upgrading.
-
-We've also launched a [bug bounty program](#bug-bounty-program) and Veria Labs is continuing to audit the proxy. More fixes will ship in upcoming versions.
-
-The two high-severity issues ([CVE-2026-35029](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789) and [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)) **both require the attacker to already have a valid API key for the proxy**. These are not exploitable by unauthenticated users. 
-
-The critical-severity issue ([CVE-2026-35030](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)) is an authentication bypass, but only affects deployments with `enable_jwt_auth` explicitly enabled, which is off by default. **The default LiteLLM configuration is not affected, and no LiteLLM Cloud customers had this feature enabled.**
-
-{/* truncate */}
-
-## Vulnerabilities
-
-### CVE-2026-35030: Authentication bypass via OIDC cache collision (Critical)
-
-Found by Veria Labs.
-
-When `enable_jwt_auth` is enabled, LiteLLM cached OIDC userinfo using `token[:20]` as the cache key. JWTs from the same signing algorithm share the same header prefix, so an attacker could forge a token that hits another user's cache entry and inherit their session. We fixed this by keying the cache on `sha256(token)` instead.
-
-**Most deployments are not affected.** This requires `enable_jwt_auth: true`, which is off by default. If you can't upgrade, disable JWT auth as a workaround.
-
-Full advisory: [GHSA-jjhc-v7c2-5hh6](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)
-
-### CVE-2026-35029: Privilege escalation via `/config/update` (High)
-
-Found by Lakera.
-
-`/config/update` didn't check the caller's role. Any authenticated user could modify the proxy's runtime configuration, which could lead to arbitrary file read, admin account takeover, or remote code execution. We now require the `proxy_admin` role on this endpoint.
-
-Full advisory: [GHSA-53mr-6c8q-9789](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789)
-
-### Password hash exposure and pass-the-hash login (High)
-
-Weak hashing originally reported by GitHub user [hamzayevmaqsud](https://github.com/hamzayevmaqsud) ([#15484](https://github.com/BerriAI/litellm/issues/15484)). The full chain was identified by Luca Vandenweghe and Maarten De Rammelaere of [iO Digital](https://www.iodigital.com/).
-
-Passwords were stored as unsalted SHA-256 hashes, and in some cases plaintext. Several API endpoints returned the hash to any authenticated user, and `/v2/login` accepted the raw hash as a credential without re-hashing it, so a stolen hash was as good as the password itself. We've moved to scrypt with random salts and stripped hashes from all API responses.
-
-Full advisory: [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)
-
-## Bug bounty program
-
-After the supply chain incident and these disclosures it was clear we needed more external eyes on the project. We've set up a bug bounty program so researchers have a way to report issues. 
-
-Bounties are currently paid for P0 (supply chain) and P1 (unauthenticated proxy access) vulnerabilities:
-
-| Severity | Bounty | Example |
-|----------|--------|---------|
-| Critical | $1,500 – $3,000 | Supply chain compromise |
-| High | $500 – $1,500 | Unauthenticated access to protected data |
-
-We plan on expanding the program further in the coming months. More info about the bug bounty program is available [here](https://github.com/BerriAI/litellm/security).
-
-## What's next
-
-Veria Labs is continuing to work with us on a broader audit of the proxy. Security advisories sent through Github will be responded to within five business days. We'll publish advisories as issues are confirmed and fixed.

docs/my-website/blog/security_townhall_updates/index.md

@@ -1,223 +0,0 @@
----
-slug: security-townhall-updates
-title: "Security Townhall Updates"
-date: 2026-03-27T12:00:00
-authors:
-  - krrish
-  - ishaan-alt
-description: "What happened, what we've done, and what comes next for LiteLLM's release and security processes."
-tags: [security, incident-report]
-hide_table_of_contents: false
----
-
-import Image from '@theme/IdealImage';
-
-Thank you to everyone who joined our town hall.
-
-We wanted to use that time to walk through what we know, what we've done so far, and how we're improving LiteLLM's release and security processes going forward. This post is a written version of that update. [Slides available here](https://drive.google.com/file/d/17hsSG7nk-OYL7VRCTbTa7McrWREtS9OO/view?usp=sharing)
-
-{/* truncate */}
-
-## What happened
-
-On March 24, 2026 at 10:39 UTC, LiteLLM v1.82.7 was pushed to PyPI. Version v1.82.8 was published soon after. Those packages were live for about 40 minutes before being quarantined by PyPI. By 16:00 UTC, the LiteLLM team had worked with PyPI to delete the affected packages.
-
-At this point, our understanding is that this was a supply-chain incident affecting those two published versions.
-
-## How did this happen?
-
-Our understanding is that the issue came from the [compromised Trivy security scanner](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) dependency in our CI/CD pipeline.
-
-<Image 
-  img={require('../../img/shared_ci_cd_environment.png')}
-  style={{width: '500px', height: '400px', display: 'block'}}
-/>
-
-There were three major contributing factors:
-
-### 1. Shared CI/CD environment
-
-At the time, everything was running on CircleCI, and all steps shared a common environment. That increased blast radius: if one component was compromised, it could potentially access credentials or context intended for other parts of the pipeline.
-
-### 2. Static credentials in environment variables
-
-Release credentials, including credentials for PyPI, GHCR, and Docker publishing, were available as static secrets in the environment. That meant a compromised step could access long-lived release credentials.
-
-### 3. Unpinned Trivy dependency
-
-In our security scanning component, we had an unpinned Trivy dependency. Our present understanding is that a compromised Trivy package ran during the scan, had access to environment variables, and enabled attackers to obtain those credentials.
-
-**In summary:** a compromised package in CI had access to secrets it should not have had, and those secrets were then used in the release path.
-
-## What we've already done
-
-
-In the last 3 days, we've taken the following steps:
-
-### 1. Minimize Scope of Impact
-
-#### Prevented further key abuse
-
-We deleted or rotated all impacted or adjacent secret keys, including PyPI, GitHub, Docker, and related credentials. Out of an abundance of caution, we've also rotated LiteLLM maintainer accounts. 
-
-#### Prevent branch attacks
-
-We removed roughly 6,000 open branches and added an auto-deletion policy for branches merged into `main`. This reduces the surface area for branch-based abuse.
-
-#### Pinned CI/CD dependencies
-
-We've pinned all Github Actions, and are working on pinning all CircleCI dependencies as well.
-
-#### Paused releases
-
-We've paused new releases until we've confirmed codebase security and put stronger release controls in place.
-
-### 2. Secured LiteLLM
-
-#### Forensic analysis
-
-We are working with Google's Mandiant cybersecurity team to confirm the source of the attack and verify the security of the codebase. We also confirmed that no malicious code was pushed to `main`.
-
-#### Confirm Application Security
-
-In parallel, we are working with whitehat hackers at [Veria Labs](https://verialabs.com/) to verify application security and review improvements to our CI/CD process.
-
-We have also confirmed that the last 20 LiteLLM releases contain no indicators of compromise, and that no unauthenticated attacks can be made against LiteLLM Proxy based on our current investigation. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions)
-
-#### Created a security working group
-
-We created a new security working group inside LiteLLM focused on:
-
-- Building threat models
-- Auditing the build process and dependencies
-
-If you're interested in joining the security working group, please file an issue [here](https://github.com/BerriAI/litellm-security-wg).
-
-### 3. Improved CI/CD
-
-We've already begun making structural changes to how releases are built and published. These align with our goals (covered in the next section) around isolated environments, ephemeral credentials, and release auditing.
-
-## Roadmap
-
-We plan on following 4 guiding principles for our new CI/CD pipeline:
-
-1. **Limit** what each package can access
-2. **Reduce** the number of sensitive environment variables
-3. **Avoid** compromised packages
-4. **Prevent** release tampering
-
-
-### Isolated environments
-
-<Image 
-  img={require('../../img/isolated_ci_cd_environments.png')}
-  style={{width: '400px', height: 'auto'}}
-/>
-
-We are breaking our CI/CD into 4 semantic concepts:
-
-1. Unit tests
-2. Integration tests
-3. Security scans
-4. Release publishing
-
-And will be running each of these in isolated environments.
-
-This will limit the damage that any single compromised component can cause.
-
-### Ephemeral credentials
-
-We plan to move to ephemeral credentials for PyPI (Trusted Publisher) and GHCR (Token-based authentication) releases. This will reduce the risk of credentials being leaked or compromised.
-
-We have already begun doing this: 
-
-- PyPI Trusted Publisher on GitHub Actions [PR](https://github.com/BerriAI/litellm/pull/24654)
-- GHCR Token-based authentication on GitHub Actions [PR](https://github.com/BerriAI/litellm/pull/24683)
-
-### Release auditing
-
-Our goal is to allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published.
-
-This will ensure, your releases are safe, even when: 
-- Stolen PyPI/GHCR credentials are used to publish malicious releases
-- Tampered registry artifacts are published
-- Tag mutations are made after the release is published
-
-We believe that [Cosign](https://github.com/sigstore/cosign) is a good fit for this, and have shipped it in [PR #24683](https://github.com/BerriAI/litellm/pull/24683).
-
-#### How to verify a Docker image with Cosign
-
-Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key that was introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
-
-**Verify using the pinned commit hash (recommended):**
-
-A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-**Verify using a release tag (convenience):**
-
-Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
-
-Expected output:
-
-```
-The following checks were performed on each of these signatures:
-  - The cosign claims were validated
-  - The signatures were verified against the specified public key
-```
-
-### Avoid Compromised Packages
-
-- Move to pinned, verified SHAs for packages and actions used in CI/CD, avoiding `latest` wherever possible. 
-- Add a cooldown period before upgrading to a new version of a package - allows more time to investigate and verify the new version. 
-
-We've added zizmor to help us catch issues such as unpinned dependencies and credential leakage. [commit](https://github.com/BerriAI/litellm/commit/a671275f5c5b0e1fb1adacdf3b6ef779aaa5d56c).
-
-
-## Frequently Asked Questions
-
-**Q: Did you observe any lateral movement into your corporate environment during this incident?**
-
-A: No. Our investigation to date, conducted in coordination with external security experts, has found no evidence of lateral movement into our internal corporate systems. The incident was isolated to the CI/CD pipeline and the release path for specific versions (v1.82.7 and v1.82.8). As a proactive measure, we have rotated all potentially impacted or adjacent secrets—including PyPI, GitHub, and Docker credentials—and updated maintainer account security to ensure continued isolation.
-
-**Q: Do you expect delays in future product releases due to these new security measures?**
-
-A: We are committed to balancing security with speed. While we have temporarily paused releases to implement stronger controls, we are moving quickly to automate our new security protocols. We are currently implementing isolated CI/CD environments, ephemeral credentials (via Trusted Publishers), and release auditing with Cosign. These improvements are designed to be integrated into our automated pipeline, allowing us to maintain a fast release cadence while ensuring every package is verified and secure.
-
-**Q: Were older packages impacted?**
-
-Our current findings show no indicators of compromise in the last 20 versions of LiteLLM. This was manually verified by our team and independently reviewed by Veria Labs.
-
-We have also published the verified versions for users to use. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions)
-
-
-
-## Questions & Support 
-
-If you believe your systems may be affected, contact us immediately:
-
-- **Security:** security@berri.ai
-- **Support:** support@berri.ai
-- **Slack:** Reach out to the LiteLLM team directly [here](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA)
-
-## Hiring 
-
-We are currently hiring for: 
-
-- DevOps Engineer - to keep ci/cd secure and running smoothly
-- Security Engineer - to keep the application secure
-
-If you're interest in joining, please apply [here](https://jobs.ashbyhq.com/litellm)

docs/my-website/blog/security_update_march_2026/index.md

@@ -1,820 +0,0 @@
----
-slug: security-update-march-2026
-title: "Security Update: Suspected Supply Chain Incident"
-date: 2026-03-24T14:00:00
-authors:
-  - krrish
-  - ishaan-alt
-description: "As of 2:00 PM ET on March 24, 2026"
-tags: [security, incident-report]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-import VersionVerificationTable from '@site/src/components/VersionVerificationTable';
-
-> **Status:** Active investigation
-> **Last updated:** March 27, 2026
-
-> **Update (March 30):** A new **clean** version of LiteLLM is now available (v1.83.0). This was released by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM.
-
-> **Update (March 27):** Review Townhall updates, including explanation of the incident, what we've done, and what comes next. [Learn more](https://docs.litellm.ai/blog/security-townhall-updates)
-
-> **Update (March 27):** Added [Verified safe versions](#verified-safe-versions) section with SHA-256 checksums for all audited PyPI and Docker releases.
-
-> **Update (March 26):** Added `checkmarx[.]zone` to [Indicators of compromise](#indicators-of-compromise-iocs)
-
-> **Update (March 25):** Added community-contributed scripts for scanning GitHub Actions and GitLab CI pipelines for the compromised versions. See [How to check if you are affected](#how-to-check-if-you-are-affected). s/o [@Zach Fury](https://www.linkedin.com/in/fryware/) for these scripts.
-
-
-## TLDR; 
-- The compromised PyPI packages were **litellm==1.82.7** and **litellm==1.82.8**. Those packages were live on March 24, 2026 from 10:39 UTC for about 40 minutes before being quarantined by PyPI.
-- We believe that the compromise originated from the [Trivy dependency](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) used in our CI/CD security scanning workflow.
-- Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages.
-- ~~We have paused all new LiteLLM releases until we complete a broader supply-chain review and confirm the release path is safe.~~ **Updated:** We have now released a new **safe** version of LiteLLM (v1.83.0) by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM. We have also verified the codebase is safe and no malicious code was pushed to `main`.
-
-
-## Overview
-
-LiteLLM AI Gateway is investigating a suspected supply chain attack involving unauthorized PyPI package publishes. Current evidence suggests a maintainer's PyPI account may have been compromised and used to distribute malicious code.
-
-At this time, we believe this incident may be linked to the broader [Trivy security compromise](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/), in which stolen credentials were reportedly used to gain unauthorized access to the LiteLLM publishing pipeline.
-
-This investigation is ongoing. Details below may change as we confirm additional findings.
-
-## Confirmed affected versions
-
-The following LiteLLM versions published to PyPI were impacted:
-
-- **v1.82.7**: contained a malicious payload in the LiteLLM AI Gateway `proxy_server.py`
-- **v1.82.8**: contained `litellm_init.pth` and a malicious payload in the LiteLLM AI Gateway `proxy_server.py`
-
-If you installed or ran either of these versions, review the recommendations below immediately.
-
-Note: These versions have already been removed from PyPI.
-
-## What happened
-
-Initial evidence suggests the attacker bypassed official CI/CD workflows and uploaded malicious packages directly to PyPI.
-
-These compromised versions appear to have included a credential stealer designed to:
-
-- Harvest secrets by scanning for:
-  - environment variables
-  - SSH keys
-  - cloud provider credentials (AWS, GCP, Azure)
-  - Kubernetes tokens
-  - database passwords
-- Encrypt and exfiltrate data via a `POST` request to `models.litellm.cloud`, which is **not** an official BerriAI / LiteLLM domain
-
-## Who is affected
-
-You may be affected if **any** of the following are true:
-
-- You installed or upgraded LiteLLM via `pip` on **March 24, 2026**, between **10:39 UTC and 16:00 UTC**
-- You ran `pip install litellm` without pinning a version and received **v1.82.7** or **v1.82.8**
-- You built a Docker image during this window that included `pip install litellm` without a pinned version
-- A dependency in your project pulled in LiteLLM as a transitive, unpinned dependency
-  (for example through AI agent frameworks, MCP servers, or LLM orchestration tools)
-
-You are **not** affected if any of the following are true:
-
-**LiteLLM AI Gateway/Proxy users:** Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages.
-
-- You are using **LiteLLM Cloud**
-- You are using the official LiteLLM AI Gateway Docker image: `ghcr.io/berriai/litellm`
-- You are on **v1.82.6 or earlier** and did not upgrade during the affected window
-- You installed LiteLLM from source via the GitHub repository, which was **not** compromised
-
-
-### How to check if you are affected
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-```bash
-pip show litellm
-```
-</TabItem>
-<TabItem value="proxy" label="PROXY">
-
-Go to the proxy base url, and check the version of the installed LiteLLM.
-
-![Proxy version check](../../img/security_update_march_2026/proxy_version.png)
-</TabItem>
-<TabItem value="github" label="GitHub Actions">
-
-Scans all repositories in a GitHub organization for workflow jobs that installed the compromised versions.
-
-**Requirements:** Python 3 and `requests` (`pip install requests`).
-
-**Setup:**
-
-```bash
-export GITHUB_TOKEN="your-github-pat"
-```
-
-**Run:**
-
-```bash
-python find_litellm_github.py
-```
-
-Set the `ORG` variable in the script to your GitHub organization name.
-
-Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day.
-
-<details>
-<summary>View full script (find_litellm_github.py)</summary>
-
-```python
-#!/usr/bin/env python3
-"""
-Scan all GitHub Actions jobs in a GitHub org that ran between
-0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8.
-
-Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later.
-"""
-
-import io
-import os
-import re
-import sys
-import zipfile
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime, timezone
-
-import requests
-
-GITHUB_URL   = "https://api.github.com"
-ORG          = "your-org"  # <-- set to your GitHub organization
-TOKEN        = os.environ.get("GITHUB_TOKEN", "")
-
-TODAY        = datetime.now(timezone.utc).date()
-WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day,  8,  0, 0, tzinfo=timezone.utc)
-WINDOW_END   = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc)
-
-TARGET_VERSIONS = {"1.82.7", "1.82.8"}
-VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE)
-
-SESSION = requests.Session()
-SESSION.headers.update({
-    "Authorization": f"Bearer {TOKEN}",
-    "Accept": "application/vnd.github+json",
-    "X-GitHub-Api-Version": "2022-11-28",
-})
-
-
-def get_paginated(url, params=None):
-    params = dict(params or {})
-    params.setdefault("per_page", 100)
-    page = 1
-    while True:
-        params["page"] = page
-        resp = SESSION.get(url, params=params, timeout=30)
-        if resp.status_code == 404:
-            return
-        resp.raise_for_status()
-        data = resp.json()
-        if isinstance(data, dict):
-            items = next((v for v in data.values() if isinstance(v, list)), [])
-        else:
-            items = data
-        if not items:
-            break
-        yield from items
-        if len(items) < params["per_page"]:
-            break
-        page += 1
-
-
-def parse_ts(ts_str):
-    if not ts_str:
-        return None
-    return datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
-
-
-def get_repos():
-    repos = []
-    for r in get_paginated(f"{GITHUB_URL}/orgs/{ORG}/repos", {"type": "all"}):
-        repos.append({"id": r["id"], "name": r["name"], "full_name": r["full_name"]})
-    return repos
-
-
-def get_runs_in_window(repo_full_name):
-    created_filter = (
-        f"{WINDOW_START.strftime('%Y-%m-%dT%H:%M:%SZ')}"
-        f"..{WINDOW_END.strftime('%Y-%m-%dT%H:%M:%SZ')}"
-    )
-    url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs"
-    runs = []
-    for run in get_paginated(url, {"created": created_filter, "per_page": 100}):
-        ts = parse_ts(run.get("run_started_at") or run.get("created_at"))
-        if ts and WINDOW_START <= ts <= WINDOW_END:
-            runs.append(run)
-    return runs
-
-
-def get_jobs_for_run(repo_full_name, run_id):
-    url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs/{run_id}/jobs"
-    jobs = []
-    for job in get_paginated(url, {"filter": "all"}):
-        ts = parse_ts(job.get("started_at"))
-        if ts and WINDOW_START <= ts <= WINDOW_END:
-            jobs.append(job)
-    return jobs
-
-
-def fetch_job_log(repo_full_name, job_id):
-    url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/jobs/{job_id}/logs"
-    resp = SESSION.get(url, timeout=60, allow_redirects=True)
-    if resp.status_code in (403, 404, 410):
-        return ""
-    resp.raise_for_status()
-
-    content_type = resp.headers.get("Content-Type", "")
-    if "zip" in content_type or resp.content[:2] == b"PK":
-        try:
-            with zipfile.ZipFile(io.BytesIO(resp.content)) as zf:
-                parts = []
-                for name in sorted(zf.namelist()):
-                    with zf.open(name) as f:
-                        parts.append(f.read().decode("utf-8", errors="replace"))
-                return "\n".join(parts)
-        except zipfile.BadZipFile:
-            pass
-    return resp.text
-
-
-def check_job(repo_full_name, job):
-    job_id   = job["id"]
-    job_name = job["name"]
-    run_id   = job["run_id"]
-    started  = job.get("started_at", "")
-
-    log_text = fetch_job_log(repo_full_name, job_id)
-    if not log_text:
-        return None
-
-    found_versions = set()
-    context_lines  = []
-    for line in log_text.splitlines():
-        m = VERSION_PATTERN.search(line)
-        if m:
-            ver = m.group(1)
-            if ver in TARGET_VERSIONS:
-                found_versions.add(ver)
-                context_lines.append(line.strip())
-
-    if not found_versions:
-        return None
-
-    return {
-        "repo":       repo_full_name,
-        "run_id":     run_id,
-        "job_id":     job_id,
-        "job_name":   job_name,
-        "started_at": started,
-        "versions":   sorted(found_versions),
-        "context":    context_lines[:10],
-        "job_url":    job.get("html_url", f"https://github.com/{repo_full_name}/actions/runs/{run_id}"),
-    }
-
-
-def main():
-    if not TOKEN:
-        print("ERROR: Set GITHUB_TOKEN environment variable.", file=sys.stderr)
-        sys.exit(1)
-
-    print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}")
-    print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}")
-    print()
-
-    print(f"Fetching repositories for org '{ORG}'...")
-    repos = get_repos()
-    print(f"  Found {len(repos)} repositories")
-    print()
-
-    jobs_to_check = []
-
-    print("Scanning workflow runs for time window...")
-    for repo in repos:
-        full_name = repo["full_name"]
-        try:
-            runs = get_runs_in_window(full_name)
-        except requests.HTTPError as e:
-            print(f"  WARN: {full_name} - {e}", file=sys.stderr)
-            continue
-        if not runs:
-            continue
-        print(f"  {full_name}: {len(runs)} run(s) in window")
-        for run in runs:
-            try:
-                jobs = get_jobs_for_run(full_name, run["id"])
-            except requests.HTTPError as e:
-                print(f"    WARN: run {run['id']} - {e}", file=sys.stderr)
-                continue
-            for job in jobs:
-                jobs_to_check.append((full_name, job))
-
-    total = len(jobs_to_check)
-    print(f"\nFetching logs for {total} job(s)...")
-    print()
-
-    hits = []
-    with ThreadPoolExecutor(max_workers=8) as pool:
-        futures = {
-            pool.submit(check_job, full_name, job): (full_name, job["id"])
-            for full_name, job in jobs_to_check
-        }
-        done = 0
-        for future in as_completed(futures):
-            done += 1
-            full_name, jid = futures[future]
-            try:
-                result = future.result()
-            except Exception as e:
-                print(f"  ERROR {full_name} job {jid}: {e}", file=sys.stderr)
-                continue
-            if result:
-                hits.append(result)
-            print(
-                f"  [{done}/{total}] {full_name} job {jid}" +
-                (f"  *** HIT: litellm {result['versions']} ***" if result else ""),
-                flush=True,
-            )
-
-    print()
-    print("=" * 72)
-    print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}")
-    print("=" * 72)
-
-    if not hits:
-        print("No matches found.")
-        return
-
-    for h in sorted(hits, key=lambda x: x["started_at"]):
-        print()
-        print(f"  Repo      : {h['repo']}")
-        print(f"  Job       : {h['job_name']} (#{h['job_id']})")
-        print(f"  Run ID    : {h['run_id']}")
-        print(f"  Started   : {h['started_at']}")
-        print(f"  Versions  : litellm {', '.join(h['versions'])}")
-        print(f"  URL       : {h['job_url']}")
-        print(f"  Log lines :")
-        for line in h["context"]:
-            print(f"    {line}")
-
-
-if __name__ == "__main__":
-    main()
-```
-
-</details>
-
-</TabItem>
-<TabItem value="gitlab" label="GitLab CI">
-
-Scans all projects in a GitLab group (including subgroups) for CI/CD jobs that installed the compromised versions.
-
-**Requirements:** Python 3 and `requests` (`pip install requests`).
-
-**Setup:**
-
-```bash
-export GITLAB_TOKEN="your-gitlab-pat"
-```
-
-**Run:**
-
-```bash
-python find_litellm_jobs.py
-```
-
-Set the `GROUP_NAME` variable in the script to your GitLab group name.
-
-Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day.
-
-<details>
-<summary>View full script (find_litellm_jobs.py)</summary>
-
-```python
-#!/usr/bin/env python3
-"""
-Scan all GitLab CI/CD jobs in a GitLab group that ran between
-0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8.
-
-Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later.
-"""
-
-import os
-import re
-import sys
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime, timezone
-
-import requests
-
-GITLAB_URL = "https://gitlab.com"
-GROUP_NAME = "YourGroup"  # <-- set to your GitLab group name
-TOKEN = os.environ.get("GITLAB_TOKEN", "")
-
-TODAY = datetime.now(timezone.utc).date()
-WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day, 8, 0, 0, tzinfo=timezone.utc)
-WINDOW_END   = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc)
-
-TARGET_VERSIONS = {"1.82.7", "1.82.8"}
-VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE)
-
-HEADERS = {"PRIVATE-TOKEN": TOKEN}
-SESSION = requests.Session()
-SESSION.headers.update(HEADERS)
-
-
-def get_paginated(url, params=None):
-    params = dict(params or {})
-    params.setdefault("per_page", 100)
-    page = 1
-    while True:
-        params["page"] = page
-        resp = SESSION.get(url, params=params, timeout=30)
-        resp.raise_for_status()
-        data = resp.json()
-        if not data:
-            break
-        yield from data
-        if len(data) < params["per_page"]:
-            break
-        page += 1
-
-
-def get_group_id(group_name):
-    resp = SESSION.get(f"{GITLAB_URL}/api/v4/groups/{group_name}", timeout=30)
-    resp.raise_for_status()
-    return resp.json()["id"]
-
-
-def get_all_projects(group_id):
-    projects = []
-    for p in get_paginated(
-        f"{GITLAB_URL}/api/v4/groups/{group_id}/projects",
-        {"include_subgroups": "true", "archived": "false"},
-    ):
-        projects.append({"id": p["id"], "name": p["path_with_namespace"]})
-    return projects
-
-
-def parse_ts(ts_str):
-    if not ts_str:
-        return None
-    ts_str = ts_str.replace("Z", "+00:00")
-    return datetime.fromisoformat(ts_str)
-
-
-def jobs_in_window(project_id):
-    matching = []
-    url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs"
-    params = {"per_page": 100, "scope[]": ["success", "failed", "canceled", "running"]}
-
-    page = 1
-    while True:
-        params["page"] = page
-        resp = SESSION.get(url, params=params, timeout=30)
-        if resp.status_code == 403:
-            return matching
-        resp.raise_for_status()
-        jobs = resp.json()
-        if not jobs:
-            break
-
-        stop_early = False
-        for job in jobs:
-            ts = parse_ts(job.get("started_at") or job.get("created_at"))
-            if ts is None:
-                continue
-            if ts > WINDOW_END:
-                continue
-            if ts < WINDOW_START:
-                stop_early = True
-                continue
-            matching.append(job)
-
-        if stop_early or len(jobs) < 100:
-            break
-        page += 1
-
-    return matching
-
-
-def fetch_trace(project_id, job_id):
-    url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs/{job_id}/trace"
-    resp = SESSION.get(url, timeout=60)
-    if resp.status_code in (403, 404):
-        return ""
-    resp.raise_for_status()
-    return resp.text
-
-
-def check_job(project_name, project_id, job):
-    job_id   = job["id"]
-    job_name = job["name"]
-    ref      = job.get("ref", "")
-    started  = job.get("started_at", job.get("created_at", ""))
-
-    trace = fetch_trace(project_id, job_id)
-    if not trace:
-        return None
-
-    found_versions = set()
-    for match in VERSION_PATTERN.finditer(trace):
-        ver = match.group(1)
-        if ver in TARGET_VERSIONS:
-            found_versions.add(ver)
-
-    if not found_versions:
-        return None
-
-    context_lines = []
-    for line in trace.splitlines():
-        if VERSION_PATTERN.search(line):
-            ver_match = VERSION_PATTERN.search(line)
-            if ver_match and ver_match.group(1) in TARGET_VERSIONS:
-                context_lines.append(line.strip())
-
-    return {
-        "project":    project_name,
-        "project_id": project_id,
-        "job_id":     job_id,
-        "job_name":   job_name,
-        "ref":        ref,
-        "started_at": started,
-        "versions":   sorted(found_versions),
-        "context":    context_lines[:10],
-        "job_url":    f"{GITLAB_URL}/{project_name}/-/jobs/{job_id}",
-    }
-
-
-def main():
-    if not TOKEN:
-        print("ERROR: Set GITLAB_TOKEN environment variable.", file=sys.stderr)
-        sys.exit(1)
-
-    print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}")
-    print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}")
-    print()
-
-    print(f"Resolving group '{GROUP_NAME}'...")
-    group_id = get_group_id(GROUP_NAME)
-
-    print("Fetching projects...")
-    projects = get_all_projects(group_id)
-    print(f"  Found {len(projects)} projects")
-    print()
-
-    all_jobs_to_check = []
-
-    print("Scanning job listings for time window...")
-    for proj in projects:
-        try:
-            jobs = jobs_in_window(proj["id"])
-        except requests.HTTPError as e:
-            print(f"  WARN: {proj['name']} - {e}", file=sys.stderr)
-            continue
-        if jobs:
-            print(f"  {proj['name']}: {len(jobs)} job(s) in window")
-        for j in jobs:
-            all_jobs_to_check.append((proj["name"], proj["id"], j))
-
-    total = len(all_jobs_to_check)
-    print(f"\nFetching traces for {total} job(s)...")
-    print()
-
-    hits = []
-    with ThreadPoolExecutor(max_workers=10) as pool:
-        futures = {
-            pool.submit(check_job, pname, pid, job): (pname, job["id"])
-            for pname, pid, job in all_jobs_to_check
-        }
-        done = 0
-        for future in as_completed(futures):
-            done += 1
-            pname, jid = futures[future]
-            try:
-                result = future.result()
-            except Exception as e:
-                print(f"  ERROR checking {pname} job {jid}: {e}", file=sys.stderr)
-                continue
-            if result:
-                hits.append(result)
-            print(f"  [{done}/{total}] checked {pname} job {jid}" +
-                  (f"  *** HIT: litellm {result['versions']} ***" if result else ""),
-                  flush=True)
-
-    print()
-    print("=" * 72)
-    print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}")
-    print("=" * 72)
-
-    if not hits:
-        print("No matches found.")
-        return
-
-    for h in sorted(hits, key=lambda x: x["started_at"]):
-        print()
-        print(f"  Project   : {h['project']}")
-        print(f"  Job       : {h['job_name']} (#{h['job_id']})")
-        print(f"  Branch/tag: {h['ref']}")
-        print(f"  Started   : {h['started_at']}")
-        print(f"  Versions  : litellm {', '.join(h['versions'])}")
-        print(f"  URL       : {h['job_url']}")
-        print(f"  Log lines :")
-        for line in h["context"]:
-            print(f"    {line}")
-
-
-if __name__ == "__main__":
-    main()
-```
-
-</details>
-
-</TabItem>
-</Tabs>
-
-*CI/CD scripts contributed by the community ([original gist](https://gist.github.com/fryz/93ec8d4898ffe5b5ac5706a208823ef3)). Review before running.*
-
-
-## Indicators of compromise (IoCs)
-
-Review affected systems for the following indicators:
-
-- `litellm_init.pth` present in your `site-packages`
-- Outbound traffic or requests to `models.litellm[.]cloud`
-  This domain is **not** affiliated with LiteLLM
-- Outbound traffic or requests to `checkmarx[.]zone`
-  This domain is **not** affiliated with LiteLLM
-
-
-## Immediate actions for affected users
-
-If you installed or ran **v1.82.7** or **v1.82.8**, take the following actions immediately.
-
-### 1. Rotate all secrets
-
-Treat any credentials present on the affected systems as compromised, including:
-
-- API keys
-- Cloud access keys
-- Database passwords
-- SSH keys
-- Kubernetes tokens
-- Any secrets stored in environment variables or configuration files
-
-### 2. Inspect your filesystem
-
-Check your `site-packages` directory for a file named `litellm_init.pth`:
-
-```bash
-find /usr/lib/python3.13/site-packages/ -name "litellm_init.pth"
-```
-
-If present:
-
-- remove it immediately
-- investigate the host for further compromise
-- preserve relevant artifacts if your security team is performing forensics
-
-### 3. Audit version history
-
-Review your:
-
-- Local environments
-- CI/CD pipelines
-- Docker builds
-- Deployment logs
-
-Confirm whether **v1.82.7** or **v1.82.8** was installed anywhere.
-
-Pin LiteLLM to a known safe version such as **v1.82.6 or earlier**, or to a later verified release once announced.
-
-
-## Response and remediation
-
-The LiteLLM AI Gateway team has already taken the following steps:
-
-- Removed compromised packages from PyPI
-- Rotated maintainer credentials and established new authorized maintainers
-- Engaged Google's Mandiant security team to assist with forensic analysis of the build and publishing chain
-
-
-## Verify Docker image signatures
-
-Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
-
-**Verify using the pinned commit hash (recommended):**
-
-A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-**Verify using a release tag (convenience):**
-
-Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
-
-Expected output:
-
-```
-The following checks were performed on each of these signatures:
-  - The cosign claims were validated
-  - The signatures were verified against the specified public key
-```
-
-## Verified safe versions
-
-We have audited every LiteLLM release published between v1.78.0 and v1.82.6 across both PyPI and Docker. Each artifact was verified by:
-
-1. Downloading the published artifact and computing its SHA-256 digest
-2. Scanning for the known [indicators of compromise](#indicators-of-compromise-iocs) (IOCs)
-3. Comparing the artifact contents against the corresponding Git commit in the BerriAI/litellm repository
-
-**All versions listed below are confirmed clean.**
-
-<Tabs>
-<TabItem value="pypi" label="PyPI Releases">
-
-<VersionVerificationTable entries={[
-  { version: "1.82.6", sha256: "164a3ef3e19f309e3cabc199bef3d2045212712fefdfa25fc7f75884a5b5b205", gitCommit: "38d477507dad" },
-  { version: "1.82.5", sha256: "e1012ab816352215c4e00776dd48b0c68058b537888a8ff82cca62af19e6fb11", gitCommit: "1998c4f3703f" },
-  { version: "1.82.4", sha256: "d37c34a847e7952a146ed0e2888a24d3edec7787955c6826337395e755ad5c4b", gitCommit: "cfeafbe38811" },
-  { version: "1.82.3", sha256: "609901f6c5a5cf8c24386e4e3f50738bb8a9db719709fd76b208c8ee6d00f7a7", gitCommit: "61409275c8d8" },
-  { version: "1.82.2", sha256: "641ed024774fa3d5b4dd9347f0efb1e31fa422fba2a6500aabedee085d1194cb", gitCommit: "f351bbdb3683" },
-  { version: "1.82.1", sha256: "a9ec3fe42eccb1611883caaf8b1bf33c9f4e12163f94c7d1004095b14c379eb2", gitCommit: "94b002066e3a" },
-  { version: "1.82.0", sha256: "5496b5d4532cccdc7a095c21cbac4042f7662021c57bc1d17be4e39838929e80", gitCommit: "6c6585af568e" },
-  { version: "1.81.16", sha256: "d6bcc13acbd26719e07bfa6b9923740e88409cbf1f9d626d85fc9ae0e0eec88c", gitCommit: "678200ee4887" },
-  { version: "1.81.15", sha256: "2fa253658702509ce09fe0e172e5a47baaadf697fb0f784c7fd4ff665ae76ae1", gitCommit: "2e819656cee9" },
-  { version: "1.81.14", sha256: "6394e61bbdef7121e5e3800349f6b01e9369e7cf611e034f1832750c481abfed", gitCommit: "96bcee0b0af7" },
-  { version: "1.81.13", sha256: "ae4aea2a55e85993f5f6dd36d036519422d24812a1a3e8540d9e987f2d7a4304", gitCommit: "cc957a19a560" },
-  { version: "1.81.12", sha256: "219cf9729e5ea30c6d3f75aa43fef3c56a717369939a6d717cbad0fd78e3c146", gitCommit: "ba0d541b1982" },
-  { version: "1.81.11", sha256: "06a66c24742e082ddd2813c87f40f5c12fe7baa73ce1f9457eaf453dc44a0f65", gitCommit: "231aedeeff7e" },
-  { version: "1.81.10", sha256: "9efa1cbe61ac051f6500c267b173d988ff2d511c2eecf1c8f2ee546c0870747c", gitCommit: "7488abece8e7" },
-  { version: "1.81.9", sha256: "24ee273bc8a62299fbb754035f83fb7d8d44329c383701a2bd034f4fd1c19084", gitCommit: "a09d3e9162eb" },
-  { version: "1.81.8", sha256: "78cca92f36bc6c267c191d1fe1e2630c812bff6daec32c58cade75748c2692f6", gitCommit: "4fea649f519b" },
-  { version: "1.81.7", sha256: "58466c88c3289c6a3830d88768cf8f307581d9e6c87861de874d1128bb2de90d", gitCommit: "3f6a281d0f7a" },
-  { version: "1.81.6", sha256: "573206ba194d49a1691370ba33f781671609ac77c35347f8a0411d852cf6341a", gitCommit: "8da3a93e6e63" },
-  { version: "1.81.5", sha256: "206505c5a0c6503e465154b9c979772be3ede3f5bf746d15b37dca5ae54d239f", gitCommit: "2cc3778761d4" },
-  { version: "1.81.3", sha256: "3f60fd8b727587952ad3dd18b68f5fed538d6f43d15bb0356f4c3a11bccb2b92", gitCommit: "f30742fe6e8e" },
-]} />
-
-</TabItem>
-<TabItem value="docker" label="Docker Images">
-
-<VersionVerificationTable entries={[
-  { version: "1.82.3", sha256: "0a571da849db5f9c3cf3fead2ffbf1df982eebff7e7b38b46dbec3f640dafdbb", gitCommit: "61409275c8d8" },
-  { version: "1.82.3-stable", sha256: "0c2b2a0ad3e50af1702fc493ecd07f22a5180b6d1cfb169440b429b40e340e29", gitCommit: "61409275c8d8" },
-  { version: "1.82.0-stable", sha256: "71bf7283767ca436edcfa9f1f26c1743487b5fa29736c61c3eb6977776007c42", gitCommit: "97947c254252" },
-  { version: "1.81.15", sha256: "303c31af87e7915e7b34d6c4d55a6ac753ef947a5deaa899e9ccfd3d1d58f7c2", gitCommit: "20bf3aa8070a" },
-  { version: "1.81.14-stable", sha256: "a34f9758048231817d799b703fb998e40e2a5cbabb89ab95039fc30798f01b3c", gitCommit: "0435375b1271" },
-  { version: "1.81.13", sha256: "a876f3f22f9b6fd481c9091c44a8a893d81c172d66dc2749298dcd3dc4a3d6f0", gitCommit: "cc957a19a560" },
-  { version: "1.81.12-stable", sha256: "e24022878ccc87f57d808ac9304f18b87b8359e6556746d81cc20a5dc85f423a", gitCommit: "ba0d541b1982" },
-  { version: "1.81.9-stable", sha256: "262e53d7702ed82579717faff0b08f7c0b7e9973a6406cfcc0e4af7826327627", gitCommit: "a09d3e9162eb" },
-  { version: "1.81.3-stable", sha256: "dff82ccc32fb648927c090607887401c7e8ec814fe7c951beb95fe51073ca02b", gitCommit: "61ed8f9e0355" },
-  { version: "1.81.0-stable", sha256: "f4913297d1bb3dc373eb8911a5ac816b597be9b5e08a91636b6c2786dd572aa8", gitCommit: "790a5ce0b323" },
-  { version: "1.80.15-stable", sha256: "0b4ec3861e978b4aa254f4070f292cd345496a5fb59c72e1ee21cd6db94b670b", gitCommit: "17c8d8d109b5" },
-  { version: "1.80.11-stable", sha256: "4068108d9101cd2affba3924310fd7f34f23d14e36dd4853733898b9e04d81ca", gitCommit: "57e07bddd341" },
-  { version: "1.80.8-stable", sha256: "0304c2eb1f3cf54262d1b4e0629487232bab459e95b99a21e5810231d2b27021", gitCommit: "3381d63152f8" },
-  { version: "1.80.5-stable", sha256: "a89e173135fff96af4b5b91ea31845164eadcf6497c82adeb64c36a23c8a3d11", gitCommit: "6c49b95a4ab7" },
-  { version: "1.80.0-stable", sha256: "a3416f4cd0c896c94a1f526d872ff6c19bee22ff4afcdcc6f9ff690707900176", gitCommit: "98365205acd0" },
-  { version: "1.79.3-stable", sha256: "27aae83d6ab6cb0b63bf8179e375ce0e11f5cfef51f2675b0c1e60c6f546dbc1", gitCommit: "c0548542d4a9" },
-  { version: "1.79.1-stable", sha256: "7780d29a9543c4ce762430db7dfb0640105f7357fc38e35bf3fb7bbb1e6ba63f", gitCommit: "c217bddb59ba" },
-  { version: "1.79.0-stable", sha256: "32bf6ac059a56641e11e4712f63b8467c295f988b6c160dc7229660417ee44bd", gitCommit: "8d495f56a9cc" },
-  { version: "1.78.5-stable", sha256: "d5e607648eafa15edc63b0b1a5ed01f8b31a1fa0c80f7d25b252ae18a593ee29", gitCommit: "c471bf1f16c2" },
-  { version: "1.78.0-stable", sha256: "7a56b32dc7153763d31c0a056123dc878a598959935d8c7daacb1fca5272c205", gitCommit: "5fde83d9f154" },
-]} />
-
-</TabItem>
-</Tabs>
-
-
-## Questions and support
-
-If you believe your systems may be affected, contact us immediately:
-
-- **Security:** `security@berri.ai`
-- **Support:** `support@berri.ai`
-- **Slack:** Reach out to the LiteLLM team directly
-
-For real-time updates, follow [LiteLLM (YC W23) on X](https://x.com/LiteLLM).
-

docs/my-website/blog/server_root_path/index.md

@@ -1,146 +0,0 @@
----
-slug: server-root-path-incident
-title: "Incident Report: SERVER_ROOT_PATH regression broke UI routing"
-date: 2026-02-21T10:00:00
-authors:
-  - yuneng
-  - ishaan-alt
-  - krrish
-tags: [incident-report, ui, stability]
-hide_table_of_contents: false
----
-
-**Date:** January 22, 2026
-**Duration:** ~4 days (until fix merged January 26, 2026)
-**Severity:** High
-**Status:** Resolved
-
-> **Note:** This fix is available starting from LiteLLM `v1.81.3.rc.6` or higher.
-
-## Summary
-
-A PR ([`#19467`](https://github.com/BerriAI/litellm/pull/19467)) accidentally removed the `root_path=server_root_path` parameter from the FastAPI app initialization in `proxy_server.py`. This caused the proxy to ignore the `SERVER_ROOT_PATH` environment variable when serving the UI. Users who deploy LiteLLM behind a reverse proxy with a path prefix (e.g., `/api/v1` or `/llmproxy`) found that all UI pages returned 404 Not Found.
-
-- **LLM API calls:** No impact. API routing was unaffected.
-- **UI pages:** All UI pages returned 404 for deployments using `SERVER_ROOT_PATH`.
-- **Swagger/OpenAPI docs:** Broken when accessed through the configured root path.
-
-{/* truncate */}
-
----
-
-## Background
-
-Many LiteLLM deployments run behind a reverse proxy (e.g., Nginx, Traefik, AWS ALB) that routes traffic to LiteLLM under a path prefix. FastAPI's `root_path` parameter tells the application about this prefix so it can correctly serve static files, generate URLs, and handle routing.
-
-```mermaid
-sequenceDiagram
-    participant User as User Browser
-    participant RP as Reverse Proxy
-    participant LP as LiteLLM Proxy
-
-    User->>RP: GET /llmproxy/ui/
-    RP->>LP: GET /ui/ (X-Forwarded-Prefix: /llmproxy)
-
-    Note over LP: Before regression:<br/>FastAPI root_path="/llmproxy"<br/>→ Serves UI correctly
-
-    Note over LP: After regression:<br/>FastAPI root_path=""<br/>→ UI assets resolve to wrong paths<br/>→ 404 Not Found
-```
-
-The `root_path` parameter was present in `proxy_server.py` since early versions of LiteLLM. It was removed as a side effect of PR [#19467](https://github.com/BerriAI/litellm/pull/19467), which was intended to fix a different UI 404 issue.
-
----
-
-## Root cause
-
-PR [#19467](https://github.com/BerriAI/litellm/pull/19467) (`73d49f8`) removed the `root_path=server_root_path` line from the `FastAPI()` constructor in `proxy_server.py`:
-
-```diff
- app = FastAPI(
-     docs_url=_get_docs_url(),
-     redoc_url=_get_redoc_url(),
-     title=_title,
-     description=_description,
-     version=version,
--    root_path=server_root_path,
-     lifespan=proxy_startup_event,
- )
-```
-
-Without `root_path`, FastAPI treated all requests as if the application was mounted at `/`, causing path mismatches for any deployment using `SERVER_ROOT_PATH`.
-
-The regression went undetected because:
-
-1. **No automated test** verified that `root_path` was set on the FastAPI app.
-2. **No manual test procedure** existed for `SERVER_ROOT_PATH` functionality.
-3. **Default deployments** (without `SERVER_ROOT_PATH`) were unaffected, so most CI tests passed.
-
----
-
-## Remediation
-
-| #   | Action                                                                                            | Status  | Code                                                                                                                       |
-| --- | ------------------------------------------------------------------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------- |
-| 1   | Restore `root_path=server_root_path` in FastAPI app initialization                                | ✅ Done | [`#19790`](https://github.com/BerriAI/litellm/pull/19790) (`5426b3c`)                                                      |
-| 2   | Add unit tests for `get_server_root_path()` and FastAPI app initialization                        | ✅ Done | [`test_server_root_path.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_server_root_path.py) |
-| 3   | Add CI workflow that builds Docker image and tests UI routing with `SERVER_ROOT_PATH` on every PR | ✅ Done | [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml)    |
-| 4   | Document manual test procedure for `SERVER_ROOT_PATH`                                             | ✅ Done | [Discussion #8495](https://github.com/BerriAI/litellm/discussions/8495)                                                    |
-
----
-
-## CI workflow details
-
-The new [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) workflow runs on every PR against `main`. It:
-
-1. Builds the LiteLLM Docker image
-2. Starts a container with `SERVER_ROOT_PATH` set (tests both `/api/v1` and `/llmproxy`)
-3. Verifies the UI returns valid HTML at `{ROOT_PATH}/ui/`
-4. Fails the workflow if the UI is unreachable
-
-```mermaid
-flowchart TD
-    A["PR opened/updated"] --> B["Build Docker image"]
-    B --> C["Start container with SERVER_ROOT_PATH=/api/v1"]
-    B --> D["Start container with SERVER_ROOT_PATH=/llmproxy"]
-    C --> E["curl {ROOT_PATH}/ui/ → expect HTML"]
-    D --> F["curl {ROOT_PATH}/ui/ → expect HTML"]
-    E -->|"HTML found"| G["✅ Pass"]
-    E -->|"404 or no HTML"| H["❌ Fail Workflow"]
-    F -->|"HTML found"| G
-    F -->|"404 or no HTML"| H
-
-    style G fill:#d4edda,stroke:#28a745
-    style H fill:#f8d7da,stroke:#dc3545
-```
-
-This prevents future regressions where changes to `proxy_server.py` accidentally break `SERVER_ROOT_PATH` support.
-
----
-
-## Timeline
-
-| Time (UTC)         | Event                                                                                                                                                        |
-| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| Jan 22, 2026 04:20 | PR [#19467](https://github.com/BerriAI/litellm/pull/19467) merged, removing `root_path=server_root_path`                                                     |
-| Jan 22–26          | Users on nightly builds report UI 404 errors when using `SERVER_ROOT_PATH`                                                                                   |
-| Jan 26, 2026 17:48 | Fix PR [#19790](https://github.com/BerriAI/litellm/pull/19790) merged, restoring `root_path=server_root_path`                                                |
-| Feb 18, 2026       | CI workflow [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) added to run on every PR |
-
----
-
-## Resolution steps for users
-
-For users still experiencing issues, update to the latest LiteLLM version:
-
-```bash
-pip install --upgrade litellm
-```
-
-Verify your `SERVER_ROOT_PATH` is correctly set:
-
-```bash
-# In your environment or docker-compose.yml
-SERVER_ROOT_PATH="/your-prefix"
-```
-
-Then confirm the UI is accessible at `http://your-host:4000/your-prefix/ui/`.

docs/my-website/blog/sub_millisecond_proxy_overhead/index.md

@@ -1,85 +0,0 @@
----
-slug: sub-millisecond-proxy-overhead
-title: "Achieving Sub-Millisecond Proxy Overhead"
-date: 2026-02-02T10:00:00
-authors:
-  - alexsander
-  - krrish
-  - ishaan-alt
-description: "Our Q1 performance target and architectural direction for achieving sub-millisecond proxy overhead on modest hardware."
-tags: [performance, architecture]
-hide_table_of_contents: false
----
-
-![Sidecar architecture: Python control plane vs. sidecar hot path](https://raw.githubusercontent.com/AlexsanderHamir/assets/main/Screenshot%202026-02-02%20172554.png)
-
-# Achieving Sub-Millisecond Proxy Overhead
-
-## Introduction
-
-Our Q1 performance target is to aggressively move toward sub-millisecond proxy overhead on a single instance with 4 CPUs and 8 GB of RAM, and to continue pushing that boundary over time. Our broader goal is to make LiteLLM inexpensive to deploy, lightweight, and fast. This post outlines the architectural direction behind that effort.
-
-Proxy overhead refers to the latency introduced by LiteLLM itself, independent of the upstream provider.
-
-To measure it, we run the same workload directly against the provider and through LiteLLM at identical QPS (for example, 1,000 QPS) and compare the latency delta. To reduce noise, the load generator, LiteLLM, and a mock LLM endpoint all run on the same machine, ensuring the difference reflects proxy overhead rather than network latency.
-
-{/* truncate */}
-
----
-
-## Where We're Coming From
-
-Under the same benchmark originally conducted by [TensorZero](https://www.tensorzero.com/docs/gateway/benchmarks), LiteLLM previously failed at around 1,000 QPS.
-
-That is no longer the case. Today, LiteLLM can be stress-tested at 1,000 QPS with no failures and can scale up to 5,000 QPS without failures on a 4-CPU, 8-GB RAM single instance setup.
-
-This establishes a more up to date baseline and provides useful context as we continue working on proxy overhead and overall performance.
-
----
-
-## Design Choice
-
-Achieving sub-millisecond proxy overhead with a Python-based system requires being deliberate about where work happens.
-
-Python is a strong fit for flexibility and extensibility: provider abstraction, configuration-driven routing, and a rich callback ecosystem. These are areas where development velocity and correctness matter more than raw throughput.
-
-At higher request rates, however, certain classes of work become expensive when executed inside the Python process on every request. Rather than rewriting LiteLLM or introducing complex deployment requirements, we adopt an optional **sidecar architecture**.
-
-This architectural change is how we intend to make LiteLLM **permanently fast**. While it supports our near-term performance targets, it is a long-term investment.
-
-Python continues to own:
-
-- Request validation and normalization
-- Model and provider selection
-- Callbacks and integrations
-
-The sidecar owns **performance-critical execution**, such as:
-
-- Efficient request forwarding
-- Connection reuse and pooling
-- Enforcing timeouts and limits
-- Aggregating high-frequency metrics
-
-This separation allows each component to focus on what it does best: Python acts as the control plane, while the sidecar handles the hot path.
-
----
-
-### Why the Sidecar Is Optional
-
-The sidecar is intentionally **optional**.
-
-This allows us to ship it incrementally, validate it under real-world workloads, and avoid making it a hard dependency before it is fully battle-tested across all LiteLLM features.
-
-Just as importantly, this ensures that self-hosting LiteLLM remains simple. The sidecar is bundled and started automatically, requires no additional infrastructure, and can be disabled entirely. From a user's perspective, LiteLLM continues to behave like a single service.
-
-As of today, the sidecar is an optimization, not a requirement.
-
----
-
-## Conclusion
-
-Sub-millisecond proxy overhead is not achieved through a single optimization, but through architectural changes.
-
-By keeping Python focused on orchestration and extensibility, and offloading performance-critical execution to a sidecar, we establish a foundation for making LiteLLM **permanently fast over time**—even on modest hardware such as a 1-CPU, 2-GB RAM instance, while keeping deployment and self-hosting simple.
-
-This work extends beyond Q1, and we will continue sharing benchmarks and updates as the architecture evolves.

docs/my-website/blog/vanta_compliance_recertification/index.md

@@ -1,18 +0,0 @@
----
-slug: vanta-compliance-recertification
-title: "LiteLLM + Vanta: SOC 2 Type 2 and ISO 27001 Recertification"
-date: 2026-03-30T10:00:00
-authors:
-  - krrish
-description: "LiteLLM is partnering with Vanta on SOC 2 Type 2 and ISO 27001 recertification and engaging independent auditors for verification."
-tags: [security, compliance]
-hide_table_of_contents: true
----
-
-![LiteLLM x Vanta SOC-2 Recertification](/img/blog/vanta_soc2_recertification.png)
-
-We are partnering with [Vanta](https://www.vanta.com/) to recertify LiteLLM's compliance for SOC 2 Type 2 and ISO 27001.
-
-As part of this process, we are also identifying independent auditors to validate and verify our compliance posture.
-
-This is part of our commitment to being the most secure and transparent AI Gateway possible.

docs/my-website/blog/video_characters_litellm/index.md

@@ -1,121 +0,0 @@
----
-slug: video_characters_api
-title: "New Video Characters, Edit and Extension API support"
-date: 2026-03-16T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "LiteLLM now supports creating, retrieving, and managing reusable video characters across multiple video generations."
-tags: [videos, characters, proxy, routing]
-hide_table_of_contents: false
----
-
-LiteLLM now supoports videos character, edit and extension apis.
-
-{/* truncate */}
-
-## What's New
-
-Four new endpoints for video character operations:
-- **Create character** - Upload a video to create a reusable asset
-- **Get character** - Retrieve character metadata
-- **Edit video** - Modify generated videos
-- **Extend video** - Continue clips with character consistency
-
-**Available from:** LiteLLM v1.83.0+
-
-## Quick Example
-
-```python
-import litellm
-
-# Create character from video
-character = litellm.avideo_create_character(
-    name="Luna",
-    video=open("luna.mp4", "rb"),
-    custom_llm_provider="openai",
-    model="sora-2"
-)
-print(f"Character: {character.id}")
-
-# Use in generation
-video = litellm.avideo(
-    model="sora-2",
-    prompt="Luna dances through a magical forest.",
-    characters=[{"id": character.id}],
-    seconds="8"
-)
-
-# Get character info
-fetched = litellm.avideo_get_character(
-    character_id=character.id,
-    custom_llm_provider="openai"
-)
-
-# Edit with character preserved
-edited = litellm.avideo_edit(
-    video_id=video.id,
-    prompt="Add warm golden lighting"
-)
-
-# Extend sequence
-extended = litellm.avideo_extension(
-    video_id=video.id,
-    prompt="Luna waves goodbye",
-    seconds="5"
-)
-```
-
-## Via Proxy
-
-```bash
-# Create character
-curl -X POST "http://localhost:4000/v1/videos/characters" \
-  -H "Authorization: Bearer sk-litellm-key" \
-  -F "video=@luna.mp4" \
-  -F "name=Luna"
-
-# Get character
-curl -X GET "http://localhost:4000/v1/videos/characters/char_abc123def456" \
-  -H "Authorization: Bearer sk-litellm-key"
-
-# Edit video
-curl -X POST "http://localhost:4000/v1/videos/edits" \
-  -H "Authorization: Bearer sk-litellm-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "video": {"id": "video_xyz789"},
-    "prompt": "Add warm golden lighting and enhance colors"
-  }'
-
-# Extend video
-curl -X POST "http://localhost:4000/v1/videos/extensions" \
-  -H "Authorization: Bearer sk-litellm-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "video": {"id": "video_xyz789"},
-    "prompt": "Luna waves goodbye and walks into the sunset",
-    "seconds": "5"
-  }'
-```
-
-## Managed Character IDs
-
-LiteLLM automatically encodes provider and model metadata into character IDs:
-
-**What happens:**
-```
-Upload character "Luna" with model "sora-2" on OpenAI
-  ↓
-LiteLLM creates: char_abc123def456 (contains provider + model_id)
-  ↓
-When you reference it later, LiteLLM decodes automatically
-  ↓
-Router knows exactly which deployment to use
-```
-
-**Behind the scenes:**
-- Character ID format: `character_<base64_encoded_metadata>`
-- Metadata includes: provider, model_id, original_character_id
-- Transparent to you - just use the ID, LiteLLM handles routing

docs/my-website/blog/vllm_embeddings_incident/index.md

@@ -1,108 +0,0 @@
----
-slug: vllm-embeddings-incident
-title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter"
-date: 2026-02-18T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-tags: [incident-report, embeddings, vllm]
-hide_table_of_contents: false
----
-
-**Date:** Feb 16, 2026
-**Duration:** ~3 hours
-**Severity:** High (for vLLM embedding users)
-**Status:** Resolved
-
-## Summary
-
-A commit ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)) intended to fix OpenAI SDK behavior broke vLLM embeddings by explicitly passing `encoding_format=None` in API requests. vLLM rejects this with error: `"unknown variant \`\`, expected float or base64"`.
-
-- **vLLM embedding calls:** Complete failure - all requests rejected
-- **Other providers:** No impact - OpenAI and other providers functioned normally
-- **Other vLLM functionality:** No impact - only embeddings were affected
-
-{/* truncate */}
-
----
-
-## Background
-
-The `encoding_format` parameter for embeddings specifies whether vectors should be returned as `float` arrays or `base64` encoded strings. Different providers have different expectations:
-
-- **OpenAI SDK:** If `encoding_format` is omitted, the SDK adds a default value of `"float"`
-- **vLLM:** Strictly validates `encoding_format` - only accepts `"float"`, `"base64"`, or complete omission. Rejects `None` or empty string values.
-
-```mermaid
-flowchart TD
-    A["1. User calls litellm.embedding()
-    litellm/main.py"] --> B["2. Transform request for provider
-    litellm/llms/openai_like/embedding/handler.py"]
-    B --> C["3. Send request to vLLM endpoint"]
-    C -->|"encoding_format omitted"| D["4a. ✅ vLLM processes request"]
-    C -->|"encoding_format='float' or 'base64'"| D
-    C -->|"encoding_format=None or ''"| E["4b. ❌ vLLM rejects with error:
-    'unknown variant, expected float or base64'"]
-
-    style D fill:#d4edda,stroke:#28a745
-    style E fill:#f8d7da,stroke:#dc3545
-    style B fill:#fff3cd,stroke:#ffc107
-```
-
----
-
-## Root cause
-
-A well-intentioned fix for OpenAI SDK behavior inadvertently broke vLLM embeddings:
-
-**The Breaking Change ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)):**
-
-In `litellm/main.py`, the code was changed to explicitly set `encoding_format=None` instead of omitting it:
-
-```python
-# Added in dbcae4a
-if encoding_format is not None:
-    optional_params["encoding_format"] = encoding_format
-else:
-    # Omitting causes openai sdk to add default value of "float"
-    optional_params["encoding_format"] = None
-```
-
-This fix worked correctly for OpenAI - explicitly passing `None` prevented the SDK from adding its default value. However, vLLM's strict parameter validation rejected `None` values, causing all embedding requests to fail.
-
----
-
-## The Fix
-
-Fix deployed ([`55348dd`](https://github.com/BerriAI/litellm/commit/55348dd9c51b5b028f676d25ad023b8f052fc071)). The solution filters out `None` and empty string values from `optional_params` before sending requests to OpenAI-like providers (including vLLM).
-
-**In `litellm/llms/openai_like/embedding/handler.py`:**
-
-```python
-# Before (broken)
-data = {"model": model, "input": input, **optional_params}
-
-# After (fixed)
-filtered_optional_params = {k: v for k, v in optional_params.items() if v not in (None, '')}
-data = {"model": model, "input": input, **filtered_optional_params}
-```
-
-This ensures:
-- Valid values (`"float"`, `"base64"`) are preserved and sent
-- `None` and empty string values are filtered out (parameter omitted entirely)
-- OpenAI SDK no longer adds defaults because liteLLM handles the parameter upstream
-
----
-
-## Remediation
-
-| # | Action | Status | Code |
-|---|---|---|---|
-| 1 | Filter `None` and empty string values in OpenAI-like embedding handler | ✅ Done | [`handler.py#L108`](https://github.com/BerriAI/litellm/blob/main/litellm/llms/openai_like/embedding/handler.py#L108) |
-| 2 | Unit tests for parameter filtering (None, empty string, valid values) | ✅ Done | [`test_openai_like_embedding.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/openai_like/embedding/test_openai_like_embedding.py) |
-| 3 | Transformation tests for hosted_vllm embedding config | ✅ Done | [`test_hosted_vllm_embedding_transformation.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py) |
-| 4 | E2E tests with actual vLLM endpoint | ✅ Done | [`test_hosted_vllm_embedding_e2e.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_e2e.py) |
-| 5 | Validate JSON payload structure matches vLLM expectations | ✅ Done | Tests verify exact JSON sent to endpoint |
-
----

docs/my-website/docs/a2a.md

@@ -1,265 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-import Image from '@theme/IdealImage';
-
-# Agent Gateway (A2A Protocol) - Overview
-
-Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track request/response logs in LiteLLM Logs. Manage which Teams, Keys can access which Agents onboarded.
-
-<Image 
-  img={require('../img/a2a_gateway.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-<br />
-<br />
-
-| Feature | Supported | 
-|---------|-----------|
-| Supported Agent Providers | A2A, Vertex AI Agent Engine, LangGraph, Azure AI Foundry, Bedrock AgentCore, Pydantic AI |
-| Logging | ✅ |
-| Load Balancing | ✅ |
-| Streaming | ✅ |
-| [Iteration Budgets](a2a_iteration_budgets) | ✅ |
-
-
-:::tip
-
-LiteLLM follows the [A2A (Agent-to-Agent) Protocol](https://github.com/google/A2A) for invoking agents.
-
-:::
-
-## Adding your Agent
-
-### Add A2A Agents
-
-You can add A2A-compatible agents through the LiteLLM Admin UI.
-
-1. Navigate to the **Agents** tab
-2. Click **Add Agent**
-3. Enter the agent name (e.g., `ij-local`) and the URL of your A2A agent
-
-<Image 
-  img={require('../img/add_agent_1.png')}
-  style={{width: '80%', display: 'block', margin: '0'}}
-/>
-
-The URL should be the invocation URL for your A2A agent (e.g., `http://localhost:10001`).
-
-
-### Add Azure AI Foundry Agents
-
-Follow [this guide, to add your azure ai foundry agent to LiteLLM Agent Gateway](./providers/azure_ai_agents#litellm-a2a-gateway)
-
-### Add Vertex AI Agent Engine
-
-Follow [this guide, to add your Vertex AI Agent Engine to LiteLLM Agent Gateway](./providers/vertex_ai_agent_engine)
-
-### Add Bedrock AgentCore Agents
-
-Follow [this guide, to add your bedrock agentcore agent to LiteLLM Agent Gateway](./providers/bedrock_agentcore#litellm-a2a-gateway)
-
-### Add LangGraph Agents
-
-Follow [this guide, to add your langgraph agent to LiteLLM Agent Gateway](./providers/langgraph#litellm-a2a-gateway)
-
-### Add Pydantic AI Agents
-
-Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./providers/pydantic_ai_agent#litellm-a2a-gateway)
-
-## Invoking your Agents
-
-See the [Invoking A2A Agents](./a2a_invoking_agents) guide to learn how to call your agents using:
-- **A2A SDK** - Native A2A protocol with full support for tasks and artifacts
-- **OpenAI SDK** - Familiar `/chat/completions` interface with `a2a/` model prefix
-
-## Tracking Agent Logs
-
-After invoking an agent, you can view the request logs in the LiteLLM **Logs** tab.
-
-The logs show:
-- **Request/Response content** sent to and received from the agent
-- **User, Key, Team** information for tracking who made the request
-- **Latency and cost** metrics
-
-<Image 
-  img={require('../img/agent2.png')}
-  style={{width: '100%', display: 'block', margin: '2rem auto'}}
-/>
-
-
-## Forwarding LiteLLM Context Headers
-
-When LiteLLM invokes your A2A agent, it sends special headers that enable:
-- **Trace Grouping**: All LLM calls from the same agent execution appear under one trace
-- **Agent Spend Tracking**: Costs are attributed to the specific agent
-
-| Header | Purpose |
-|--------|---------|
-| `X-LiteLLM-Trace-Id` | Links all LLM calls to the same execution flow |
-| `X-LiteLLM-Agent-Id` | Attributes spend to the correct agent |
-
-
-To enable these features, your A2A server must **forward these headers** to any LLM calls it makes back to LiteLLM.
-
-### Implementation Steps
-
-**Step 1: Extract headers from incoming A2A request**
-```python def get_litellm_headers(request) -> dict:
-    """Extract X-LiteLLM-* headers from incoming A2A request."""
-    all_headers = request.call_context.state.get('headers', {})
-    return {
-        k: v for k, v in all_headers.items() 
-        if k.lower().startswith('x-litellm-')
-    }
-```
-
-**Step 2: Forward headers to your LLM calls**
-Pass the extracted headers when making calls back to LiteLLM:
-<Tabs>
-<TabItem value="openai" label="OpenAI SDK" default>
-
-```python from openai import OpenAI
-
-headers = get_litellm_headers(request)
-
-client = OpenAI(
-    api_key="sk-your-litellm-key",
-    base_url="http://localhost:4000",
-    default_headers=headers,  # Forward headers
-)
-
-response = client.chat.completions.create(
-    model="gpt-4o",
-    messages=[{"role": "user", "content": "Hello"}]
-)
-```
-</TabItem>
-
-<TabItem value="langchain" label="LangChain">
-
-```python
-from langchain_openai import ChatOpenAI
-
-headers = get_litellm_headers(request)
-
-llm = ChatOpenAI(
-    model="gpt-4o",
-    openai_api_key="sk-your-litellm-key",
-    base_url="http://localhost:4000",
-    default_headers=headers,  # Forward headers
-)
-```
-</TabItem>
-<TabItem value="litellm" label="LiteLLM SDK">
-
-```python
-import litellm
-
-headers = get_litellm_headers(request)
-
-response = litellm.completion(
-    model="gpt-4o",
-    messages=[{"role": "user", "content": "Hello"}],
-    api_base="http://localhost:4000",
-    extra_headers=headers,  # Forward headers
-)
-```
-</TabItem>
-<TabItem value="requests" label="HTTP (requests/httpx)">
-
-```python
-import httpx
-
-headers = get_litellm_headers(request)
-headers["Authorization"] = "Bearer sk-your-litellm-key"
-
-response = httpx.post(
-    "http://localhost:4000/v1/chat/completions",
-    headers=headers,
-    json={"model": "gpt-4o", "messages": [{"role": "user", "content": "Hello"}]}
-)
-```
-</TabItem>
-</Tabs>
-
-### Result
-
-With header forwarding enabled, you'll see:
-
-**Trace Grouping in Langfuse:**
-
-<Image
-  img={require('../img/a2a_trace_grouping.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-**Agent Spend Attribution:**
-
-<Image
-  img={require('../img/a2a_agent_spend.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-## API Reference
-
-### Endpoint
-
-```
-POST /a2a/{agent_name}/message/send
-```
-
-### Authentication
-
-Include your LiteLLM Virtual Key in the `Authorization` header:
-
-```
-Authorization: Bearer sk-your-litellm-key
-```
-
-### Request Format
-
-LiteLLM follows the [A2A JSON-RPC 2.0 specification](https://github.com/google/A2A):
-
-```json title="Request Body"
-{
-  "jsonrpc": "2.0",
-  "id": "unique-request-id",
-  "method": "message/send",
-  "params": {
-    "message": {
-      "role": "user",
-      "parts": [{"kind": "text", "text": "Your message here"}],
-      "messageId": "unique-message-id"
-    }
-  }
-}
-```
-
-### Response Format
-
-```json title="Response"
-{
-  "jsonrpc": "2.0",
-  "id": "unique-request-id",
-  "result": {
-    "kind": "task",
-    "id": "task-id",
-    "contextId": "context-id",
-    "status": {"state": "completed", "timestamp": "2025-01-01T00:00:00Z"},
-    "artifacts": [
-      {
-        "artifactId": "artifact-id",
-        "name": "response",
-        "parts": [{"kind": "text", "text": "Agent response here"}]
-      }
-    ]
-  }
-}
-```
-
-## Agent Registry
-
-Want to create a central registry so your team can discover what agents are available within your company?
-
-Use the [AI Hub](./proxy/ai_hub) to make agents public and discoverable across your organization. This allows developers to browse available agents without needing to rebuild them.

docs/my-website/docs/a2a_agent_headers.md

@@ -1,252 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# A2A Agent Authentication Headers
-
-Forward authentication credentials (Bearer tokens, API keys, etc.) from clients to backend A2A agents.
-
-## Overview
-
-When LiteLLM proxies a request to a backend A2A agent, the agent may require its own authentication headers. There are three ways to supply them:
-
-| Method | Who configures | How it works |
-|---|---|---|
-| **Static headers** | Admin (UI / API) | Always sent, regardless of client request |
-| **Forward client headers** | Admin (UI / API) | Header names to extract from client request and forward |
-| **Convention-based** | Client (no admin config) | Client sends `x-a2a-{agent_name}-{header}` — automatically routed |
-
-All three methods can be combined. **Static headers always win** on key conflicts.
-
----
-
-## Method 1 — Static Headers
-
-Admin-configured headers that are always sent to the backend agent. Use this for server-to-server tokens or internal credentials that clients should never see or override.
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Agents** in the LiteLLM dashboard.
-2. Create or edit an agent.
-3. Open the **Authentication Headers** panel.
-4. Under **Static Headers**, click **Add Static Header** and fill in the header name and value.
-
-</TabItem>
-<TabItem value="api" label="REST API">
-
-```bash
-curl -X POST http://localhost:4000/v1/agents \
-  -H "Authorization: Bearer sk-admin" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "agent_name": "my-agent",
-    "agent_card_params": { ... },
-    "static_headers": {
-      "Authorization": "Bearer internal-server-token",
-      "X-Internal-Service": "litellm-proxy"
-    }
-  }'
-```
-
-To update an existing agent:
-
-```bash
-curl -X PATCH http://localhost:4000/v1/agents/{agent_id} \
-  -H "Authorization: Bearer sk-admin" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "static_headers": {
-      "Authorization": "Bearer new-token"
-    }
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-**Client call — no special headers needed:**
-
-```bash
-curl -X POST http://localhost:4000/a2a/my-agent \
-  -H "Authorization: Bearer sk-client-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "jsonrpc": "2.0", "id": "1", "method": "message/send",
-    "params": { "message": { "role": "user", "parts": [{"kind": "text", "text": "Hello"}], "messageId": "msg-1" } }
-  }'
-```
-
-The backend agent receives `Authorization: Bearer internal-server-token` without the client ever knowing the value.
-
----
-
-## Method 2 — Forward Client Headers
-
-Admin specifies a list of header **names**. When the client sends a request that includes those headers, LiteLLM extracts their values and forwards them to the backend agent. The client controls the values; the admin controls which headers are eligible to be forwarded.
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Agents** in the LiteLLM dashboard.
-2. Create or edit an agent.
-3. Open the **Authentication Headers** panel.
-4. Under **Forward Client Headers**, type header names and press **Enter** (e.g. `x-api-key`, `Authorization`).
-
-</TabItem>
-<TabItem value="api" label="REST API">
-
-```bash
-curl -X POST http://localhost:4000/v1/agents \
-  -H "Authorization: Bearer sk-admin" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "agent_name": "my-agent",
-    "agent_card_params": { ... },
-    "extra_headers": ["x-api-key", "x-user-token"]
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-**Client call — include the forwarded headers:**
-
-```bash
-curl -X POST http://localhost:4000/a2a/my-agent \
-  -H "Authorization: Bearer sk-client-key" \
-  -H "x-api-key: user-secret-value" \
-  -H "Content-Type: application/json" \
-  -d '{ ... }'
-```
-
-The backend agent receives `x-api-key: user-secret-value`.
-
-:::note
-Header name matching is **case-insensitive**. If the client sends `X-API-Key` and `extra_headers` lists `x-api-key`, they match.
-:::
-
----
-
-## Method 3 — Convention-Based Forwarding
-
-Clients can forward headers to a specific agent without any admin pre-configuration by using the naming convention:
-
-```
-x-a2a-{agent_name_or_id}-{header_name}: value
-```
-
-LiteLLM parses these headers automatically and routes them to the matching agent only.
-
-**Examples:**
-
-| Client header sent | Agent name/ID | Forwarded as |
-|---|---|---|
-| `x-a2a-my-agent-authorization: Bearer tok` | `my-agent` | `authorization: Bearer tok` |
-| `x-a2a-my-agent-x-api-key: secret` | `my-agent` | `x-api-key: secret` |
-| `x-a2a-abc123-authorization: Bearer tok` | agent ID `abc123` | `authorization: Bearer tok` |
-
-```bash
-curl -X POST http://localhost:4000/a2a/my-agent \
-  -H "Authorization: Bearer sk-client-key" \
-  -H "x-a2a-my-agent-authorization: Bearer agent-specific-token" \
-  -H "Content-Type: application/json" \
-  -d '{ ... }'
-```
-
-The `x-a2a-other-agent-authorization` header sent in the same request is **not** forwarded to `my-agent` — it is silently ignored.
-
-:::tip Matches both agent name and agent ID
-Both the human-readable name (e.g. `my-agent`) and the UUID (e.g. `abc123-...`) are valid. Use whichever is convenient for the client.
-:::
-
----
-
-## Merge Precedence
-
-When multiple methods supply the same header name, **static headers win**:
-
-```
-dynamic (forwarded/convention)  →  merged  ←  static (overlays, wins)
-```
-
-Example:
-
-| Source | `Authorization` value |
-|---|---|
-| Client sends (via `extra_headers` or convention) | `Bearer client-token` |
-| Admin-configured `static_headers` | `Bearer server-token` |
-| **What the backend agent receives** | **`Bearer server-token`** |
-
-This ensures admin-controlled credentials cannot be overridden by client requests.
-
----
-
-## Combining All Three Methods
-
-```bash
-# Register agent with static + forwarded headers
-curl -X POST http://localhost:4000/v1/agents \
-  -H "Authorization: Bearer sk-admin" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "agent_name": "my-agent",
-    "agent_card_params": { ... },
-    "static_headers": {
-      "X-Internal-Token": "secret123"
-    },
-    "extra_headers": ["x-user-id"]
-  }'
-
-# Client call using all three mechanisms
-curl -X POST http://localhost:4000/a2a/my-agent \
-  -H "Authorization: Bearer sk-client-key" \
-  -H "x-user-id: user-42" \
-  -H "x-a2a-my-agent-x-request-id: req-abc" \
-  -H "Content-Type: application/json" \
-  -d '{ ... }'
-```
-
-The backend agent receives:
-
-```
-X-Internal-Token: secret123          ← static header (always)
-x-user-id: user-42                   ← forwarded (in extra_headers)
-x-request-id: req-abc                ← convention-based (x-a2a-my-agent-*)
-X-LiteLLM-Trace-Id: <uuid>           ← LiteLLM internal
-X-LiteLLM-Agent-Id: <agent-id>       ← LiteLLM internal
-```
-
----
-
-## Header Isolation
-
-Each agent invocation uses an isolated HTTP connection. Headers configured for agent A are **never** sent to agent B, even if both agents are running and receiving requests simultaneously.
-
----
-
-## API Reference
-
-### `POST /v1/agents` / `PATCH /v1/agents/{agent_id}`
-
-| Field | Type | Description |
-|---|---|---|
-| `static_headers` | `object` | `{"Header-Name": "value"}` — always forwarded |
-| `extra_headers` | `string[]` | Header names to extract from client request and forward |
-
-### Agent Response
-
-Both fields are returned in `GET /v1/agents` and `GET /v1/agents/{agent_id}`:
-
-```json
-{
-  "agent_id": "...",
-  "agent_name": "my-agent",
-  "static_headers": { "X-Internal-Token": "secret123" },
-  "extra_headers": ["x-user-id"],
-  ...
-}
-```
-
-:::caution
-`static_headers` values are stored in the database and returned by the API. Treat them as you would any credential — do not store sensitive long-lived tokens here if your API is publicly accessible. Consider using short-lived tokens or environment-injected secrets instead.
-:::

docs/my-website/docs/a2a_agent_permissions.md

@@ -1,259 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-import Image from '@theme/IdealImage';
-
-# Agent Permission Management
-
-Control which A2A agents can be accessed by specific keys or teams in LiteLLM.
-
-## Overview
-
-Agent Permission Management lets you restrict which agents a LiteLLM Virtual Key or Team can access. This is useful for:
-
-- **Multi-tenant environments**: Give different teams access to different agents
-- **Security**: Prevent keys from invoking agents they shouldn't have access to
-- **Compliance**: Enforce access policies for sensitive agent workflows
-
-When permissions are configured:
-- `GET /v1/agents` only returns agents the key/team can access
-- `POST /a2a/{agent_id}` (Invoking an agent) returns `403 Forbidden` if access is denied
-
-## Setting Permissions on a Key
-
-This example shows how to create a key with agent permissions and test access.
-
-### 1. Get Your Agent ID
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Agents** in the sidebar
-2. Click into the agent you want
-3. Copy the **Agent ID**
-
-<Image 
-  img={require('../img/agent_id.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-</TabItem>
-<TabItem value="api" label="API">
-
-```bash title="List all agents" showLineNumbers
-curl "http://localhost:4000/v1/agents" \
-  -H "Authorization: Bearer sk-master-key"
-```
-
-Response:
-```json title="Response" showLineNumbers
-{
-  "agents": [
-    {"agent_id": "agent-123", "name": "Support Agent"},
-    {"agent_id": "agent-456", "name": "Sales Agent"}
-  ]
-}
-```
-
-</TabItem>
-</Tabs>
-
-### 2. Create a Key with Agent Permissions
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Keys** → **Create Key**
-2. Expand **Agent Settings**
-3. Select the agents you want to allow
-
-<Image 
-  img={require('../img/agent_key.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-</TabItem>
-<TabItem value="api" label="API">
-
-```bash title="Create key with agent permissions" showLineNumbers
-curl -X POST "http://localhost:4000/key/generate" \
-  -H "Authorization: Bearer sk-master-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "object_permission": {
-      "agents": ["agent-123"]
-    }
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-### 3. Test Access
-
-**Allowed agent (succeeds):**
-```bash title="Invoke allowed agent" showLineNumbers
-curl -X POST "http://localhost:4000/a2a/agent-123" \
-  -H "Authorization: Bearer sk-your-new-key" \
-  -H "Content-Type: application/json" \
-  -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
-```
-
-**Blocked agent (fails with 403):**
-```bash title="Invoke blocked agent" showLineNumbers
-curl -X POST "http://localhost:4000/a2a/agent-456" \
-  -H "Authorization: Bearer sk-your-new-key" \
-  -H "Content-Type: application/json" \
-  -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
-```
-
-Response:
-```json title="403 Forbidden Response" showLineNumbers
-{
-  "error": {
-    "message": "Access denied to agent: agent-456",
-    "code": 403
-  }
-}
-```
-
-## Setting Permissions on a Team
-
-Restrict all keys belonging to a team to only access specific agents.
-
-### 1. Create a Team with Agent Permissions
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Teams** → **Create Team**
-2. Expand **Agent Settings**
-3. Select the agents you want to allow for this team
-
-<Image 
-  img={require('../img/agent_key.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-</TabItem>
-<TabItem value="api" label="API">
-
-```bash title="Create team with agent permissions" showLineNumbers
-curl -X POST "http://localhost:4000/team/new" \
-  -H "Authorization: Bearer sk-master-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "team_alias": "support-team",
-    "object_permission": {
-      "agents": ["agent-123"]
-    }
-  }'
-```
-
-Response:
-```json title="Response" showLineNumbers
-{
-  "team_id": "team-abc-123",
-  "team_alias": "support-team"
-}
-```
-
-</TabItem>
-</Tabs>
-
-### 2. Create a Key for the Team
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Keys** → **Create Key**
-2. Select the **Team** from the dropdown
-
-<Image 
-  img={require('../img/agent_team.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-</TabItem>
-<TabItem value="api" label="API">
-
-```bash title="Create key for team" showLineNumbers
-curl -X POST "http://localhost:4000/key/generate" \
-  -H "Authorization: Bearer sk-master-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "team_id": "team-abc-123"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-### 3. Test Access
-
-The key inherits agent permissions from the team.
-
-**Allowed agent (succeeds):**
-```bash title="Invoke allowed agent" showLineNumbers
-curl -X POST "http://localhost:4000/a2a/agent-123" \
-  -H "Authorization: Bearer sk-team-key" \
-  -H "Content-Type: application/json" \
-  -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
-```
-
-**Blocked agent (fails with 403):**
-```bash title="Invoke blocked agent" showLineNumbers
-curl -X POST "http://localhost:4000/a2a/agent-456" \
-  -H "Authorization: Bearer sk-team-key" \
-  -H "Content-Type: application/json" \
-  -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
-```
-
-## How It Works
-
-```mermaid
-flowchart TD
-    A[Request to invoke agent] --> B{LiteLLM Virtual Key has agent restrictions?}
-    B -->|Yes| C{LiteLLM Team has agent restrictions?}
-    B -->|No| D{LiteLLM Team has agent restrictions?}
-    
-    C -->|Yes| E[Use intersection of key + team permissions]
-    C -->|No| F[Use key permissions only]
-    
-    D -->|Yes| G[Inherit team permissions]
-    D -->|No| H[Allow ALL agents]
-    
-    E --> I{Agent in allowed list?}
-    F --> I
-    G --> I
-    H --> J[Allow request]
-    
-    I -->|Yes| J
-    I -->|No| K[Return 403 Forbidden]
-```
-
-| Key Permissions | Team Permissions | Result | Notes |
-|-----------------|------------------|--------|-------|
-| None | None | Key can access **all** agents | Open access by default when no restrictions are set |
-| `["agent-1", "agent-2"]` | None | Key can access `agent-1` and `agent-2` | Key uses its own permissions |
-| None | `["agent-1", "agent-3"]` | Key can access `agent-1` and `agent-3` | Key inherits team's permissions |
-| `["agent-1", "agent-2"]` | `["agent-1", "agent-3"]` | Key can access `agent-1` only | Intersection of both lists (most restrictive wins) |
-
-## Viewing Permissions
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Keys** or **Teams**
-2. Click into the key/team you want to view
-3. Agent permissions are displayed in the info view
-
-</TabItem>
-<TabItem value="api" label="API">
-
-```bash title="Get key info" showLineNumbers
-curl "http://localhost:4000/key/info?key=sk-your-key" \
-  -H "Authorization: Bearer sk-master-key"
-```
-
-</TabItem>
-</Tabs>

docs/my-website/docs/a2a_cost_tracking.md

@@ -1,147 +0,0 @@
-import Image from '@theme/IdealImage';
-
-# A2A Agent Cost Tracking
-
-LiteLLM supports adding custom cost tracking for A2A agents. You can configure:
-
-- **Flat cost per query** - A fixed cost charged for each agent request
-- **Cost by input/output tokens** - Variable cost based on token usage
-
-This allows you to track and attribute costs for agent usage across your organization, making it easy to see how much each team or project is spending on agent calls.
-
-## Quick Start
-
-### 1. Navigate to Agents
-
-From the sidebar, click on "Agents" to open the agent management page.
-
-![Navigate to Agents](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/f9ac0752-6936-4dda-b7ed-f536fefcc79a/ascreenshot.jpeg?tl_px=208,326&br_px=2409,1557&force_format=jpeg&q=100&width=1120.0)
-
-### 2. Create a New Agent
-
-Click "+ Add New Agent" to open the creation form. You'll need to provide a few basic details:
-
-- **Agent Name** - A unique identifier for your agent (used in API calls)
-- **Display Name** - A human-readable name shown in the UI
-
-![Enter Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/f5bacfeb-67a0-4644-a400-b3d50b6b9ce5/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1463&force_format=jpeg&q=100&width=1120.0)
-
-![Enter Display Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/6db6422b-fe85-4a8b-aa5c-39319f0d4621/ascreenshot.jpeg?tl_px=0,27&br_px=2617,1490&force_format=jpeg&q=100&width=1120.0)
-
-### 3. Configure Cost Settings
-
-Scroll down and click on "Cost Configuration" to expand the cost settings panel. This is where you define how much to charge for agent usage.
-
-![Click Cost Configuration](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/a3019ae8-629c-431b-b2d8-2743cc517be7/ascreenshot.jpeg?tl_px=0,653&br_px=2201,1883&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=388,416)
-
-### 4. Set Cost Per Query
-
-Enter the cost per query amount (in dollars). For example, entering `0.05` means each request to this agent will be charged $0.05.
-
-![Set Cost Per Query](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/91159f8a-1f66-4555-a166-600e4bdecc68/ascreenshot.jpeg?tl_px=0,653&br_px=2201,1883&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=372,281)
-
-![Enter Cost Amount](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/2add2f69-fd72-462e-9335-1e228c7150da/ascreenshot.jpeg?tl_px=0,420&br_px=2617,1884&force_format=jpeg&q=100&width=1120.0)
-
-### 5. Create the Agent
-
-Once you've configured everything, click "Create Agent" to save. Your agent is now ready to use with cost tracking enabled.
-
-![Create Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/1876cf29-b8a7-4662-b944-2b86a8b7cd2e/ascreenshot.jpeg?tl_px=416,653&br_px=2618,1883&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=706,523)
-
-## Testing Cost Tracking
-
-Let's verify that cost tracking is working by sending a test request through the Playground.
-
-### 1. Go to Playground
-
-Click "Playground" in the sidebar to open the interactive testing interface.
-
-![Go to Playground](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/7d5d8338-6393-49a5-b255-86aef5bf5dfa/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=41,98)
-
-### 2. Select A2A Endpoint
-
-By default, the Playground uses the chat completions endpoint. To test your agent, click "Endpoint Type" and select `/v1/a2a/message/send` from the dropdown.
-
-![Select Endpoint Type](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/4d066510-0878-4e0b-8abf-0b074fe2a560/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=325,238)
-
-![Select A2A Endpoint](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/fe2f8957-4e8a-4331-b177-d5093480cf60/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=333,261)
-
-### 3. Select Your Agent
-
-Now pick the agent you just created from the agent dropdown. You should see it listed by its display name.
-
-![Select Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/8c7add70-fe72-48cb-ba33-9f53b989fcad/ascreenshot.jpeg?tl_px=0,150&br_px=2201,1381&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=287,277)
-
-### 4. Send a Test Message
-
-Type a message and hit send. You can use the suggested prompts or write your own.
-
-![Send Message](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/2c16acb1-4016-447e-88e9-c4522e408ea2/ascreenshot.jpeg?tl_px=15,653&br_px=2216,1883&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,443)
-
-Once the agent responds, the request is logged with the cost you configured.
-
-![Agent Response](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/2dcf7109-0be4-4d03-8333-ef45759c70c9/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=494,273)
-
-## Viewing Cost in Logs
-
-Now let's confirm the cost was actually tracked.
-
-### 1. Navigate to Logs
-
-Click "Logs" in the sidebar to see all recent requests.
-
-![Go to Logs](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/c96abf3c-f06a-4401-ada6-04b6e8040453/ascreenshot.jpeg?tl_px=0,118&br_px=2201,1349&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=41,277)
-
-### 2. View Cost Attribution
-
-Find your agent request in the list. You'll see the cost column showing the amount you configured. This cost is now attributed to the API key that made the request, so you can track spend per team or project.
-
-![View Cost in Logs](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-13/1ae167ec-1a43-48a3-9251-43d4cb3e57f5/ascreenshot.jpeg?tl_px=335,11&br_px=2536,1242&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,277)
-
-## View Spend in Usage Page
-
-Navigate to the Agent Usage tab in the Admin UI to view agent-level spend analytics:
-
-### 1. Access Agent Usage
-
-Go to the Usage page in the Admin UI (`PROXY_BASE_URL/ui/?login=success&page=new_usage`) and click on the **Agent Usage** tab.
-
-<Image img={require('../img/agent_usage_ui_navigation.png')} />
-
-### 2. View Agent Analytics
-
-The Agent Usage dashboard provides:
-
-- **Total spend per agent**: View aggregated spend across all agents
-- **Daily spend trends**: See how agent spend changes over time
-- **Model usage breakdown**: Understand which models each agent uses
-- **Activity metrics**: Track requests, tokens, and success rates per agent
-
-<Image img={require('../img/agent_usage_analytics.png')} />
-
-### 3. Filter by Agent
-
-Use the agent filter dropdown to view spend for specific agents:
-
-- Select one or more agent IDs from the dropdown
-- View filtered analytics, spend logs, and activity metrics
-- Compare spend across different agents
-
-<Image img={require('../img/agent_usage_filter.png')} />
-
-## Cost Configuration Options
-
-You can mix and match these options depending on your pricing model:
-
-| Field                         | Description                               |
-| ----------------------------- | ----------------------------------------- |
-| **Cost Per Query ($)**        | Fixed cost charged for each agent request |
-| **Input Cost Per Token ($)**  | Cost per input token processed            |
-| **Output Cost Per Token ($)** | Cost per output token generated           |
-
-For most use cases, a flat cost per query is simplest. Use token-based pricing if your agent costs vary significantly based on input/output length.
-
-## Related
-
-- [A2A Agent Gateway](./a2a.md)
-- [Spend Tracking](./proxy/cost_tracking.md)

docs/my-website/docs/a2a_invoking_agents.md

@@ -1,280 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Invoking A2A Agents
-
-Learn how to invoke A2A agents through LiteLLM using different methods.
-
-:::tip Deploy Your Own A2A Agent
-
-Want to test with your own agent? Deploy this template A2A agent powered by Google Gemini:
-
-[**shin-bot-litellm/a2a-gemini-agent**](https://github.com/shin-bot-litellm/a2a-gemini-agent) - Simple deployable A2A agent with streaming support
-
-:::
-
-## A2A SDK
-
-Use the [A2A Python SDK](https://pypi.org/project/a2a-sdk) to invoke agents through LiteLLM using the A2A protocol.
-
-### Non-Streaming
-
-This example shows how to:
-1. **List available agents** - Query `/v1/agents` to see which agents your key can access
-2. **Select an agent** - Pick an agent from the list
-3. **Invoke via A2A** - Use the A2A protocol to send messages to the agent
-
-```python showLineNumbers title="invoke_a2a_agent.py"
-from uuid import uuid4
-import httpx
-import asyncio
-from a2a.client import A2ACardResolver, A2AClient
-from a2a.types import MessageSendParams, SendMessageRequest
-
-# === CONFIGURE THESE ===
-LITELLM_BASE_URL = "http://localhost:4000"  # Your LiteLLM proxy URL
-LITELLM_VIRTUAL_KEY = "sk-1234"             # Your LiteLLM Virtual Key
-# =======================
-
-async def main():
-    headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
-    
-    async with httpx.AsyncClient(headers=headers) as client:
-        # Step 1: List available agents
-        response = await client.get(f"{LITELLM_BASE_URL}/v1/agents")
-        agents = response.json()
-        
-        print("Available agents:")
-        for agent in agents:
-            print(f"  - {agent['agent_name']} (ID: {agent['agent_id']})")
-        
-        if not agents:
-            print("No agents available for this key")
-            return
-        
-        # Step 2: Select an agent and invoke it
-        selected_agent = agents[0]
-        agent_id = selected_agent["agent_id"]
-        agent_name = selected_agent["agent_name"]
-        print(f"\nInvoking: {agent_name}")
-        
-        # Step 3: Use A2A protocol to invoke the agent
-        base_url = f"{LITELLM_BASE_URL}/a2a/{agent_id}"
-        resolver = A2ACardResolver(httpx_client=client, base_url=base_url)
-        agent_card = await resolver.get_agent_card()
-        a2a_client = A2AClient(httpx_client=client, agent_card=agent_card)
-        
-        request = SendMessageRequest(
-            id=str(uuid4()),
-            params=MessageSendParams(
-                message={
-                    "role": "user",
-                    "parts": [{"kind": "text", "text": "Hello, what can you do?"}],
-                    "messageId": uuid4().hex,
-                }
-            ),
-        )
-        response = await a2a_client.send_message(request)
-        print(f"Response: {response.model_dump(mode='json', exclude_none=True, indent=4)}")
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-### Streaming
-
-For streaming responses, use `send_message_streaming`:
-
-```python showLineNumbers title="invoke_a2a_agent_streaming.py"
-from uuid import uuid4
-import httpx
-import asyncio
-from a2a.client import A2ACardResolver, A2AClient
-from a2a.types import MessageSendParams, SendStreamingMessageRequest
-
-# === CONFIGURE THESE ===
-LITELLM_BASE_URL = "http://localhost:4000"  # Your LiteLLM proxy URL
-LITELLM_VIRTUAL_KEY = "sk-1234"             # Your LiteLLM Virtual Key
-LITELLM_AGENT_NAME = "ij-local"             # Agent name registered in LiteLLM
-# =======================
-
-async def main():
-    base_url = f"{LITELLM_BASE_URL}/a2a/{LITELLM_AGENT_NAME}"
-    headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
-    
-    async with httpx.AsyncClient(headers=headers) as httpx_client:
-        # Resolve agent card and create client
-        resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
-        agent_card = await resolver.get_agent_card()
-        client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
-
-        # Send a streaming message
-        request = SendStreamingMessageRequest(
-            id=str(uuid4()),
-            params=MessageSendParams(
-                message={
-                    "role": "user",
-                    "parts": [{"kind": "text", "text": "Tell me a long story"}],
-                    "messageId": uuid4().hex,
-                }
-            ),
-        )
-        
-        # Stream the response
-        async for chunk in client.send_message_streaming(request):
-            print(chunk.model_dump(mode="json", exclude_none=True))
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-## /chat/completions API (OpenAI SDK)
-
-You can also invoke A2A agents using the familiar OpenAI SDK by using the `a2a/` model prefix.
-
-### Non-Streaming
-
-<Tabs>
-<TabItem value="python" label="Python" default>
-
-```python showLineNumbers title="openai_non_streaming.py"
-import openai
-
-client = openai.OpenAI(
-    api_key="sk-1234",  # Your LiteLLM Virtual Key
-    base_url="http://localhost:4000"  # Your LiteLLM proxy URL
-)
-
-response = client.chat.completions.create(
-    model="a2a/my-agent",  # Use a2a/ prefix with your agent name
-    messages=[
-        {"role": "user", "content": "Hello, what can you do?"}
-    ]
-)
-
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-<TabItem value="typescript" label="TypeScript">
-
-```typescript showLineNumbers title="openai_non_streaming.ts"
-import OpenAI from 'openai';
-
-const client = new OpenAI({
-  apiKey: 'sk-1234',  // Your LiteLLM Virtual Key
-  baseURL: 'http://localhost:4000'  // Your LiteLLM proxy URL
-});
-
-const response = await client.chat.completions.create({
-  model: 'a2a/my-agent',  // Use a2a/ prefix with your agent name
-  messages: [
-    { role: 'user', content: 'Hello, what can you do?' }
-  ]
-});
-
-console.log(response.choices[0].message.content);
-```
-
-</TabItem>
-<TabItem value="curl" label="cURL">
-
-```bash showLineNumbers title="curl_non_streaming.sh"
-curl -X POST http://localhost:4000/v1/chat/completions \
-  -H "Authorization: Bearer sk-1234" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "a2a/my-agent",
-    "messages": [
-      {"role": "user", "content": "Hello, what can you do?"}
-    ]
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-### Streaming
-
-<Tabs>
-<TabItem value="python" label="Python" default>
-
-```python showLineNumbers title="openai_streaming.py"
-import openai
-
-client = openai.OpenAI(
-    api_key="sk-1234",  # Your LiteLLM Virtual Key
-    base_url="http://localhost:4000"  # Your LiteLLM proxy URL
-)
-
-stream = client.chat.completions.create(
-    model="a2a/my-agent",  # Use a2a/ prefix with your agent name
-    messages=[
-        {"role": "user", "content": "Tell me a long story"}
-    ],
-    stream=True
-)
-
-for chunk in stream:
-    if chunk.choices[0].delta.content:
-        print(chunk.choices[0].delta.content, end="", flush=True)
-```
-
-</TabItem>
-<TabItem value="typescript" label="TypeScript">
-
-```typescript showLineNumbers title="openai_streaming.ts"
-import OpenAI from 'openai';
-
-const client = new OpenAI({
-  apiKey: 'sk-1234',  // Your LiteLLM Virtual Key
-  baseURL: 'http://localhost:4000'  // Your LiteLLM proxy URL
-});
-
-const stream = await client.chat.completions.create({
-  model: 'a2a/my-agent',  // Use a2a/ prefix with your agent name
-  messages: [
-    { role: 'user', content: 'Tell me a long story' }
-  ],
-  stream: true
-});
-
-for await (const chunk of stream) {
-  const content = chunk.choices[0]?.delta?.content;
-  if (content) {
-    process.stdout.write(content);
-  }
-}
-```
-
-</TabItem>
-<TabItem value="curl" label="cURL">
-
-```bash showLineNumbers title="curl_streaming.sh"
-curl -X POST http://localhost:4000/v1/chat/completions \
-  -H "Authorization: Bearer sk-1234" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "a2a/my-agent",
-    "messages": [
-      {"role": "user", "content": "Tell me a long story"}
-    ],
-    "stream": true
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-## Key Differences
-
-| Method | Use Case | Advantages |
-|--------|----------|------------|
-| **A2A SDK** | Native A2A protocol integration | • Full A2A protocol support<br/>• Access to task states and artifacts<br/>• Context management |
-| **OpenAI SDK** | Familiar OpenAI-style interface | • Drop-in replacement for OpenAI calls<br/>• Easier migration from LLM to agent workflows<br/>• Works with existing OpenAI tooling |
-
-:::tip Model Prefix
-
-When using the OpenAI SDK, always prefix your agent name with `a2a/` (e.g., `a2a/my-agent`) to route requests to the A2A agent instead of an LLM provider.
-
-:::

docs/my-website/docs/a2a_iteration_budgets.md

@@ -1,188 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Agent Iteration Budgets
-
-Control runaway costs from agentic loops with per-session iteration and budget caps.
-
-## Overview
-
-When agents run agentic loops, they can make unbounded LLM calls, causing unexpected costs. LiteLLM provides two controls:
-
-| Control | Description |
-|---------|-------------|
-| **Max Iterations** | Hard cap on the number of LLM calls per session |
-| **Max Budget Per Session** | Dollar cap per session (identified by `x-litellm-trace-id`) |
-
-Both controls require a `session_id` (sent via `x-litellm-trace-id` header or `metadata.session_id`) to track calls within a session.
-
-## Trace-ID Enforcement
-
-LiteLLM supports two independent trace-id flags, configured in `litellm_params` on the agent:
-
-| Flag | Description |
-|------|-------------|
-| `require_trace_id_on_calls_to_agent` | Requires callers invoking this agent to include `x-litellm-trace-id`. Use when the agent should only be called as a sub-agent with a trace context. Returns **400** if missing. |
-| `require_trace_id_on_calls_by_agent` | Requires all LLM/MCP calls made **by** this agent (via its virtual key) to include `x-litellm-trace-id`. This is what enables `max_iterations` and `max_budget_per_session` tracking. Returns **400** if missing. |
-
-## Configuring via UI
-
-When creating an agent in the LiteLLM Admin UI:
-
-1. Navigate to the **Agents** tab and click **Add Agent**
-2. In the **Agent Settings** step, expand the **Tracing** section
-3. Toggle **Require x-litellm-trace-id on calls BY this agent** to enable session tracking
-4. Set **Max Iterations** to cap the number of LLM calls per session
-5. Set **Max Budget Per Session ($)** to cap spend per session
-
-The trace-id flags are stored on the agent's `litellm_params`. Budget controls (`max_iterations`, `max_budget_per_session`) are stored in the virtual key's metadata.
-
-## Configuring via API
-
-Set trace-id enforcement on the agent itself:
-
-```bash
-curl -X POST 'http://localhost:4000/v1/agents' \
-  -H 'Authorization: Bearer sk-1234' \
-  -H 'Content-Type: application/json' \
-  -d '{
-    "agent_name": "my-research-agent",
-    "agent_card_params": {
-      "name": "my-research-agent",
-      "description": "A research agent with budget controls",
-      "url": "http://my-agent:8080",
-      "version": "1.0.0"
-    },
-    "litellm_params": {
-      "require_trace_id_on_calls_to_agent": true,
-      "require_trace_id_on_calls_by_agent": true
-    }
-  }'
-```
-
-Budget controls are set on the agent's `litellm_params` (not on individual keys), so they apply across all keys for the agent:
-
-```bash
-curl -X POST 'http://localhost:4000/v1/agents' \
-  -H 'Authorization: Bearer sk-1234' \
-  -H 'Content-Type: application/json' \
-  -d '{
-    "agent_name": "my-research-agent",
-    "agent_card_params": {
-      "name": "my-research-agent",
-      "description": "A research agent with budget controls",
-      "url": "http://my-agent:8080",
-      "version": "1.0.0"
-    },
-    "litellm_params": {
-      "require_trace_id_on_calls_by_agent": true,
-      "max_iterations": 25,
-      "max_budget_per_session": 5.00
-    }
-  }'
-```
-
-## How It Works
-
-### Session Tracking
-
-Callers identify their session by including a `session_id` in one of these ways:
-- **Header**: `x-litellm-trace-id: my-session-123`
-- **Metadata**: `{"metadata": {"session_id": "my-session-123"}}`
-
-### Max Iterations
-
-When `max_iterations` is set in agent `litellm_params`:
-- Each LLM call for a session increments a counter
-- When the counter exceeds `max_iterations`, the request receives a **429 Too Many Requests**
-- Counters expire after 1 hour by default (configurable via `LITELLM_MAX_ITERATIONS_TTL` env var)
-
-### Max Budget Per Session
-
-When `max_budget_per_session` is set in agent `litellm_params`:
-- After each successful LLM call, the response cost is accumulated for the session
-- Before each call, the accumulated spend is checked against the budget
-- When spend exceeds the budget, the request receives a **429 Too Many Requests**
-- Session spend counters expire after 1 hour by default (configurable via `LITELLM_MAX_BUDGET_PER_SESSION_TTL` env var)
-
-## Example
-
-Create an agent with max 25 iterations and a $5 budget cap:
-
-<Tabs>
-<TabItem value="ui" label="Via UI">
-
-1. Go to **Agents** → **Add Agent**
-2. Configure your agent (name, model, etc.)
-3. In **Agent Settings**, expand the **Tracing** section
-4. Toggle on **Require x-litellm-trace-id on calls BY this agent**
-5. Set **Max Iterations** to `25`
-6. Set **Max Budget Per Session** to `5.00`
-7. Proceed to create a new key for the agent
-8. Click **Create Agent**
-
-</TabItem>
-<TabItem value="api" label="Via API">
-
-```bash
-# 1. Create the agent with trace-id enforcement
-curl -X POST 'http://localhost:4000/v1/agents' \
-  -H 'Authorization: Bearer sk-1234' \
-  -H 'Content-Type: application/json' \
-  -d '{
-    "agent_name": "my-research-agent",
-    "agent_card_params": {
-      "name": "my-research-agent",
-      "description": "A research agent with budget controls",
-      "url": "http://my-agent:8080",
-      "version": "1.0.0"
-    },
-    "litellm_params": {
-      "require_trace_id_on_calls_by_agent": true
-    }
-  }'
-
-# 2. Create a key for the agent
-curl -X POST 'http://localhost:4000/key/generate' \
-  -H 'Authorization: Bearer sk-1234' \
-  -H 'Content-Type: application/json' \
-  -d '{
-    "agent_id": "<agent_id_from_step_1>",
-    "key_alias": "my-research-agent-key"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-### Making Calls with Session Tracking
-
-```bash
-curl -X POST 'http://localhost:4000/chat/completions' \
-  -H 'Authorization: Bearer sk-agent-key-xxx' \
-  -H 'x-litellm-trace-id: session-abc-123' \
-  -H 'Content-Type: application/json' \
-  -d '{
-    "model": "gpt-4o",
-    "messages": [{"role": "user", "content": "Hello"}]
-  }'
-```
-
-After 25 calls or $5 spent within this session, subsequent requests will receive:
-
-```json
-{
-  "error": {
-    "message": "Session budget exceeded for session session-abc-123. Current spend: $5.0032, max_budget_per_session: $5.00.",
-    "type": "budget_exceeded",
-    "code": 429
-  }
-}
-```
-
-## Environment Variables
-
-| Variable | Default | Description |
-|----------|---------|-------------|
-| `LITELLM_MAX_ITERATIONS_TTL` | `3600` (1 hour) | TTL in seconds for session iteration counters |
-| `LITELLM_MAX_BUDGET_PER_SESSION_TTL` | `3600` (1 hour) | TTL in seconds for session budget counters |

docs/my-website/docs/adding_provider/adding_guardrail_support.md

@@ -1,412 +0,0 @@
-# Adding Guardrail Support to Endpoints
-
-This guide explains how to add guardrail translation support to new LiteLLM endpoints (e.g., Chat Completions, Responses API, etc.).
-
-## When to Add Guardrail Support
-
-Add guardrail support when:
-- You're creating a new LiteLLM endpoint (e.g., a new API format)
-- You want to enable guardrails for an existing endpoint that doesn't support them
-- You need custom text extraction logic for a specific message format
-
-## Directory Structure
-
-Guardrail handlers follow this structure:
-
-```
-litellm/llms/{provider}/{endpoint}/guardrail_translation/
-├── __init__.py          # Exports handler and registers call types
-├── handler.py           # Main handler implementation
-└── README.md            # Documentation (optional but recommended)
-```
-
-### Example Structures
-
-**OpenAI Chat Completions:**
-```
-litellm/llms/openai/chat/guardrail_translation/
-├── __init__.py
-├── handler.py
-└── README.md
-```
-
-**OpenAI Responses API:**
-```
-litellm/llms/openai/responses/guardrail_translation/
-├── __init__.py
-├── handler.py
-└── README.md
-```
-
-**Anthropic Messages:**
-```
-litellm/llms/anthropic/chat/guardrail_translation/
-├── __init__.py
-└── handler.py
-```
-
-## Step-by-Step Implementation
-
-### Step 1: Create the Handler Class
-
-Create `handler.py` that inherits from `BaseTranslation`:
-
-```python
-"""
-{Provider} {Endpoint} Handler for Unified Guardrails
-
-This module provides guardrail translation support for {Provider}'s {Endpoint} format.
-"""
-
-import asyncio
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
-
-from litellm._logging import verbose_proxy_logger
-from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
-
-if TYPE_CHECKING:
-    from litellm.integrations.custom_guardrail import CustomGuardrail
-    from litellm.types.utils import ModelResponse  # Or appropriate response type
-
-
-class MyEndpointHandler(BaseTranslation):
-    """
-    Handler for processing {Endpoint} with guardrails.
-
-    This class provides methods to:
-    1. Process input (pre-call hook)
-    2. Process output response (post-call hook)
-    """
-
-    async def process_input_messages(
-        self,
-        data: dict,
-        guardrail_to_apply: "CustomGuardrail",
-    ) -> Any:
-        """
-        Process input by applying guardrails to text content.
-        
-        Args:
-            data: Request data dictionary
-            guardrail_to_apply: The guardrail instance to apply
-            
-        Returns:
-            Modified data with guardrails applied
-        """
-        # Your implementation here
-        pass
-
-    async def process_output_response(
-        self,
-        response: Any,  # Use appropriate response type
-        guardrail_to_apply: "CustomGuardrail",
-    ) -> Any:
-        """
-        Process output response by applying guardrails to text content.
-        
-        Args:
-            response: API response object
-            guardrail_to_apply: The guardrail instance to apply
-            
-        Returns:
-            Modified response with guardrails applied
-        """
-        # Your implementation here
-        pass
-```
-
-### Step 2: Implement Core Methods
-
-#### A. Process Input Messages
-
-Extract text from input, apply guardrails, and map back:
-
-```python
-async def process_input_messages(
-    self,
-    data: dict,
-    guardrail_to_apply: "CustomGuardrail",
-) -> Any:
-    """Process input messages by applying guardrails to text content."""
-    # 1. Get input data from request
-    messages = data.get("messages")  # or appropriate field
-    if messages is None:
-        return data
-
-    # 2. Extract text and create tasks
-    tasks = []
-    task_mappings: List[Tuple[int, Optional[int]]] = []
-    
-    for msg_idx, message in enumerate(messages):
-        await self._extract_input_text_and_create_tasks(
-            message=message,
-            msg_idx=msg_idx,
-            tasks=tasks,
-            task_mappings=task_mappings,
-            guardrail_to_apply=guardrail_to_apply,
-        )
-
-    # 3. Run all guardrail tasks in parallel
-    if tasks:
-        responses = await asyncio.gather(*tasks)
-
-        # 4. Map responses back to original structure
-        await self._apply_guardrail_responses_to_input(
-            messages=messages,
-            responses=responses,
-            task_mappings=task_mappings,
-        )
-
-    return data
-```
-
-#### B. Process Output Response
-
-Extract text from response, apply guardrails, and update:
-
-```python
-async def process_output_response(
-    self,
-    response: "ModelResponse",
-    guardrail_to_apply: "CustomGuardrail",
-) -> Any:
-    """Process output response by applying guardrails to text content."""
-    # 1. Check if response has text to process
-    if not self._has_text_content(response):
-        return response
-
-    # 2. Extract text and create tasks
-    tasks = []
-    task_mappings: List[Tuple[int, Optional[int]]] = []
-    
-    for idx, item in enumerate(response.choices):  # or appropriate field
-        await self._extract_output_text_and_create_tasks(
-            item=item,
-            idx=idx,
-            tasks=tasks,
-            task_mappings=task_mappings,
-            guardrail_to_apply=guardrail_to_apply,
-        )
-
-    # 3. Run all guardrail tasks in parallel
-    if tasks:
-        responses = await asyncio.gather(*tasks)
-
-        # 4. Update response with guardrailed text
-        await self._apply_guardrail_responses_to_output(
-            response=response,
-            responses=responses,
-            task_mappings=task_mappings,
-        )
-
-    return response
-```
-
-### Step 3: Create Helper Methods
-
-Implement helper methods for text extraction and mapping:
-
-```python
-async def _extract_input_text_and_create_tasks(
-    self,
-    message: Dict[str, Any],
-    msg_idx: int,
-    tasks: List,
-    task_mappings: List[Tuple[int, Optional[int]]],
-    guardrail_to_apply: "CustomGuardrail",
-) -> None:
-    """Extract text content from a message and create guardrail tasks."""
-    content = message.get("content")
-    if content is None:
-        return
-
-    if isinstance(content, str):
-        # Simple string content
-        tasks.append(guardrail_to_apply.apply_guardrail(text=content))
-        task_mappings.append((msg_idx, None))
-    elif isinstance(content, list):
-        # List content (e.g., multimodal)
-        for content_idx, content_item in enumerate(content):
-            if isinstance(content_item, dict):
-                text_str = content_item.get("text")
-                if text_str:
-                    tasks.append(guardrail_to_apply.apply_guardrail(text=text_str))
-                    task_mappings.append((msg_idx, int(content_idx)))
-
-async def _apply_guardrail_responses_to_input(
-    self,
-    messages: List[Dict[str, Any]],
-    responses: List[str],
-    task_mappings: List[Tuple[int, Optional[int]]],
-) -> None:
-    """Apply guardrail responses back to input messages."""
-    for task_idx, guardrail_response in enumerate(responses):
-        msg_idx, content_idx = task_mappings[task_idx]
-        
-        if content_idx is None:
-            # String content
-            messages[msg_idx]["content"] = guardrail_response
-        else:
-            # List content
-            messages[msg_idx]["content"][content_idx]["text"] = guardrail_response
-
-def _has_text_content(self, response: Any) -> bool:
-    """Check if response has any text content to process."""
-    # Implement based on your response structure
-    return True  # or appropriate logic
-```
-
-### Step 4: Register the Handler
-
-Create `__init__.py` to register the handler with call types:
-
-```python
-"""My Endpoint handler for Unified Guardrails."""
-
-from litellm.llms.{provider}/{endpoint}/guardrail_translation.handler import (
-    MyEndpointHandler,
-)
-from litellm.types.utils import CallTypes
-
-guardrail_translation_mappings = {
-    CallTypes.my_endpoint: MyEndpointHandler,
-    CallTypes.amy_endpoint: MyEndpointHandler,  # async version if applicable
-}
-
-__all__ = ["guardrail_translation_mappings"]
-```
-
-**Important:** Make sure your `CallTypes` are defined in `litellm/types/utils.py`.
-
-### Step 5: Add Documentation
-
-Create `README.md` with usage examples and format details:
-
-```markdown
-# {Provider} {Endpoint} Guardrail Translation Handler
-
-Handler for processing {Provider}'s {Endpoint} with guardrails.
-
-## Overview
-
-This handler processes {Endpoint} input/output by:
-1. Extracting text from messages/responses
-2. Applying guardrails to text content
-3. Mapping guardrailed text back to original structure
-
-## Data Format
-
-### Input Format
-```json
-{
-  "field": "value",
-  "messages": [...]
-}
-```
-
-### Output Format
-```json
-{
-  "field": "value",
-  "output": [...]
-}
-```
-
-## Usage
-
-The handler is automatically discovered and applied when guardrails are used with this endpoint.
-
-```bash
-curl -X POST 'http://localhost:4000/{my_endpoint}' \
--H 'Content-Type: application/json' \
--H 'Authorization: Bearer your-api-key' \
--d '{
-    "model": "gpt-3.5-turbo",
-    "messages": [{"role": "user", "content": "Hello"}],
-    "guardrails": ["test"]
-}'
-
-```
-## Extension
-
-Override these methods to customize behavior:
-- `_extract_input_text_and_create_tasks()`: Custom text extraction
-- `_apply_guardrail_responses_to_input()`: Custom response mapping
-- `_has_text_content()`: Custom content detection
-```
-
-### Step 6: Add Unit Tests
-
-Create comprehensive tests in `tests/test_litellm/llms/{provider}/{endpoint}/`:
-
-```python
-"""
-Unit tests for {Provider} {Endpoint} Guardrail Translation Handler
-"""
-
-import os
-import sys
-import pytest
-
-sys.path.insert(0, os.path.abspath("../../../../../.."))
-
-from litellm.integrations.custom_guardrail import CustomGuardrail
-from litellm.llms import get_guardrail_translation_mapping
-from litellm.llms.{provider}.{endpoint}.guardrail_translation.handler import (
-    MyEndpointHandler,
-)
-from litellm.types.utils import CallTypes
-
-
-class MockGuardrail(CustomGuardrail):
-    """Mock guardrail for testing"""
-    
-    async def apply_guardrail(self, text: str) -> str:
-        return f"{text} [GUARDRAILED]"
-
-
-class TestHandlerDiscovery:
-    """Test that the handler is properly discovered"""
-    
-    def test_handler_discovered(self):
-        handler_class = get_guardrail_translation_mapping(CallTypes.my_endpoint)
-        assert handler_class == MyEndpointHandler
-
-
-class TestInputProcessing:
-    """Test input processing functionality"""
-    
-    @pytest.mark.asyncio
-    async def test_process_simple_input(self):
-        handler = MyEndpointHandler()
-        guardrail = MockGuardrail(guardrail_name="test")
-        
-        data = {"messages": [{"role": "user", "content": "Hello"}]}
-        result = await handler.process_input_messages(data, guardrail)
-        
-        assert result["messages"][0]["content"] == "Hello [GUARDRAILED]"
-
-
-class TestOutputProcessing:
-    """Test output processing functionality"""
-    
-    @pytest.mark.asyncio
-    async def test_process_simple_output(self):
-        handler = MyEndpointHandler()
-        guardrail = MockGuardrail(guardrail_name="test")
-        
-        # Create mock response
-        response = create_mock_response()
-        result = await handler.process_output_response(response, guardrail)
-        
-        # Assert guardrail was applied
-        assert "GUARDRAILED" in get_response_text(result)
-```
-
-## Support
-
-For questions or issues:
-- Check existing handler implementations for examples
-- Review the base translation class documentation
-- Create an issue on GitHub with the `guardrails` label
-

docs/my-website/docs/adding_provider/directory_structure.md

@@ -1,24 +0,0 @@
-# Directory Structure
-
-When adding a new provider, you need to create a directory for the provider that follows the following structure:
-
-```
-litellm/llms/
-└── provider_name/
-    ├── completion/ # use when endpoint is equivalent to openai's `/v1/completions`
-    │   ├── handler.py
-    │   └── transformation.py
-    ├── chat/ # use when endpoint is equivalent to openai's `/v1/chat/completions`
-    │   ├── handler.py
-    │   └── transformation.py
-    ├── embed/ # use when endpoint is equivalent to openai's `/v1/embeddings`
-    │   ├── handler.py
-    │   └── transformation.py
-    ├── audio_transcription/ # use when endpoint is equivalent to openai's `/v1/audio/transcriptions`
-    │   ├── handler.py
-    │   └── transformation.py
-    └── rerank/ # use when endpoint is equivalent to cohere's `/rerank` endpoint.
-        ├── handler.py
-        └── transformation.py
-```
-

docs/my-website/docs/adding_provider/generic_guardrail_api.md

@@ -1,430 +0,0 @@
-# [BETA] Generic Guardrail API - Integrate Without a PR
-
-## The Problem
-
-As a guardrail provider, integrating with LiteLLM traditionally requires:
-- Making a PR to the LiteLLM repository
-- Waiting for review and merge
-- Maintaining provider-specific code in LiteLLM's codebase
-- Updating the integration for changes to your API
-
-## The Solution
-
-The **Generic Guardrail API** lets you integrate with LiteLLM **instantly** by implementing a simple API endpoint. No PR required.
-
-### Key Benefits
-
-1. **No PR Needed** - Deploy and integrate immediately
-2. **Universal Support** - Works across ALL LiteLLM endpoints (chat, embeddings, image generation, etc.)
-3. **Simple Contract** - One endpoint, three response types
-4. **Multi-Modal Support** - Handle both text and images in requests/responses
-5. **Custom Parameters** - Pass provider-specific params via config
-6. **Full Control** - You own and maintain your guardrail API
-
-## Supported Endpoints
-
-The Generic Guardrail API works with the following LiteLLM endpoints:
-
-- `/v1/chat/completions` - OpenAI Chat Completions
-- `/v1/completions` - OpenAI Text Completions
-- `/v1/responses` - OpenAI Responses API
-- `/v1/images/generations` - OpenAI Image Generation
-- `/v1/audio/transcriptions` - OpenAI Audio Transcriptions
-- `/v1/audio/speech` - OpenAI Text-to-Speech
-- `/v1/messages` - Anthropic Messages
-- `/v1/rerank` - Cohere Rerank
-- Pass-through endpoints
-
-## How It Works
-
-1. LiteLLM extracts text and images from any request (chat messages, embeddings, image prompts, etc.)
-2. Sends extracted content + metadata to your API endpoint
-3. Your API responds with: `BLOCKED`, `NONE`, or `GUARDRAIL_INTERVENED`
-4. LiteLLM enforces the decision and applies any modifications
-
-## API Contract
-
-### Endpoint
-
-Implement `POST /beta/litellm_basic_guardrail_api`
-
-### Request Format
-
-```json
-{
-  "texts": ["extracted text from the request"],  // array of text strings
-  "images": ["base64_encoded_image_data"],  // optional array of images
-  "tools": [  // tool calls sent to the LLM (in the OpenAI Chat Completions spec)
-    {
-      "type": "function",
-      "function": {
-        "name": "get_weather",
-        "description": "Get the current weather",
-        "parameters": {
-          "type": "object",
-          "properties": {
-            "location": {"type": "string"}
-          }
-        }
-      }
-    }
-  ],
-  "tool_calls": [  // tool calls received from the LLM (in the OpenAI Chat Completions spec)
-    {
-      "id": "call_abc123",
-      "type": "function",
-      "function": {
-        "name": "get_weather",
-        "arguments": "{\"location\": \"San Francisco\"}"
-      }
-    }
-  ],
-  "structured_messages": [  // optional, full messages in OpenAI format (for chat endpoints)
-    {"role": "system", "content": "You are a helpful assistant"},
-    {"role": "user", "content": "Hello"}
-  ],
-  "request_data": {
-    "user_api_key_hash": "hash of the litellm virtual key used",
-    "user_api_key_alias": "alias of the litellm virtual key used",
-    "user_api_key_user_id": "user id associated with the litellm virtual key used",
-    "user_api_key_user_email": "user email associated with the litellm virtual key used",
-    "user_api_key_team_id": "team id associated with the litellm virtual key used",
-    "user_api_key_team_alias": "team alias associated with the litellm virtual key used",
-    "user_api_key_end_user_id": "end user id associated with the litellm virtual key used",
-    "user_api_key_org_id": "org id associated with the litellm virtual key used"
-  },
-  "request_headers": {  // optional: inbound request headers (allowlist). Allowed headers show their value; all others show "[present]" to indicate the header existed.
-    "User-Agent": "OpenAI/Python 2.17.0",
-    "Content-Type": "application/json",
-    "X-Request-Id": "[present]"
-  },
-  "litellm_version": "1.x.y",  // optional: LiteLLM library version running this proxy
-  "input_type": "request",  // "request" or "response"
-  "litellm_call_id": "unique_call_id",  // the call id of the individual LLM call
-  "litellm_trace_id": "trace_id",  // the trace id of the LLM call - useful if there are multiple LLM calls for the same conversation
-  "additional_provider_specific_params": {
-    // your custom params from config
-  }
-}
-```
-
-### Response Format
-
-```json
-{
-  "action": "BLOCKED" | "NONE" | "GUARDRAIL_INTERVENED",
-  "blocked_reason": "why content was blocked",  // required if action=BLOCKED
-  "texts": ["modified text"],  // optional array of modified text strings
-  "images": ["modified_base64_image"]  // optional array of modified images
-}
-```
-
-**Actions:**
-- `BLOCKED` - LiteLLM raises error and blocks request
-- `NONE` - Request proceeds unchanged  
-- `GUARDRAIL_INTERVENED` - Request proceeds with modified texts/images (provide `texts` and/or `images` fields)
-
-## Parameters
-
-### `tools` Parameter
-
-The `tools` parameter provides information about available function/tool definitions in the request.
-
-**Format:** OpenAI `ChatCompletionToolParam` format (see [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools))
-
-**Example:**
-```json
-{
-  "type": "function",
-  "function": {
-    "name": "get_weather",
-    "description": "Get the current weather in a location",
-    "parameters": {
-      "type": "object",
-      "properties": {
-        "location": {
-          "type": "string",
-          "description": "City and state, e.g. San Francisco, CA"
-        },
-        "unit": {
-          "type": "string",
-          "enum": ["celsius", "fahrenheit"]
-        }
-      },
-      "required": ["location"]
-    }
-  }
-}
-```
-
-**Availability:**
-- **Input only:** Tools are only passed for `input_type="request"` (pre-call guardrails). Output/response guardrails do not currently receive tool definitions.
-- **Supported endpoints:** The `tools` parameter is supported on: `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`. Other endpoints do not have tool support.
-
-**Use cases:**
-- Enforce tool permission policies (e.g., only allow certain users/teams to access specific tools)
-- Validate tool schemas before sending to LLM
-- Log tool usage for audit purposes
-- Block sensitive tools based on user context
-
-### `tool_calls` Parameter
-
-The `tool_calls` parameter contains actual function/tool invocations being made in the request or response.
-
-**Format:** OpenAI `ChatCompletionMessageToolCall` format (see [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/object#chat/object-tool_calls))
-
-**Example:**
-```json
-{
-  "id": "call_abc123",
-  "type": "function",
-  "function": {
-    "name": "get_weather",
-    "arguments": "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}"
-  }
-}
-```
-
-**Key Difference from `tools`:**
-- **`tools`** = Tool definitions/schemas (what tools are *available*)
-- **`tool_calls`** = Tool invocations/executions (what tools are *being called* with what arguments)
-
-**Availability:**
-- **Both input and output:** Tool calls can be present in both `input_type="request"` (assistant messages requesting tool calls) and `input_type="response"` (LLM responses with tool calls).
-- **Supported endpoints:** The `tool_calls` parameter is supported on: `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`.
-
-**Use cases:**
-- Validate tool call arguments before execution
-- Redact sensitive data from tool call arguments (e.g., PII)
-- Log tool invocations for audit/debugging
-- Block tool calls with dangerous parameters
-- Modify tool call arguments (e.g., enforce constraints, sanitize inputs)
-- Monitor tool usage patterns across users/teams
-
-### `structured_messages` Parameter
-
-The `structured_messages` parameter provides the full input in OpenAI chat completion spec format, useful for distinguishing between system and user messages.
-
-**Format:** Array of OpenAI chat completion messages (see [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages))
-
-**Example:**
-```json
-[
-  {"role": "system", "content": "You are a helpful assistant"},
-  {"role": "user", "content": "Hello"}
-]
-```
-
-**Availability:**
-- **Supported endpoints:** `/v1/chat/completions`, `/v1/messages`, `/v1/responses`
-- **Input only:** Only passed for `input_type="request"` (pre-call guardrails)
-
-**Use cases:**
-- Apply different policies for system vs user messages
-- Enforce role-based content restrictions
-- Log structured conversation context
-
-## LiteLLM Configuration
-
-Add to `config.yaml`:
-
-```yaml
-litellm_settings:
-  guardrails:
-    - guardrail_name: "my-guardrail"
-      litellm_params:
-        guardrail: generic_guardrail_api
-        mode: pre_call  # or post_call, during_call
-        api_base: https://your-guardrail-api.com
-        api_key: os.environ/YOUR_GUARDRAIL_API_KEY  # optional
-        unreachable_fallback: fail_closed  # default: fail_closed. Set to fail_open to proceed if the guardrail endpoint is unreachable (network errors, or HTTP 502/503/504 from an upstream proxy/LB).
-        additional_provider_specific_params:
-          # your custom parameters
-          threshold: 0.8
-          language: "en"
-```
-
-### Static and dynamic headers
-
-You can send two kinds of headers to your guardrail endpoint:
-
-- **Static headers** (`headers`): A key/value map sent with **every** request to your guardrail. Use this for fixed values (e.g. API keys, `X-Service-Name`). Configure in `litellm_params`:
-
-  ```yaml
-  litellm_params:
-    guardrail: generic_guardrail_api
-    api_base: https://your-guardrail-api.com
-    headers:
-      X-Service-Name: "my-app"
-      X-API-Key: "secret"
-  ```
-
-- **Dynamic headers** (`extra_headers`): A list of **header names** that are forwarded from the **client request** to your guardrail. Only headers in this list (plus a small default allowlist such as `x-litellm-*`) have their values sent; others are sent as `[present]`. Use this to pass through client-provided headers (e.g. `x-request-id`, `x-correlation-id`). Configure in `litellm_params`:
-
-  ```yaml
-  litellm_params:
-    guardrail: generic_guardrail_api
-    api_base: https://your-guardrail-api.com
-    extra_headers:
-      - x-request-id
-      - x-correlation-id
-      - x-custom-auth
-  ```
-
-This mirrors the [MCP static and extra headers](/docs/mcp#forwarding-custom-headers-to-mcp-servers) behavior.
-
-### Example: Pillar Security
-
-[Pillar Security](https://pillar.security) uses the Generic Guardrail API to provide comprehensive AI security scanning including prompt injection protection, PII/PCI detection, secret detection, and content moderation.
-
-```yaml
-guardrails:
-  - guardrail_name: "pillar-security"
-    litellm_params:
-      guardrail: generic_guardrail_api
-      mode: [pre_call, post_call]
-      api_base: https://api.pillar.security/api/v1/integrations/litellm
-      api_key: os.environ/PILLAR_API_KEY
-      default_on: true
-      additional_provider_specific_params:
-        plr_mask: true      # Enable automatic masking of sensitive data
-        plr_evidence: true  # Include detection evidence in response
-        plr_scanners: true  # Include scanner details in response
-```
-
-See the [Pillar Security documentation](../proxy/guardrails/pillar_security.md) for full configuration options.
-
-## Usage
-
-Users apply your guardrail by name:
-
-```python
-response = client.chat.completions.create(
-    model="gpt-4",
-    messages=[{"role": "user", "content": "hello"}],
-    guardrails=["my-guardrail"]
-)
-```
-
-Or with dynamic parameters:
-
-```python
-response = client.chat.completions.create(
-    model="gpt-4",
-    messages=[{"role": "user", "content": "hello"}],
-    guardrails=[{
-        "my-guardrail": {
-            "extra_body": {
-                "custom_threshold": 0.9
-            }
-        }
-    }]
-)
-```
-
-## Implementation Example
-
-See [mock_bedrock_guardrail_server.py](https://github.com/BerriAI/litellm/blob/main/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py) for a complete reference implementation.
-
-**Minimal FastAPI example:**
-
-```python
-from fastapi import FastAPI
-from pydantic import BaseModel
-from typing import List, Optional, Dict, Any
-
-app = FastAPI()
-
-class GuardrailRequest(BaseModel):
-    texts: List[str]
-    images: Optional[List[str]] = None
-    tools: Optional[List[Dict[str, Any]]] = None  # OpenAI ChatCompletionToolParam format (tool definitions)
-    tool_calls: Optional[List[Dict[str, Any]]] = None  # OpenAI ChatCompletionMessageToolCall format (tool invocations)
-    structured_messages: Optional[List[Dict[str, Any]]] = None  # OpenAI messages format (for chat endpoints)
-    request_data: Dict[str, Any]
-    input_type: str  # "request" or "response"
-    litellm_call_id: Optional[str] = None
-    litellm_trace_id: Optional[str] = None
-    additional_provider_specific_params: Dict[str, Any]
-
-class GuardrailResponse(BaseModel):
-    action: str  # BLOCKED, NONE, or GUARDRAIL_INTERVENED
-    blocked_reason: Optional[str] = None
-    texts: Optional[List[str]] = None
-    images: Optional[List[str]] = None
-
-@app.post("/beta/litellm_basic_guardrail_api")
-async def apply_guardrail(request: GuardrailRequest):
-    # Your guardrail logic here
-    
-    # Example: Check text content
-    for text in request.texts:
-        if "badword" in text.lower():
-            return GuardrailResponse(
-                action="BLOCKED",
-                blocked_reason="Content contains prohibited terms"
-            )
-    
-    # Example: Check tool definitions (if present in request)
-    if request.tools:
-        for tool in request.tools:
-            if tool.get("type") == "function":
-                function_name = tool.get("function", {}).get("name", "")
-                # Block sensitive tool definitions
-                if function_name in ["delete_data", "access_admin_panel"]:
-                    return GuardrailResponse(
-                        action="BLOCKED",
-                        blocked_reason=f"Tool '{function_name}' is not allowed"
-                    )
-    
-    # Example: Check tool calls (if present in request or response)
-    if request.tool_calls:
-        for tool_call in request.tool_calls:
-            if tool_call.get("type") == "function":
-                function_name = tool_call.get("function", {}).get("name", "")
-                arguments_str = tool_call.get("function", {}).get("arguments", "{}")
-                
-                # Parse arguments and validate
-                import json
-                try:
-                    arguments = json.loads(arguments_str)
-                    # Block dangerous arguments
-                    if "file_path" in arguments and ".." in str(arguments["file_path"]):
-                        return GuardrailResponse(
-                            action="BLOCKED",
-                            blocked_reason="Tool call contains path traversal attempt"
-                        )
-                except json.JSONDecodeError:
-                    pass
-    
-    # Example: Check structured messages (if present in request)
-    if request.structured_messages:
-        for message in request.structured_messages:
-            if message.get("role") == "system":
-                # Apply stricter policies to system messages
-                if "admin" in message.get("content", "").lower():
-                    return GuardrailResponse(
-                        action="BLOCKED",
-                        blocked_reason="System message contains restricted terms"
-                    )
-    
-    return GuardrailResponse(action="NONE")
-```
-
-## When to Use This
-
-✅ **Use Generic Guardrail API when:**
-- You want instant integration without waiting for PRs
-- You maintain your own guardrail service
-- You need full control over updates and features
-- You want to support all LiteLLM endpoints automatically
-
-❌ **Make a PR when:**
-- You want deeper integration with LiteLLM internals
-- Your guardrail requires complex LiteLLM-specific logic
-- You want to be featured as a built-in provider
-
-## Questions?
-
-This is a **beta API**. We're actively improving it based on feedback. Open an issue or PR if you need additional capabilities.
-

docs/my-website/docs/adding_provider/generic_prompt_management_api.md

@@ -1,576 +0,0 @@
-# [BETA] Generic Prompt Management API - Integrate Without a PR
-
-## The Problem
-
-As a prompt management provider, integrating with LiteLLM traditionally requires:
-- Making a PR to the LiteLLM repository
-- Waiting for review and merge
-- Maintaining provider-specific code in LiteLLM's codebase
-- Updating the integration for changes to your API
-
-## The Solution
-
-The **Generic Prompt Management API** lets you integrate with LiteLLM **instantly** by implementing a simple API endpoint. No PR required.
-
-### Key Benefits
-
-1. **No PR Needed** - Deploy and integrate immediately
-3. **Simple Contract** - One GET endpoint, standard JSON response
-4. **Variable Substitution** - Support for prompt variables with `{variable}` syntax
-5. **Custom Parameters** - Pass provider-specific query params via config
-6. **Full Control** - You own and maintain your prompt management API
-7. **Model & Parameters Override** - Optionally override model and parameters from your prompts
-
-## Get Started in 3 Steps
-
-### Step 1: Configure LiteLLM
-
-Add to your `config.yaml`:
-
-```yaml
-prompts:
-  - prompt_id: "simple_prompt"
-    litellm_params:
-      prompt_integration: "generic_prompt_management"
-      api_base: http://localhost:8080
-      api_key: os.environ/YOUR_API_KEY
-```
-
-### Step 2: Implement Your API Endpoint
-
-```python
-from fastapi import FastAPI
-from pydantic import BaseModel
-
-app = FastAPI()
-
-@app.get("/beta/litellm_prompt_management")
-async def get_prompt(prompt_id: str):
-    return {
-        "prompt_id": prompt_id,
-        "prompt_template": [
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": "Help me with {task}"}
-        ],
-        "prompt_template_model": "gpt-4",
-        "prompt_template_optional_params": {"temperature": 0.7}
-    }
-```
-
-### Step 3: Use in Your App
-
-```python
-from litellm import completion
-
-response = completion(
-    model="gpt-4",
-    prompt_id="simple_prompt",
-    prompt_variables={"task": "data analysis"},
-    messages=[{"role": "user", "content": "I have sales data"}]
-)
-```
-
-That's it! LiteLLM fetches your prompt, applies variables, and makes the request
-
-## API Contract
-
-### Endpoint
-
-Implement `GET /beta/litellm_prompt_management`
-
-### Request Format
-
-Your endpoint will receive a GET request with query parameters:
-
-```
-GET /beta/litellm_prompt_management?prompt_id={prompt_id}&{custom_params}
-```
-
-**Query Parameters:**
-- `prompt_id` (required): The ID of the prompt to fetch
-- Custom parameters: Any additional parameters you configured in `provider_specific_query_params`
-
-**Example:**
-```
-GET /beta/litellm_prompt_management?prompt_id=hello-world-prompt-2bac&project_name=litellm&slug=hello-world-prompt-2bac
-```
-
-### Response Format
-
-```json
-{
-  "prompt_id": "hello-world-prompt-2bac",
-  "prompt_template": [
-    {
-      "role": "system",
-      "content": "You are a helpful assistant specialized in {domain}."
-    },
-    {
-      "role": "user",
-      "content": "Help me with {task}"
-    }
-  ],
-  "prompt_template_model": "gpt-4",
-  "prompt_template_optional_params": {
-    "temperature": 0.7,
-    "max_tokens": 500,
-    "top_p": 0.9
-  }
-}
-```
-
-**Response Fields:**
-- `prompt_id` (string, required): The ID of the prompt
-- `prompt_template` (array, required): Array of OpenAI-format messages with optional `{variable}` placeholders
-- `prompt_template_model` (string, optional): Model to use for this prompt (overrides client model unless `ignore_prompt_manager_model: true`)
-- `prompt_template_optional_params` (object, optional): Additional parameters like temperature, max_tokens, etc. (merged with client params unless `ignore_prompt_manager_optional_params: true`)
-
-## LiteLLM Configuration
-
-Add to `config.yaml`:
-
-```yaml
-model_list:
-  - model_name: gpt-3.5-turbo
-    litellm_params:
-      model: openai/gpt-3.5-turbo
-      api_key: os.environ/OPENAI_API_KEY
-
-prompts:
-  - prompt_id: "simple_prompt"
-    litellm_params:
-      prompt_integration: "generic_prompt_management"
-      provider_specific_query_params:
-        project_name: litellm
-        slug: hello-world-prompt-2bac
-      api_base: http://localhost:8080
-      api_key: os.environ/YOUR_PROMPT_API_KEY  # optional
-      ignore_prompt_manager_model: true  # optional, keep client's model
-      ignore_prompt_manager_optional_params: true  # optional, don't merge prompt manager's params (e.g. temperature, max_tokens, etc.)
-```
-
-### Configuration Parameters
-
-- `prompt_integration`: Must be `"generic_prompt_management"`
-- `provider_specific_query_params`: Custom query parameters sent to your API (optional)
-- `api_base`: Base URL of your prompt management API
-- `api_key`: Optional API key for authentication (sent as `Bearer` token)
-- `ignore_prompt_manager_model`: If `true`, use the model specified by client instead of prompt's model (default: `false`)
-- `ignore_prompt_manager_optional_params`: If `true`, don't merge prompt's optional params with client params (default: `false`)
-
-## Usage
-
-### Using with LiteLLM SDK
-
-**Basic usage with prompt ID:**
-
-```python
-from litellm import completion
-
-response = completion(
-    model="gpt-4",
-    prompt_id="simple_prompt",
-    messages=[{"role": "user", "content": "Additional message"}]
-)
-```
-
-**With prompt variables:**
-
-```python
-response = completion(
-    model="gpt-4",
-    prompt_id="simple_prompt",
-    prompt_variables={
-        "domain": "data science",
-        "task": "analyzing customer churn"
-    },
-    messages=[{"role": "user", "content": "Please provide a detailed analysis"}]
-)
-```
-
-The prompt template will have `{domain}` replaced with "data science" and `{task}` replaced with "analyzing customer churn".
-
-### Using with LiteLLM Proxy
-
-**1. Start the proxy with your config:**
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-**2. Make requests with prompt_id:**
-
-```bash
-curl http://0.0.0.0:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gpt-4",
-    "prompt_id": "simple_prompt",
-    "prompt_variables": {
-      "domain": "healthcare",
-      "task": "patient risk assessment"
-    },
-    "messages": [
-      {"role": "user", "content": "Analyze the following data..."}
-    ]
-  }'
-```
-
-**3. Using with OpenAI SDK:**
-
-```python
-from openai import OpenAI
-
-client = OpenAI(
-    base_url="http://0.0.0.0:4000",
-    api_key="sk-1234"
-)
-
-response = client.chat.completions.create(
-    model="gpt-4",
-    messages=[
-        {"role": "user", "content": "Analyze the data"}
-    ],
-    extra_body={
-        "prompt_id": "simple_prompt",
-        "prompt_variables": {
-            "domain": "finance",
-            "task": "fraud detection"
-        }
-    }
-)
-```
-
-## Implementation Example
-
-See [mock_prompt_management_server.py](https://github.com/BerriAI/litellm/blob/main/cookbook/mock_prompt_management_server/mock_prompt_management_server.py) for a complete reference implementation with multiple example prompts, authentication, and convenience endpoints.
-
-**Minimal FastAPI example:**
-
-```python
-from fastapi import FastAPI, HTTPException, Header
-from typing import Optional, Dict, Any, List
-from pydantic import BaseModel
-
-app = FastAPI()
-
-# In-memory prompt storage (replace with your database)
-PROMPTS = {
-    "hello-world-prompt": {
-        "prompt_id": "hello-world-prompt",
-        "prompt_template": [
-            {
-                "role": "system",
-                "content": "You are a helpful assistant specialized in {domain}."
-            },
-            {
-                "role": "user", 
-                "content": "Help me with: {task}"
-            }
-        ],
-        "prompt_template_model": "gpt-4",
-        "prompt_template_optional_params": {
-            "temperature": 0.7,
-            "max_tokens": 500
-        }
-    },
-    "code-review-prompt": {
-        "prompt_id": "code-review-prompt",
-        "prompt_template": [
-            {
-                "role": "system",
-                "content": "You are an expert code reviewer. Review code for {language}."
-            },
-            {
-                "role": "user",
-                "content": "Review the following code:\n\n{code}"
-            }
-        ],
-        "prompt_template_model": "gpt-4-turbo",
-        "prompt_template_optional_params": {
-            "temperature": 0.3,
-            "max_tokens": 1000
-        }
-    }
-}
-
-class PromptResponse(BaseModel):
-    prompt_id: str
-    prompt_template: List[Dict[str, str]]
-    prompt_template_model: Optional[str] = None
-    prompt_template_optional_params: Optional[Dict[str, Any]] = None
-
-@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
-async def get_prompt(
-    prompt_id: str,
-    authorization: Optional[str] = Header(None),
-    project_name: Optional[str] = None,
-    slug: Optional[str] = None,
-):
-    """
-    Get a prompt by ID with optional filtering by project_name and slug.
-    
-    Args:
-        prompt_id: The ID of the prompt to fetch
-        authorization: Optional Bearer token for authentication
-        project_name: Optional project name filter
-        slug: Optional slug filter
-    """
-    
-    # Optional: Validate authorization
-    if authorization:
-        token = authorization.replace("Bearer ", "")
-        # Validate your token here
-        if not is_valid_token(token):
-            raise HTTPException(status_code=401, detail="Invalid API key")
-    
-    # Optional: Apply additional filtering based on custom params
-    if project_name or slug:
-        # You can use these parameters to filter or validate access
-        # For example, check if the user has access to this project
-        pass
-    
-    # Fetch the prompt from your storage
-    if prompt_id not in PROMPTS:
-        raise HTTPException(
-            status_code=404,
-            detail=f"Prompt '{prompt_id}' not found"
-        )
-    
-    prompt_data = PROMPTS[prompt_id]
-    
-    return PromptResponse(**prompt_data)
-
-def is_valid_token(token: str) -> bool:
-    """Validate API token - implement your logic here"""
-    # Example: Check against your database or secret store
-    valid_tokens = ["your-secret-token", "another-valid-token"]
-    return token in valid_tokens
-
-# Optional: Health check endpoint
-@app.get("/health")
-async def health_check():
-    return {"status": "healthy"}
-
-# Optional: List all prompts endpoint
-@app.get("/prompts")
-async def list_prompts(authorization: Optional[str] = Header(None)):
-    """List all available prompts"""
-    if authorization:
-        token = authorization.replace("Bearer ", "")
-        if not is_valid_token(token):
-            raise HTTPException(status_code=401, detail="Invalid API key")
-    
-    return {
-        "prompts": [
-            {"prompt_id": pid, "model": p.get("prompt_template_model")}
-            for pid, p in PROMPTS.items()
-        ]
-    }
-
-if __name__ == "__main__":
-    import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=8080)
-```
-
-### Running the Example Server
-
-1. Install dependencies:
-```bash
-uv add fastapi uvicorn
-```
-
-2. Save the code above to `prompt_server.py`
-
-3. Run the server:
-```bash
-python prompt_server.py
-```
-
-4. Test the endpoint:
-```bash
-curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt&project_name=litellm&slug=hello-world-prompt-2bac"
-```
-
-Expected response:
-```json
-{
-  "prompt_id": "hello-world-prompt",
-  "prompt_template": [
-    {
-      "role": "system",
-      "content": "You are a helpful assistant specialized in {domain}."
-    },
-    {
-      "role": "user",
-      "content": "Help me with: {task}"
-    }
-  ],
-  "prompt_template_model": "gpt-4",
-  "prompt_template_optional_params": {
-    "temperature": 0.7,
-    "max_tokens": 500
-  }
-}
-```
-
-## Advanced Features
-
-### Variable Substitution
-
-LiteLLM automatically substitutes variables in your prompt templates using the `{variable}` syntax. Both `{variable}` and `{{variable}}` formats are supported.
-
-**Example prompt template:**
-```json
-{
-  "prompt_template": [
-    {
-      "role": "system",
-      "content": "You are an expert in {domain} with {years} years of experience."
-    }
-  ]
-}
-```
-
-**Client request:**
-```python
-completion(
-    model="gpt-4",
-    prompt_id="expert_prompt",
-    prompt_variables={
-        "domain": "machine learning",
-        "years": "10"
-    }
-)
-```
-
-**Result:**
-```
-"You are an expert in machine learning with 10 years of experience."
-```
-
-### Caching
-
-LiteLLM automatically caches fetched prompts in memory. The cache key includes:
-- `prompt_id`
-- `prompt_label` (if provided)
-- `prompt_version` (if provided)
-
-This means your API endpoint is only called once per unique prompt configuration.
-
-### Model Override Behavior
-
-**Default behavior (without `ignore_prompt_manager_model`):**
-```yaml
-prompts:
-  - prompt_id: "my_prompt"
-    litellm_params:
-      prompt_integration: "generic_prompt_management"
-      api_base: http://localhost:8080
-```
-
-If your API returns `"prompt_template_model": "gpt-4"`, LiteLLM will use `gpt-4` regardless of what the client specified.
-
-**With `ignore_prompt_manager_model: true`:**
-```yaml
-prompts:
-  - prompt_id: "my_prompt"
-    litellm_params:
-      prompt_integration: "generic_prompt_management"
-      api_base: http://localhost:8080
-      ignore_prompt_manager_model: true
-```
-
-LiteLLM will use the model specified by the client, ignoring the prompt's model.
-
-### Parameter Merging Behavior
-
-**Default behavior (without `ignore_prompt_manager_optional_params`):**
-
-Client params are merged with prompt params, with prompt params taking precedence:
-```python
-# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
-# Client sends: {"temperature": 0.9, "top_p": 0.95}
-# Final params: {"temperature": 0.7, "max_tokens": 500, "top_p": 0.95}
-```
-
-**With `ignore_prompt_manager_optional_params: true`:**
-
-Only client params are used:
-```python
-# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
-# Client sends: {"temperature": 0.9, "top_p": 0.95}
-# Final params: {"temperature": 0.9, "top_p": 0.95}
-```
-
-## Security Considerations
-
-1. **Authentication**: Use the `api_key` parameter to secure your prompt management API
-2. **Authorization**: Implement team/user-based access control using the custom query parameters
-3. **Rate Limiting**: Add rate limiting to prevent abuse of your API
-4. **Input Validation**: Validate all query parameters before processing
-5. **HTTPS**: Always use HTTPS in production for encrypted communication
-6. **Secrets**: Store API keys in environment variables, not in config files
-
-## Use Cases
-
-✅ **Use Generic Prompt Management API when:**
-- You want instant integration without waiting for PRs
-- You maintain your own prompt management service
-- You need full control over prompt versioning and updates
-- You want to build custom prompt management features
-- You need to integrate with your internal systems
-
-✅ **Common scenarios:**
-- Internal prompt management system for your organization
-- Multi-tenant prompt management with team-based access control
-- A/B testing different prompt versions
-- Prompt experimentation and analytics
-- Integration with existing prompt engineering workflows
-
-## When to Use This
-
-✅ **Use Generic Prompt Management API when:**
-- You want instant integration without waiting for PRs
-- You maintain your own prompt management service
-- You need full control over updates and features
-- You want custom prompt storage and versioning logic
-
-❌ **Make a PR when:**
-- You want deeper integration with LiteLLM internals
-- Your integration requires complex LiteLLM-specific logic
-- You want to be featured as a built-in provider
-- You're building a reusable integration for the community
-
-## Troubleshooting
-
-### Prompt not found
-- Verify the `prompt_id` matches exactly (case-sensitive)
-- Check that your API endpoint is accessible from LiteLLM
-- Verify authentication if using `api_key`
-
-### Variables not substituted
-- Ensure variables use `{variable}` or `{{variable}}` syntax
-- Check that variable names in `prompt_variables` match template exactly
-- Variables are case-sensitive
-
-### Model not being overridden
-- Check if `ignore_prompt_manager_model: true` is set in config
-- Verify your API is returning `prompt_template_model` in the response
-
-### Parameters not being applied
-- Check if `ignore_prompt_manager_optional_params: true` is set
-- Verify your API is returning `prompt_template_optional_params`
-- Ensure parameter names match OpenAI's parameter names
-
-## Questions?
-
-This is a **beta API**. We're actively improving it based on feedback. Open an issue or PR if you need additional capabilities.
-
-## Related Documentation
-
-- [Prompt Management Overview](../proxy/prompt_management.md)
-- [Generic Guardrail API](./generic_guardrail_api.md)
-- [LiteLLM Proxy Setup](../proxy/quick_start.md)
-

docs/my-website/docs/adding_provider/new_rerank_provider.md

@@ -1,84 +0,0 @@
-# Add Rerank Provider
-
-LiteLLM **follows the Cohere Rerank API format** for all rerank providers. Here's how to add a new rerank provider:
-
-## 1. Create a transformation.py file
-
-Create a config class named `<Provider><Endpoint>Config` that inherits from [`BaseRerankConfig`](https://github.com/BerriAI/litellm/blob/main/litellm/llms/base_llm/rerank/transformation.py):
-
-```python
-from litellm.types.rerank import OptionalRerankParams, RerankRequest, RerankResponse
-class YourProviderRerankConfig(BaseRerankConfig):
-    def get_supported_cohere_rerank_params(self, model: str) -> list:
-        return [
-            "query",
-            "documents",
-            "top_n",
-            # ... other supported params
-        ]
-
-    def transform_rerank_request(self, model: str, optional_rerank_params: Dict, headers: dict) -> dict:
-        # Transform request to RerankRequest spec
-        return rerank_request.model_dump(exclude_none=True)
-
-    def transform_rerank_response(self, model: str, raw_response: httpx.Response, ...) -> RerankResponse:
-        # Transform provider response to RerankResponse
-        return RerankResponse(**raw_response_json)
-```
-
-
-## 2. Register Your Provider
-Add your provider to `litellm.utils.get_provider_rerank_config()`:
-
-```python
-elif litellm.LlmProviders.YOUR_PROVIDER == provider:
-    return litellm.YourProviderRerankConfig()
-```
-
-
-## 3. Add Provider to `rerank_api/main.py`
-
-Add a code block to handle when your provider is called. Your provider should use the `base_llm_http_handler.rerank` method
-
-
-```python
-elif _custom_llm_provider == "your_provider":
-    ...
-    response = base_llm_http_handler.rerank(
-        model=model,
-        custom_llm_provider=_custom_llm_provider,
-        optional_rerank_params=optional_rerank_params,
-        logging_obj=litellm_logging_obj,
-        timeout=optional_params.timeout,
-        api_key=dynamic_api_key or optional_params.api_key,
-        api_base=api_base,
-        _is_async=_is_async,
-        headers=headers or litellm.headers or {},
-        client=client,
-        mod el_response=model_response,
-    )
-    ...
-```
-
-## 4. Add Tests
-
-Add a test file to [`tests/llm_translation`](https://github.com/BerriAI/litellm/tree/main/tests/llm_translation)
-
-```python
-def test_basic_rerank_cohere():
-    response = litellm.rerank(
-        model="cohere/rerank-english-v3.0",
-        query="hello",
-        documents=["hello", "world"],
-        top_n=3,
-    )
-
-    print("re rank response: ", response)
-
-    assert response.id is not None
-    assert response.results is not None
-```
-
-
-## Reference PRs
-- [Add Infinity Rerank](https://github.com/BerriAI/litellm/pull/7321)

docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md

@@ -1,133 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Adding a New Guardrail Integration
-
-You're going to create a class that checks text before it goes to the LLM or after it comes back. If it violates your rules, you block it.
-
-## How It Works
-
-Request with guardrail:
-
-```bash
-curl --location 'http://localhost:4000/chat/completions' \
---header 'Authorization: Bearer sk-1234' \
---header 'Content-Type: application/json' \
---data '{
-    "model": "gpt-4",
-    "messages": [{"role": "user", "content": "How do I hack a system?"}],
-    "guardrails": ["my-guardrail"]
-}'
-```
-
-Your guardrail checks input, then output. If something's wrong, raise an exception.
-
-## Build Your Guardrail
-
-### Create Your Directory
-
-```bash
-mkdir -p litellm/proxy/guardrails/guardrail_hooks/my_guardrail
-cd litellm/proxy/guardrails/guardrail_hooks/my_guardrail
-```
-
-Two files: `my_guardrail.py` (main class) and `__init__.py` (initialization).
-
-### Write the Main Class
-
-`my_guardrail.py`:
-
-Follow from [Custom Guardrail](../proxy/guardrails/custom_guardrail#custom-guardrail) tutorial.
-
-### Create the Init File
-
-`__init__.py`:
-
-```python
-from typing import TYPE_CHECKING
-
-from litellm.types.guardrails import SupportedGuardrailIntegrations
-
-from .my_guardrail import MyGuardrail
-
-if TYPE_CHECKING:
-    from litellm.types.guardrails import Guardrail, LitellmParams
-
-
-def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"):
-    import litellm
-    
-    _my_guardrail_callback = MyGuardrail(
-        api_base=litellm_params.api_base,
-        api_key=litellm_params.api_key,
-        guardrail_name=guardrail.get("guardrail_name", ""),
-        event_hook=litellm_params.mode,
-        default_on=litellm_params.default_on,
-    )
-    
-    litellm.logging_callback_manager.add_litellm_callback(_my_guardrail_callback)
-    return _my_guardrail_callback
-
-
-guardrail_initializer_registry = {
-    SupportedGuardrailIntegrations.MY_GUARDRAIL.value: initialize_guardrail,
-}
-
-guardrail_class_registry = {
-    SupportedGuardrailIntegrations.MY_GUARDRAIL.value: MyGuardrail,
-}
-```
-
-### Register Your Guardrail Type
-
-Add to `litellm/types/guardrails.py`:
-
-```python
-class SupportedGuardrailIntegrations(str, Enum):
-    LAKERA = "lakera_prompt_injection"
-    APORIA = "aporia"
-    BEDROCK = "bedrock_guardrails"
-    PRESIDIO = "presidio"
-    ZSCALER_AI_GUARD = "zscaler_ai_guard"
-    MY_GUARDRAIL = "my_guardrail"
-```
-
-## Usage
-
-### Config File
-
-```yaml
-model_list:
-  - model_name: gpt-4
-    litellm_params:
-      model: gpt-4
-    api_key: os.environ/OPENAI_API_KEY
-
-guardrails:
-    - guardrail_name: my_guardrail
-        litellm_params:
-        guardrail: my_guardrail
-        mode: during_call
-        api_key: os.environ/MY_GUARDRAIL_API_KEY
-        api_base: https://api.myguardrail.com
-```
-
-### Per-Request
-
-```bash
-curl --location 'http://localhost:4000/chat/completions' \
---header 'Authorization: Bearer sk-1234' \
---header 'Content-Type: application/json' \
---data '{
-    "model": "gpt-4",
-    "messages": [{"role": "user", "content": "Test message"}],
-    "guardrails": ["my_guardrail"]
-}'
-```
-
-## Testing
-
-Add unit tests inside `test_litellm/` folder.
-
-
-

docs/my-website/docs/aiohttp_benchmarks.md

@@ -1,38 +0,0 @@
-# LiteLLM v1.71.1 Benchmarks
-
-## Overview
-
-This document presents performance benchmarks comparing LiteLLM's v1.71.1 to prior litellm versions.
-
-**Related PR:** [#11097](https://github.com/BerriAI/litellm/pull/11097)
-
-## Testing Methodology
-
-The load testing was conducted using the following parameters:
-- **Request Rate:** 200 RPS (Requests Per Second)
-- **User Ramp Up:** 200 concurrent users
-- **Transport Comparison:** httpx (existing) vs aiohttp (new implementation)
-- **Number of pods/instance of litellm:** 1
-- **Machine Specs:** 2 vCPUs, 4GB RAM
-- **LiteLLM Settings:**
-    - Tested against a [fake openai endpoint](https://exampleopenaiendpoint-production.up.railway.app/)
-    - Set `USE_AIOHTTP_TRANSPORT="True"` in the environment variables. This feature flag enables the aiohttp transport.
-
-
-## Benchmark Results
-
-| Metric | httpx (Existing) | aiohttp (LiteLLM v1.71.1) | Improvement | Calculation |
-|--------|------------------|-------------------|-------------|-------------|
-| **RPS** | 50.2 | 224 | **+346%** ✅ | (224 - 50.2) / 50.2 × 100 = 346% |
-| **Median Latency** | 2,500ms | 74ms | **-97%** ✅ | (74 - 2500) / 2500 × 100 = -97% |
-| **95th Percentile** | 5,600ms | 250ms | **-96%** ✅ | (250 - 5600) / 5600 × 100 = -96% |
-| **99th Percentile** | 6,200ms | 330ms | **-95%** ✅ | (330 - 6200) / 6200 × 100 = -95% |
-
-## Key Improvements
-
-- **4.5x increase** in requests per second (from 50.2 to 224 RPS)
-- **97% reduction** in median response time (from 2.5 seconds to 74ms)
-- **96% reduction** in 95th percentile latency (from 5.6 seconds to 250ms)
-- **95% reduction** in 99th percentile latency (from 6.2 seconds to 330ms)
-
-

docs/my-website/docs/anthropic_count_tokens.md

@@ -1,233 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# /v1/messages/count_tokens
-
-## Overview
-
-Anthropic-compatible token counting endpoint. Count tokens for messages before sending them to the model.
-
-| Feature | Supported | Notes |
-|---------|-----------|-------|
-| Cost Tracking | ❌ | Token counting only, no cost incurred |
-| Logging | ✅ | Works across all integrations |
-| End-user Tracking | ✅ | |
-| Supported Providers | Anthropic, Vertex AI (Claude), Bedrock (Claude), Gemini, Vertex AI | Auto-routes to provider-specific token counting APIs |
-
-## Quick Start
-
-### 1. Start LiteLLM Proxy
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING on http://0.0.0.0:4000
-```
-
-### 2. Count Tokens
-
-<Tabs>
-<TabItem value="curl" label="curl">
-
-```bash
-curl -X POST "http://localhost:4000/v1/messages/count_tokens" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "claude-3-5-sonnet-20241022",
-    "messages": [
-      {"role": "user", "content": "Hello, how are you?"}
-    ]
-  }'
-```
-
-</TabItem>
-<TabItem value="python" label="Python (httpx)">
-
-```python
-import httpx
-
-response = httpx.post(
-    "http://localhost:4000/v1/messages/count_tokens",
-    headers={
-        "Content-Type": "application/json",
-        "Authorization": "Bearer sk-1234"
-    },
-    json={
-        "model": "claude-3-5-sonnet-20241022",
-        "messages": [
-            {"role": "user", "content": "Hello, how are you?"}
-        ]
-    }
-)
-
-print(response.json())
-# {"input_tokens": 14}
-```
-
-</TabItem>
-</Tabs>
-
-**Expected Response:**
-
-```json
-{
-  "input_tokens": 14
-}
-```
-
-## LiteLLM Proxy Configuration
-
-Add models to your `config.yaml`:
-
-```yaml
-model_list:
-  - model_name: claude-3-5-sonnet
-    litellm_params:
-      model: anthropic/claude-3-5-sonnet-20241022
-      api_key: os.environ/ANTHROPIC_API_KEY
-
-  - model_name: claude-vertex
-    litellm_params:
-      model: vertex_ai/claude-3-5-sonnet-v2@20241022
-      vertex_project: my-project
-      vertex_location: us-east5
-      vertex_count_tokens_location: us-east5 # Optional: Override location for token counting (count_tokens not available on global location)
-
-  - model_name: claude-bedrock
-    litellm_params:
-      model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
-      aws_region_name: us-west-2
-```
-
-## Request Parameters
-
-| Parameter | Type | Required | Description |
-|-----------|------|----------|-------------|
-| `model` | string | ✅ | The model to use for token counting |
-| `messages` | array | ✅ | Array of messages in Anthropic format |
-
-### Messages Format
-
-```json
-{
-  "messages": [
-    {"role": "user", "content": "Hello!"},
-    {"role": "assistant", "content": "Hi there!"},
-    {"role": "user", "content": "How are you?"}
-  ]
-}
-```
-
-## Response Format
-
-```json
-{
-  "input_tokens": <number>
-}
-```
-
-| Field | Type | Description |
-|-------|------|-------------|
-| `input_tokens` | integer | Number of tokens in the input messages |
-
-## Supported Providers
-
-The `/v1/messages/count_tokens` endpoint automatically routes to the appropriate provider-specific token counting API:
-
-| Provider | Token Counting Method |
-|----------|----------------------|
-| Anthropic | [Anthropic Token Counting API](https://docs.anthropic.com/en/docs/build-with-claude/token-counting) |
-| OpenAI | [OpenAI Responses API `/input_tokens`](https://platform.openai.com/docs/api-reference/responses/input-tokens) — see [Token Counting](./count_tokens.md) |
-| Vertex AI (Claude) | Vertex AI Partner Models Token Counter |
-| Bedrock (Claude) | AWS Bedrock CountTokens API |
-| Gemini | Google AI Studio countTokens API |
-| Vertex AI (Gemini) | Vertex AI countTokens API |
-
-## Examples
-
-### Count Tokens with System Message
-
-```bash
-curl -X POST "http://localhost:4000/v1/messages/count_tokens" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "claude-3-5-sonnet-20241022",
-    "messages": [
-      {"role": "user", "content": "You are a helpful assistant. Please help me write a haiku about programming."}
-    ]
-  }'
-```
-
-### Count Tokens for Multi-turn Conversation
-
-```bash
-curl -X POST "http://localhost:4000/v1/messages/count_tokens" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "claude-3-5-sonnet-20241022",
-    "messages": [
-      {"role": "user", "content": "What is the capital of France?"},
-      {"role": "assistant", "content": "The capital of France is Paris."},
-      {"role": "user", "content": "What is its population?"}
-    ]
-  }'
-```
-
-### Using with Vertex AI Claude
-
-```bash
-curl -X POST "http://localhost:4000/v1/messages/count_tokens" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "claude-vertex",
-    "messages": [
-      {"role": "user", "content": "Hello, world!"}
-    ]
-  }'
-```
-
-### Using with Bedrock Claude
-
-```bash
-curl -X POST "http://localhost:4000/v1/messages/count_tokens" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "claude-bedrock",
-    "messages": [
-      {"role": "user", "content": "Hello, world!"}
-    ]
-  }'
-```
-
-## Comparison with Anthropic Passthrough
-
-LiteLLM provides two ways to count tokens:
-
-| Endpoint | Description | Use Case |
-|----------|-------------|----------|
-| `/v1/messages/count_tokens` | LiteLLM's Anthropic-compatible endpoint | Works with all supported providers (Anthropic, Vertex AI, Bedrock, etc.) |
-| `/anthropic/v1/messages/count_tokens` | [Pass-through to Anthropic API](./pass_through/anthropic_completion.md#example-2-token-counting-api) | Direct Anthropic API access with native headers |
-
-### Pass-through Example
-
-For direct Anthropic API access with full native headers:
-
-```bash
-curl --request POST \
-    --url http://0.0.0.0:4000/anthropic/v1/messages/count_tokens \
-    --header "x-api-key: $LITELLM_API_KEY" \
-    --header "anthropic-version: 2023-06-01" \
-    --header "anthropic-beta: token-counting-2024-11-01" \
-    --header "content-type: application/json" \
-    --data '{
-        "model": "claude-3-5-sonnet-20241022",
-        "messages": [
-            {"role": "user", "content": "Hello, world"}
-        ]
-    }'
-```

docs/my-website/docs/anthropic_unified/index.md

@@ -1,620 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# /v1/messages
-
-Use LiteLLM to call all your LLM APIs in the Anthropic `v1/messages` format. 
-
-
-## Overview 
-
-| Feature | Supported | Notes | 
-|-------|-------|-------|
-| Cost Tracking | ✅ | Works with all supported models |
-| Logging | ✅ | Works across all integrations |
-| End-user Tracking | ✅ | |
-| Streaming | ✅ | |
-| Fallbacks | ✅ | Works between supported models |
-| Loadbalancing | ✅ | Works between supported models |
-| Guardrails | ✅ | Applies to input and output text (non-streaming only) |
-| Supported Providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai`, etc. |
-
-## Usage 
----
-
-### LiteLLM Python SDK 
-
-<Tabs>
-<TabItem value="anthropic" label="Anthropic">
-
-#### Non-streaming example
-```python showLineNumbers title="Anthropic Example using LiteLLM Python SDK"
-import litellm
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    api_key=api_key,
-    model="anthropic/claude-3-haiku-20240307",
-    max_tokens=100,
-)
-```
-
-#### Streaming example
-```python showLineNumbers title="Anthropic Streaming Example using LiteLLM Python SDK"
-import litellm
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    api_key=api_key,
-    model="anthropic/claude-3-haiku-20240307",
-    max_tokens=100,
-    stream=True,
-)
-async for chunk in response:
-    print(chunk)
-```
-
-</TabItem>
-
-<TabItem value="openai" label="OpenAI">
-
-#### Non-streaming example
-```python showLineNumbers title="OpenAI Example using LiteLLM Python SDK"
-import litellm
-import os
-
-# Set API key
-os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
-
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="openai/gpt-4",
-    max_tokens=100,
-)
-```
-
-#### Streaming example
-```python showLineNumbers title="OpenAI Streaming Example using LiteLLM Python SDK"
-import litellm
-import os
-
-# Set API key
-os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
-
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="openai/gpt-4",
-    max_tokens=100,
-    stream=True,
-)
-async for chunk in response:
-    print(chunk)
-```
-
-</TabItem>
-
-<TabItem value="gemini" label="Google AI Studio">
-
-#### Non-streaming example
-```python showLineNumbers title="Google Gemini Example using LiteLLM Python SDK"
-import litellm
-import os
-
-# Set API key
-os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
-
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="gemini/gemini-2.0-flash-exp",
-    max_tokens=100,
-)
-```
-
-#### Streaming example
-```python showLineNumbers title="Google Gemini Streaming Example using LiteLLM Python SDK"
-import litellm
-import os
-
-# Set API key
-os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
-
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="gemini/gemini-2.0-flash-exp",
-    max_tokens=100,
-    stream=True,
-)
-async for chunk in response:
-    print(chunk)
-```
-
-</TabItem>
-
-<TabItem value="vertex" label="Vertex AI">
-
-#### Non-streaming example
-```python showLineNumbers title="Vertex AI Example using LiteLLM Python SDK"
-import litellm
-import os
-
-# Set credentials - Vertex AI uses application default credentials
-# Run 'gcloud auth application-default login' to authenticate
-os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
-os.environ["VERTEXAI_LOCATION"] = "us-central1"
-
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="vertex_ai/gemini-2.0-flash-exp",
-    max_tokens=100,
-)
-```
-
-#### Streaming example
-```python showLineNumbers title="Vertex AI Streaming Example using LiteLLM Python SDK"
-import litellm
-import os
-
-# Set credentials - Vertex AI uses application default credentials
-# Run 'gcloud auth application-default login' to authenticate
-os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
-os.environ["VERTEXAI_LOCATION"] = "us-central1"
-
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="vertex_ai/gemini-2.0-flash-exp",
-    max_tokens=100,
-    stream=True,
-)
-async for chunk in response:
-    print(chunk)
-```
-
-</TabItem>
-
-<TabItem value="bedrock" label="AWS Bedrock">
-
-#### Non-streaming example
-```python showLineNumbers title="AWS Bedrock Example using LiteLLM Python SDK"
-import litellm
-import os
-
-# Set AWS credentials
-os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key-id"
-os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-access-key"
-os.environ["AWS_REGION_NAME"] = "us-west-2"  # or your AWS region
-
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
-    max_tokens=100,
-)
-```
-
-#### Streaming example
-```python showLineNumbers title="AWS Bedrock Streaming Example using LiteLLM Python SDK"
-import litellm
-import os
-
-# Set AWS credentials
-os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key-id"
-os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-access-key"
-os.environ["AWS_REGION_NAME"] = "us-west-2"  # or your AWS region
-
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
-    max_tokens=100,
-    stream=True,
-)
-async for chunk in response:
-    print(chunk)
-```
-
-</TabItem>
-</Tabs>
-
-Example response:
-```json
-{
-  "content": [
-    {
-      "text": "Hi! this is a very short joke",
-      "type": "text"
-    }
-  ],
-  "id": "msg_013Zva2CMHLNnXjNJJKqJ2EF",
-  "model": "claude-3-7-sonnet-20250219",
-  "role": "assistant",
-  "stop_reason": "end_turn",
-  "stop_sequence": null,
-  "type": "message",
-  "usage": {
-    "input_tokens": 2095,
-    "output_tokens": 503,
-    "cache_creation_input_tokens": 2095,
-    "cache_read_input_tokens": 0
-  }
-}
-```
-
-### LiteLLM Proxy Server 
-
-<Tabs>
-<TabItem value="anthropic-proxy" label="Anthropic">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-    - model_name: anthropic-claude
-      litellm_params:
-        model: claude-3-7-sonnet-latest
-        api_key: os.environ/ANTHROPIC_API_KEY
-```
-
-2. Start proxy 
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it! 
-
-```python showLineNumbers title="Anthropic Example using LiteLLM Proxy Server"
-import anthropic
-
-# point anthropic sdk to litellm proxy 
-client = anthropic.Anthropic(
-    base_url="http://0.0.0.0:4000",
-    api_key="sk-1234",
-)
-
-response = client.messages.create(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="anthropic-claude",
-    max_tokens=100,
-)
-```
-
-</TabItem>
-
-<TabItem value="openai-proxy" label="OpenAI">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-    - model_name: openai-gpt4
-      litellm_params:
-        model: openai/gpt-4
-        api_key: os.environ/OPENAI_API_KEY
-```
-
-2. Start proxy 
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it! 
-
-```python showLineNumbers title="OpenAI Example using LiteLLM Proxy Server"
-import anthropic
-
-# point anthropic sdk to litellm proxy 
-client = anthropic.Anthropic(
-    base_url="http://0.0.0.0:4000",
-    api_key="sk-1234",
-)
-
-response = client.messages.create(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="openai-gpt4",
-    max_tokens=100,
-)
-```
-
-</TabItem>
-
-<TabItem value="gemini-proxy" label="Google AI Studio">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-    - model_name: gemini-2-flash
-      litellm_params:
-        model: gemini/gemini-2.0-flash-exp
-        api_key: os.environ/GEMINI_API_KEY
-```
-
-2. Start proxy 
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it! 
-
-```python showLineNumbers title="Google Gemini Example using LiteLLM Proxy Server"
-import anthropic
-
-# point anthropic sdk to litellm proxy 
-client = anthropic.Anthropic(
-    base_url="http://0.0.0.0:4000",
-    api_key="sk-1234",
-)
-
-response = client.messages.create(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="gemini-2-flash",
-    max_tokens=100,
-)
-```
-
-</TabItem>
-
-<TabItem value="vertex-proxy" label="Vertex AI">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-    - model_name: vertex-gemini
-      litellm_params:
-        model: vertex_ai/gemini-2.0-flash-exp
-        vertex_project: your-gcp-project-id
-        vertex_location: us-central1
-```
-
-2. Start proxy 
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it! 
-
-```python showLineNumbers title="Vertex AI Example using LiteLLM Proxy Server"
-import anthropic
-
-# point anthropic sdk to litellm proxy 
-client = anthropic.Anthropic(
-    base_url="http://0.0.0.0:4000",
-    api_key="sk-1234",
-)
-
-response = client.messages.create(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="vertex-gemini",
-    max_tokens=100,
-)
-```
-
-</TabItem>
-
-<TabItem value="bedrock-proxy" label="AWS Bedrock">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-    - model_name: bedrock-claude
-      litellm_params:
-        model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
-        aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
-        aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
-        aws_region_name: us-west-2
-```
-
-2. Start proxy 
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it! 
-
-```python showLineNumbers title="AWS Bedrock Example using LiteLLM Proxy Server"
-import anthropic
-
-# point anthropic sdk to litellm proxy 
-client = anthropic.Anthropic(
-    base_url="http://0.0.0.0:4000",
-    api_key="sk-1234",
-)
-
-response = client.messages.create(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    model="bedrock-claude",
-    max_tokens=100,
-)
-```
-
-</TabItem>
-
-<TabItem value="curl-proxy" label="curl">
-
-```bash showLineNumbers title="Example using LiteLLM Proxy Server"
-curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
--H 'content-type: application/json' \
--H 'x-api-key: $LITELLM_API_KEY' \
--H 'anthropic-version: 2023-06-01' \
--d '{
-  "model": "anthropic-claude",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Hello, can you tell me a short joke?"
-    }
-  ],
-  "max_tokens": 100
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Request Format
----
-
-Request body will be in the Anthropic messages API format. **litellm follows the Anthropic messages specification for this endpoint.**
-
-#### Example request body
-
-```json
-{
-  "model": "claude-3-7-sonnet-20250219",
-  "max_tokens": 1024,
-  "messages": [
-    {
-      "role": "user",
-      "content": "Hello, world"
-    }
-  ]
-}
-```
-
-#### Required Fields
-- **model** (string):  
-  The model identifier (e.g., `"claude-3-7-sonnet-20250219"`).
-- **max_tokens** (integer):  
-  The maximum number of tokens to generate before stopping.  
-  _Note: The model may stop before reaching this limit; value must be greater than 1._
-- **messages** (array of objects):  
-  An ordered list of conversational turns.  
-  Each message object must include:
-  - **role** (enum: `"user"` or `"assistant"`):  
-    Specifies the speaker of the message.
-  - **content** (string or array of content blocks):  
-    The text or content blocks (e.g., an array containing objects with a `type` such as `"text"`) that form the message.  
-    _Example equivalence:_
-    ```json
-    {"role": "user", "content": "Hello, Claude"}
-    ```
-    is equivalent to:
-    ```json
-    {"role": "user", "content": [{"type": "text", "text": "Hello, Claude"}]}
-    ```
-
-#### Optional Fields
-- **metadata** (object):  
-  Contains additional metadata about the request (e.g., `user_id` as an opaque identifier).
-- **stop_sequences** (array of strings):  
-  Custom sequences that, when encountered in the generated text, cause the model to stop.
-- **stream** (boolean):  
-  Indicates whether to stream the response using server-sent events.
-- **system** (string or array):  
-  A system prompt providing context or specific instructions to the model.
-- **temperature** (number):  
-  Controls randomness in the model's responses. Valid range: `0 < temperature < 1`.
-- **thinking** (object):
-  Configuration for enabling extended thinking. If enabled, it includes:
-  - **budget_tokens** (integer):
-    Minimum of 1024 tokens (and less than `max_tokens`).
-  - **type** (enum):
-    E.g., `"enabled"`.
-  - **summary** (string, optional):
-    Enables the summary style for thinking blocks. Possible values: `"auto"`, `"concise"`, `"detailed"`, `"disabled"`.
-    When routing to non-Anthropic providers (e.g., `openai/gpt-5.1`), the `summary` value is preserved and forwarded to the downstream API.
-- **tool_choice** (object):  
-  Instructs how the model should utilize any provided tools.
-- **tools** (array of objects):  
-  Definitions for tools available to the model. Each tool includes:
-  - **name** (string):  
-    The tool's name.
-  - **description** (string):  
-    A detailed description of the tool.
-  - **input_schema** (object):  
-    A JSON schema describing the expected input format for the tool.
-- **top_k** (integer):  
-  Limits sampling to the top K options.
-- **top_p** (number):  
-  Enables nucleus sampling with a cumulative probability cutoff. Valid range: `0 < top_p < 1`.
-
-
-## Response Format
----
-
-Responses will be in the Anthropic messages API format.
-
-#### Example Response
-
-```json
-{
-  "content": [
-    {
-      "text": "Hi! My name is Claude.",
-      "type": "text"
-    }
-  ],
-  "id": "msg_013Zva2CMHLNnXjNJJKqJ2EF",
-  "model": "claude-3-7-sonnet-20250219",
-  "role": "assistant",
-  "stop_reason": "end_turn",
-  "stop_sequence": null,
-  "type": "message",
-  "usage": {
-    "input_tokens": 2095,
-    "output_tokens": 503,
-    "cache_creation_input_tokens": 2095,
-    "cache_read_input_tokens": 0
-  }
-}
-```
-
-#### Response fields
-
-- **content** (array of objects):  
-  Contains the generated content blocks from the model. Each block includes:
-  - **type** (string):  
-    Indicates the type of content (e.g., `"text"`, `"tool_use"`, `"thinking"`, or `"redacted_thinking"`).
-  - **text** (string):  
-    The generated text from the model.  
-    _Note: Maximum length is 5,000,000 characters._
-  - **citations** (array of objects or `null`):  
-    Optional field providing citation details. Each citation includes:
-    - **cited_text** (string):  
-      The excerpt being cited.
-    - **document_index** (integer):  
-      An index referencing the cited document.
-    - **document_title** (string or `null`):  
-      The title of the cited document.
-    - **start_char_index** (integer):  
-      The starting character index for the citation.
-    - **end_char_index** (integer):  
-      The ending character index for the citation.
-    - **type** (string):  
-      Typically `"char_location"`.
-
-- **id** (string):  
-  A unique identifier for the response message.  
-  _Note: The format and length of IDs may change over time._
-
-- **model** (string):  
-  Specifies the model that generated the response.
-
-- **role** (string):  
-  Indicates the role of the generated message. For responses, this is always `"assistant"`.
-
-- **stop_reason** (string):  
-  Explains why the model stopped generating text. Possible values include:
-  - `"end_turn"`: The model reached a natural stopping point.
-  - `"max_tokens"`: The generation stopped because the maximum token limit was reached.
-  - `"stop_sequence"`: A custom stop sequence was encountered.
-  - `"tool_use"`: The model invoked one or more tools.
-
-- **stop_sequence** (string or `null`):  
-  Contains the specific stop sequence that caused the generation to halt, if applicable; otherwise, it is `null`.
-
-- **type** (string):  
-  Denotes the type of response object, which is always `"message"`.
-
-- **usage** (object):  
-  Provides details on token usage for billing and rate limiting. This includes:
-  - **input_tokens** (integer):  
-    Total number of input tokens processed.
-  - **output_tokens** (integer):  
-    Total number of output tokens generated.
-  - **cache_creation_input_tokens** (integer or `null`):  
-    Number of tokens used to create a cache entry.
-  - **cache_read_input_tokens** (integer or `null`):  
-    Number of tokens read from the cache.

docs/my-website/docs/anthropic_unified/messages_to_responses_mapping.md

@@ -1,120 +0,0 @@
-# v1/messages → /responses Parameter Mapping
-
-When you send a request to `/v1/messages` targeting an OpenAI or Azure model, LiteLLM internally routes it through the OpenAI Responses API. This page documents exactly how every parameter gets translated in both directions.
-
-The transformation lives in `litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py`.
-
-
-## Request: Anthropic → Responses API
-
-### Top-level parameters
-
-| Anthropic (`/v1/messages`) | Responses API | Notes |
-|---|---|---|
-| `model` | `model` | Passed through as-is |
-| `messages` | `input` | Structurally transformed — see the messages section below |
-| `system` (string) | `instructions` | Passed as a plain string |
-| `system` (list of content blocks) | `instructions` | Text blocks are joined with `\n`; non-text blocks are ignored |
-| `max_tokens` | `max_output_tokens` | Renamed |
-| `temperature` | `temperature` | Passed through as-is |
-| `top_p` | `top_p` | Passed through as-is |
-| `tools` | `tools` | Format-translated — see the tools section below |
-| `tool_choice` | `tool_choice` | Type-remapped — see the tool_choice section below |
-| `thinking` | `reasoning` | Budget tokens mapped to effort level — see the thinking section below |
-| `output_format` or `output_config.format` | `text` | Wrapped as `{"format": {"type": "json_schema", "name": "structured_output", "schema": ..., "strict": true}}` |
-| `context_management` | `context_management` | Converted from Anthropic dict to OpenAI array format — see the context_management section below |
-| `metadata.user_id` | `user` | Extracted from the metadata object and truncated to 64 characters |
-| `stop_sequences` | ❌ Not mapped | Dropped silently |
-| `top_k` | ❌ Not mapped | Dropped silently |
-| `speed` | ❌ Not mapped | Only used to set Anthropic beta headers on the native path |
-
-
-### How messages get converted
-
-Each Anthropic message is expanded into one or more Responses API input items. The key difference is that `tool_result` and `tool_use` blocks become **top-level items** in the input array rather than being nested inside a message.
-
-| Anthropic message | Responses API input item |
-|---|---|
-| `user` role, string content | `{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "..."}]}` |
-| `user` role, `{"type": "text"}` block | `{"type": "input_text", "text": "..."}` inside a user message |
-| `user` role, `{"type": "image", "source": {"type": "base64"}}` | `{"type": "input_image", "image_url": "data:<media_type>;base64,<data>"}` inside a user message |
-| `user` role, `{"type": "image", "source": {"type": "url"}}` | `{"type": "input_image", "image_url": "<url>"}` inside a user message |
-| `user` role, `{"type": "tool_result"}` block | Top-level `{"type": "function_call_output", "call_id": "...", "output": "..."}` — pulled out of the message entirely |
-| `assistant` role, string content | `{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "..."}]}` |
-| `assistant` role, `{"type": "text"}` block | `{"type": "output_text", "text": "..."}` inside an assistant message |
-| `assistant` role, `{"type": "tool_use"}` block | Top-level `{"type": "function_call", "call_id": "<id>", "name": "...", "arguments": "<JSON string>"}` — pulled out of the message entirely |
-| `assistant` role, `{"type": "thinking"}` block | `{"type": "output_text", "text": "<thinking text>"}` inside an assistant message |
-
-
-### tools
-
-| Anthropic tool | Responses API tool |
-|---|---|
-| Any tool where `type` starts with `"web_search"` or `name == "web_search"` | `{"type": "web_search_preview"}` |
-| All other tools | `{"type": "function", "name": "...", "description": "...", "parameters": <input_schema>}` |
-
-
-### tool_choice
-
-| Anthropic `tool_choice.type` | Responses API `tool_choice` |
-|---|---|
-| `"auto"` | `{"type": "auto"}` |
-| `"any"` | `{"type": "required"}` |
-| `"tool"` | `{"type": "function", "name": "<tool name>"}` |
-
-
-### thinking → reasoning
-
-The `budget_tokens` value is mapped to a string effort level. `summary` is always set to `"detailed"`.
-
-| `thinking.budget_tokens` | `reasoning.effort` |
-|---|---|
-| >= 10000 | `"high"` |
-| >= 5000 | `"medium"` |
-| >= 2000 | `"low"` |
-| < 2000 | `"minimal"` |
-
-If `thinking.type` is anything other than `"enabled"`, the `reasoning` field is not sent at all.
-
-
-### context_management
-
-Anthropic uses a nested dict with an `edits` array. OpenAI uses a flat array of compaction objects.
-
-```
-Anthropic input:
-{
-  "edits": [
-    {
-      "type": "compact_20260112",
-      "trigger": {"type": "input_tokens", "value": 150000}
-    }
-  ]
-}
-
-Responses API output:
-[
-  {"type": "compaction", "compact_threshold": 150000}
-]
-```
-
-
-## Response: Responses API → Anthropic
-
-When the Responses API reply comes back, LiteLLM converts it into an Anthropic `AnthropicMessagesResponse`.
-
-| Responses API field | Anthropic response field | Notes |
-|---|---|---|
-| `response.id` | `id` | |
-| `response.model` | `model` | Falls back to `"unknown-model"` if missing |
-| `ResponseReasoningItem` — `summary[*].text` | `content` block `{"type": "thinking", "thinking": "..."}` | Each non-empty summary text becomes a thinking block |
-| `ResponseOutputMessage` — `content[*]` where `type == "output_text"` | `content` block `{"type": "text", "text": "..."}` | |
-| `ResponseFunctionToolCall` — `{call_id, name, arguments}` | `content` block `{"type": "tool_use", "id": "...", "name": "...", "input": {...}}` | `arguments` is JSON-parsed back into a dict |
-| Any `function_call` present in output | `stop_reason: "tool_use"` | |
-| `response.status == "incomplete"` | `stop_reason: "max_tokens"` | Takes precedence over the default |
-| Everything else | `stop_reason: "end_turn"` | Default |
-| `response.usage.input_tokens` | `usage.input_tokens` | |
-| `response.usage.output_tokens` | `usage.output_tokens` | |
-| *(hardcoded)* | `type: "message"` | Always set |
-| *(hardcoded)* | `role: "assistant"` | Always set |
-| *(hardcoded)* | `stop_sequence: null` | Always null on this path |

docs/my-website/docs/anthropic_unified/structured_output.md

@@ -1,294 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Structured Output /v1/messages
-
-Use LiteLLM to call Anthropic's structured output feature via the `/v1/messages` endpoint.
-
-## Supported Providers
-
-| Provider | Supported | Notes |
-|----------|-----------|-------|
-| Anthropic | ✅ | Native support |
-| Azure AI (Anthropic models) | ✅ | Claude models on Azure AI |
-| Bedrock (Converse Anthropic models) | ✅ | Claude models via Bedrock Converse API |
-| Bedrock (Invoke Anthropic models) | ✅ | Claude models via Bedrock Invoke API |
-
-## Usage
-
-### LiteLLM Proxy Server
-
-<Tabs>
-<TabItem value="anthropic" label="Anthropic">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-  - model_name: claude-sonnet
-    litellm_params:
-      model: anthropic/claude-sonnet-4-5-20250514
-      api_key: os.environ/ANTHROPIC_API_KEY
-```
-
-2. Start proxy
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it!
-
-```bash
-curl http://localhost:4000/v1/messages \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_API_KEY" \
-  -H "anthropic-version: 2023-06-01" \
-  -d '{
-    "model": "claude-sonnet",
-    "max_tokens": 1024,
-    "messages": [
-      {
-        "role": "user",
-        "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
-      }
-    ],
-    "output_format": {
-      "type": "json_schema",
-      "schema": {
-        "type": "object",
-        "properties": {
-          "name": {"type": "string"},
-          "email": {"type": "string"},
-          "plan_interest": {"type": "string"},
-          "demo_requested": {"type": "boolean"}
-        },
-        "required": ["name", "email", "plan_interest", "demo_requested"],
-        "additionalProperties": false
-      }
-    }
-  }'
-```
-
-</TabItem>
-
-<TabItem value="azure_ai" label="Azure AI (Anthropic)">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-  - model_name: azure-claude-sonnet
-    litellm_params:
-      model: azure_ai/claude-sonnet-4-5-20250514
-      api_key: os.environ/AZURE_AI_API_KEY
-      api_base: https://your-endpoint.inference.ai.azure.com
-```
-
-2. Start proxy
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it!
-
-```bash
-curl http://localhost:4000/v1/messages \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_API_KEY" \
-  -H "anthropic-version: 2023-06-01" \
-  -d '{
-    "model": "azure-claude-sonnet",
-    "max_tokens": 1024,
-    "messages": [
-      {
-        "role": "user",
-        "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
-      }
-    ],
-    "output_format": {
-      "type": "json_schema",
-      "schema": {
-        "type": "object",
-        "properties": {
-          "name": {"type": "string"},
-          "email": {"type": "string"},
-          "plan_interest": {"type": "string"},
-          "demo_requested": {"type": "boolean"}
-        },
-        "required": ["name", "email", "plan_interest", "demo_requested"],
-        "additionalProperties": false
-      }
-    }
-  }'
-```
-
-</TabItem>
-
-<TabItem value="bedrock" label="Bedrock (Converse)">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-  - model_name: bedrock-claude-sonnet
-    litellm_params:
-      model: bedrock/global.anthropic.claude-sonnet-4-5-20250929-v1:0
-      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
-      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
-      aws_region_name: us-west-2
-```
-
-2. Start proxy
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it!
-
-```bash
-curl http://localhost:4000/v1/messages \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_API_KEY" \
-  -H "anthropic-version: 2023-06-01" \
-  -d '{
-    "model": "bedrock-claude-sonnet",
-    "max_tokens": 1024,
-    "messages": [
-      {
-        "role": "user",
-        "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
-      }
-    ],
-    "output_format": {
-      "type": "json_schema",
-      "schema": {
-        "type": "object",
-        "properties": {
-          "name": {"type": "string"},
-          "email": {"type": "string"},
-          "plan_interest": {"type": "string"},
-          "demo_requested": {"type": "boolean"}
-        },
-        "required": ["name", "email", "plan_interest", "demo_requested"],
-        "additionalProperties": false
-      }
-    }
-  }'
-```
-
-</TabItem>
-
-<TabItem value="bedrock_invoke" label="Bedrock (Invoke)">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-  - model_name: bedrock-claude-invoke
-    litellm_params:
-      model: bedrock/invoke/global.anthropic.claude-sonnet-4-5-20250929-v1:0
-      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
-      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
-      aws_region_name: us-west-2
-```
-
-2. Start proxy
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it!
-
-```bash
-curl http://localhost:4000/v1/messages \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_API_KEY" \
-  -H "anthropic-version: 2023-06-01" \
-  -d '{
-    "model": "bedrock-claude-invoke",
-    "max_tokens": 1024,
-    "messages": [
-      {
-        "role": "user",
-        "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
-      }
-    ],
-    "output_format": {
-      "type": "json_schema",
-      "schema": {
-        "type": "object",
-        "properties": {
-          "name": {"type": "string"},
-          "email": {"type": "string"},
-          "plan_interest": {"type": "string"},
-          "demo_requested": {"type": "boolean"}
-        },
-        "required": ["name", "email", "plan_interest", "demo_requested"],
-        "additionalProperties": false
-      }
-    }
-  }'
-```
-
-
-</TabItem>
-</Tabs>
-
-## Example Response
-
-```json
-{
-  "id": "msg_01XFDUDYJgAACzvnptvVoYEL",
-  "type": "message",
-  "role": "assistant",
-  "content": [
-    {
-      "type": "text",
-      "text": "{\"name\":\"John Smith\",\"email\":\"john@example.com\",\"plan_interest\":\"Enterprise\",\"demo_requested\":true}"
-    }
-  ],
-  "model": "claude-sonnet-4-5-20250514",
-  "stop_reason": "end_turn",
-  "stop_sequence": null,
-  "usage": {
-    "input_tokens": 75,
-    "output_tokens": 28
-  }
-}
-```
-
-## Request Format
-
-### output_format
-
-The `output_format` parameter specifies the structured output format.
-
-```json
-{
-  "output_format": {
-    "type": "json_schema",
-    "schema": {
-      "type": "object",
-      "properties": {
-        "field_name": {"type": "string"},
-        "another_field": {"type": "integer"}
-      },
-      "required": ["field_name", "another_field"],
-      "additionalProperties": false
-    }
-  }
-}
-```
-
-#### Fields
-
-- **type** (string): Must be `"json_schema"`
-- **schema** (object): A JSON Schema object defining the expected output structure
-  - **type** (string): The root type, typically `"object"`
-  - **properties** (object): Defines the fields and their types
-  - **required** (array): List of required field names
-  - **additionalProperties** (boolean): Set to `false` to enforce strict schema adherence

docs/my-website/docs/apply_guardrail.md

@@ -1,155 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# /guardrails/apply_guardrail
-
-Use this endpoint to directly call a guardrail configured on your LiteLLM instance. This is useful when you have services that need to directly call a guardrail.
-
-## Supported Guardrail Types
-
-This endpoint supports various guardrail types including:
-- **Presidio** - PII detection and masking
-- **Bedrock** - AWS Bedrock guardrails for content moderation
-- **Lakera** - AI safety guardrails
-- **PANW Prisma AIRS** - Threat detection, DLP, and policy enforcement
-- **Custom guardrails** - User-defined guardrails
-
-## Configuration
-
-### Bedrock Guardrail Configuration
-
-To use Bedrock guardrails with the apply_guardrail endpoint, configure your guardrail in your LiteLLM config.yaml:
-
-```yaml
-guardrails:
-  - guardrail_name: "bedrock-content-guard"
-    litellm_params:
-      guardrail: bedrock
-      mode: "pre_call"
-      guardrailIdentifier: "your-guardrail-id"  # Your actual Bedrock guardrail ID
-      guardrailVersion: "DRAFT"  # or your version number
-      aws_region_name: "us-east-1"  # Your AWS region
-      aws_role_name: "your-role-arn"  # Your AWS role with Bedrock permissions
-      default_on: true
-```
-
-**Required AWS Setup:**
-1. Create a Bedrock guardrail in AWS Console
-2. Get the guardrail ID and version
-3. Ensure your AWS credentials have Bedrock permissions
-4. Configure the guardrail in your LiteLLM config 
-
-
-## Usage
----
-
-<Tabs>
-<TabItem value="presidio" label="Presidio PII Guardrail" default>
-
-In this example `mask_pii` is a Presidio guardrail configured on LiteLLM.
-
-```bash showLineNumbers title="Example calling the endpoint"
-curl -X POST 'http://localhost:4000/guardrails/apply_guardrail' \
--H 'Content-Type: application/json' \
--H 'Authorization: Bearer your-api-key' \
--d '{
-    "guardrail_name": "mask_pii",
-    "text": "My name is John Doe and my email is john@example.com",
-    "language": "en",
-    "entities": ["NAME", "EMAIL"]
-}'
-```
-
-</TabItem>
-<TabItem value="bedrock" label="Bedrock Guardrail">
-
-In this example `bedrock-content-guard` is a Bedrock guardrail configured on LiteLLM.
-
-```bash showLineNumbers title="Example calling the endpoint"
-curl -X POST 'http://localhost:4000/guardrails/apply_guardrail' \
--H 'Content-Type: application/json' \
--H 'Authorization: Bearer your-api-key' \
--d '{
-    "guardrail_name": "bedrock-content-guard",
-    "text": "This is potentially harmful content that should be blocked",
-    "language": "en"
-}'
-```
-
-**Note**: For Bedrock guardrails, the `entities` parameter is not used as Bedrock handles content moderation based on its own policies.
-
-</TabItem>
-</Tabs>
-
-
-## Request Format
----
-
-The request body should follow the ApplyGuardrailRequest format.
-
-#### Example Request Body
-
-```json
-{
-    "guardrail_name": "mask_pii",
-    "text": "My name is John Doe and my email is john@example.com",
-    "language": "en",
-    "entities": ["NAME", "EMAIL"]
-}
-```
-
-#### Required Fields
-- **guardrail_name** (string):  
-  The identifier for the guardrail to apply (e.g., "mask_pii").
-- **text** (string):  
-  The input text to process through the guardrail.
-
-#### Optional Fields
-- **language** (string):  
-  The language of the input text (e.g., "en" for English).
-- **entities** (array of strings):  
-  Specific entities to process or filter (e.g., ["NAME", "EMAIL"]).
-
-## Response Format
----
-
-The response will contain the processed text after applying the guardrail.
-
-#### Example Response
-
-<Tabs>
-<TabItem value="presidio" label="Presidio Response" default>
-
-```json
-{
-    "response_text": "My name is [REDACTED] and my email is [REDACTED]"
-}
-```
-
-</TabItem>
-<TabItem value="bedrock" label="Bedrock Response">
-
-```json
-{
-    "response_text": "This is potentially harmful content that should be blocked"
-}
-```
-
-**Note**: If Bedrock guardrail blocks the content, the endpoint will return an error with the blocking reason.
-
-</TabItem>
-</Tabs>
-
-#### Response Fields
-- **response_text** (string):  
-  The text after applying the guardrail.
-
-#### Error Responses
-
-If a guardrail blocks content (e.g., Bedrock guardrail), the endpoint will return an error:
-
-```json
-{
-    "detail": "Content blocked by Bedrock guardrail: Content violates policy"
-}
-```

docs/my-website/docs/assistants.md

@@ -1,353 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# /assistants
-
-:::warning Deprecation Notice
-
-OpenAI has deprecated the Assistants API. It will shut down on **August 26, 2026**.
-
-Consider migrating to the [Responses API](/docs/response_api) instead. See [OpenAI's migration guide](https://platform.openai.com/docs/guides/responses-vs-assistants) for details.
-
-:::
-
-Covers Threads, Messages, Assistants. 
-
-LiteLLM currently covers: 
-- Create Assistants 
-- Delete Assistants
-- Get Assistants
-- Create Thread
-- Get Thread
-- Add Messages
-- Get Messages
-- Run Thread
-
-
-## **Supported Providers**:
-- [OpenAI](#quick-start)
-- [Azure OpenAI](#azure-openai)
-- [OpenAI-Compatible APIs](#openai-compatible-apis)
-
-## Quick Start 
-
-Call an existing Assistant. 
-
-- Get the Assistant 
-
-- Create a Thread when a user starts a conversation.
-
-- Add Messages to the Thread as the user asks questions.
-
-- Run the Assistant on the Thread to generate a response by calling the model and the tools.
-
-### SDK + PROXY
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-**Create an Assistant**
-
-
-```python
-import litellm
-import os 
-
-# setup env
-os.environ["OPENAI_API_KEY"] = "sk-.."
-
-assistant = litellm.create_assistants(
-            custom_llm_provider="openai",
-            model="gpt-4-turbo",
-            instructions="You are a personal math tutor. When asked a question, write and run Python code to answer the question.",
-            name="Math Tutor",
-            tools=[{"type": "code_interpreter"}],
-)
-
-### ASYNC USAGE ### 
-# assistant = await litellm.acreate_assistants(
-#             custom_llm_provider="openai",
-#             model="gpt-4-turbo",
-#             instructions="You are a personal math tutor. When asked a question, write and run Python code to answer the question.",
-#             name="Math Tutor",
-#             tools=[{"type": "code_interpreter"}],
-# )
-```
-
-**Get the Assistant**
-
-```python
-from litellm import get_assistants, aget_assistants
-import os 
-
-# setup env
-os.environ["OPENAI_API_KEY"] = "sk-.."
-
-assistants = get_assistants(custom_llm_provider="openai")
-
-### ASYNC USAGE ### 
-# assistants = await aget_assistants(custom_llm_provider="openai")
-```
-
-**Create a Thread**
-
-```python
-from litellm import create_thread, acreate_thread
-import os 
-
-os.environ["OPENAI_API_KEY"] = "sk-.."
-
-new_thread = create_thread(
-            custom_llm_provider="openai",
-            messages=[{"role": "user", "content": "Hey, how's it going?"}],  # type: ignore
-        )
-
-### ASYNC USAGE ### 
-# new_thread = await acreate_thread(custom_llm_provider="openai",messages=[{"role": "user", "content": "Hey, how's it going?"}])
-```
-
-**Add Messages to the Thread**
-
-```python
-from litellm import create_thread, get_thread, aget_thread, add_message, a_add_message
-import os 
-
-os.environ["OPENAI_API_KEY"] = "sk-.."
-
-## CREATE A THREAD
-_new_thread = create_thread(
-            custom_llm_provider="openai",
-            messages=[{"role": "user", "content": "Hey, how's it going?"}],  # type: ignore
-        )
-
-## OR retrieve existing thread
-received_thread = get_thread(
-            custom_llm_provider="openai",
-            thread_id=_new_thread.id,
-        )
-
-### ASYNC USAGE ### 
-# received_thread = await aget_thread(custom_llm_provider="openai", thread_id=_new_thread.id,)
-
-## ADD MESSAGE TO THREAD
-message = {"role": "user", "content": "Hey, how's it going?"}
-added_message = add_message(
-            thread_id=_new_thread.id, custom_llm_provider="openai", **message
-        )
-
-### ASYNC USAGE ### 
-# added_message = await a_add_message(thread_id=_new_thread.id, custom_llm_provider="openai", **message)
-```
-
-**Run the Assistant on the Thread**
-
-```python
-from litellm import get_assistants, create_thread, add_message, run_thread, arun_thread
-import os 
-
-os.environ["OPENAI_API_KEY"] = "sk-.."
-assistants = get_assistants(custom_llm_provider="openai")
-
-## get the first assistant ###
-assistant_id = assistants.data[0].id
-
-## GET A THREAD
-_new_thread = create_thread(
-            custom_llm_provider="openai",
-            messages=[{"role": "user", "content": "Hey, how's it going?"}],  # type: ignore
-        )
-
-## ADD MESSAGE
-message = {"role": "user", "content": "Hey, how's it going?"}
-added_message = add_message(
-            thread_id=_new_thread.id, custom_llm_provider="openai", **message
-        )
-
-## 🚨 RUN THREAD
-response = run_thread(
-            custom_llm_provider="openai", thread_id=thread_id, assistant_id=assistant_id
-        )
-
-### ASYNC USAGE ### 
-# response = await arun_thread(custom_llm_provider="openai", thread_id=thread_id, assistant_id=assistant_id)
-
-print(f"run_thread: {run_thread}")
-```
-</TabItem>
-<TabItem value="proxy" label="PROXY">
-
-```yaml
-assistant_settings:
-  custom_llm_provider: azure
-  litellm_params: 
-    api_key: os.environ/AZURE_API_KEY
-    api_base: os.environ/AZURE_API_BASE
-    api_version: os.environ/AZURE_API_VERSION
-```
-
-```bash
-$ litellm --config /path/to/config.yaml
-
-# RUNNING on http://0.0.0.0:4000
-```
-
-
-**Create the Assistant**
-
-```bash
-curl "http://localhost:4000/v1/assistants" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.",
-    "name": "Math Tutor",
-    "tools": [{"type": "code_interpreter"}],
-    "model": "gpt-4-turbo"
-  }'
-```
-
-
-**Get the Assistant**
-
-```bash
-curl "http://0.0.0.0:4000/v1/assistants?order=desc&limit=20" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234"
-```
-
-**Create a Thread**
-
-```bash
-curl http://0.0.0.0:4000/v1/threads \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d ''
-```
-
-**Get a Thread**
-
-```bash
-curl http://0.0.0.0:4000/v1/threads/{thread_id} \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234"
-```
-
-**Add Messages to the Thread**
-
-```bash
-curl http://0.0.0.0:4000/v1/threads/{thread_id}/messages \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-      "role": "user",
-      "content": "How does AI work? Explain it in simple terms."
-    }'
-```
-
-**Run the Assistant on the Thread**
-
-```bash
-curl http://0.0.0.0:4000/v1/threads/thread_abc123/runs \
-  -H "Authorization: Bearer sk-1234" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "assistant_id": "asst_abc123"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-## Streaming 
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-```python
-from litellm import run_thread_stream 
-import os
-
-os.environ["OPENAI_API_KEY"] = "sk-.."
-
-message = {"role": "user", "content": "Hey, how's it going?"}  
-
-data = {"custom_llm_provider": "openai", "thread_id": _new_thread.id, "assistant_id": assistant_id, **message}
-
-run = run_thread_stream(**data)
-with run as run:
-    assert isinstance(run, AssistantEventHandler)
-    for chunk in run: 
-      print(f"chunk: {chunk}")
-    run.until_done()
-```
-
-</TabItem>
-<TabItem value="proxy" label="PROXY">
-
-```bash
-curl -X POST 'http://0.0.0.0:4000/threads/{thread_id}/runs' \
--H 'Authorization: Bearer sk-1234' \
--H 'Content-Type: application/json' \
--d '{
-      "assistant_id": "asst_6xVZQFFy1Kw87NbnYeNebxTf",
-      "stream": true
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## [👉 Proxy API Reference](https://litellm-api.up.railway.app/#/assistants)
-
-
-## Azure OpenAI
-
-**config**
-```yaml
-assistant_settings:
-  custom_llm_provider: azure
-  litellm_params: 
-    api_key: os.environ/AZURE_API_KEY
-    api_base: os.environ/AZURE_API_BASE
-```
-
-**curl**
-
-```bash
-curl -X POST "http://localhost:4000/v1/assistants" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.",
-    "name": "Math Tutor",
-    "tools": [{"type": "code_interpreter"}],
-    "model": "<my-azure-deployment-name>"
-  }'
-```
-
-## OpenAI-Compatible APIs 
-
-To call openai-compatible Assistants API's (eg. Astra Assistants API), just add `openai/` to the model name: 
-
-
-**config**
-```yaml
-assistant_settings:
-  custom_llm_provider: openai
-  litellm_params: 
-    api_key: os.environ/ASTRA_API_KEY
-    api_base: os.environ/ASTRA_API_BASE
-```
-
-**curl**
-
-```bash
-curl -X POST "http://localhost:4000/v1/assistants" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.",
-    "name": "Math Tutor",
-    "tools": [{"type": "code_interpreter"}],
-    "model": "openai/<my-astra-model-name>"
-  }'
-```

docs/my-website/docs/audio_transcription.md

@@ -1,208 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# /audio/transcriptions
-
-## Overview 
-
-| Feature | Supported | Notes | 
-|-------|-------|-------|
-| Cost Tracking | ✅ | Works with all supported models |
-| Logging | ✅ | Works across all integrations |
-| End-user Tracking | ✅ | |
-| Fallbacks | ✅ | Works between supported models |
-| Loadbalancing | ✅ | Works between supported models |
-| Guardrails | ✅ | Applies to output transcribed text (non-streaming only) |
-| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud`, `mistral` | |
-
-## Quick Start
-
-### LiteLLM Python SDK
-
-```python showLineNumbers title="Python SDK Example"
-from litellm import transcription
-import os 
-
-# set api keys 
-os.environ["OPENAI_API_KEY"] = ""
-audio_file = open("/path/to/audio.mp3", "rb")
-
-response = transcription(model="whisper", file=audio_file)
-
-print(f"response: {response}")
-```
-
-### LiteLLM Proxy
-
-### Add model to config 
-
-
-<Tabs>
-<TabItem value="openai" label="OpenAI">
-
-```yaml showLineNumbers title="OpenAI Configuration"
-model_list:
-- model_name: whisper
-  litellm_params:
-    model: whisper-1
-    api_key: os.environ/OPENAI_API_KEY
-  model_info:
-    mode: audio_transcription
-    
-general_settings:
-  master_key: sk-1234
-```
-</TabItem>
-<TabItem value="openai+azure" label="OpenAI + Azure">
-
-```yaml showLineNumbers title="OpenAI + Azure Configuration"
-model_list:
-- model_name: whisper
-  litellm_params:
-    model: whisper-1
-    api_key: os.environ/OPENAI_API_KEY
-  model_info:
-    mode: audio_transcription
-- model_name: whisper
-  litellm_params:
-    model: azure/azure-whisper
-    api_version: 2024-02-15-preview
-    api_base: os.environ/AZURE_EUROPE_API_BASE
-    api_key: os.environ/AZURE_EUROPE_API_KEY
-  model_info:
-    mode: audio_transcription
-
-general_settings:
-  master_key: sk-1234
-```
-
-</TabItem>
-</Tabs>
-
-### Start proxy 
-
-```bash showLineNumbers title="Start Proxy Server"
-litellm --config /path/to/config.yaml 
-
-# RUNNING on http://0.0.0.0:8000
-```
-
-### Test 
-
-<Tabs>
-<TabItem value="curl" label="Curl">
-
-```bash showLineNumbers title="Test with cURL"
-curl --location 'http://0.0.0.0:8000/v1/audio/transcriptions' \
---header 'Authorization: Bearer sk-1234' \
---form 'file=@"/Users/krrishdholakia/Downloads/gettysburg.wav"' \
---form 'model="whisper"'
-```
-
-</TabItem>
-<TabItem value="openai" label="OpenAI Python SDK">
-
-```python showLineNumbers title="Test with OpenAI Python SDK"
-from openai import OpenAI
-client = openai.OpenAI(
-    api_key="sk-1234",
-    base_url="http://0.0.0.0:8000"
-)
-
-
-audio_file = open("speech.mp3", "rb")
-transcript = client.audio.transcriptions.create(
-  model="whisper",
-  file=audio_file
-)
-```
-</TabItem>
-</Tabs>
-
-## Supported Providers
-
-- OpenAI
-- Azure
-- [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
-- [Groq](./providers/groq.md#speech-to-text---whisper)
-- [Deepgram](./providers/deepgram.md)
-- [Mistral (Voxtral)](./providers/mistral.md#audio-transcription)
-- [OVHcloud AI Endpoints](./providers/ovhcloud.md)
-
----
-
-## Fallbacks
-
-You can configure fallbacks for audio transcription to automatically retry with different models if the primary model fails.
-
-<Tabs>
-<TabItem value="curl" label="Curl">
-
-```bash showLineNumbers title="Test with cURL and Fallbacks"
-curl --location 'http://0.0.0.0:4000/v1/audio/transcriptions' \
---header 'Authorization: Bearer sk-1234' \
---form 'file=@"gettysburg.wav"' \
---form 'model="groq/whisper-large-v3"' \
---form 'fallbacks[]="openai/whisper-1"'
-```
-
-</TabItem>
-<TabItem value="openai" label="OpenAI Python SDK">
-
-```python showLineNumbers title="Test with OpenAI Python SDK and Fallbacks"
-from openai import OpenAI
-client = OpenAI(
-    api_key="sk-1234",
-    base_url="http://0.0.0.0:4000"
-)
-
-audio_file = open("gettysburg.wav", "rb")
-transcript = client.audio.transcriptions.create(
-    model="groq/whisper-large-v3",
-    file=audio_file,
-    extra_body={
-        "fallbacks": ["openai/whisper-1"]
-    }
-)
-```
-</TabItem>
-</Tabs>
-
-### Testing Fallbacks
-
-You can test your fallback configuration using `mock_testing_fallbacks=true` to simulate failures:
-
-<Tabs>
-<TabItem value="curl" label="Curl">
-
-```bash showLineNumbers title="Test Fallbacks with Mock Testing"
-curl --location 'http://0.0.0.0:4000/v1/audio/transcriptions' \
---header 'Authorization: Bearer sk-1234' \
---form 'file=@"gettysburg.wav"' \
---form 'model="groq/whisper-large-v3"' \
---form 'fallbacks[]="openai/whisper-1"' \
---form 'mock_testing_fallbacks=true'
-```
-
-</TabItem>
-<TabItem value="openai" label="OpenAI Python SDK">
-
-```python showLineNumbers title="Test Fallbacks with Mock Testing"
-from openai import OpenAI
-client = OpenAI(
-    api_key="sk-1234",
-    base_url="http://0.0.0.0:4000"
-)
-
-audio_file = open("gettysburg.wav", "rb")
-transcript = client.audio.transcriptions.create(
-    model="groq/whisper-large-v3",
-    file=audio_file,
-    extra_body={
-        "fallbacks": ["openai/whisper-1"],
-        "mock_testing_fallbacks": True
-    }
-)
-```
-</TabItem>
-</Tabs>

docs/my-website/docs/batches.md

@@ -1,455 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# /batches
-
-Covers Batches, Files
-
-| Feature | Supported | Notes | 
-|-------|-------|-------|
-| Supported Providers | OpenAI, Azure, Vertex, Bedrock, vLLM | - |
-| ✨ Cost Tracking | ✅ | LiteLLM Enterprise only |
-| Logging | ✅ | Works across all logging integrations |
-
-## Quick Start 
-
-- Create File for Batch Completion
-
-- Create Batch Request
-
-- List Batches
-
-- Retrieve the Specific Batch and File Content
-
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM PROXY Server">
-
-```bash
-$ export OPENAI_API_KEY="sk-..."
-
-$ litellm
-
-# RUNNING on http://0.0.0.0:4000
-```
-
-**Create File for Batch Completion**
-
-```shell
-curl http://localhost:4000/v1/files \
-    -H "Authorization: Bearer sk-1234" \
-    -F purpose="batch" \
-    -F file="@mydata.jsonl"
-```
-
-**Create Batch Request**
-
-```bash
-curl http://localhost:4000/v1/batches \
-        -H "Authorization: Bearer sk-1234" \
-        -H "Content-Type: application/json" \
-        -d '{
-            "input_file_id": "file-abc123",
-            "endpoint": "/v1/chat/completions",
-            "completion_window": "24h"
-    }'
-```
-
-**Retrieve the Specific Batch**
-
-```bash
-curl http://localhost:4000/v1/batches/batch_abc123 \
-    -H "Authorization: Bearer sk-1234" \
-    -H "Content-Type: application/json" \
-```
-
-
-**List Batches**
-
-```bash
-curl http://localhost:4000/v1/batches \
-    -H "Authorization: Bearer sk-1234" \
-    -H "Content-Type: application/json" \
-```
-
-</TabItem>
-<TabItem value="sdk" label="SDK">
-
-**Create File for Batch Completion**
-
-```python
-import litellm
-import os 
-import asyncio
-
-os.environ["OPENAI_API_KEY"] = "sk-.."
-
-file_name = "openai_batch_completions.jsonl"
-_current_dir = os.path.dirname(os.path.abspath(__file__))
-file_path = os.path.join(_current_dir, file_name)
-file_obj = await litellm.acreate_file(
-    file=open(file_path, "rb"),
-    purpose="batch",
-    custom_llm_provider="openai",
-)
-print("Response from creating file=", file_obj)
-```
-
-**Create Batch Request**
-
-```python
-import litellm
-import os 
-import asyncio
-
-create_batch_response = await litellm.acreate_batch(
-    completion_window="24h",
-    endpoint="/v1/chat/completions",
-    input_file_id=batch_input_file_id,
-    custom_llm_provider="openai",
-    metadata={"key1": "value1", "key2": "value2"},
-)
-
-print("response from litellm.create_batch=", create_batch_response)
-```
-
-**Retrieve the Specific Batch and File Content**
-
-```python
-    # Maximum wait time before we give up
-    MAX_WAIT_TIME = 300  
-
-    # Time to wait between each status check
-    POLL_INTERVAL = 5
-    
-    #Time waited till now 
-    waited = 0
-
-    # Wait for the batch to finish processing before trying to retrieve output
-    # This loop checks the batch status every few seconds (polling)
-
-    while True:
-        retrieved_batch = await litellm.aretrieve_batch(
-            batch_id=create_batch_response.id,
-            custom_llm_provider="openai"
-        )
-        
-        status = retrieved_batch.status
-        print(f"⏳ Batch status: {status}")
-        
-        if status == "completed" and retrieved_batch.output_file_id:
-            print("✅ Batch complete. Output file ID:", retrieved_batch.output_file_id)
-            break
-        elif status in ["failed", "cancelled", "expired"]:
-            raise RuntimeError(f"❌ Batch failed with status: {status}")
-        
-        await asyncio.sleep(POLL_INTERVAL)
-        waited += POLL_INTERVAL
-        if waited > MAX_WAIT_TIME:
-            raise TimeoutError("❌ Timed out waiting for batch to complete.")
-
-print("retrieved batch=", retrieved_batch)
-# just assert that we retrieved a non None batch
-
-assert retrieved_batch.id == create_batch_response.id
-
-# try to get file content for our original file
-
-file_content = await litellm.afile_content(
-    file_id=batch_input_file_id, custom_llm_provider="openai"
-)
-
-print("file content = ", file_content)
-```
-
-**List Batches**
-
-```python
-list_batches_response = litellm.list_batches(custom_llm_provider="openai", limit=2)
-print("list_batches_response=", list_batches_response)
-```
-
-</TabItem>
-
-</Tabs>
-
-
-## Multi-Account / Model-Based Routing
-
-Route batch operations to different provider accounts using model-specific credentials from your `config.yaml`. This eliminates the need for environment variables and enables multi-tenant batch processing.
-
-### How It Works
-
-**Priority Order:**
-1. **Encoded Batch/File ID** (highest) - Model info embedded in the ID
-2. **Model Parameter** - Via header (`x-litellm-model`), query param, or request body
-3. **Custom Provider** (fallback) - Uses environment variables
-
-### Configuration
-
-```yaml
-model_list:
-  - model_name: gpt-4o-account-1
-    litellm_params:
-      model: openai/gpt-4o
-      api_key: sk-account-1-key
-      api_base: https://api.openai.com/v1
-  
-  - model_name: gpt-4o-account-2
-    litellm_params:
-      model: openai/gpt-4o
-      api_key: sk-account-2-key
-      api_base: https://api.openai.com/v1
-  
-  - model_name: azure-batches
-    litellm_params:
-      model: azure/gpt-4
-      api_key: azure-key-123
-      api_base: https://my-resource.openai.azure.com
-      api_version: "2024-02-01"
-```
-
-### Usage Examples
-
-#### Scenario 1: Encoded File ID with Model
-
-When you upload a file with a model parameter, LiteLLM encodes the model information in the file ID. All subsequent operations automatically use those credentials.
-
-```bash
-# Step 1: Upload file with model
-curl http://localhost:4000/v1/files \
-  -H "Authorization: Bearer sk-1234" \
-  -H "x-litellm-model: gpt-4o-account-1" \
-  -F purpose="batch" \
-  -F file="@batch.jsonl"
-
-# Response includes encoded file ID:
-# {
-#   "id": "file-bGl0ZWxsbTpmaWxlLUxkaUwzaVYxNGZRVlpYcU5KVEdkSjk7bW9kZWwsZ3B0LTRvLWFjY291bnQtMQ",
-#   ...
-# }
-
-# Step 2: Create batch - automatically routes to gpt-4o-account-1
-curl http://localhost:4000/v1/batches \
-  -H "Authorization: Bearer sk-1234" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "input_file_id": "file-bGl0ZWxsbTpmaWxlLUxkaUwzaVYxNGZRVlpYcU5KVEdkSjk7bW9kZWwsZ3B0LTRvLWFjY291bnQtMQ",
-    "endpoint": "/v1/chat/completions",
-    "completion_window": "24h"
-  }'
-
-# Batch ID is also encoded with model:
-# {
-#   "id": "batch_bGl0ZWxsbTpiYXRjaF82OTIwM2IzNjg0MDQ4MTkwYTA3ODQ5NDY3YTFjMDJkYTttb2RlbCxncHQtNG8tYWNjb3VudC0x",
-#   "input_file_id": "file-bGl0ZWxsbTpmaWxlLUxkaUwzaVYxNGZRVlpYcU5KVEdkSjk7bW9kZWwsZ3B0LTRvLWFjY291bnQtMQ",
-#   ...
-# }
-
-# Step 3: Retrieve batch - automatically routes to gpt-4o-account-1
-curl http://localhost:4000/v1/batches/batch_bGl0ZWxsbTpiYXRjaF82OTIwM2IzNjg0MDQ4MTkwYTA3ODQ5NDY3YTFjMDJkYTttb2RlbCxncHQtNG8tYWNjb3VudC0x \
-  -H "Authorization: Bearer sk-1234"
-```
-
-**✅ Benefits:**
-- No need to specify model on every request
-- File and batch IDs "remember" which account created them
-- Automatic routing for retrieve, cancel, and file content operations
-
-#### Scenario 2: Model via Header/Query Parameter
-
-Specify the model for each request without encoding it in the ID.
-
-```bash
-# Create batch with model header
-curl http://localhost:4000/v1/batches \
-  -H "Authorization: Bearer sk-1234" \
-  -H "x-litellm-model: gpt-4o-account-2" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "input_file_id": "file-abc123",
-    "endpoint": "/v1/chat/completions",
-    "completion_window": "24h"
-  }'
-
-# Or use query parameter
-curl "http://localhost:4000/v1/batches?model=gpt-4o-account-2" \
-  -H "Authorization: Bearer sk-1234" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "input_file_id": "file-abc123",
-    "endpoint": "/v1/chat/completions",
-    "completion_window": "24h"
-  }'
-
-# List batches for specific model
-curl "http://localhost:4000/v1/batches?model=gpt-4o-account-2" \
-  -H "Authorization: Bearer sk-1234"
-```
-
-**✅ Use Case:**
-- One-off batch operations
-- Different models for different operations
-- Explicit control over routing
-
-#### Scenario 3: Environment Variables (Fallback)
-
-Traditional approach using environment variables when no model is specified.
-
-```bash
-export OPENAI_API_KEY="sk-env-key"
-
-curl http://localhost:4000/v1/batches \
-  -H "Authorization: Bearer sk-1234" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "input_file_id": "file-abc123",
-    "endpoint": "/v1/chat/completions",
-    "completion_window": "24h"
-  }'
-```
-
-**✅ Use Case:**
-- Backward compatibility
-- Simple single-account setups
-- Quick prototyping
-
-### Complete Multi-Account Example
-
-```bash
-# Upload file to Account 1
-FILE_1=$(curl -s http://localhost:4000/v1/files \
-  -H "x-litellm-model: gpt-4o-account-1" \
-  -F purpose="batch" \
-  -F file="@batch1.jsonl" | jq -r '.id')
-
-# Upload file to Account 2
-FILE_2=$(curl -s http://localhost:4000/v1/files \
-  -H "x-litellm-model: gpt-4o-account-2" \
-  -F purpose="batch" \
-  -F file="@batch2.jsonl" | jq -r '.id')
-
-# Create batch on Account 1 (auto-routed via encoded file ID)
-BATCH_1=$(curl -s http://localhost:4000/v1/batches \
-  -d "{\"input_file_id\": \"$FILE_1\", \"endpoint\": \"/v1/chat/completions\", \"completion_window\": \"24h\"}" | jq -r '.id')
-
-# Create batch on Account 2 (auto-routed via encoded file ID)
-BATCH_2=$(curl -s http://localhost:4000/v1/batches \
-  -d "{\"input_file_id\": \"$FILE_2\", \"endpoint\": \"/v1/chat/completions\", \"completion_window\": \"24h\"}" | jq -r '.id')
-
-# Retrieve both batches (auto-routed to correct accounts)
-curl http://localhost:4000/v1/batches/$BATCH_1
-curl http://localhost:4000/v1/batches/$BATCH_2
-
-# List batches per account
-curl "http://localhost:4000/v1/batches?model=gpt-4o-account-1"
-curl "http://localhost:4000/v1/batches?model=gpt-4o-account-2"
-```
-
-### SDK Usage with Model Routing
-
-```python
-import litellm
-import asyncio
-
-# Upload file with model routing
-file_obj = await litellm.acreate_file(
-    file=open("batch.jsonl", "rb"),
-    purpose="batch",
-    model="gpt-4o-account-1",  # Route to specific account
-)
-
-print(f"File ID: {file_obj.id}")
-# File ID is encoded with model info
-
-# Create batch - automatically uses gpt-4o-account-1 credentials
-batch = await litellm.acreate_batch(
-    completion_window="24h",
-    endpoint="/v1/chat/completions",
-    input_file_id=file_obj.id,  # Model info embedded in ID
-)
-
-print(f"Batch ID: {batch.id}")
-# Batch ID is also encoded
-
-# Retrieve batch - automatically routes to correct account
-retrieved = await litellm.aretrieve_batch(
-    batch_id=batch.id,  # Model info embedded in ID
-)
-
-print(f"Batch status: {retrieved.status}")
-
-# Or explicitly specify model
-batch2 = await litellm.acreate_batch(
-    completion_window="24h",
-    endpoint="/v1/chat/completions",
-    input_file_id="file-regular-id",
-    model="gpt-4o-account-2",  # Explicit routing
-)
-```
-
-### How ID Encoding Works
-
-LiteLLM encodes model information into file and batch IDs using base64:
-
-```
-Original:  file-abc123
-Encoded:   file-bGl0ZWxsbTpmaWxlLWFiYzEyMzttb2RlbCxncHQtNG8tdGVzdA
-           └─┬─┘ └──────────────────┬──────────────────────┘
-          prefix      base64(litellm:file-abc123;model,gpt-4o-test)
-
-Original:  batch_xyz789
-Encoded:   batch_bGl0ZWxsbTpiYXRjaF94eXo3ODk7bW9kZWwsZ3B0LTRvLXRlc3Q
-           └──┬──┘ └──────────────────┬──────────────────────┘
-           prefix       base64(litellm:batch_xyz789;model,gpt-4o-test)
-```
-
-The encoding:
-- ✅ Preserves OpenAI-compatible prefixes (`file-`, `batch_`)
-- ✅ Is transparent to clients
-- ✅ Enables automatic routing without additional parameters
-- ✅ Works across all batch and file endpoints
-
-### Supported Endpoints
-
-All batch and file endpoints support model-based routing:
-
-| Endpoint | Method | Model Routing |
-|----------|--------|---------------|
-| `/v1/files` | POST | ✅ Via header/query/body |
-| `/v1/files/{file_id}` | GET | ✅ Auto from encoded ID + header/query |
-| `/v1/files/{file_id}/content` | GET | ✅ Auto from encoded ID + header/query |
-| `/v1/files/{file_id}` | DELETE | ✅ Auto from encoded ID |
-| `/v1/batches` | POST | ✅ Auto from file ID + header/query/body |
-| `/v1/batches` | GET | ✅ Via header/query |
-| `/v1/batches/{batch_id}` | GET | ✅ Auto from encoded ID |
-| `/v1/batches/{batch_id}/cancel` | POST | ✅ Auto from encoded ID |
-
-## **Supported Providers**:
-### [Azure OpenAI](./providers/azure#azure-batches-api)
-### [OpenAI](#quick-start)
-### [Vertex AI](./providers/vertex#batch-apis)
-### [Bedrock](./providers/bedrock_batches)
-### [vLLM](./providers/vllm_batches)
-
-
-## How Cost Tracking for Batches API Works
-
-LiteLLM tracks batch processing costs by logging two key events:
-
-| Event Type | Description | When it's Logged |
-|------------|-------------|------------------|
-| `acreate_batch` | Initial batch creation | When batch request is submitted |
-| `batch_success` | Final usage and cost | When batch processing completes |
-
-Cost calculation:
-
-- LiteLLM polls the batch status until completion
-- Upon completion, it aggregates usage and costs from all responses in the output file
-- Total `token` and `response_cost` reflect the combined metrics across all batch responses
-
-
-
-
-
-## [Swagger API Reference](https://litellm-api.up.railway.app/#/batch)